def validUsefulProxy(params):
"""
检验代理是否可用
"""
marks = params.split('&&')
if isinstance(marks[1], bytes):
marks[1] = marks[1].decode('utf8')
proxies = {"http": "http://{proxy}".format(proxy=marks[1])}
flag = None
try:
# 超过20秒的代理就不要了
r = requests.get('http://httpbin.org/ip',
proxies=proxies,
timeout=10,
verify=False)
if r.status_code == 200 and r.json().get("origin"):
# logger.info('%s is ok' % proxy)
flag = True
except Exception as e:
flag = False
if not flag:
sqlite = Sqlite(address + 'ip.db')
sqlite.update_data('delete from ip_house where id = {}'.format(
marks[0]))
Log.d("删除无效代理:" + marks[1])
def validUsefulProxy(params):
"""
检验代理是否可用
"""
marks = params.split('&&')
if isinstance(marks[1], bytes):
marks[1] = marks[1].decode('utf8')
if 'HTTP' == marks[2]:
proxies = {"http": "http://{proxy}".format(proxy=marks[1])}
else:
proxies = {"https": "https://{proxy}".format(proxy=marks[1])}
flag = None
try:
# 超过3秒的代理就不要了
r = requests.get('http://httpbin.org/ip',
proxies=proxies,
timeout=3,
verify=False)
if r.status_code == 200 and r.json().get("origin"):
# print('请求IP:'+r.json().get("origin")+', 代理为:'+json.dumps(proxies))
Log.v('有效代理:' + marks[1])
flag = True
except Exception as e:
flag = False
if not flag:
sqlite = Sqlite(address + 'ip.db')
sqlite.update_data('delete from ip_house where id = {}'.format(
marks[0]))
Log.d("无效代理:" + marks[1])
def getAllProxy(pool_size=10, thread_or_process=True, is_refash=True):
Log.v('正在更新ip池,请稍后...')
# address = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + '/'
if is_refash:
proxys = GetFreeProxy.get_list_proxys()
proxys = list(set(proxys))
sqlite = Sqlite(address + 'ip.db')
sqlite.update_data('DELETE FROM ip_house')
sqlite = Sqlite(address + 'ip.db')
for i in range(len(proxys)):
if proxys[i] and GetFreeProxy.verifyProxyFormat(proxys[i]):
sqlite.cursor.execute(
"INSERT INTO ip_house VALUES (?,?,?,?);",
[i + 1, proxys[i], 'HTTP', 'true'])
sqlite.conn.commit()
sqlite.close_conn()
GetFreeProxy.get_proxys_by_website()
else:
sqlite = Sqlite(address + 'ip.db')
results = sqlite.query_data(
'select count(proxy_adress) from ip_house')
if int(results[0][0]) == 0:
proxys = GetFreeProxy.get_list_proxys()
proxys = list(set(proxys))
sqlite = Sqlite(address + 'ip.db')
for i in range(len(proxys)):
if proxys[i] and GetFreeProxy.verifyProxyFormat(proxys[i]):
sqlite.cursor.execute(
"INSERT INTO ip_house VALUES (?,?,?,?);",
[i + 1, proxys[i], 'HTTP', 'true'])
sqlite.conn.commit()
sqlite.close_conn()
GetFreeProxy.get_proxys_by_website()
sqlite = Sqlite(address + 'ip.db')
results = sqlite.query_data(
'select id,proxy_adress,type from ip_house')
params = []
for result in results:
param = str(result[0]) + '&&' + result[1] + '&&' + result[2]
params.append(param)
Log.v("发现ip代理数量:" + str(len(params)))
Log.v('正在检查ip可用性...')
if thread_or_process:
GetFreeProxy.exec_multi_threading(pool_size, params)
else:
GetFreeProxy.exec_multi_process(pool_size, params)
Log.v('更新完成')
