def validUsefulProxy(params): """ 检验代理是否可用 """ marks = params.split('&&') if isinstance(marks[1], bytes): marks[1] = marks[1].decode('utf8') proxies = {"http": "http://{proxy}".format(proxy=marks[1])} flag = None try: # 超过20秒的代理就不要了 r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=10, verify=False) if r.status_code == 200 and r.json().get("origin"): # logger.info('%s is ok' % proxy) flag = True except Exception as e: flag = False if not flag: sqlite = Sqlite(address + 'ip.db') sqlite.update_data('delete from ip_house where id = {}'.format( marks[0])) Log.d("删除无效代理:" + marks[1])
def validUsefulProxy(params): """ 检验代理是否可用 """ marks = params.split('&&') if isinstance(marks[1], bytes): marks[1] = marks[1].decode('utf8') if 'HTTP' == marks[2]: proxies = {"http": "http://{proxy}".format(proxy=marks[1])} else: proxies = {"https": "https://{proxy}".format(proxy=marks[1])} flag = None try: # 超过3秒的代理就不要了 r = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=3, verify=False) if r.status_code == 200 and r.json().get("origin"): # print('请求IP:'+r.json().get("origin")+', 代理为:'+json.dumps(proxies)) Log.v('有效代理:' + marks[1]) flag = True except Exception as e: flag = False if not flag: sqlite = Sqlite(address + 'ip.db') sqlite.update_data('delete from ip_house where id = {}'.format( marks[0])) Log.d("无效代理:" + marks[1])
def getAllProxy(pool_size=10, thread_or_process=True, is_refash=True): Log.v('正在更新ip池,请稍后...') # address = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + '/' if is_refash: proxys = GetFreeProxy.get_list_proxys() proxys = list(set(proxys)) sqlite = Sqlite(address + 'ip.db') sqlite.update_data('DELETE FROM ip_house') sqlite = Sqlite(address + 'ip.db') for i in range(len(proxys)): if proxys[i] and GetFreeProxy.verifyProxyFormat(proxys[i]): sqlite.cursor.execute( "INSERT INTO ip_house VALUES (?,?,?,?);", [i + 1, proxys[i], 'HTTP', 'true']) sqlite.conn.commit() sqlite.close_conn() GetFreeProxy.get_proxys_by_website() else: sqlite = Sqlite(address + 'ip.db') results = sqlite.query_data( 'select count(proxy_adress) from ip_house') if int(results[0][0]) == 0: proxys = GetFreeProxy.get_list_proxys() proxys = list(set(proxys)) sqlite = Sqlite(address + 'ip.db') for i in range(len(proxys)): if proxys[i] and GetFreeProxy.verifyProxyFormat(proxys[i]): sqlite.cursor.execute( "INSERT INTO ip_house VALUES (?,?,?,?);", [i + 1, proxys[i], 'HTTP', 'true']) sqlite.conn.commit() sqlite.close_conn() GetFreeProxy.get_proxys_by_website() sqlite = Sqlite(address + 'ip.db') results = sqlite.query_data( 'select id,proxy_adress,type from ip_house') params = [] for result in results: param = str(result[0]) + '&&' + result[1] + '&&' + result[2] params.append(param) Log.v("发现ip代理数量:" + str(len(params))) Log.v('正在检查ip可用性...') if thread_or_process: GetFreeProxy.exec_multi_threading(pool_size, params) else: GetFreeProxy.exec_multi_process(pool_size, params) Log.v('更新完成')