def store_proxies(proxies): conn = RedisClient(name='certificate_proxies') now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) conn_check = connect_check(proxies) if conn_check: proxies = json.dumps(proxies) duplicate_check = conn.exist(proxies) if not duplicate_check: conn.set(proxies, 1) info_logger.info(str(now) + ' New proxies: ' + str(proxies)) print(str(now) + ' New proxies: ' + str(proxies)) else: info_logger.info( str(now) + ' Already exist proxies: ' + str(proxies)) else: err_logger.error( str(now) + ' Can not connect baidu.com -- proxies: ' + str(proxies))
def get_uniform_proxy(name, url): proxy_url = 'http://166.188.20.55:3000/api/proxy/web' headers = { 'Content-Type': 'application/json', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/' '537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' } data = {'name': name, 'url': url} while True: resp = requests.post(url=proxy_url, headers=headers, data=json.dumps(data)).json() region = resp['proxy']['region'] if region == 'R1' or region == 'R2': # print(region, 'err') continue else: conn = RedisClient(name='certificate_proxies') ip = resp['proxy']['ip'] port = '30000' proxies = { 'http': 'http://%s:%s' % (ip, port), 'https': 'http://%s:%s' % (ip, port), } ping_url = 'http://www.baidu.com' status_code = requests.get(ping_url, proxies=proxies, timeout=(3.1, 15)).status_code if status_code == 200: p = json.dumps(proxies) check = conn.exist(p) if not check: conn.set(p, 1) # conn.lpush(p) print('New proxies: ', proxies) break else: print('already exist proxies: ', p) else: print('connection error...')
def download_proxies(): # url = 'http://www.xdaili.cn/ipagent/privateProxy/applyStaticProxy?' \ # 'count=1&spiderId=fd4708592c97444c9f42060c500649ac&returnType=2' url = 'http://www.xdaili.cn/ipagent/privateProxy/applyStaticProxy' \ '?count=1&spiderId=8407fdf0311d4eb7b30f9e39699795e5&returnType=2' second = datetime.datetime.now().second if second == 0 or second == 24 or second == 48: #print(second) time.sleep(1) conn = RedisClient(name='certificate_proxies') content = requests.get(url, timeout=(3.1, 15)).json() result = content['RESULT'] if result == '提取太频繁,请按规定频率提取!': err_logger.error(result) else: for proxy in content['RESULT']: ip = proxy['ip'] port = proxy['port'] proxies = { 'http': 'http://%s:%s' % (ip, port), 'https': 'http://%s:%s' % (ip, port), } ping_url = 'http://www.baidu.com' status_code = requests.get(ping_url, timeout=(3.1, 15), proxies=proxies).status_code if status_code == 200: p = json.dumps(proxies) now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) check = conn.exist(p) if not check: conn.set(p, 1) conn.lpush(p) now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) print(now, ' New proxies: ', p) else: print(now, ' already exist proxies: ', p)