def request_url_by_requests(url, proxies): """ Request a url by requests :param url: :param proxies: :return: """ headers = {'User-Agent': get_random_user_agent()} try: res = requests.get(url, headers=headers, timeout=CONFIG.TEST_URL['timeout'], proxies=proxies) res.raise_for_status() except requests.exceptions.ConnectTimeout as e: logger.error(type='Request timeout!', message=proxies.get('http', ) + ' Request ' + url + ' failed') res = None except Exception as e: logger.error(type='Request failed!', message=proxies.get('http', ) + ' Request ' + url + ' failed') res = None return res
async def valid_proxy(proxy, nums=1): if nums > 3: await db_client.delete(proxy) logger.error(type='无效代理', message="{0} 已丢弃".format(proxy)) else: ip, port = proxy.split(':') isOk = await get_proxy_info(ip, port) if not isOk: logger.error(type='无效代理', message="{0}:第 {1} 次重试".format(proxy, nums)) await valid_proxy(proxy, nums=nums + 1) else: logger.info(type='有效代理', message="{0} 有效".format(proxy))
async def valid_proxy(proxy, nums=1): if nums > 5: await db_client.delete(proxy) logger.error(type='Invalid proxy', message="{0} had been abandoned".format(proxy)) return False else: ip, port = proxy.split(':') is_ok = await get_proxy_info(ip, port) if not is_ok: logger.error(type='Invalid proxy', message="{0}:retry times = {1}".format(proxy, nums)) res = await valid_proxy(proxy, nums=nums + 1) return res else: logger.info(type='Valid proxy', message="{0} is valid".format(proxy)) return True