def __init__(self):
"""
init db and crawlers
"""
self.redis = RedisClient()
self.crawlers_cls = crawlers_cls
self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
def __init__(self):
"""
初始化 db 和 爬虫
"""
self.redis = RedisClient()
self.crawlers_cls = crawlers_cls
self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
def get_conn():
"""
get redis client object
:return:
"""
if not hasattr(g, 'redis'):
g.redis = RedisClient()
return g.redis
def __init__(self):
"""
init redis
"""
self.redis = RedisClient()
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko'
') Chrome/83.0.4103.97 Safari/537.36'
}
def checking(proxy):
ocj = RedisClient()
proxies = {
'http': 'http://{}'.format(proxy),
'https': 'https://{}'.format(proxy)
}
try:
res_proxy = requests.get(TEST_URL,
headers=TEST_HEADERS,
proxies=proxies,
verify=False,
timeout=100)
if res_proxy.status_code == 200:
return proxy
else:
ocj.decrease(proxy)
return None
except ConnectionError and Timeout and HTTPError:
ocj.decrease(proxy)
return None
def __init__(self):
"""
init redis
"""
self.redis = RedisClient()
self.loop = asyncio.get_event_loop()
def clien():
client = RedisClient()
return client.random()