def __init__(self): """ init db and crawlers """ self.redis = RedisClient() self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
def __init__(self): """ 初始化 db 和 爬虫 """ self.redis = RedisClient() self.crawlers_cls = crawlers_cls self.crawlers = [crawler_cls() for crawler_cls in self.crawlers_cls]
def get_conn(): """ get redis client object :return: """ if not hasattr(g, 'redis'): g.redis = RedisClient() return g.redis
def __init__(self): """ init redis """ self.redis = RedisClient() self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko' ') Chrome/83.0.4103.97 Safari/537.36' }
def checking(proxy): ocj = RedisClient() proxies = { 'http': 'http://{}'.format(proxy), 'https': 'https://{}'.format(proxy) } try: res_proxy = requests.get(TEST_URL, headers=TEST_HEADERS, proxies=proxies, verify=False, timeout=100) if res_proxy.status_code == 200: return proxy else: ocj.decrease(proxy) return None except ConnectionError and Timeout and HTTPError: ocj.decrease(proxy) return None
def __init__(self): """ init redis """ self.redis = RedisClient() self.loop = asyncio.get_event_loop()
def clien(): client = RedisClient() return client.random()