def __init__(self):
self.log = logging.getLogger("proxy.spider")
self.sched = BlockingScheduler()
self.client = RedisClient(host=REDIS['host'],
port=REDIS['port'],
db=REDIS['db'],
password=REDIS['password'],
max_conns=REDIS['max_conns'])
self._config_schedule()
def __init__(self, thread_id):
super(Validator, self).__init__()
# id从1开始
self.thread_id = thread_id
self.log = logging.getLogger('proxy.validator_{}'.format(thread_id))
self.client = RedisClient(host=REDIS['host'],
port=REDIS['port'],
db=REDIS['db'],
password=REDIS['password'],
max_conns=REDIS['max_conns'])
def __init__(self):
self.proxy_queue = Queue()
self.logger = logger
self.html_request = HtmlRequest()
self.db = RedisClient(config.NAME, config.HOST, config.PORT, config.PASSWORD)
def __init__(self):
self.logger = logger
self.db = RedisClient(config.NAME, config.HOST, config.PORT,
config.PASSWORD)
self.html_request = HtmlRequest()
self.html_parser = HtmlParser()
# -*- coding: utf-8 -*-
__author__ = 'ada'
# Created by ada on 13/10/2017
from flask import Flask
from config import REDIS
from db.RedisClient import RedisClient
app = Flask(__name__)
client = RedisClient(host=REDIS['host'],
port=REDIS['port'],
db=REDIS['db'],
password=REDIS['password'],
max_conns=REDIS['max_conns'])
from .ProxyApi import *
def __init__(self):
self.redis = RedisClient()
self.crawler = Crawler()
def get_conn():
if not hasattr(g, 'redis'):
g.redis = RedisClient()
return g.redis
def __init__(self):
self.db = RedisClient(config.NAME, config.HOST, config.PORT, config.PASSWORD)