class Core(Request, threading.Thread): def __init__(self, queue=None): Request.__init__(self) threading.Thread.__init__(self) self.logger = Logger().logger self.config = Config().config self.proxy_pool = ProxyPool() self.__queue = queue self.db = DB().db self.proxy_ip, self.proxy_port = None, None self.proxy() def proxy(self): if self.config.get('PROXY', 'use') == 'TRUE': if not self.proxy_ip: self.proxy_ip, self.proxy_port = self.proxy_pool.get_proxy( type=self.config.get('PROXY', 'type'), seed_num=int(self.config.get('PROXY', 'seed_num')), distinct=self.config.get('PROXY', 'distinct') == 'TRUE' ) else: self.proxy_pool.delete_proxy(ip=self.proxy_ip, type=self.proxy_type) self.proxy_ip, self.proxy_port = self.proxy_pool.get_proxy( type=self.config.get('PROXY', 'type'), seed_num=int(self.config.get('PROXY', 'seed_num')), distinct=self.config.get('PROXY', 'distinct') == 'TRUE' ) else: self.proxy_ip, self.proxy_port = None, None @use_proxy @load_params def get(self, url, params=None, **kwargs): return super().get(url, params, **kwargs) @use_proxy @load_params def post(self, url, data, **kwargs): return super().post(url, data, **kwargs) def task(self): pass def run(self): self.logger.info('go') try: self.task() self.logger.info('ok') except Exception as e: self.logger.info('something wrong') finally: if self.__queue: assert isinstance(self.__queue, queue.Queue) self.__queue.task_done()
class Brisk(Request): def __init__(self): super(Brisk, self).__init__() self.config = Config().config self.logger = Logger().logger self.proxy_manager = ProxyPool() self.db = DB().db self.__proxy_status = self.config.get('PROXY', 'empty') if self.__proxy_status == 'TRUE': self.proxy_manager.delete_valid_proxy() self.__hook_name = 'hook' self.__walk_name = 'walk' self.__flow_name = 'flow' self.__brisk_type = self.config.get('RUN', 'type') self.__func_filter = lambda m: not m.startswith("__") and \ not m.startswith(self.__hook_name) and \ not m.startswith(self.__walk_name) and \ not m.startswith(self.__flow_name) self.__flow_num = int(self.config.get('RUN', 'num')) self.__hook = None self.__flow_queue = queue.Queue() self.__walk_queue = queue.Queue() self.__go_init() def __go_init(self): for method_name in list( filter(lambda m: m.startswith(self.__hook_name) and callable(getattr(self, m)), dir(self))): method = self.__class__.__dict__[method_name] obj = Core() obj.task = types.MethodType(method, obj) for func_name in filter(self.__func_filter, self.__class__.__dict__): func = self.__class__.__dict__[func_name] setattr(obj, func_name, types.MethodType(func, obj)) self.__hook = obj break if self.__brisk_type == 'WALK': for method_name in list( filter(lambda m: m.startswith(self.__walk_name) and callable(getattr(self, m)), dir(self))): self.__walk_queue.put(method_name) if self.__brisk_type == 'FLOW': for method_name in list( filter(lambda m: m.startswith(self.__flow_name) and callable(getattr(self, m)), dir(self))): self.__flow_queue.put(method_name) def go(self): self.logger.info('brisk go') self.logger.info('brisk create {} task(s)'.format(self.__flow_queue.qsize())) if self.__hook: self.__hook_attr_base = dir(self.__hook) self.logger.info('brisk create hook') self.__hook.start() self.__hook.join() self.logger.info('brisk complete hook') self.__hook: Core self.__hook_attr = [] for method_name in dir(self.__hook): if method_name not in self.__hook_attr_base: self.__hook_attr.append(method_name) while not self.__walk_queue.empty(): method_name = self.__walk_queue.get() method = self.__class__.__dict__[method_name] t = Core(self.__walk_queue) for attr_name in self.__hook_attr: setattr(t, attr_name, self.__hook.__dict__[attr_name]) t.task = types.MethodType(method, t) for func_name in filter(self.__func_filter, self.__class__.__dict__): func = self.__class__.__dict__[func_name] setattr(t, func_name, types.MethodType(func, t)) if self.__hook: t.make(self.__hook.headers, self.__hook.cookies) t.start() t.join() self.__walk_queue.join() while not self.__flow_queue.empty(): if (threading.activeCount() - 1) < self.__flow_num: method_name = self.__flow_queue.get() method = self.__class__.__dict__[method_name] t = Core(self.__flow_queue) for attr_name in self.__hook_attr: setattr(t, attr_name, self.__hook.__dict__[attr_name]) t.task = types.MethodType(method, t) for func_name in filter(self.__func_filter, self.__class__.__dict__): func = self.__class__.__dict__[func_name] setattr(t, func_name, types.MethodType(func, t)) if self.__hook: t.make(self.__hook.headers, self.__hook.cookies) t.start() self.__flow_queue.join() self.logger.info('brisk ok')