def ready(self): #self.http = HTTPClient() #self.http.set_proxy(settings.PROXY) self.logger = logging.getLogger("webrobot") self._web_clients = {} self._load_actions() self.robot_api = Robot() #thread.start_new_thread(self.robot_task_tacker, ()) self.task_pool = None if settings.WORK_THREAD_COUNT > 1: self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
def ready(self): # self.http = HTTPClient() # self.http.set_proxy(settings.PROXY) self.logger = logging.getLogger("webrobot") self._web_clients = {} self._load_actions() self.robot_api = Robot() # thread.start_new_thread(self.robot_task_tacker, ()) self.task_pool = None if settings.WORK_THREAD_COUNT > 1: self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
class Webrobot(Sailor): def ready(self): # self.http = HTTPClient() # self.http.set_proxy(settings.PROXY) self.logger = logging.getLogger("webrobot") self._web_clients = {} self._load_actions() self.robot_api = Robot() # thread.start_new_thread(self.robot_task_tacker, ()) self.task_pool = None if settings.WORK_THREAD_COUNT > 1: self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger) def _load_actions(self,): self.actions = settings.ACTIONS for k, v in self.actions.iteritems(): self.actions[k] = self._init_actions(*v) def _init_actions(self, cls, param={}): cls = import_class(cls) if cls is None: raise RuntimeError, "Could not loading crawler '%s'." % cls return cls(**param) def start(self, t): client_id = t.header("Client_id") for l in t.list_actions(): client_id, action_id, action, args = self._parse_action(l, None) if self.task_pool is None: self._process_task(client_id, action_id, action, args) else: self.task_pool.add_task(self._process_task, (client_id, action_id, action, args), client_id) if self.task_pool is not None: self.logger.info("waiting all task done") while self.task_pool.pending_count() > 0: import time time.sleep(3) self.logger.info("all task done") def _process_task(self, client_id, action_id, action, args): self.logger.info("start to process task for id:%s" % client_id) client = self._web_clients.get(client_id, None) if client is None: client = WebClient(client_id, "", "clients") client.start_client() self._web_clients[client_id] = client try: handler = self.actions.get(action, None) self.logger.debug("[%s] %s, args:%s" % (client_id, action, str(args))) if handler is not None: result = handler(client, self.robot_api, **args) self.action_done(action_id, client, result, self.robot_api) else: self.logger.error("not found sipder action:'%s'", action) except Exception, e: self.logger.exception(e)
class Webrobot(Sailor): def ready(self): #self.http = HTTPClient() #self.http.set_proxy(settings.PROXY) self.logger = logging.getLogger("webrobot") self._web_clients = {} self._load_actions() self.robot_api = Robot() #thread.start_new_thread(self.robot_task_tacker, ()) self.task_pool = None if settings.WORK_THREAD_COUNT > 1: self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger) def _load_actions(self, ): self.actions = settings.ACTIONS for k, v in self.actions.iteritems(): self.actions[k] = self._init_actions(*v) def _init_actions(self, cls, param={}): cls = import_class(cls) if cls is None: raise RuntimeError, "Could not loading crawler '%s'." % cls return cls(**param) def start(self, t): client_id = t.header('Client_id') for l in t.list_actions(): client_id, action_id, action, args = self._parse_action(l, None) if self.task_pool is None: self._process_task(client_id, action_id, action, args) else: self.task_pool.add_task(self._process_task, (client_id, action_id, action, args), client_id) if self.task_pool is not None: self.logger.info("waiting all task done") while self.task_pool.pending_count() > 0: import time time.sleep(3) self.logger.info("all task done") def _process_task(self, client_id, action_id, action, args): self.logger.info("start to process task for id:%s" % client_id) client = self._web_clients.get(client_id, None) if client is None: client = WebClient(client_id, "", "clients") client.start_client() self._web_clients[client_id] = client try: handler = self.actions.get(action, None) self.logger.debug("[%s] %s, args:%s" % (client_id, action, str(args))) if handler is not None: result = handler(client, self.robot_api, **args) self.action_done(action_id, client, result, self.robot_api) else: self.logger.error("not found sipder action:'%s'", action) except Exception, e: self.logger.exception(e)