def ready(self):
#self.http = HTTPClient()
#self.http.set_proxy(settings.PROXY)
self.logger = logging.getLogger("webrobot")
self._web_clients = {}
self._load_actions()
self.robot_api = Robot()
#thread.start_new_thread(self.robot_task_tacker, ())
self.task_pool = None
if settings.WORK_THREAD_COUNT > 1:
self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
def ready(self):
# self.http = HTTPClient()
# self.http.set_proxy(settings.PROXY)
self.logger = logging.getLogger("webrobot")
self._web_clients = {}
self._load_actions()
self.robot_api = Robot()
# thread.start_new_thread(self.robot_task_tacker, ())
self.task_pool = None
if settings.WORK_THREAD_COUNT > 1:
self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
class Webrobot(Sailor):
def ready(self):
# self.http = HTTPClient()
# self.http.set_proxy(settings.PROXY)
self.logger = logging.getLogger("webrobot")
self._web_clients = {}
self._load_actions()
self.robot_api = Robot()
# thread.start_new_thread(self.robot_task_tacker, ())
self.task_pool = None
if settings.WORK_THREAD_COUNT > 1:
self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT, self.logger)
def _load_actions(self,):
self.actions = settings.ACTIONS
for k, v in self.actions.iteritems():
self.actions[k] = self._init_actions(*v)
def _init_actions(self, cls, param={}):
cls = import_class(cls)
if cls is None:
raise RuntimeError, "Could not loading crawler '%s'." % cls
return cls(**param)
def start(self, t):
client_id = t.header("Client_id")
for l in t.list_actions():
client_id, action_id, action, args = self._parse_action(l, None)
if self.task_pool is None:
self._process_task(client_id, action_id, action, args)
else:
self.task_pool.add_task(self._process_task, (client_id, action_id, action, args), client_id)
if self.task_pool is not None:
self.logger.info("waiting all task done")
while self.task_pool.pending_count() > 0:
import time
time.sleep(3)
self.logger.info("all task done")
def _process_task(self, client_id, action_id, action, args):
self.logger.info("start to process task for id:%s" % client_id)
client = self._web_clients.get(client_id, None)
if client is None:
client = WebClient(client_id, "", "clients")
client.start_client()
self._web_clients[client_id] = client
try:
handler = self.actions.get(action, None)
self.logger.debug("[%s] %s, args:%s" % (client_id, action, str(args)))
if handler is not None:
result = handler(client, self.robot_api, **args)
self.action_done(action_id, client, result, self.robot_api)
else:
self.logger.error("not found sipder action:'%s'", action)
except Exception, e:
self.logger.exception(e)
class Webrobot(Sailor):
def ready(self):
#self.http = HTTPClient()
#self.http.set_proxy(settings.PROXY)
self.logger = logging.getLogger("webrobot")
self._web_clients = {}
self._load_actions()
self.robot_api = Robot()
#thread.start_new_thread(self.robot_task_tacker, ())
self.task_pool = None
if settings.WORK_THREAD_COUNT > 1:
self.task_pool = TaskScheduler(settings.WORK_THREAD_COUNT,
self.logger)
def _load_actions(self, ):
self.actions = settings.ACTIONS
for k, v in self.actions.iteritems():
self.actions[k] = self._init_actions(*v)
def _init_actions(self, cls, param={}):
cls = import_class(cls)
if cls is None:
raise RuntimeError, "Could not loading crawler '%s'." % cls
return cls(**param)
def start(self, t):
client_id = t.header('Client_id')
for l in t.list_actions():
client_id, action_id, action, args = self._parse_action(l, None)
if self.task_pool is None:
self._process_task(client_id, action_id, action, args)
else:
self.task_pool.add_task(self._process_task,
(client_id, action_id, action, args),
client_id)
if self.task_pool is not None:
self.logger.info("waiting all task done")
while self.task_pool.pending_count() > 0:
import time
time.sleep(3)
self.logger.info("all task done")
def _process_task(self, client_id, action_id, action, args):
self.logger.info("start to process task for id:%s" % client_id)
client = self._web_clients.get(client_id, None)
if client is None:
client = WebClient(client_id, "", "clients")
client.start_client()
self._web_clients[client_id] = client
try:
handler = self.actions.get(action, None)
self.logger.debug("[%s] %s, args:%s" %
(client_id, action, str(args)))
if handler is not None:
result = handler(client, self.robot_api, **args)
self.action_done(action_id, client, result, self.robot_api)
else:
self.logger.error("not found sipder action:'%s'", action)
except Exception, e:
self.logger.exception(e)
