def on_result(self, message):
self.debug("got message: %s" % message)
try:
name = message.get('mode', HANDLER_MODE_DEFAULT)
handler = load_handler(name, self.ctx, None)
self.debug("Spider loaded handler: %s" % handler)
handler.newtask(message)
del handler
except Exception as e:
self.error(e)
def __init__(self, context, task, no_sync=False):
logger = logging.getLogger('handler')
log_level = logging.WARN
if context.obj.get('debug', False):
log_level = logging.DEBUG
super(Loader, self).__init__(logger, log_level)
mode = task.get('mode', HANDLER_MODE_DEFAULT)
self.params = {"context": context, "task": task, "no_sync": no_sync}
try:
self.handler = load_handler(mode, **self.params)
except ImportError:
raise CDSpiderHandlerError("handler not exists")
def schedule(self, message = None):
self.info("%s route starting..." % self.__class__.__name__)
def handler_schedule(handler, mode, frequency):
self.info("%s loaded handler: %s by %s" % (self.__class__.__name__, handler, frequency))
save = {}
while True:
has_item = False
for item in handler.route(mode, frequency, save):
self.info("%s got route parameter: %s", self.__class__.__name__, json.dumps(item))
if item:
has_item = True
message = {
"frequency": frequency,
"mode": mode,
**item
}
self.debug("%s route message: %s" % (self.__class__.__name__, str(message)))
if not self.testing_mode:
self.outqueue.put_nowait(message)
if not has_item:
break
time.sleep(0.1)
threads = []
frequencymap = self.frequency if self.frequency else self.ctx.obj.get('app_config', {}).get('frequencymap', {}).keys()
if self.mode:
for key in self.mode:
handler = load_handler(key, context=self.ctx, task=None)
for frequency in frequencymap:
threads.append(run_in_thread(handler_schedule, handler, key, frequency))
else:
def execut(ext, data):
for frequency in data["frequencymap"]:
threads.append(run_in_thread(handler_schedule, ext.obj, ext.name, frequency))
call_extension("handler", execut, {"ctx": self.ctx, "frequencymap": frequencymap}, context=self.ctx, task=None)
for each in threads:
if not each.is_alive():
continue
if hasattr(each, 'terminate'):
each.terminate()
each.join()
self.info("%s route end, %s threads was run" % (self.__class__.__name__, len(threads)))
def schedule(self, message):
self.debug("%s schedule got message: %s" % (self.__class__.__name__, str(message)))
if 'mode' not in message or not message['mode']:
raise CDSpiderError("%s handler mode is missing" % self.__class__.__name__)
self.info("%s schedule starting..." % self.__class__.__name__)
handler_mode = message['mode']
handler = load_handler(handler_mode, context=self.ctx, task=None)
self.info("%s loaded handler: %s" % (self.__class__.__name__, handler))
save = {"now": int(time.time())}
while True:
has_item = False
for item in handler.schedule(message, save):
if item:
item['mode'] = handler_mode
self.debug("%s schedule task: %s" % (self.__class__.__name__, str(item)))
if not self.testing_mode:
self.send_task(item)
has_item = True
if not has_item or 'count' in message:
break
time.sleep(0.1)
del handler
self.info("%s schedule end" % self.__class__.__name__)
def status(self, message):
name = message.get('mode', HANDLER_MODE_DEFAULT)
handler = load_handler(name, self.ctx, None)
self.info("Spider loaded handler: %s" % handler)
handler.status(message)
del handler