def result_worker(ctx):
g = ctx.obj
from pyspider.result import ResultWorker
result_worker = ResultWorker(resultdb=g.resultdb, inqueue=g.processor2result)
g.instances.append(result_worker)
result_worker.run()
def result_worker(ctx, ResultWorker=ResultWorker):
g = ctx.obj
result_worker = ResultWorker(resultdb=g.resultdb, inqueue=g.processor2result)
g.instances.append(result_worker)
if g.get('testing_mode'):
return result_worker
result_worker.run()
def on_result(self, task, res):
assert task['taskid']
assert task['project']
assert task['url']
assert res
logger.info("[*] on_result: %s"%task['url'])
# your processing code goes here
if isinstance(res, list):
for result in res:
log_dict = {'title': result['Title'], 'type': result['Type']}
ResultWorker.on_result(self, task, log_dict)
self.dump(result)
elif isinstance(res, dict):
log_dict = {'title': res['Title'], 'type': res['Type']}
ResultWorker.on_result(self, task, log_dict)
self.dump(res)
else:
raise Exception("Wrong result type")
def run_result_worker(g=g):
from pyspider.result import ResultWorker
result_worker = ResultWorker(resultdb=g.resultdb,
inqueue=g.processor2result)
g.result_worker = result_worker
result_worker.run()
def run_result_worker(g=g):
from pyspider.result import ResultWorker
result_worker = ResultWorker(resultdb=g.resultdb, inqueue=g.processor2result)
g.result_worker = result_worker
result_worker.run()