Esempio n. 1
0
def result_worker(ctx):
    g = ctx.obj
    from pyspider.result import ResultWorker
    result_worker = ResultWorker(resultdb=g.resultdb, inqueue=g.processor2result)
    g.instances.append(result_worker)

    result_worker.run()
Esempio n. 2
0
def result_worker(ctx, ResultWorker=ResultWorker):
    g = ctx.obj
    result_worker = ResultWorker(resultdb=g.resultdb, inqueue=g.processor2result)

    g.instances.append(result_worker)
    if g.get('testing_mode'):
        return result_worker

    result_worker.run()
Esempio n. 3
0
    def on_result(self, task, res):
        assert task['taskid']
        assert task['project']
        assert task['url']
        assert res
        logger.info("[*] on_result: %s"%task['url'])

        # your processing code goes here
        if isinstance(res, list):
            for result in res:
                log_dict = {'title': result['Title'], 'type': result['Type']}
                ResultWorker.on_result(self, task, log_dict)
                self.dump(result)
        elif isinstance(res, dict):
            log_dict = {'title': res['Title'], 'type': res['Type']}
            ResultWorker.on_result(self, task, log_dict)
            self.dump(res)
        else:
            raise Exception("Wrong result type")
Esempio n. 4
0
def run_result_worker(g=g):
    from pyspider.result import ResultWorker
    result_worker = ResultWorker(resultdb=g.resultdb,
                                 inqueue=g.processor2result)
    g.result_worker = result_worker
    result_worker.run()
Esempio n. 5
0
def run_result_worker(g=g):
    from pyspider.result import ResultWorker
    result_worker = ResultWorker(resultdb=g.resultdb, inqueue=g.processor2result)
    g.result_worker = result_worker
    result_worker.run()