def __init__(self): settings = Settings() settings.set('SPIDER_FEED_PARTITIONS', 1) settings.set('QUEUE_HOSTNAME_PARTITIONING', True) self.mb = MessageBus(settings) sl = self.mb.spider_log() # sw self.sw_sl_c = sl.consumer(partition_id=0, type='sw') us = self.mb.scoring_log() self.sw_us_p = us.producer() sleep(0.1) # db self.db_sl_c = sl.consumer(partition_id=None, type='db') self.db_us_c = us.consumer() sf = self.mb.spider_feed() self.db_sf_p = sf.producer() sleep(0.1) # spider self.sp_sl_p = sl.producer() self.sp_sf_c = sf.consumer(0) sleep(0.1)
def __init__(self): settings = Settings() settings.set('SPIDER_FEED_PARTITIONS', 1) settings.set('QUEUE_HOSTNAME_PARTITIONING', True) self.mb = MessageBus(settings) sl = self.mb.spider_log() # sw self.sw_sl_c = sl.consumer(partition_id=0, type='sw') us = self.mb.scoring_log() self.sw_us_p = us.producer() sleep(0.1) # db self.db_sl_c = sl.consumer(partition_id=None, type='db') self.db_us_c = us.consumer() sf = self.mb.spider_feed() self.db_sf_p = sf.producer() sleep(0.1) # spider self.sp_sl_p = sl.producer() self.sp_sf_c = sf.consumer(0) sleep(0.1)
help='Disables periodical generation of new batches.') parser.add_argument('--no-incoming', action='store_true', help='Disables periodical incoming topic consumption.') parser.add_argument( '--config', type=str, required=True, help='Settings module name, should be accessible by import.') parser.add_argument( '--log-level', '-L', type=str, default='INFO', help="Log level, for ex. DEBUG, INFO, WARN, ERROR, FATAL.") parser.add_argument('--port', type=int, help="Json Rpc service port to listen.") args = parser.parse_args() logger.setLevel(args.log_level) logger.addHandler(CONSOLE) settings = Settings(module=args.config) if args.port: settings.set("JSONRPC_PORT", [args.port]) worker = FrontierWorker(settings, args.no_batches, args.no_incoming) server = WorkerJsonRpcService(worker, settings) server.start_listening() worker.run()
self.stats.setdefault('batches_after_start', 0) self.stats['batches_after_start'] += 1 self.stats['last_batch_generated'] = asctime() return count if __name__ == '__main__': parser = ArgumentParser(description="Frontera DB worker.") parser.add_argument('--no-batches', action='store_true', help='Disables periodical generation of new batches.') parser.add_argument('--no-incoming', action='store_true', help='Disables periodical incoming topic consumption.') parser.add_argument('--config', type=str, required=True, help='Settings module name, should be accessible by import.') parser.add_argument('--log-level', '-L', type=str, default='INFO', help="Log level, for ex. DEBUG, INFO, WARN, ERROR, FATAL.") parser.add_argument('--port', type=int, help="Json Rpc service port to listen.") args = parser.parse_args() logger.setLevel(args.log_level) logger.addHandler(CONSOLE) settings = Settings(module=args.config) if args.port: settings.set("JSONRPC_PORT", [args.port]) worker = FrontierWorker(settings, args.no_batches, args.no_incoming) server = WorkerJsonRpcService(worker, settings) server.start_listening() worker.run()