def run(self): logger.debug("initializing spool") config = self.config_parser self.spool = spool = Spooler(config) num_gather_workers = None if config.has_option('core', 'gather_workers'): num_gather_workers = abs(config.getint('core', 'gather_workers')) if not num_gather_workers: num_gather_workers = GATHER_POOL_WORKERS self._gather_pool = pool = Pool(num_gather_workers) persist_queue = self.persist_queue self.collect_manager = CollectPlugins( base_class=Collect, config=config, init_args=(config, persist_queue, spool, pool), entry_points='arke_plugins', ) self.collect_manager.load(pool=self._gather_pool) try: self.persist_runner() except KeyboardInterrupt: pass self.shutdown()
def run(self): logging.debug("initializing spool") self.spool = spool = Spooler(self.config_parser) self._gather_pool = pool = Pool(GATHER_POOL_WORKERS) config = self.config_parser persist_queue = self.persist_queue self.collect_manager = CollectPlugins( base_class=Collect, config=config, init_args=(config, persist_queue, spool, pool), entry_points='arke_plugins', ) self.collect_manager.load(pool=self._gather_pool) try: self.persist_runner() except KeyboardInterrupt: pass self.shutdown()
def __init__(self, pool_count=10): super(Persist, self).__init__() self._pool_count = pool_count self.queue = Queue() self.spool = Spooler().register(self)