def _signal_kill(self, signum, _): install_shutdown_handlers(signal.SIG_IGN) signame = signal_names[signum] log.msg(format='Received %(signame)s twice, forcing unclean shutdown', level=log.INFO, signame=signame) reactor.callFromThread(self._stop_reactor)
def startService(self): log.msg("Start pooler ...") IApp = service.IServiceCollection(self.app, self.app) self.task_storage = IApp.getServiceNamed('task_storage') self.signal_manager = get_signal_manager(self.app) self.signal_manager.connect(self.update_tasks, signal=signals.tasks_updated) self.update_tasks()
def _signal_shutdown(self, signum, _): install_shutdown_handlers(self._signal_kill) signame = signal_names[signum] log.msg( format= "Received %(signame)s, shutting down gracefully. Send again to force ", level=log.INFO, signame=signame) reactor.callFromThread(self.stop)
def startService(self): log.msg("Start pooler ...") IApp = service.IServiceCollection(self.app, self.app) self.task_storage = IApp.getServiceNamed('task_storage') self.signal_manager = get_signal_manager(self.app) self.signal_manager.connect(self.update_tasks, signal=signals.tasks_updated) self.update_tasks()
def startService(self): app = IServiceCollection(self.app, self.app) self.poller = app.getServiceNamed('poller') self.fetcher = app.getServiceNamed('fetcher') self.amqp = app.getServiceNamed('amqp') self.task_storage = app.getServiceNamed('task_storage') self.check_storage_path() for slot in range(self.max_proc): self._wait_for_project(slot) log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r', version=__version__, max_proc=self.max_proc, system='Launcher')
def startService(self): app = IServiceCollection(self.app, self.app) self.poller = app.getServiceNamed('poller') self.fetcher = app.getServiceNamed('fetcher') self.amqp = app.getServiceNamed('amqp') self.task_storage = app.getServiceNamed('task_storage') self.check_storage_path() for slot in range(self.max_proc): self._wait_for_project(slot) log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r', version=__version__, max_proc=self.max_proc, system='Launcher')
def poll(self): if self.dq.pending or \ not self.task_storage or \ not self.task_storage.ready: return c = yield maybeDeferred(self.task_storage.count) if c: for task in self.tasks_list: if task['status'] != self.task_storage.TASK_STANDBY: continue task['status'] = self.task_storage.TASK_HOLD log.msg("add task to queue %s." % task['job_id']) self.put(task) break
def poll(self): if self.dq.pending or \ not self.task_storage or \ not self.task_storage.ready: return c = yield maybeDeferred(self.task_storage.count) if c: for task in self.tasks_list: if task['status'] != self.task_storage.TASK_STANDBY: continue task['status'] = self.task_storage.TASK_HOLD log.msg("add task to queue %s." % task['job_id']) self.put(task) break
def ready(self, connection): self._in_retry['connection'] = False self._in_retry['channel'] = False if not connection: raise ConnectionClosed self._conn = connection log.msg("AMQP Connection created") if self._channel: yield self.check_connection() self._channel = yield self._conn.channel() self.conn_retry_interval = 0 log.msg("Setting up exchange and queue") # Exchange yield self._channel.exchange_declare( exchange=self.exchange_name, type=self.settings.get("exchange_type", 'topic', section='amqp'), passive=True, ) # Queue self._queue_in = yield self._channel.queue_declare( queue=self.queue_in_name, auto_delete=False, exclusive=False, durable=True, ) self._queue_out = yield self._channel.queue_declare( queue=self.queue_out_name, auto_delete=False, exclusive=False, durable=True, ) # Queue-in > Exchange yield self._channel.queue_bind( exchange=self.exchange_name, queue=self.queue_in_name, routing_key=self.queue_in_routing_key) yield self._channel.queue_bind( exchange=self.exchange_name, queue=self.queue_out_name, routing_key=self.queue_out_routing_key) yield self._channel.basic_qos(prefetch_count=1) self._running = True log.msg("Start Consuming %s" % self.queue_in_name) log.msg("Start Producing %s" % self.queue_out_name) # Consume queue_in queue queue_obj, consumer_tag = yield self._channel.basic_consume(queue=self.queue_in_name, no_ack=False) l = task.LoopingCall(self.read, queue_obj) l.start(0.01)
def ready(self, connection): self._in_retry['connection'] = False self._in_retry['channel'] = False if not connection: raise ConnectionClosed self._conn = connection log.msg("AMQP Connection created") if self._channel: yield self.check_connection() self._channel = yield self._conn.channel() self.conn_retry_interval = 0 log.msg("Setting up exchange and queue") # Exchange yield self._channel.exchange_declare( exchange=self.exchange_name, type=self.settings.get("exchange_type", 'topic', section='amqp'), passive=True, ) # Queue self._queue_in = yield self._channel.queue_declare( queue=self.queue_in_name, auto_delete=False, exclusive=False, durable=True, ) self._queue_out = yield self._channel.queue_declare( queue=self.queue_out_name, auto_delete=False, exclusive=False, durable=True, ) # Queue-in > Exchange yield self._channel.queue_bind(exchange=self.exchange_name, queue=self.queue_in_name, routing_key=self.queue_in_routing_key) yield self._channel.queue_bind(exchange=self.exchange_name, queue=self.queue_out_name, routing_key=self.queue_out_routing_key) yield self._channel.basic_qos(prefetch_count=1) self._running = True log.msg("Start Consuming %s" % self.queue_in_name) log.msg("Start Producing %s" % self.queue_out_name) # Consume queue_in queue queue_obj, consumer_tag = yield self._channel.basic_consume( queue=self.queue_in_name, no_ack=False) l = task.LoopingCall(self.read, queue_obj) l.start(0.01)
def startService(self): log.msg("Starting Fetcher service ...")
def startService(self): log.msg("Starting Fetcher service ...")
def startService(self): log.msg("Start connecting to Database ...") self.start()
def _signal_kill(self, signum, _): install_shutdown_handlers(signal.SIG_IGN) signame = signal_names[signum] log.msg(format='Received %(signame)s twice, forcing unclean shutdown', level=log.INFO, signame=signame) reactor.callFromThread(self._stop_reactor)
def _signal_shutdown(self, signum, _): install_shutdown_handlers(self._signal_kill) signame = signal_names[signum] log.msg(format="Received %(signame)s, shutting down gracefully. Send again to force ", level=log.INFO, signame=signame) reactor.callFromThread(self.stop)
def startService(self): log.msg("Start connecting to Database ...") self.start()