def _signal_kill(self, signum, _):
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
level=log.INFO,
signame=signame)
reactor.callFromThread(self._stop_reactor)
def startService(self):
log.msg("Start pooler ...")
IApp = service.IServiceCollection(self.app, self.app)
self.task_storage = IApp.getServiceNamed('task_storage')
self.signal_manager = get_signal_manager(self.app)
self.signal_manager.connect(self.update_tasks, signal=signals.tasks_updated)
self.update_tasks()
def _signal_shutdown(self, signum, _):
install_shutdown_handlers(self._signal_kill)
signame = signal_names[signum]
log.msg(
format=
"Received %(signame)s, shutting down gracefully. Send again to force ",
level=log.INFO,
signame=signame)
reactor.callFromThread(self.stop)
def startService(self):
log.msg("Start pooler ...")
IApp = service.IServiceCollection(self.app, self.app)
self.task_storage = IApp.getServiceNamed('task_storage')
self.signal_manager = get_signal_manager(self.app)
self.signal_manager.connect(self.update_tasks,
signal=signals.tasks_updated)
self.update_tasks()
def startService(self):
app = IServiceCollection(self.app, self.app)
self.poller = app.getServiceNamed('poller')
self.fetcher = app.getServiceNamed('fetcher')
self.amqp = app.getServiceNamed('amqp')
self.task_storage = app.getServiceNamed('task_storage')
self.check_storage_path()
for slot in range(self.max_proc):
self._wait_for_project(slot)
log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r',
version=__version__, max_proc=self.max_proc, system='Launcher')
def startService(self):
app = IServiceCollection(self.app, self.app)
self.poller = app.getServiceNamed('poller')
self.fetcher = app.getServiceNamed('fetcher')
self.amqp = app.getServiceNamed('amqp')
self.task_storage = app.getServiceNamed('task_storage')
self.check_storage_path()
for slot in range(self.max_proc):
self._wait_for_project(slot)
log.msg(format='Flowder %(version)s started: max_proc=%(max_proc)r',
version=__version__,
max_proc=self.max_proc,
system='Launcher')
def poll(self):
if self.dq.pending or \
not self.task_storage or \
not self.task_storage.ready:
return
c = yield maybeDeferred(self.task_storage.count)
if c:
for task in self.tasks_list:
if task['status'] != self.task_storage.TASK_STANDBY:
continue
task['status'] = self.task_storage.TASK_HOLD
log.msg("add task to queue %s." % task['job_id'])
self.put(task)
break
def poll(self):
if self.dq.pending or \
not self.task_storage or \
not self.task_storage.ready:
return
c = yield maybeDeferred(self.task_storage.count)
if c:
for task in self.tasks_list:
if task['status'] != self.task_storage.TASK_STANDBY:
continue
task['status'] = self.task_storage.TASK_HOLD
log.msg("add task to queue %s." % task['job_id'])
self.put(task)
break
def ready(self, connection):
self._in_retry['connection'] = False
self._in_retry['channel'] = False
if not connection:
raise ConnectionClosed
self._conn = connection
log.msg("AMQP Connection created")
if self._channel:
yield self.check_connection()
self._channel = yield self._conn.channel()
self.conn_retry_interval = 0
log.msg("Setting up exchange and queue")
# Exchange
yield self._channel.exchange_declare(
exchange=self.exchange_name,
type=self.settings.get("exchange_type", 'topic', section='amqp'),
passive=True,
)
# Queue
self._queue_in = yield self._channel.queue_declare(
queue=self.queue_in_name,
auto_delete=False,
exclusive=False,
durable=True,
)
self._queue_out = yield self._channel.queue_declare(
queue=self.queue_out_name,
auto_delete=False,
exclusive=False,
durable=True,
)
# Queue-in > Exchange
yield self._channel.queue_bind(
exchange=self.exchange_name,
queue=self.queue_in_name,
routing_key=self.queue_in_routing_key)
yield self._channel.queue_bind(
exchange=self.exchange_name,
queue=self.queue_out_name,
routing_key=self.queue_out_routing_key)
yield self._channel.basic_qos(prefetch_count=1)
self._running = True
log.msg("Start Consuming %s" % self.queue_in_name)
log.msg("Start Producing %s" % self.queue_out_name)
# Consume queue_in queue
queue_obj, consumer_tag = yield self._channel.basic_consume(queue=self.queue_in_name, no_ack=False)
l = task.LoopingCall(self.read, queue_obj)
l.start(0.01)
def ready(self, connection):
self._in_retry['connection'] = False
self._in_retry['channel'] = False
if not connection:
raise ConnectionClosed
self._conn = connection
log.msg("AMQP Connection created")
if self._channel:
yield self.check_connection()
self._channel = yield self._conn.channel()
self.conn_retry_interval = 0
log.msg("Setting up exchange and queue")
# Exchange
yield self._channel.exchange_declare(
exchange=self.exchange_name,
type=self.settings.get("exchange_type", 'topic', section='amqp'),
passive=True,
)
# Queue
self._queue_in = yield self._channel.queue_declare(
queue=self.queue_in_name,
auto_delete=False,
exclusive=False,
durable=True,
)
self._queue_out = yield self._channel.queue_declare(
queue=self.queue_out_name,
auto_delete=False,
exclusive=False,
durable=True,
)
# Queue-in > Exchange
yield self._channel.queue_bind(exchange=self.exchange_name,
queue=self.queue_in_name,
routing_key=self.queue_in_routing_key)
yield self._channel.queue_bind(exchange=self.exchange_name,
queue=self.queue_out_name,
routing_key=self.queue_out_routing_key)
yield self._channel.basic_qos(prefetch_count=1)
self._running = True
log.msg("Start Consuming %s" % self.queue_in_name)
log.msg("Start Producing %s" % self.queue_out_name)
# Consume queue_in queue
queue_obj, consumer_tag = yield self._channel.basic_consume(
queue=self.queue_in_name, no_ack=False)
l = task.LoopingCall(self.read, queue_obj)
l.start(0.01)
def startService(self):
log.msg("Starting Fetcher service ...")
def startService(self):
log.msg("Starting Fetcher service ...")
def startService(self):
log.msg("Start connecting to Database ...")
self.start()
def _signal_kill(self, signum, _):
install_shutdown_handlers(signal.SIG_IGN)
signame = signal_names[signum]
log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
level=log.INFO, signame=signame)
reactor.callFromThread(self._stop_reactor)
def _signal_shutdown(self, signum, _):
install_shutdown_handlers(self._signal_kill)
signame = signal_names[signum]
log.msg(format="Received %(signame)s, shutting down gracefully. Send again to force ",
level=log.INFO, signame=signame)
reactor.callFromThread(self.stop)
def startService(self):
log.msg("Start connecting to Database ...")
self.start()
