def _drain(self):
with self._current_work_items_lock:
if not self._current_work_items:
self.log.info('EXIT_DRAIN:0')
return
result_store = forge.get_datastore()
dispatch_queue = forge.get_dispatch_queue()
self.log.info('EXIT_DRAIN:%s', len(self._current_work_items))
for item in self._current_work_items:
work = Task(item)
task = Task({})
task.sid = work.sid
task.srl = work.srl
task.dispatch_queue = work.dispatch_queue
task.classification = work.classification
self.log.info("DRAIN: %s/%s", task.sid, task.srl)
task.watermark(self.service_cls.SERVICE_NAME, None)
task.recoverable_failure(
'Task was pre-empted (shutdown, vm revert or cull)')
task.cache_key = result_store.save_error(
self.service_cls.SERVICE_NAME, None, None, task)
dispatch_queue.send_raw(task.as_dispatcher_response())
logger.info("Monitoring the following service queues: %s", threshold)
while True:
queue_lengths = get_service_queue_lengths()
over = {
k: v for k, v in queue_lengths.iteritems() if v > (threshold.get(k, 0) or v)
}
for name, size in over.iteritems():
excess = size - threshold.get(name, size)
if excess <= 0:
continue
for msg in get_queue(name).unpush(excess):
# noinspection PyBroadException
try:
t = Task(msg)
t.watermark(name, '')
t.nonrecoverable_failure('Service busy.')
t.cache_key = store.save_error(name, None, None, t)
dispatch_queue.send_raw(t.as_dispatcher_response())
logger.info("%s is too busy to process %s.", name, t.srl)
except: # pylint:disable=W0702
logger.exception('Problem sending response:')
time.sleep(config.system.update_interval)