Ejemplo n.º 1
0
    def _drain(self):

        with self._current_work_items_lock:
            if not self._current_work_items:
                self.log.info('EXIT_DRAIN:0')
                return

            result_store = forge.get_datastore()
            dispatch_queue = forge.get_dispatch_queue()
            self.log.info('EXIT_DRAIN:%s', len(self._current_work_items))
            for item in self._current_work_items:
                work = Task(item)
                task = Task({})
                task.sid = work.sid
                task.srl = work.srl
                task.dispatch_queue = work.dispatch_queue
                task.classification = work.classification
                self.log.info("DRAIN: %s/%s", task.sid, task.srl)
                task.watermark(self.service_cls.SERVICE_NAME, None)
                task.recoverable_failure(
                    'Task was pre-empted (shutdown, vm revert or cull)')
                task.cache_key = result_store.save_error(
                    self.service_cls.SERVICE_NAME, None, None, task)
                dispatch_queue.send_raw(task.as_dispatcher_response())
Ejemplo n.º 2
0
logger.info("Monitoring the following service queues: %s", threshold)

while True:
    queue_lengths = get_service_queue_lengths()

    over = {
        k: v for k, v in queue_lengths.iteritems() if v > (threshold.get(k, 0) or v)
    }

    for name, size in over.iteritems():
        excess = size - threshold.get(name, size)
        if excess <= 0:
            continue

        for msg in get_queue(name).unpush(excess):
            # noinspection PyBroadException
            try:
                t = Task(msg)

                t.watermark(name, '')
                t.nonrecoverable_failure('Service busy.')
                t.cache_key = store.save_error(name, None, None, t)

                dispatch_queue.send_raw(t.as_dispatcher_response())
                logger.info("%s is too busy to process %s.", name, t.srl)
            except:  # pylint:disable=W0702
                logger.exception('Problem sending response:')

    time.sleep(config.system.update_interval)