class Queue(LightQueue): '''Create a queue object with a given maximum size. If *maxsize* is less than zero or ``None``, the queue size is infinite. ``Queue(0)`` is a channel, that is, its :meth:`put` method always blocks until the item is delivered. (This is unlike the standard :class:`Queue`, where 0 means infinite size). In all other respects, this Queue class resembled the standard library, :class:`Queue`. ''' def __init__(self, maxsize=None): LightQueue.__init__(self, maxsize) self.unfinished_tasks = 0 self._cond = Event() def _format(self): result = LightQueue._format(self) if self.unfinished_tasks: result += ' tasks=%s _cond=%s' % (self.unfinished_tasks, self._cond) return result def _put(self, item): LightQueue._put(self, item) self._put_bookkeeping() def _put_bookkeeping(self): self.unfinished_tasks += 1 if self._cond.ready(): self._cond.reset() def task_done(self): '''Indicate that a formerly enqueued task is complete. Used by queue consumer threads. For each :meth:`get <Queue.get>` used to fetch a task, a subsequent call to :meth:`task_done` tells the queue that the processing on the task is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put <Queue.put>` into the queue). Raises a :exc:`ValueError` if called more times than there were items placed in the queue. ''' if self.unfinished_tasks <= 0: raise ValueError('task_done() called too many times') self.unfinished_tasks -= 1 if self.unfinished_tasks == 0: self._cond.send(None) def join(self): '''Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. ''' self._cond.wait()
class Queue(LightQueue): '''Create a queue object with a given maximum size. If *maxsize* is less than zero or ``None``, the queue size is infinite. ``Queue(0)`` is a channel, that is, its :meth:`put` method always blocks until the item is delivered. (This is unlike the standard :class:`Queue`, where 0 means infinite size). In all other respects, this Queue class resembled the standard library, :class:`Queue`. ''' def __init__(self, maxsize=None): LightQueue.__init__(self, maxsize) self.unfinished_tasks = 0 self._cond = Event() def _format(self): result = LightQueue._format(self) if self.unfinished_tasks: result += ' tasks=%s _cond=%s' % (self.unfinished_tasks, self._cond) return result def _put(self, item): LightQueue._put(self, item) self._put_bookkeeping() def _put_bookkeeping(self): self.unfinished_tasks += 1 if self._cond.ready(): self._cond.reset() def task_done(self): '''Indicate that a formerly enqueued task is complete. Used by queue consumer threads. For each :meth:`get <Queue.get>` used to fetch a task, a subsequent call to :meth:`task_done` tells the queue that the processing on the task is complete. If a :meth:`join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put <Queue.put>` into the queue). Raises a :exc:`ValueError` if called more times than there were items placed in the queue. ''' if self.unfinished_tasks <= 0: raise ValueError('task_done() called too many times') self.unfinished_tasks -= 1 if self.unfinished_tasks == 0: self._cond.send(None) def join(self): '''Block until all items in the queue have been gotten and processed. The count of unfinished tasks goes up whenever an item is added to the queue. The count goes down whenever a consumer thread calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. ''' if self.unfinished_tasks > 0: self._cond.wait()
def test_container_doesnt_exhaust_max_workers(container): spam_called = Event() spam_continue = Event() class Service(object): name = 'max-workers' @foobar def spam(self, a): spam_called.send(a) spam_continue.wait() container = ServiceContainer(Service, config={MAX_WORKERS_CONFIG_KEY: 1}) dep = get_extension(container, Entrypoint) # start the first worker, which should wait for spam_continue container.spawn_worker(dep, ['ham'], {}) # start the next worker in a speparate thread, # because it should block until the first one completed gt = spawn(container.spawn_worker, dep, ['eggs'], {}) with Timeout(1): assert spam_called.wait() == 'ham' # if the container had spawned the second worker, we would see # an error indicating that spam_called was fired twice, and the # greenthread would now be dead. assert not gt.dead # reset the calls and allow the waiting worker to complete. spam_called.reset() spam_continue.send(None) # the second worker should now run and complete assert spam_called.wait() == 'eggs' assert gt.dead
class Timer(Entrypoint): def __init__(self, interval, eager=True, **kwargs): self.gt = None self.eager = eager self.interval = interval self.stopping_event = Event() self.finished_event = Event() super(Timer, self).__init__(**kwargs) def start(self): self.gt = self.container.spawn_manage_thread(self._run) def stop(self): self.stopping_event.send(True) self.gt.wait() def kill(self): self.gt.kill() def _run(self): def gen_interval(): start_time = time.time() start = 1 if self.eager else 0 for n in count(start=start): i = max(start_time + n * self.interval - time.time(), 0) yield i interval = gen_interval() to_sleep = next(interval) while True: with Timeout(to_sleep, exception=False): self.stopping_event.wait() break self.container.spawn_worker_thread(self, (), {}, res_handler=self.res_handler) self.finished_event.wait() self.finished_event.reset() to_sleep = next(interval) def res_handler(self, context, result, exc_info): self.finished_event.send(True) return result, exc_info
class Timer(Entrypoint): def __init__(self, interval, eager=False, **kwargs): """ Timer entrypoint. Fires every `interval` seconds or as soon as the previous worker completes if that took longer. The default behaviour is to wait `interval` seconds before firing for the first time. If you want the entrypoint to fire as soon as the service starts, pass `eager=True`. Example:: timer = Timer.decorator class Service(object): name = "service" @timer(interval=5) def tick(self): pass """ self.interval = interval self.eager = eager self.should_stop = Event() self.worker_complete = Event() self.gt = None super(Timer, self).__init__(**kwargs) def start(self): _log.debug('starting %s', self) self.gt = self.container.spawn_managed_thread(self._run) def stop(self): _log.debug('stopping %s', self) self.should_stop.send(True) self.gt.wait() def kill(self): _log.debug('killing %s', self) self.gt.kill() def _run(self): """ Runs the interval loop. """ sleep_time = 0 if self.eager else self.interval while True: # sleep for `sleep_time`, unless `should_stop` fires, in which # case we leave the while loop and stop entirely with Timeout(sleep_time, exception=False): self.should_stop.wait() break start = time.time() self.handle_timer_tick() self.worker_complete.wait() self.worker_complete.reset() elapsed_time = (time.time() - start) # next time, sleep however long is left of our interval, taking # off the time we took to run sleep_time = max(self.interval - elapsed_time, 0) def handle_timer_tick(self): args = () kwargs = {} # Note that we don't catch ContainerBeingKilled here. If that's raised, # there is nothing for us to do anyway. The exception bubbles, and is # caught by :meth:`Container._handle_thread_exited`, though the # triggered `kill` is a no-op, since the container is already # `_being_killed`. self.container.spawn_worker( self, args, kwargs, handle_result=self.handle_result) def handle_result(self, worker_ctx, result, exc_info): self.worker_complete.send() return result, exc_info
class ScheduledMessageService(object): MIN_SLEEP_TIME = timedelta(seconds=1) MAX_SLEEP_TIME = timedelta(minutes=60) MAX_CLAIM_TIME = timedelta(minutes=5) def __init__(self, context): self.context = context self.service_queue = Event() self._listener = None self._dispatch = None def run(self): try: with self.context: self._listener = self._start_listener() self._dispatcher = spawn(self.run_dispatcher) procs = [self._listener, self._dispatcher] waitall(procs) except GreenletExit: pass finally: killall(procs) waitall(procs) ################################################################ # The listener consumes messages on the scheduled message queue # and stores the deferred messages in the database. ################################################################ def _start_listener(self): @always_ack def cb(message_data, message): with self.context: _handle_scheduler_command(message_data, message, self.context) self.wakeup_dispatcher() dispatch = MessageDispatch(self.context) return dispatch.start_worker(SCHEDULER_COMMAND, cb) ############################################################## # The dispatcher consumes deferred messages from the database # when their scheduled time arrives and spits them out # to the message broker ############################################################## def run_dispatcher(self): try: # cleanup any mess left over last time... with self.context: self.cleanup() while(True): log.info("checking for ready messages...") last_time = self.send_ready_messages() sleep_time = self._calc_sleep(last_time) log.info("sleeping for %s" % sleep_time) sleep_secs = sleep_time.days*84600 + sleep_time.seconds try: with_timeout(sleep_secs, self.service_queue.wait) except TimeoutError: pass if self.service_queue.ready(): self.service_queue.reset() except GreenletExit: log.debug("ScheduledMessageService dispatcher exiting...") def wakeup_dispatcher(self): if not self.service_queue.ready(): self.service_queue.send(True) def _calc_sleep(self, after=None): next_time = self.find_next_send_time(after=after) if next_time is None: sleep_time = self.MAX_SLEEP_TIME else: sleep_time = next_time - datetime.utcnow() sleep_time += timedelta(seconds=1) sleep_time -= timedelta(microseconds=sleep_time.microseconds) if sleep_time < self.MIN_SLEEP_TIME: sleep_time = self.MIN_SLEEP_TIME if sleep_time > self.MAX_SLEEP_TIME: sleep_time = self.MAX_SLEEP_TIME return sleep_time def find_next_send_time(self, after=None): if after is None: after = datetime.utcnow() after_str = DateTimeField()._to_json(after) next_query = dict( startkey = [False, after_str, {}], endkey = [True, None], include_docs = False, descending = False, limit = 1 ) next_send = None for r in view_deferred_messages_by_timestamp(self.context.db, **next_query): next_send = DateTimeField()._to_python(r.key[1]) break return next_send def send_ready_messages(self): while True: now = datetime.utcnow() now_str = DateTimeField()._to_json(now) query = dict( startkey = [False, None], endkey = [False, now_str, {}], include_docs = True, descending = False, limit = 100 ) vr = view_deferred_messages_by_timestamp(self.context.db, **query) batch = [] for r in vr: batch.append(DeferredAMQPMessage.wrap(r.doc)) if len(batch) == 0: break dispatch_count = 0 for message in batch: try: if self._dispatch_message(message): dispatch_count += 1 except GreenletExit: # asked to stop, go ahead and quit. raise except: log.error("Unexected error dispatching message %s: %s" % (message, traceback.format_exc())) log.info("Dispatched %d messages" % dispatch_count) return now def _dispatch_message(self, message): if not message.claim(self.context.db): return try: publisher = Publisher(self.context.broker, exchange=message.options.exchange, exchange_type=message.options.exchange_type) publisher.send(message.message, routing_key = message.options.routing_key, delivery_mode = message.options.delivery_mode, mandatory = message.options.mandatory, priority = message.options.priority) publisher.close() except: log.error("Error dispatching deferred message %s: %s" % (message, traceback.format_exc())) self.error_reschedule(message) return False else: log.debug("Dispatched message %s" % message) # sent with no problems, done with it. self.context.db.delete(message) return True def error_reschedule(self, message): message.error_count += 1 if message.error_count < 10: delay = 2**message.error_count else: delay = 60*10 resched_time = datetime.utcnow() + timedelta(seconds=delay) message.unclaim(self.context.db, resched_time) log.warn("Rescheduled message %s for %s" % (message.id, resched_time)) def cleanup(self): log.info("Performing cleanup of claimed items...") # anything older than this has held the claim for too long # and is considered dead. cutoff = datetime.utcnow() - self.MAX_CLAIM_TIME cutoff_str = DateTimeField()._to_json(cutoff) query = dict( startkey = [True, cutoff_str, {}], endkey = [True], limit = 100, include_docs = True, descending = True ) unclaim_count = 0 while(True): vr = view_deferred_messages_by_timestamp(self.context.db, **query) batch = [DeferredAMQPMessage.wrap(r.doc) for r in vr] if len(batch) == 0: break for message in batch: self.error_reschedule(message) unclaim_count += 1 if unclaim_count > 0: log.warn('Cleanup unclaimed %d items' % unclaim_count)
class Service(ConsumerMixin): def __init__(self, controllercls, connection, exchange, topic, pool=None, poolsize=1000): self.nodeid = UIDGEN() if pool is None: self.procpool = GreenPool(size=poolsize) else: self.procpool = pool self.connection = connection self.controller = controllercls() self.topic = topic self.greenlet = None self.messagesem = Semaphore() self.consume_ready = Event() node_topic = "{}.{}".format(self.topic, self.nodeid) self.queues = [entities.get_topic_queue(exchange, topic), entities.get_topic_queue(exchange, node_topic), entities.get_fanout_queue(topic), ] self._channel = None self._consumers = None def start(self): # self.connection = newrpc.create_connection() if self.greenlet is not None and not self.greenlet.dead: raise RuntimeError() self.greenlet = eventlet.spawn(self.run) def get_consumers(self, Consumer, channel): return [Consumer(self.queues, callbacks=[self.on_message, ]), ] def on_consume_ready(self, connection, channel, consumers, **kwargs): self._consumers = consumers self._channel = channel self.consume_ready.send(None) def on_consume_end(self, connection, channel): self.consume_ready.reset() def on_message(self, body, message): # need a semaphore to stop killing between message ack() # and spawning process. with self.messagesem: self.procpool.spawn(self.handle_request, body) message.ack() def handle_request(self, body): newrpc.process_message(self.connection, self.controller, body) def wait(self): try: self.greenlet.wait() except greenlet.GreenletExit: pass return self.procpool.waitall() def kill(self): if self.greenlet is not None and not self.greenlet.dead: self.should_stop = True #with self.messagesem: #self.greenlet.kill() self.greenlet.wait() if self._consumers: for c in self._consumers: c.cancel() if self._channel is not None: self._channel.close() def link(self, *args, **kwargs): return self.greenlet.link(*args, **kwargs) def kill_processes(self): for g in self.procpool.coroutines_running: g.kill()
class AbstractChannel(object): """ Superclass for both the Connection, which is treated as channel 0, and other user-created Channel objects. The subclasses must have a _METHOD_MAP class property, mapping between AMQP method signatures and Python methods. """ def __init__(self, connection, channel_id): self.connection = connection self.channel_id = channel_id connection.channels[channel_id] = self self.method_queue = [] # Higher level queue for methods self.new_method = Event() self.auto_decode = False def __enter__(self): """ Support for Python >= 2.5 'with' statements. """ return self def __exit__(self, type, value, traceback): """ Support for Python >= 2.5 'with' statements. """ self.close() def _send_method(self, method_sig, args='', content=None): """ Send a method for our channel. """ if isinstance(args, AMQPWriter): args = args.getvalue() self.connection.method_writer.write_method(self.channel_id, method_sig, args, content) def close(self): """ Close this Channel or Connection """ raise NotImplementedError('Must be overriden in subclass') def received_method(self, method_sig, args, content=None): meth = (method_sig, args, content) self.method_queue.append(meth) self.new_method.send(meth) self.new_method.reset() def _wait_method(self, allowed_methods): """ Wait for a method from the server destined for this channel. """ # # Check the channel's deferred methods # while(True): for queued_method in self.method_queue: method_sig = queued_method[0] if (allowed_methods is None) \ or (method_sig in allowed_methods) \ or (method_sig == (20, 40)): self.method_queue.remove(queued_method) return queued_method # nope, wait for it... self.new_method.wait() def wait(self, allowed_methods=None): """ Wait for a method that matches our allowed_methods parameter (the default value of None means match any method), and dispatch to it. """ method_sig, args, content = self._wait_method(allowed_methods) if content \ and self.auto_decode \ and hasattr(content, 'content_encoding'): try: content.body = content.body.decode(content.content_encoding) except Exception: pass amqp_method = self._METHOD_MAP.get(method_sig, None) if amqp_method is None: raise Exception('Unknown AMQP method (%d, %d)' % method_sig) if content is None: return amqp_method(self, args) else: return amqp_method(self, args, content) # # Placeholder, the concrete implementations will have to # supply their own versions of _METHOD_MAP # _METHOD_MAP = {}
class Timer(Entrypoint): def __init__(self, interval, eager=False, **kwargs): """ Timer entrypoint. Fires every `interval` seconds or as soon as the previous worker completes if that took longer. The default behaviour is to wait `interval` seconds before firing for the first time. If you want the entrypoint to fire as soon as the service starts, pass `eager=True`. Example:: timer = Timer.decorator class Service(object): name = "service" @timer(interval=5) def tick(self): pass """ self.interval = interval self.eager = eager self.should_stop = Event() self.worker_complete = Event() self.gt = None super(Timer, self).__init__(**kwargs) def start(self): _log.debug('starting %s', self) self.gt = self.container.spawn_managed_thread(self._run) def stop(self): _log.debug('stopping %s', self) self.should_stop.send(True) self.gt.wait() def kill(self): _log.debug('killing %s', self) self.gt.kill() def _run(self): """ Runs the interval loop. """ def get_next_interval(): start_time = time.time() start = 0 if self.eager else 1 for count in itertools.count(start=start): yield max(start_time + count * self.interval - time.time(), 0) interval = get_next_interval() sleep_time = next(interval) while True: # sleep for `sleep_time`, unless `should_stop` fires, in which # case we leave the while loop and stop entirely with Timeout(sleep_time, exception=False): self.should_stop.wait() break self.handle_timer_tick() self.worker_complete.wait() self.worker_complete.reset() sleep_time = next(interval) def handle_timer_tick(self): args = () kwargs = {} # Note that we don't catch ContainerBeingKilled here. If that's raised, # there is nothing for us to do anyway. The exception bubbles, and is # caught by :meth:`Container._handle_thread_exited`, though the # triggered `kill` is a no-op, since the container is already # `_being_killed`. self.container.spawn_worker( self, args, kwargs, handle_result=self.handle_result) def handle_result(self, worker_ctx, result, exc_info): self.worker_complete.send() return result, exc_info
class Service(ConsumerMixin): def __init__( self, controllercls, connection_factory, exchange, topic, pool=None, poolsize=1000): self.nodeid = UIDGEN() self.max_workers = poolsize if pool is None: self.procpool = GreenPool(size=poolsize) else: self.procpool = pool self.controller = controllercls() self.service = self.controller self.topic = topic self.greenlet = None self.consume_ready = Event() node_topic = "{}.{}".format(self.topic, self.nodeid) self.nova_queues = [ entities.get_topic_queue(exchange, topic), entities.get_topic_queue(exchange, node_topic), entities.get_fanout_queue(topic), ] self._channel = None self._consumers = None self.connection = connection_factory() self.connection_factory = connection_factory inject_dependencies(self.controller, self) self._connection_pool = Pool( max_size=self.procpool.size, create=connection_factory ) self.workers = set() self._pending_ack_messages = [] self._pending_requeue_messages = [] self._do_cancel_consumers = False self._consumers_cancelled = Event() self._timers = list(get_timers(self.controller)) def start(self): self.start_timers() # greenlet has a magic attribute ``dead`` - pylint: disable=E1101 if self.greenlet is not None and not self.greenlet.dead: raise RuntimeError() self.greenlet = eventlet.spawn(self.run) def start_timers(self): for timer in self._timers: timer.start() def get_consumers(self, Consumer, channel): nova_consumer = Consumer( self.nova_queues, callbacks=[self.on_nova_message, ]) consume_consumers = get_consumers( Consumer, self, self.on_consume_message) consumers = [nova_consumer] + list(consume_consumers) prefetch_count = self.procpool.size for consumer in consumers: consumer.qos(prefetch_count=prefetch_count) return consumers def on_consume_ready(self, connection, channel, consumers, **kwargs): self._consumers = consumers self._channel = channel self.consume_ready.send(None) def on_consume_end(self, connection, channel): self.consume_ready.reset() def on_nova_message(self, body, message): _log.debug('spawning RPC worker (%d free)', self.procpool.free()) gt = self.procpool.spawn(self.handle_rpc_message, body) gt.link(self.handle_rpc_message_processed, message) self.workers.add(gt) def on_consume_message(self, consumer_method_config, body, message): _log.debug('spawning consume worker (%d free)', self.procpool.free()) gt = self.procpool.spawn( self.handle_consume_message, consumer_method_config, body, message) gt.link(self.handle_consume_message_processed) self.workers.add(gt) def handle_rpc_message(self, body): # item is patched on for python with ``with``, pylint can't find it # pylint: disable=E1102 with self._connection_pool.item() as connection: process_rpc_message(connection, self.controller, body) def handle_rpc_message_processed(self, gt, message): self.workers.discard(gt) self._pending_ack_messages.append(message) def handle_consume_message(self, consumer_method_config, body, message): with log_time(_log.debug, 'processed consume message in %0.3fsec'): consumer_method, consumer_config = consumer_method_config try: consumer_method(body) except Exception as e: if consumer_config.requeue_on_error: _log.exception( 'failed to consume message, requeueing message: ' '%s(): %s', consumer_method, e) self._pending_requeue_messages.append(message) else: _log.exception( 'failed to consume message, ignoring message: ' '%s(): %s', consumer_method, e) self._pending_ack_messages.append(message) else: self._pending_ack_messages.append(message) def handle_consume_message_processed(self, gt): self.workers.discard(gt) def on_iteration(self): self.process_consumer_cancellation() # we need to make sure we process any pending messages before shutdown self.process_pending_message_acks() self.process_shutdown() def process_consumer_cancellation(self): if self._do_cancel_consumers: self._do_cancel_consumers = False if self._consumers: _log.debug('cancelling consumers') for consumer in self._consumers: consumer.cancel() self._consumers_cancelled.send(True) def process_pending_message_acks(self): messages = self._pending_ack_messages if messages: _log.debug('ack() %d processed messages', len(messages)) while messages: msg = messages.pop() msg.ack() eventlet.sleep() messages = self._pending_requeue_messages if messages: _log.debug('requeue() %d processed messages', len(messages)) while messages: msg = messages.pop() msg.requeue() eventlet.sleep() def consume(self, limit=None, timeout=None, safety_interval=0.1, **kwargs): """ Lifted from kombu so we are able to break the loop immediately after a shutdown is triggered rather than waiting for the timeout. """ elapsed = 0 with self.Consumer() as (connection, channel, consumers): with self.extra_context(connection, channel): self.on_consume_ready(connection, channel, consumers, **kwargs) for i in limit and xrange(limit) or count(): # moved from after the following `should_stop` condition to # avoid waiting on a drain_events timeout before breaking # the loop. self.on_iteration() if self.should_stop: break try: connection.drain_events(timeout=safety_interval) except socket.timeout: elapsed += safety_interval # Excluding the following clause from coverage, # as timeout never appears to be set - This method # is a lift from kombu so will leave in place for now. if timeout and elapsed >= timeout: # pragma: no cover raise socket.timeout() except socket.error: if not self.should_stop: raise else: yield elapsed = 0 def process_shutdown(self): consumers_cancelled = self._consumers_cancelled.ready() no_active_timers = (len(self._timers) == 0) no_active_workers = (self.procpool.running() < 1) no_pending_message_acks = not ( self._pending_ack_messages or self._pending_requeue_messages ) ready_to_stop = ( consumers_cancelled and no_active_timers and no_active_workers and no_pending_message_acks ) if ready_to_stop: _log.debug('notifying service to stop') self.should_stop = True def cancel_consumers(self): # greenlet has a magic attribute ``dead`` - pylint: disable=E1101 if self.greenlet is not None and not self.greenlet.dead: # since consumers were started in a separate thread, # we will just notify the thread to avoid getting # "Second simultaneous read" errors _log.debug('notifying consumers to be cancelled') self._do_cancel_consumers = True self._consumers_cancelled.wait() else: _log.debug('consumer thread already dead') def cancel_timers(self): if self._timers: _log.debug('stopping %d timers', len(self._timers)) while self._timers: self._timers.pop().stop() def kill_workers(self): _log.debug('force killing %d workers', len(self.workers)) while self.workers: self.workers.pop().kill() def wait_for_workers(self): pool = self.procpool _log.debug('waiting for %d workers to complete', pool.running()) pool.waitall() def shut_down(self): # greenlet has a magic attribute ``dead`` - pylint: disable=E1101 if self.greenlet is not None and not self.greenlet.dead: _log.debug('stopping service') self.greenlet.wait() # TODO: when is this ever not None? if self._channel is not None: _log.debug('closing channel') self._channel.close() def kill(self, force=False): _log.debug('killing service') self.cancel_consumers() self.cancel_timers() if force: self.kill_workers() else: self.wait_for_workers() self.shut_down() def link(self, *args, **kwargs): return self.greenlet.link(*args, **kwargs)
class Service(ConsumerMixin): def __init__(self, controllercls, connection, exchange, topic, pool=None, poolsize=1000): self.nodeid = UIDGEN() if pool is None: self.procpool = GreenPool(size=poolsize) else: self.procpool = pool self.connection = connection self.controller = controllercls() self.topic = topic self.greenlet = None self.messagesem = Semaphore() self.consume_ready = Event() node_topic = "{}.{}".format(self.topic, self.nodeid) self.queues = [ entities.get_topic_queue(exchange, topic), entities.get_topic_queue(exchange, node_topic), entities.get_fanout_queue(topic), ] self._channel = None self._consumers = None def start(self): # self.connection = newrpc.create_connection() if self.greenlet is not None and not self.greenlet.dead: raise RuntimeError() self.greenlet = eventlet.spawn(self.run) def get_consumers(self, Consumer, channel): return [ Consumer(self.queues, callbacks=[ self.on_message, ]), ] def on_consume_ready(self, connection, channel, consumers, **kwargs): self._consumers = consumers self._channel = channel self.consume_ready.send(None) def on_consume_end(self, connection, channel): self.consume_ready.reset() def on_message(self, body, message): # need a semaphore to stop killing between message ack() # and spawning process. with self.messagesem: self.procpool.spawn(self.handle_request, body) message.ack() def handle_request(self, body): newrpc.process_message(self.connection, self.controller, body) def wait(self): try: self.greenlet.wait() except greenlet.GreenletExit: pass return self.procpool.waitall() def kill(self): if self.greenlet is not None and not self.greenlet.dead: self.should_stop = True #with self.messagesem: #self.greenlet.kill() self.greenlet.wait() if self._consumers: for c in self._consumers: c.cancel() if self._channel is not None: self._channel.close() def link(self, *args, **kwargs): return self.greenlet.link(*args, **kwargs) def kill_processes(self): for g in self.procpool.coroutines_running: g.kill()
class Cron(Entrypoint): def __init__(self, schedule: str, tz: str = None, concurrency: str = ConcurrencyPolicy.WAIT, **kwargs): """ Cron entrypoint. Fires according to a (possibly timezone-aware) cron schedule. If no timezone info is passed, the default is UTC. Set ``concurrency`` to ``ConcurrencyPolicy.ALLOW`` to allow multiple workers to run simultaneously. Set ``concurrency`` to ``ConcurrencyPolicy.SKIP`` to skip lapsed scheduled runs. The default behavior (``ConcurrencyPolicy.WAIT``) is to wait until the running worker completes and immediately spawn another if the schedule has lapsed. Example:: class Service(object): name = "service" @cron(schedule='0 12 * * *', tz='America/Chicago') def ping(self): # method executes every day at noon America/Chicago time print("pong") """ self.schedule = schedule self.tz = tz self.concurrency = concurrency self.should_stop = Event() self.worker_complete = Event() self.gt = None super().__init__(**kwargs) def start(self): _log.debug('starting %s', self) self.gt = self.container.spawn_managed_thread(self._run) def stop(self): _log.debug('stopping %s', self) self.should_stop.send(True) self.gt.wait() def kill(self): _log.debug('killing %s', self) self.gt.kill() def _get_next_interval(self): now_utc = datetime.datetime.now(tz=pytz.UTC) if self.tz: tz = pytz.timezone(self.tz) base = now_utc.astimezone(tz) else: base = now_utc cron_schedule = croniter(self.schedule, base) while True: yield max(cron_schedule.get_next() - time.time(), 0) def _run(self): """ Runs the schedule loop. """ interval = self._get_next_interval() sleep_time = next(interval) while True: # sleep for `sleep_time`, unless `should_stop` fires, in which # case we leave the while loop and stop entirely with Timeout(sleep_time, exception=False): self.should_stop.wait() break self.handle_timer_tick() if self.concurrency != ConcurrencyPolicy.ALLOW: self.worker_complete.wait() self.worker_complete.reset() sleep_time = next(interval) # a sleep time of zero represents that we've elapsed the next start time, so # if the user set the policy to skip, we need to update the interval again. if self.concurrency == ConcurrencyPolicy.SKIP and sleep_time == 0: sleep_time = next(interval) def handle_timer_tick(self): args = () kwargs = {} # Note that we don't catch ContainerBeingKilled here. If that's raised, # there is nothing for us to do anyway. The exception bubbles, and is # caught by :meth:`Container._handle_thread_exited`, though the # triggered `kill` is a no-op, since the container is already # `_being_killed`. self.container.spawn_worker(self, args, kwargs, handle_result=self.handle_result) def handle_result(self, worker_ctx, result, exc_info): # we only care about the worker completion if we're going to be waiting for it. if self.concurrency != ConcurrencyPolicy.ALLOW: self.worker_complete.send() return result, exc_info