def run_continuously(): scheduler = Scheduler() for task in settings.CELERYBEAT_SCHEDULE.values(): apply_async = TaskWorker.resolve_callable(task['task']).apply_async total_seconds = task['schedule'].total_seconds() scheduler.every(total_seconds).seconds.do(apply_async) scheduler.run_continuously()
def handle(self, *arg, **options): if options.get('status'): print(Control('dispatcher').status()) return if options.get('running'): print(Control('dispatcher').running()) return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and cache connections (that way lies race conditions) django_connection.close() django_cache.close() # spawn a daemon thread to periodically enqueues scheduled tasks # (like the node heartbeat) periodic.run_continuously() consumer = None try: queues = ['tower_broadcast_all', get_local_queuename()] consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4)) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def apply_async(self, entry, publisher, **kwargs): task = TaskWorker.resolve_callable(entry.task) result, queue = task.apply_async() class TaskResult(object): id = result['uuid'] return TaskResult()
def handle(self, *arg, **options): if options.get('status'): print(Control('dispatcher').status()) return if options.get('running'): print(Control('dispatcher').running()) return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and memcached connections (that way lies race conditions) django_connection.close() django_cache.close() # spawn a daemon thread to periodically enqueues scheduled tasks # (like the node heartbeat) periodic.run_continuously() reaper.reap() consumer = None # don't ship external logs inside the dispatcher's parent process # this exists to work around a race condition + deadlock bug on fork # in cpython itself: # https://bugs.python.org/issue37429 AWXProxyHandler.disable() with Connection(settings.BROKER_URL) as conn: try: bcast = 'tower_broadcast_all' queues = [ Queue(q, Exchange(q), routing_key=q) for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()]) ] queues.append( Queue( construct_bcast_queue_name(bcast), exchange=Exchange(bcast, type='fanout'), routing_key=bcast, reply=True ) ) consumer = AWXConsumer( 'dispatcher', conn, TaskWorker(), queues, AutoscalePool(min_workers=4) ) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def apply_async(self, entry, producer=None, advance=True, **kwargs): for conn in connections.all(): # If the database connection has a hiccup, re-establish a new # connection conn.close_if_unusable_or_obsolete() task = TaskWorker.resolve_callable(entry.task) result, queue = task.apply_async() class TaskResult(object): id = result['uuid'] return TaskResult()
def apply_async(self, entry, producer=None, advance=True, **kwargs): task = TaskWorker.resolve_callable(entry.task) result, queue = task.apply_async() class TaskResult(object): id = result['uuid'] return TaskResult()
def handle(self, *arg, **options): if options.get('status'): print Control('dispatcher').status() return if options.get('running'): print Control('dispatcher').running() return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and memcached connections (that way lies race conditions) django_connection.close() django_cache.close() beat = Process(target=self.beat) beat.daemon = True beat.start() reaper.reap() consumer = None with Connection(settings.BROKER_URL) as conn: try: bcast = 'tower_broadcast_all' queues = [ Queue(q, Exchange(q), routing_key=q) for q in (settings.AWX_CELERY_QUEUES_STATIC + [get_local_queuename()]) ] queues.append( Queue( construct_bcast_queue_name(bcast), exchange=Exchange(bcast, type='fanout'), routing_key=bcast, reply=True ) ) consumer = AWXConsumer( 'dispatcher', conn, TaskWorker(), queues, AutoscalePool(min_workers=4) ) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def handle(self, *arg, **options): if options.get('status'): print(Control('dispatcher').status()) return if options.get('running'): print(Control('dispatcher').running()) return if options.get('reload'): return Control('dispatcher').control({'control': 'reload'}) # It's important to close these because we're _about_ to fork, and we # don't want the forked processes to inherit the open sockets # for the DB and memcached connections (that way lies race conditions) django_connection.close() django_cache.close() # spawn a daemon thread to periodically enqueues scheduled tasks # (like the node heartbeat) periodic.run_continuously() reaper.reap() consumer = None # don't ship external logs inside the dispatcher's parent process # this exists to work around a race condition + deadlock bug on fork # in cpython itself: # https://bugs.python.org/issue37429 AWXProxyHandler.disable() try: queues = ['tower_broadcast_all', get_local_queuename()] consumer = AWXConsumerPG('dispatcher', TaskWorker(), queues, AutoscalePool(min_workers=4)) consumer.run() except KeyboardInterrupt: logger.debug('Terminating Task Dispatcher') if consumer: consumer.stop()
def tm(self): return TaskWorker()