def run(self, *args, **kwargs): super(AWXConsumerPG, self).run(*args, **kwargs) logger.info( f"Running worker {self.name} listening to queues {self.queues}") init = False while True: try: with pg_bus_conn(new_connection=True) as conn: for queue in self.queues: conn.listen(queue) if init is False: self.worker.on_start() init = True for e in conn.events(): self.process_task(json.loads(e.payload)) self.pg_is_down = False if self.should_stop: return except psycopg2.InterfaceError: logger.warning( "Stale Postgres message bus connection, reconnecting") continue except (db.DatabaseError, psycopg2.OperationalError): # If we have attained stady state operation, tolerate short-term database hickups if not self.pg_is_down: logger.exception( f"Error consuming new events from postgres, will retry for {self.pg_max_wait} s" ) self.pg_down_time = time.time() self.pg_is_down = True current_downtime = time.time() - self.pg_down_time if current_downtime > self.pg_max_wait: logger.exception( f"Postgres event consumer has not recovered in {current_downtime} s, exiting" ) raise # Wait for a second before next attempt, but still listen for any shutdown signals for i in range(10): if self.should_stop: return time.sleep(0.1) for conn in db.connections.all(): conn.close_if_unusable_or_obsolete() except Exception: # Log unanticipated exception in addition to writing to stderr to get timestamps and other metadata logger.exception( 'Encountered unhandled error in dispatcher main loop') raise
def run(self, *args, **kwargs): super(AWXConsumerPG, self).run(*args, **kwargs) logger.warn( f"Running worker {self.name} listening to queues {self.queues}") while True: try: with pg_bus_conn() as conn: for queue in self.queues: conn.listen(queue) for e in conn.events(): self.process_task(json.loads(e.payload)) if self.should_stop: return except psycopg2.InterfaceError: logger.warn( "Stale Postgres message bus connection, reconnecting") continue
def control(self, body): logger.warning(f'Received control signal:\n{body}') control = body.get('control') if control in ('status', 'running'): reply_queue = body['reply_to'] if control == 'status': msg = '\n'.join([self.listening_on, self.pool.debug()]) elif control == 'running': msg = [] for worker in self.pool.workers: worker.calculate_managed_tasks() msg.extend(worker.managed_tasks.keys()) with pg_bus_conn() as conn: conn.notify(reply_queue, json.dumps(msg)) elif control == 'reload': for worker in self.pool.workers: worker.quit() else: logger.error('unrecognized control message: {}'.format(control))