def run(self, *args, **kwargs):
super(AWXConsumerPG, self).run(*args, **kwargs)
logger.info(
f"Running worker {self.name} listening to queues {self.queues}")
init = False
while True:
try:
with pg_bus_conn(new_connection=True) as conn:
for queue in self.queues:
conn.listen(queue)
if init is False:
self.worker.on_start()
init = True
for e in conn.events():
self.process_task(json.loads(e.payload))
self.pg_is_down = False
if self.should_stop:
return
except psycopg2.InterfaceError:
logger.warning(
"Stale Postgres message bus connection, reconnecting")
continue
except (db.DatabaseError, psycopg2.OperationalError):
# If we have attained stady state operation, tolerate short-term database hickups
if not self.pg_is_down:
logger.exception(
f"Error consuming new events from postgres, will retry for {self.pg_max_wait} s"
)
self.pg_down_time = time.time()
self.pg_is_down = True
current_downtime = time.time() - self.pg_down_time
if current_downtime > self.pg_max_wait:
logger.exception(
f"Postgres event consumer has not recovered in {current_downtime} s, exiting"
)
raise
# Wait for a second before next attempt, but still listen for any shutdown signals
for i in range(10):
if self.should_stop:
return
time.sleep(0.1)
for conn in db.connections.all():
conn.close_if_unusable_or_obsolete()
except Exception:
# Log unanticipated exception in addition to writing to stderr to get timestamps and other metadata
logger.exception(
'Encountered unhandled error in dispatcher main loop')
raise
def run(self, *args, **kwargs):
super(AWXConsumerPG, self).run(*args, **kwargs)
logger.warn(
f"Running worker {self.name} listening to queues {self.queues}")
while True:
try:
with pg_bus_conn() as conn:
for queue in self.queues:
conn.listen(queue)
for e in conn.events():
self.process_task(json.loads(e.payload))
if self.should_stop:
return
except psycopg2.InterfaceError:
logger.warn(
"Stale Postgres message bus connection, reconnecting")
continue
def control(self, body):
logger.warning(f'Received control signal:\n{body}')
control = body.get('control')
if control in ('status', 'running'):
reply_queue = body['reply_to']
if control == 'status':
msg = '\n'.join([self.listening_on, self.pool.debug()])
elif control == 'running':
msg = []
for worker in self.pool.workers:
worker.calculate_managed_tasks()
msg.extend(worker.managed_tasks.keys())
with pg_bus_conn() as conn:
conn.notify(reply_queue, json.dumps(msg))
elif control == 'reload':
for worker in self.pool.workers:
worker.quit()
else:
logger.error('unrecognized control message: {}'.format(control))
