def __init__(self, prefix, options, stats, datastore):
self.prefix = prefix
self.options = options
self.stats = stats
self.redis = datastore
self.coalescer = CoalescingMachine(prefix,
datastore,
stats=stats)
route_key = "route." + prefix + "#"
self.consumer_args['topic'] = [route_key] * len(self.exchanges)
self.consumer_args['user'] = self.options['user']
self.consumer_args['password'] = self.options['passwd']
log.info("Binding to queue with route key: %s" % (route_key))
self.listener = TcPulseConsumer(self.exchanges,
callback=self._route_callback_handler,
**self.consumer_args)
class TaskEventApp(object):
# ampq/pulse listener
listener = None
# State transitions
# pending --> running --> (completed|exception|failed)
# \-> exception
exchanges = ['exchange/taskcluster-queue/v1/task-pending',
'exchange/taskcluster-queue/v1/task-completed',
'exchange/taskcluster-queue/v1/task-exception',
'exchange/taskcluster-queue/v1/task-failed']
# TODO: move these to args and env options
# TODO: make perm coalescer service pulse creds
consumer_args = {
'applabel': 'releng-tc-coalesce',
'topic': ['#', '#', '#', '#'],
'durable': True,
'user': '******',
'password': '******'
}
options = None
# Coalesing machine
coalescer = None
def __init__(self, prefix, options, stats, datastore):
self.prefix = prefix
self.options = options
self.stats = stats
self.redis = datastore
self.coalescer = CoalescingMachine(prefix,
datastore,
stats=stats)
route_key = "route." + prefix + "#"
self.consumer_args['topic'] = [route_key] * len(self.exchanges)
self.consumer_args['user'] = self.options['user']
self.consumer_args['password'] = self.options['passwd']
log.info("Binding to queue with route key: %s" % (route_key))
self.listener = TcPulseConsumer(self.exchanges,
callback=self._route_callback_handler,
**self.consumer_args)
def run(self):
while True:
try:
self.listener.listen()
except KeyboardInterrupt:
# Handle both SIGTERM and SIGINT
self._graceful_shutdown()
except:
traceback.print_exc()
def _graceful_shutdown(self):
log.info("Gracefully shutting down")
log.info("Deleting Pulse queue")
self.listener.delete_queue()
sys.exit(1)
def _route_callback_handler(self, body, message):
"""
Route call body and msg to proper callback handler
"""
# Ignore tasks with non-zero runId (for now)
if not body['runId'] == 0:
message.ack()
return
taskState = body['status']['state']
taskId = body['status']['taskId']
# Extract first coalesce key that matches
for route in message.headers['CC']:
route = route[6:]
if self.prefix == route[:len(self.prefix)]:
coalesce_key = route[len(self.prefix):]
break
if taskState == 'pending':
self.coalescer.insert_task(taskId, coalesce_key)
elif taskState == 'completed' or \
taskState == 'exception' or \
taskState == 'failed':
self.coalescer.remove_task(taskId, coalesce_key)
else:
raise StateError
message.ack()
self.stats.notch('total_msgs_handled')
log.debug("taskId: %s (%s)" % (taskId, taskState))