async def test_conversion():
'''
Test message conversion between 2 queues
'''
bus = MessageBus()
bus.add_queue('input')
bus.add_queue(
'output',
maxsize=3) # limit size to immediately stop execution for unit test
assert isinstance(bus.queues['input'], asyncio.Queue)
assert isinstance(bus.queues['output'], asyncio.Queue)
assert bus.queues['input'].qsize() == 0
assert bus.queues['output'].qsize() == 0
await bus.send('input', 'test x')
await bus.send('input', 'hello world.')
await bus.send('output', 'lowercase')
# Convert all strings from input in uppercase
assert bus.queues['input'].qsize() == 2
task = asyncio.create_task(bus.run(lambda x: x.upper(), 'input', 'output'))
await bus.receive('output') == 'lowercase'
await bus.receive('output') == 'TEST X'
await bus.receive('output') == 'HELLO WORLD.'
task.cancel()
assert bus.queues['input'].qsize() == 0
assert bus.queues['output'].qsize() == 0
class EventListener(object):
'''
Listen to external events and trigger new tasks
'''
def __init__(self, cache_root):
# Create message bus shared amongst process
self.bus = MessageBus()
# Build client applications configuration
# TODO: use simpler secret structure per client
clients_conf = {h['type']: h for h in taskcluster.secrets['HOOKS']}
code_review_conf = clients_conf.get('static-analysis-phabricator')
code_coverage_conf = clients_conf.get('code-coverage')
# Code Review Workflow
if code_review_conf:
self.code_review = CodeReview(
api_key=taskcluster.secrets['PHABRICATOR']['token'],
url=taskcluster.secrets['PHABRICATOR']['url'],
publish=taskcluster.secrets['PHABRICATOR'].get(
'publish', False),
risk_analysis_reviewers=code_review_conf.get(
'risk_analysis_reviewers', []))
self.code_review.register(self.bus)
# Build mercurial worker & queue
self.mercurial = MercurialWorker(
QUEUE_MERCURIAL,
QUEUE_PHABRICATOR_RESULTS,
repositories=self.code_review.get_repositories(
taskcluster.secrets['repositories'], cache_root),
)
self.mercurial.register(self.bus)
# Create web server
self.webserver = WebServer(QUEUE_WEB_BUILDS)
self.webserver.register(self.bus)
else:
self.code_review = None
self.mercurial = None
self.webserver = None
# Code Coverage Workflow
if code_coverage_conf:
self.code_coverage = CodeCoverage(code_coverage_conf, self.bus)
# Setup monitoring for newly created tasks
self.monitoring = Monitoring(QUEUE_MONITORING,
taskcluster.secrets['ADMINS'],
7 * 3600)
self.monitoring.register(self.bus)
# Create pulse listener for code coverage
self.pulse = PulseListener(
QUEUE_PULSE_CODECOV,
'exchange/taskcluster-queue/v1/task-group-resolved',
'#',
taskcluster.secrets['PULSE_USER'],
taskcluster.secrets['PULSE_PASSWORD'],
)
self.pulse.register(self.bus)
else:
self.code_coverage = None
self.monitoring = None
self.pulse = None
assert self.code_review or self.code_coverage, 'No client applications to run !'
def run(self):
consumers = []
if self.code_review:
consumers += [
# Code review main workflow
self.code_review.run(),
# Add mercurial task
self.mercurial.run(),
]
# Publish results on Phabricator
if self.code_review.publish:
consumers.append(
self.bus.run(self.code_review.publish_results,
QUEUE_PHABRICATOR_RESULTS))
# Start the web server in its own process
self.webserver.start()
if self.code_coverage:
consumers += [
# Code coverage main workflow
self.code_coverage.run(),
# Add monitoring task
self.monitoring.run(),
# Add pulse task
self.pulse.run(),
]
# Run all tasks concurrently
run_tasks(consumers)
# Stop the webserver when other async process are stopped
if self.webserver:
self.webserver.stop()