async def test_failure_general(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
Use a Python common exception to trigger a broken build
'''
diff = {
'phid': 'PHID-DIFF-test123',
'id': 1234,
'baseRevision': None,
'revisionPHID': 'PHID-DREV-deadbeef'
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-somehash', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_mc.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
# Raise an exception during the workflow to trigger a broken build
def boom(*args):
raise Exception('Boom')
mock_mc.apply_build = boom
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc}
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the unit result was published
mode, out_build, details = await bus.receive('phabricator')
assert mode == 'fail:general'
assert out_build == build
assert details['duration'] > 0
assert details['message'] == 'Boom'
task.cancel()
# Clone should not be modified
tip = mock_mc.repo.tip()
assert tip.node == initial.node
async def test_failure_mercurial(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
Apply a bad mercurial patch to trigger a mercurial fail
'''
diff = {
'revisionPHID': 'PHID-DREV-666',
'baseRevision': 'missing',
'phid': 'PHID-DIFF-666',
'id': 666,
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-build-666', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_mc.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc}
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
assert mode == 'fail:mercurial'
assert out_build == build
assert details['duration'] > 0
assert details['message'] == MERCURIAL_FAILURE
task.cancel()
# Clone should not be modified
tip = mock_mc.repo.tip()
assert tip.node == initial.node
async def test_treeherder_link(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
'''
# Preload the build
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
'baseRevision': 'abcdef12345',
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-HMBT-somehash', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_mc.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc},
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
tip = mock_mc.repo.tip()
assert mode == 'success'
assert out_build == build
assert details['treeherder_url'] == 'https://treeherder.mozilla.org/#/jobs?repo=try&revision={}'.format(tip.node.decode('utf-8'))
task.cancel()
# Tip should be updated
assert tip.node != initial.node
async def test_monitoring_retry_exceptions(QueueMock, NotifyMock,
mock_taskcluster):
bus = MessageBus()
monitoring = Monitoring('testqueue', ['pinco@pallino'], 1)
monitoring.register(bus)
await bus.send('testqueue', ('Group1', 'Hook1', 'Task-exception-retry:2'))
await bus.send('testqueue', ('Group1', 'Hook2', 'Task-exception-retry:0'))
assert bus.queues['testqueue'].qsize() == 2
monitoring.queue = QueueMock
assert len(monitoring.queue.created_tasks) == 0
monitoring.notify = NotifyMock
# Task exception with 2 retries
await monitoring.check_task()
assert monitoring.stats['Hook1']['exception'] == ['Task-exception-retry:2']
assert len(monitoring.queue.created_tasks) == 1
assert bus.queues['testqueue'].qsize() == 2
# The retried task should maintain the original taskGroupId
old_task = monitoring.queue.task('Task-exception-retry:2')
new_task_id, new_task = monitoring.queue.created_tasks[0]
assert new_task_id != 'Task-exception-retry:2'
assert new_task != old_task
assert new_task['taskGroupId'] == old_task['taskGroupId']
assert new_task['payload'] == old_task['payload']
assert new_task['created'] != old_task['created']
# Task exception with 0 retries
# No new task should be created
await monitoring.check_task()
assert monitoring.stats['Hook2']['exception'] == ['Task-exception-retry:0']
assert len(monitoring.queue.created_tasks) == 1
assert bus.queues['testqueue'].qsize() == 1
async def test_parse(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
hook.triggered_groups.add('already-triggered-group')
assert await hook.parse({'taskGroupId': 'already-triggered-group'}) is None
def test_is_coverage_task(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
cov_task = {'task': {'metadata': {'name': 'build-linux64-ccov'}}}
assert hook.is_coverage_task(cov_task)
cov_task = {'task': {'metadata': {'name': 'build-linux64-ccov/opt'}}}
assert hook.is_coverage_task(cov_task)
cov_task = {'task': {'metadata': {'name': 'build-win64-ccov/debug'}}}
assert hook.is_coverage_task(cov_task)
nocov_task = {
'task': {
'metadata': {
'name': 'test-linux64-ccov/opt-mochitest-1'
}
}
}
assert not hook.is_coverage_task(nocov_task)
nocov_task = {
'task': {
'metadata': {
'name': 'test-linux64/opt-mochitest-1'
}
}
}
assert not hook.is_coverage_task(nocov_task)
async def test_monitoring_restartable(QueueMock, IndexMock, mock_taskcluster):
bus = MessageBus()
monitoring = Monitoring('testqueue', ['pinco@pallino'], 1)
monitoring.register(bus)
monitoring.index = IndexMock
monitoring.queue = QueueMock
# Check a failed task is restartable
# when the monitoring flag is set
assert monitoring.is_restartable('Task-failed-restart')
# Check a failed task is not restartable
# when the monitoring flag is not set
assert not monitoring.is_restartable('Task-failed-nope')
# Check a completed task is not restartable
assert not monitoring.is_restartable('Task-completed-restart')
assert not monitoring.is_restartable('Task-completed-nope')
# Check a failed task is restarted by the monitoring flow
assert len(monitoring.queue.created_tasks) == 0
await bus.send('testqueue',
('Group', 'Hook-staticanalysis/bot', 'Task-failed-restart'))
await monitoring.check_task()
assert len(monitoring.queue.created_tasks) == 1
async def test_get_build_task_in_group(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
hook.triggered_groups.add('already-triggered-group')
assert await hook.get_build_task_in_group('already-triggered-group'
) is None
async def test_conversion():
'''
Test message conversion between 2 queues
'''
bus = MessageBus()
bus.add_queue('input')
bus.add_queue(
'output',
maxsize=3) # limit size to immediately stop execution for unit test
assert isinstance(bus.queues['input'], asyncio.Queue)
assert isinstance(bus.queues['output'], asyncio.Queue)
assert bus.queues['input'].qsize() == 0
assert bus.queues['output'].qsize() == 0
await bus.send('input', 'test x')
await bus.send('input', 'hello world.')
await bus.send('output', 'lowercase')
# Convert all strings from input in uppercase
assert bus.queues['input'].qsize() == 2
task = asyncio.create_task(bus.run(lambda x: x.upper(), 'input', 'output'))
await bus.receive('output') == 'lowercase'
await bus.receive('output') == 'TEST X'
await bus.receive('output') == 'HELLO WORLD.'
task.cancel()
assert bus.queues['input'].qsize() == 0
assert bus.queues['output'].qsize() == 0
def test_success(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
assert run_async_parser(hook, 'RS0UwZahQ_qAcdZzEb_Y9g') == [{
'REPOSITORY':
'https://hg.mozilla.org/mozilla-central',
'REVISION':
'ec3dd3ee2ae4b3a63529a912816a110e925eb2d0'
}]
def test_success_windows(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
assert run_async_parser(hook, 'MibGDsa4Q7uFNzDf7EV6nw') == [{
'REPOSITORY':
'https://hg.mozilla.org/mozilla-central',
'REVISION':
'63519bfd42ee379f597c0357af2e712ec3cd9f50'
}]
def test_success_try(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
assert run_async_parser(hook, 'FG3goVnCQfif8ZEOaM_4IA') == [{
'REPOSITORY':
'https://hg.mozilla.org/try',
'REVISION':
'066cb18ba95a7efe144e729713c429e422d9f95b'
}]
async def test_message_passing_mp():
'''
Test sending & receiving messages on a multiprocessing queueu
'''
bus = MessageBus()
bus.add_queue('test', mp=True)
assert isinstance(bus.queues['test'], multiprocessing.queues.Queue)
await bus.send('test', {'payload': 1234})
await bus.send('test', {'another': 'deadbeef'})
await bus.send('test', 'covfefe')
assert bus.queues['test'].qsize() == 3
msg = await bus.receive('test')
assert msg == {'payload': 1234}
msg = await bus.receive('test')
assert msg == {'another': 'deadbeef'}
msg = await bus.receive('test')
assert msg == 'covfefe'
assert bus.queues['test'].qsize() == 0
def test_success_windows(mock_taskcluster):
bus = MessageBus()
with open(os.path.join(FIXTURES_DIR, 'MibGDsa4Q7uFNzDf7EV6nw.json')) as f:
responses.add(responses.GET, 'http://taskcluster.test/queue/v1/task-group/MibGDsa4Q7uFNzDf7EV6nw/list', json=json.load(f), status=200, match_querystring=True) # noqa
hook = CodeCoverage({
'hookId': 'services-staging-codecoverage/bot'
}, bus)
assert hook.parse({
'taskGroupId': 'MibGDsa4Q7uFNzDf7EV6nw'
}) == [{'REPOSITORY': 'https://hg.mozilla.org/mozilla-central', 'REVISION': '63519bfd42ee379f597c0357af2e712ec3cd9f50'}]
def test_wrong_branch(mock_taskcluster):
bus = MessageBus()
with open(os.path.join(FIXTURES_DIR, 'bNq-VIT-Q12o6nXcaUmYNQ.json')) as f:
responses.add(responses.GET, 'http://taskcluster.test/queue/v1/task-group/bNq-VIT-Q12o6nXcaUmYNQ/list', json=json.load(f), status=200, match_querystring=True) # noqa
hook = CodeCoverage({
'hookId': 'services-staging-codecoverage/bot'
}, bus)
assert hook.parse({
'taskGroupId': 'bNq-VIT-Q12o6nXcaUmYNQ'
}) is None
def test_success(mock_taskcluster):
bus = MessageBus()
with open(os.path.join(FIXTURES_DIR, 'RS0UwZahQ_qAcdZzEb_Y9g.json')) as f:
responses.add(responses.GET, 'http://taskcluster.test/queue/v1/task-group/RS0UwZahQ_qAcdZzEb_Y9g/list', json=json.load(f), status=200, match_querystring=True) # noqa
hook = CodeCoverage({
'hookId': 'services-staging-codecoverage/bot'
}, bus)
assert hook.parse({
'taskGroupId': 'RS0UwZahQ_qAcdZzEb_Y9g'
}) == [{'REPOSITORY': 'https://hg.mozilla.org/mozilla-central', 'REVISION': 'ec3dd3ee2ae4b3a63529a912816a110e925eb2d0'}]
def test_success_try(mock_taskcluster):
bus = MessageBus()
with open(os.path.join(FIXTURES_DIR, 'FG3goVnCQfif8ZEOaM_4IA.json')) as f:
responses.add(responses.GET, 'http://taskcluster.test/queue/v1/task-group/FG3goVnCQfif8ZEOaM_4IA/list', json=json.load(f), status=200, match_querystring=True) # noqa
hook = CodeCoverage({
'hookId': 'services-staging-codecoverage/bot'
}, bus)
assert hook.parse({
'taskGroupId': 'FG3goVnCQfif8ZEOaM_4IA'
}) == [{'REPOSITORY': 'https://hg.mozilla.org/try', 'REVISION': '066cb18ba95a7efe144e729713c429e422d9f95b'}]
def test_hook_group(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
assert hook.group_id == 'project-releng'
assert hook.hook_id == 'services-staging-codecoverage/bot'
hook = CodeCoverage(
{
'hookGroupId': 'anotherProject',
'hookId': 'anotherHook',
}, bus)
assert hook.group_id == 'anotherProject'
assert hook.hook_id == 'anotherHook'
def test_queue_creation():
'''
Test queues creation with different types
'''
bus = MessageBus()
assert len(bus.queues) == 0
bus.add_queue('test')
assert len(bus.queues) == 1
with pytest.raises(AssertionError) as e:
bus.add_queue('test')
assert str(e.value) == 'Queue test already setup'
assert len(bus.queues) == 1
bus.add_queue('another')
assert len(bus.queues) == 2
bus.add_queue('different', mp=True)
assert len(bus.queues) == 3
assert isinstance(bus.queues['test'], asyncio.Queue)
assert isinstance(bus.queues['another'], asyncio.Queue)
assert isinstance(bus.queues['different'], multiprocessing.queues.Queue)
async def test_report_all_completed(QueueMock, NotifyMock, mock_taskcluster):
bus = MessageBus()
monitoring = Monitoring('testqueue', ['pinco@pallino'], 1)
monitoring.register(bus)
await bus.send('testqueue', ('Group1', 'Hook1', 'Task1-completed'))
await bus.send('testqueue', ('Group1', 'Hook1', 'Task2-completed'))
assert bus.queues['testqueue'].qsize() == 2
monitoring.queue = QueueMock
monitoring.notify = NotifyMock
await monitoring.check_task()
await monitoring.check_task()
# No email sent, since all tasks were successful.
monitoring.send_report()
assert NotifyMock.email_obj == {}
assert monitoring.stats == {}
async def test_create_task(HooksMock, QueueMock, mock_taskcluster):
bus = MessageBus()
bus.add_queue(QUEUE_MONITORING, maxsize=1)
bus.add_queue(QUEUE_PULSE_CODECOV)
conf = {
'hookGroupId': 'aGroup',
'hookId': 'aHook',
}
hook = CodeCoverage(conf, bus)
hook.hooks = HooksMock
hook.queue = QueueMock
# Add a dummy task in the target group
QueueMock.add_task_in_group('aGroup',
name='whatever',
env={'key': 'value'})
# Add a coverage task in the target group
env = {
'GECKO_HEAD_REPOSITORY': 'https://hg.mozilla.org/mozilla-central',
'GECKO_HEAD_REV': 'deadbeef',
}
QueueMock.add_task_in_group('aGroup',
name='build-linux64-ccov/test',
env=env)
# Send a pulse message with the target group
pulse_payload = {
'taskGroupId': 'aGroup',
}
await bus.send(QUEUE_PULSE_CODECOV, pulse_payload)
# Run the code coverage event listener as a task
task = asyncio.create_task(hook.run())
# Stop as soon as a message is sent to monitoring
group_id, hook_id, task_id = await bus.queues[QUEUE_MONITORING].get()
task.cancel()
assert bus.queues[QUEUE_MONITORING].qsize() == 0
assert group_id == 'aGroup'
assert hook_id == 'aHook'
assert task_id == 'fake_task_id'
assert HooksMock.obj['group_id'] == 'aGroup'
assert HooksMock.obj['hook_id'] == 'aHook'
assert HooksMock.obj['payload'] == {
'REPOSITORY': 'https://hg.mozilla.org/mozilla-central',
'REVISION': 'deadbeef',
}
async def test_risk_analysis_should_trigger(PhabricatorMock, mock_taskcluster):
bus = MessageBus()
build = PhabricatorBuild(
MockRequest(diff='125397',
repo='PHID-REPO-saax4qdxlbbhahhp2kg5',
revision='36474',
target='PHID-HMBT-icusvlfibcebizyd33op'))
with PhabricatorMock as phab:
client = CodeReview(
risk_analysis_reviewers=['ehsan', 'heycam'],
url='http://phabricator.test/api/',
api_key='fakekey',
)
client.register(bus)
phab.update_state(build)
phab.load_reviewers(build)
assert client.should_run_risk_analysis(build)
async def test_maxsize():
'''
Test a queue maxsize behaves as expected
Maxsize=-1 is enabled by default
'''
bus = MessageBus()
bus.add_queue('async')
bus.add_queue('mp', mp=True)
assert bus.queues['async'].maxsize == -1
# No maxsize getter on mp queues
assert bus.queues['async'].empty()
assert bus.queues['mp'].empty()
for i in range(1000):
await bus.send('async', i)
await bus.send('mp', i)
assert not bus.queues['async'].full()
assert not bus.queues['mp'].full()
async def test_monitoring(QueueMock, NotifyMock, mock_taskcluster):
bus = MessageBus()
monitoring = Monitoring('testqueue', ['pinco@pallino'], 1)
monitoring.register(bus)
await bus.send('testqueue', ('Group1', 'Hook1', 'Task-invalid'))
await bus.send('testqueue', ('Group1', 'Hook1', 'Task-pending'))
await bus.send('testqueue', ('Group1', 'Hook1', 'Task1-completed'))
await bus.send('testqueue', ('Group1', 'Hook1', 'Task2-completed'))
await bus.send('testqueue', ('Group1', 'Hook2', 'Task-exception'))
await bus.send('testqueue', ('Group2', 'Hook1', 'Task-failed'))
assert bus.queues['testqueue'].qsize() == 6
monitoring.queue = QueueMock
monitoring.notify = NotifyMock
# No report sent, since we haven't collected any stats yet.
monitoring.send_report()
assert NotifyMock.email_obj == {}
# Queue throws exception, remove task from queue.
await monitoring.check_task()
assert bus.queues['testqueue'].qsize() == 5
# Task is pending, put it back in the queue.
await monitoring.check_task()
assert bus.queues['testqueue'].qsize() == 5
# No report sent, since we haven't collected any stats yet.
monitoring.send_report()
assert NotifyMock.email_obj == {}
# Task is completed.
await monitoring.check_task()
assert monitoring.stats['Hook1']['completed'] == ['Task1-completed']
assert bus.queues['testqueue'].qsize() == 4
# Another task is completed.
await monitoring.check_task()
assert monitoring.stats['Hook1']['completed'] == [
'Task1-completed', 'Task2-completed'
]
assert bus.queues['testqueue'].qsize() == 3
# Task exception.
assert len(monitoring.queue.created_tasks) == 0
await monitoring.check_task()
assert monitoring.stats['Hook1']['exception'] == []
assert monitoring.stats['Hook2']['exception'] == ['Task-exception']
# A new task has been retried, replacing the exception
assert len(monitoring.queue.created_tasks) == 1
assert bus.queues['testqueue'].qsize() == 3
# Task failed.
await monitoring.check_task()
assert monitoring.stats['Hook1']['failed'] == ['Task-failed']
assert bus.queues['testqueue'].qsize() == 2
# Task is pending, put it back in the queue.
await monitoring.check_task()
assert bus.queues['testqueue'].qsize() == 2
content = '''# Hook1 tasks for the last period
## completed
66.67% of all tasks (2/3)
* [Task1-completed](https://tools.taskcluster.net/task-inspector/#Task1-completed)
* [Task2-completed](https://tools.taskcluster.net/task-inspector/#Task2-completed)
## exception
0.00% of all tasks (0/3)
## failed
33.33% of all tasks (1/3)
* [Task-failed](https://tools.taskcluster.net/task-inspector/#Task-failed)
# Hook2 tasks for the last period
## completed
0.00% of all tasks (0/1)
## exception
100.00% of all tasks (1/1)
* [Task-exception](https://tools.taskcluster.net/task-inspector/#Task-exception)
## failed
0.00% of all tasks (0/1)
'''
monitoring.send_report()
assert NotifyMock.email_obj['address'] == 'pinco@pallino'
assert NotifyMock.email_obj['subject'] == 'Pulse listener tasks'
assert NotifyMock.email_obj['content'] == content
assert NotifyMock.email_obj['template'] == 'fullscreen'
assert monitoring.stats == {}
async def test_push_to_try_existing_rev(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
with a push to try server
but applying on an existing revision
'''
bus = MessageBus()
bus.add_queue('phabricator')
repo_dir = mock_mc.repo.root().decode('utf-8')
def _readme(content):
# Make a commit on README.md in the repo
readme = os.path.join(repo_dir, 'README.md')
with open(readme, 'a') as f:
f.write(content)
_, rev = mock_mc.repo.commit(message=content.encode('utf-8'), user=b'test')
return rev
# Make two commits, the first one is our base
base = _readme('Local base for diffs')
extra = _readme('EXTRA')
# Preload the build
diff = {
'phid': 'PHID-DIFF-solo',
'revisionPHID': 'PHID-DREV-solo',
'id': 9876,
# Revision does not exist, will apply on tip
'baseRevision': base,
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-HMBT-deadbeef', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
# The patched and config files should not exist at first
target = os.path.join(repo_dir, 'solo.txt')
config = os.path.join(repo_dir, 'try_task_config.json')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc},
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
tip = mock_mc.repo.tip()
assert mode == 'success'
assert out_build == build
assert details['treeherder_url'] == 'https://treeherder.mozilla.org/#/jobs?repo=try&revision={}'.format(tip.node.decode('utf-8'))
task.cancel()
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'Solo PATCH\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-HMBT-deadbeef',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches and config have applied)
assert tip.node != base
assert tip.desc == b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-solo'
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(mock_mc.ssh_key_path)
mock_mc.repo.push.assert_called_with(
dest=b'http://mozilla-central/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
# Check the parent is the solo patch commit
parents = mock_mc.repo.parents(tip.node)
assert len(parents) == 1
parent = parents[0]
assert parent.desc == b'A nice human readable commit message\nDifferential Diff: PHID-DIFF-solo'
# Check the grand parent is the base, not extra
great_parents = mock_mc.repo.parents(parent.node)
assert len(great_parents) == 1
great_parent = great_parents[0]
assert great_parent.node == base
# Extra commit should not appear
assert parent.node != extra
assert great_parent.node != extra
assert 'EXTRA' not in open(os.path.join(repo_dir, 'README.md')).read()
def test_wrong_branch(mock_taskcluster):
bus = MessageBus()
hook = CodeCoverage({'hookId': 'services-staging-codecoverage/bot'}, bus)
assert run_async_parser(hook, 'bNq-VIT-Q12o6nXcaUmYNQ') is None
async def test_monitoring_whiteline_between_failed_and_hook(
QueueMock, NotifyMock, mock_taskcluster):
bus = MessageBus()
monitoring = Monitoring('testqueue', ['pinco@pallino'], 1)
monitoring.register(bus)
await bus.send('testqueue', ('Group1', 'Hook1', 'Task-failed'))
await bus.send('testqueue', ('Group1', 'Hook2', 'Task-failed'))
assert bus.queues['testqueue'].qsize() == 2
monitoring.queue = QueueMock
monitoring.notify = NotifyMock
# Task exception.
await monitoring.check_task()
assert monitoring.stats['Hook1']['failed'] == ['Task-failed']
assert bus.queues['testqueue'].qsize() == 1
# Task failed.
await monitoring.check_task()
assert monitoring.stats['Hook2']['failed'] == ['Task-failed']
assert bus.queues['testqueue'].qsize() == 0
content = '''# Hook1 tasks for the last period
## completed
0.00% of all tasks (0/1)
## exception
0.00% of all tasks (0/1)
## failed
100.00% of all tasks (1/1)
* [Task-failed](https://tools.taskcluster.net/task-inspector/#Task-failed)
# Hook2 tasks for the last period
## completed
0.00% of all tasks (0/1)
## exception
0.00% of all tasks (0/1)
## failed
100.00% of all tasks (1/1)
* [Task-failed](https://tools.taskcluster.net/task-inspector/#Task-failed)'''
monitoring.send_report()
assert NotifyMock.email_obj['address'] == 'pinco@pallino'
assert NotifyMock.email_obj['subject'] == 'Pulse listener tasks'
assert NotifyMock.email_obj['content'] == content
assert NotifyMock.email_obj['template'] == 'fullscreen'
assert monitoring.stats == {}
def __init__(self, cache_root):
# Create message bus shared amongst process
self.bus = MessageBus()
# Build client applications configuration
# TODO: use simpler secret structure per client
clients_conf = {h['type']: h for h in taskcluster.secrets['HOOKS']}
code_review_conf = clients_conf.get('static-analysis-phabricator')
code_coverage_conf = clients_conf.get('code-coverage')
# Code Review Workflow
if code_review_conf:
self.code_review = CodeReview(
api_key=taskcluster.secrets['PHABRICATOR']['token'],
url=taskcluster.secrets['PHABRICATOR']['url'],
publish=taskcluster.secrets['PHABRICATOR'].get(
'publish', False),
risk_analysis_reviewers=code_review_conf.get(
'risk_analysis_reviewers', []))
self.code_review.register(self.bus)
# Build mercurial worker & queue
self.mercurial = MercurialWorker(
QUEUE_MERCURIAL,
QUEUE_PHABRICATOR_RESULTS,
repositories=self.code_review.get_repositories(
taskcluster.secrets['repositories'], cache_root),
)
self.mercurial.register(self.bus)
# Create web server
self.webserver = WebServer(QUEUE_WEB_BUILDS)
self.webserver.register(self.bus)
else:
self.code_review = None
self.mercurial = None
self.webserver = None
# Code Coverage Workflow
if code_coverage_conf:
self.code_coverage = CodeCoverage(code_coverage_conf, self.bus)
# Setup monitoring for newly created tasks
self.monitoring = Monitoring(QUEUE_MONITORING,
taskcluster.secrets['ADMINS'],
7 * 3600)
self.monitoring.register(self.bus)
# Create pulse listener for code coverage
self.pulse = PulseListener(
QUEUE_PULSE_CODECOV,
'exchange/taskcluster-queue/v1/task-group-resolved',
'#',
taskcluster.secrets['PULSE_USER'],
taskcluster.secrets['PULSE_PASSWORD'],
)
self.pulse.register(self.bus)
else:
self.code_coverage = None
self.monitoring = None
self.pulse = None
assert self.code_review or self.code_coverage, 'No client applications to run !'
class EventListener(object):
'''
Listen to external events and trigger new tasks
'''
def __init__(self, cache_root):
# Create message bus shared amongst process
self.bus = MessageBus()
# Build client applications configuration
# TODO: use simpler secret structure per client
clients_conf = {h['type']: h for h in taskcluster.secrets['HOOKS']}
code_review_conf = clients_conf.get('static-analysis-phabricator')
code_coverage_conf = clients_conf.get('code-coverage')
# Code Review Workflow
if code_review_conf:
self.code_review = CodeReview(
api_key=taskcluster.secrets['PHABRICATOR']['token'],
url=taskcluster.secrets['PHABRICATOR']['url'],
publish=taskcluster.secrets['PHABRICATOR'].get(
'publish', False),
risk_analysis_reviewers=code_review_conf.get(
'risk_analysis_reviewers', []))
self.code_review.register(self.bus)
# Build mercurial worker & queue
self.mercurial = MercurialWorker(
QUEUE_MERCURIAL,
QUEUE_PHABRICATOR_RESULTS,
repositories=self.code_review.get_repositories(
taskcluster.secrets['repositories'], cache_root),
)
self.mercurial.register(self.bus)
# Create web server
self.webserver = WebServer(QUEUE_WEB_BUILDS)
self.webserver.register(self.bus)
else:
self.code_review = None
self.mercurial = None
self.webserver = None
# Code Coverage Workflow
if code_coverage_conf:
self.code_coverage = CodeCoverage(code_coverage_conf, self.bus)
# Setup monitoring for newly created tasks
self.monitoring = Monitoring(QUEUE_MONITORING,
taskcluster.secrets['ADMINS'],
7 * 3600)
self.monitoring.register(self.bus)
# Create pulse listener for code coverage
self.pulse = PulseListener(
QUEUE_PULSE_CODECOV,
'exchange/taskcluster-queue/v1/task-group-resolved',
'#',
taskcluster.secrets['PULSE_USER'],
taskcluster.secrets['PULSE_PASSWORD'],
)
self.pulse.register(self.bus)
else:
self.code_coverage = None
self.monitoring = None
self.pulse = None
assert self.code_review or self.code_coverage, 'No client applications to run !'
def run(self):
consumers = []
if self.code_review:
consumers += [
# Code review main workflow
self.code_review.run(),
# Add mercurial task
self.mercurial.run(),
]
# Publish results on Phabricator
if self.code_review.publish:
consumers.append(
self.bus.run(self.code_review.publish_results,
QUEUE_PHABRICATOR_RESULTS))
# Start the web server in its own process
self.webserver.start()
if self.code_coverage:
consumers += [
# Code coverage main workflow
self.code_coverage.run(),
# Add monitoring task
self.monitoring.run(),
# Add pulse task
self.pulse.run(),
]
# Run all tasks concurrently
run_tasks(consumers)
# Stop the webserver when other async process are stopped
if self.webserver:
self.webserver.stop()
async def test_push_to_try_nss(PhabricatorMock, mock_nss):
'''
Run mercurial worker on a single diff
with a push to try server, but with NSS support (try syntax)
'''
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
# Revision does not exist, will apply on tip
'baseRevision': 'abcdef12345',
}
build = MockBuild(1234, 'PHID-REPO-nss', 5678, 'PHID-HMBT-deadbeef', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_nss.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_nss.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-nss': mock_nss}
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
tip = mock_nss.repo.tip()
assert mode == 'success'
assert out_build == build
assert details['treeherder_url'] == 'https://treeherder.mozilla.org/#/jobs?repo=try&revision={}'.format(tip.node.decode('utf-8'))
task.cancel()
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'First Line\nSecond Line\n'
# The config should have content now
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'code-review': {
'phabricator-build-target': 'PHID-HMBT-deadbeef',
}
},
}
# Get tip commit in repo
# It should be different from the initial one (patches + config have applied)
assert tip.node != initial.node
# Check all commits messages
assert [c.desc for c in mock_nss.repo.log()] == [
b'try: -a -b XXX -c YYY',
b'Bug XXX - A second commit message\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A first commit message\nDifferential Diff: PHID-DIFF-xxxx',
b'Readme'
]
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(mock_nss.ssh_key_path)
mock_nss.repo.push.assert_called_with(
dest=b'http://nss/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)