async def test_conversion():
'''
Test message conversion between 2 queues
'''
bus = MessageBus()
bus.add_queue('input')
bus.add_queue(
'output',
maxsize=3) # limit size to immediately stop execution for unit test
assert isinstance(bus.queues['input'], asyncio.Queue)
assert isinstance(bus.queues['output'], asyncio.Queue)
assert bus.queues['input'].qsize() == 0
assert bus.queues['output'].qsize() == 0
await bus.send('input', 'test x')
await bus.send('input', 'hello world.')
await bus.send('output', 'lowercase')
# Convert all strings from input in uppercase
assert bus.queues['input'].qsize() == 2
task = asyncio.create_task(bus.run(lambda x: x.upper(), 'input', 'output'))
await bus.receive('output') == 'lowercase'
await bus.receive('output') == 'TEST X'
await bus.receive('output') == 'HELLO WORLD.'
task.cancel()
assert bus.queues['input'].qsize() == 0
assert bus.queues['output'].qsize() == 0
async def test_failure_general(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
Use a Python common exception to trigger a broken build
'''
diff = {
'phid': 'PHID-DIFF-test123',
'id': 1234,
'baseRevision': None,
'revisionPHID': 'PHID-DREV-deadbeef'
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-somehash', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_mc.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
# Raise an exception during the workflow to trigger a broken build
def boom(*args):
raise Exception('Boom')
mock_mc.apply_build = boom
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc}
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the unit result was published
mode, out_build, details = await bus.receive('phabricator')
assert mode == 'fail:general'
assert out_build == build
assert details['duration'] > 0
assert details['message'] == 'Boom'
task.cancel()
# Clone should not be modified
tip = mock_mc.repo.tip()
assert tip.node == initial.node
async def test_failure_mercurial(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
Apply a bad mercurial patch to trigger a mercurial fail
'''
diff = {
'revisionPHID': 'PHID-DREV-666',
'baseRevision': 'missing',
'phid': 'PHID-DIFF-666',
'id': 666,
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-build-666', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_mc.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc}
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
assert mode == 'fail:mercurial'
assert out_build == build
assert details['duration'] > 0
assert details['message'] == MERCURIAL_FAILURE
task.cancel()
# Clone should not be modified
tip = mock_mc.repo.tip()
assert tip.node == initial.node
async def test_create_task(HooksMock, QueueMock, mock_taskcluster):
bus = MessageBus()
bus.add_queue(QUEUE_MONITORING, maxsize=1)
bus.add_queue(QUEUE_PULSE_CODECOV)
conf = {
'hookGroupId': 'aGroup',
'hookId': 'aHook',
}
hook = CodeCoverage(conf, bus)
hook.hooks = HooksMock
hook.queue = QueueMock
# Add a dummy task in the target group
QueueMock.add_task_in_group('aGroup',
name='whatever',
env={'key': 'value'})
# Add a coverage task in the target group
env = {
'GECKO_HEAD_REPOSITORY': 'https://hg.mozilla.org/mozilla-central',
'GECKO_HEAD_REV': 'deadbeef',
}
QueueMock.add_task_in_group('aGroup',
name='build-linux64-ccov/test',
env=env)
# Send a pulse message with the target group
pulse_payload = {
'taskGroupId': 'aGroup',
}
await bus.send(QUEUE_PULSE_CODECOV, pulse_payload)
# Run the code coverage event listener as a task
task = asyncio.create_task(hook.run())
# Stop as soon as a message is sent to monitoring
group_id, hook_id, task_id = await bus.queues[QUEUE_MONITORING].get()
task.cancel()
assert bus.queues[QUEUE_MONITORING].qsize() == 0
assert group_id == 'aGroup'
assert hook_id == 'aHook'
assert task_id == 'fake_task_id'
assert HooksMock.obj['group_id'] == 'aGroup'
assert HooksMock.obj['hook_id'] == 'aHook'
assert HooksMock.obj['payload'] == {
'REPOSITORY': 'https://hg.mozilla.org/mozilla-central',
'REVISION': 'deadbeef',
}
async def test_treeherder_link(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
'''
# Preload the build
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
'baseRevision': 'abcdef12345',
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-HMBT-somehash', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_mc.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc},
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
tip = mock_mc.repo.tip()
assert mode == 'success'
assert out_build == build
assert details['treeherder_url'] == 'https://treeherder.mozilla.org/#/jobs?repo=try&revision={}'.format(tip.node.decode('utf-8'))
task.cancel()
# Tip should be updated
assert tip.node != initial.node
async def test_message_passing_mp():
'''
Test sending & receiving messages on a multiprocessing queueu
'''
bus = MessageBus()
bus.add_queue('test', mp=True)
assert isinstance(bus.queues['test'], multiprocessing.queues.Queue)
await bus.send('test', {'payload': 1234})
await bus.send('test', {'another': 'deadbeef'})
await bus.send('test', 'covfefe')
assert bus.queues['test'].qsize() == 3
msg = await bus.receive('test')
assert msg == {'payload': 1234}
msg = await bus.receive('test')
assert msg == {'another': 'deadbeef'}
msg = await bus.receive('test')
assert msg == 'covfefe'
assert bus.queues['test'].qsize() == 0
async def test_maxsize():
'''
Test a queue maxsize behaves as expected
Maxsize=-1 is enabled by default
'''
bus = MessageBus()
bus.add_queue('async')
bus.add_queue('mp', mp=True)
assert bus.queues['async'].maxsize == -1
# No maxsize getter on mp queues
assert bus.queues['async'].empty()
assert bus.queues['mp'].empty()
for i in range(1000):
await bus.send('async', i)
await bus.send('mp', i)
assert not bus.queues['async'].full()
assert not bus.queues['mp'].full()
def test_queue_creation():
'''
Test queues creation with different types
'''
bus = MessageBus()
assert len(bus.queues) == 0
bus.add_queue('test')
assert len(bus.queues) == 1
with pytest.raises(AssertionError) as e:
bus.add_queue('test')
assert str(e.value) == 'Queue test already setup'
assert len(bus.queues) == 1
bus.add_queue('another')
assert len(bus.queues) == 2
bus.add_queue('different', mp=True)
assert len(bus.queues) == 3
assert isinstance(bus.queues['test'], asyncio.Queue)
assert isinstance(bus.queues['another'], asyncio.Queue)
assert isinstance(bus.queues['different'], multiprocessing.queues.Queue)
async def test_push_to_try_nss(PhabricatorMock, mock_nss):
'''
Run mercurial worker on a single diff
with a push to try server, but with NSS support (try syntax)
'''
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
# Revision does not exist, will apply on tip
'baseRevision': 'abcdef12345',
}
build = MockBuild(1234, 'PHID-REPO-nss', 5678, 'PHID-HMBT-deadbeef', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
bus = MessageBus()
bus.add_queue('phabricator')
# Get initial tip commit in repo
initial = mock_nss.repo.tip()
# The patched and config files should not exist at first
repo_dir = mock_nss.repo.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-nss': mock_nss}
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
tip = mock_nss.repo.tip()
assert mode == 'success'
assert out_build == build
assert details['treeherder_url'] == 'https://treeherder.mozilla.org/#/jobs?repo=try&revision={}'.format(tip.node.decode('utf-8'))
task.cancel()
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'First Line\nSecond Line\n'
# The config should have content now
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'code-review': {
'phabricator-build-target': 'PHID-HMBT-deadbeef',
}
},
}
# Get tip commit in repo
# It should be different from the initial one (patches + config have applied)
assert tip.node != initial.node
# Check all commits messages
assert [c.desc for c in mock_nss.repo.log()] == [
b'try: -a -b XXX -c YYY',
b'Bug XXX - A second commit message\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A first commit message\nDifferential Diff: PHID-DIFF-xxxx',
b'Readme'
]
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(mock_nss.ssh_key_path)
mock_nss.repo.push.assert_called_with(
dest=b'http://nss/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
async def test_push_to_try_existing_rev(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
with a push to try server
but applying on an existing revision
'''
bus = MessageBus()
bus.add_queue('phabricator')
repo_dir = mock_mc.repo.root().decode('utf-8')
def _readme(content):
# Make a commit on README.md in the repo
readme = os.path.join(repo_dir, 'README.md')
with open(readme, 'a') as f:
f.write(content)
_, rev = mock_mc.repo.commit(message=content.encode('utf-8'), user=b'test')
return rev
# Make two commits, the first one is our base
base = _readme('Local base for diffs')
extra = _readme('EXTRA')
# Preload the build
diff = {
'phid': 'PHID-DIFF-solo',
'revisionPHID': 'PHID-DREV-solo',
'id': 9876,
# Revision does not exist, will apply on tip
'baseRevision': base,
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-HMBT-deadbeef', diff)
with PhabricatorMock as phab:
phab.load_patches_stack(build)
# The patched and config files should not exist at first
target = os.path.join(repo_dir, 'solo.txt')
config = os.path.join(repo_dir, 'try_task_config.json')
assert not os.path.exists(target)
assert not os.path.exists(config)
worker = MercurialWorker(
'mercurial',
'phabricator',
repositories={'PHID-REPO-mc': mock_mc},
)
worker.register(bus)
assert len(worker.repositories) == 1
await bus.send('mercurial', build)
assert bus.queues['mercurial'].qsize() == 1
task = asyncio.create_task(worker.run())
# Check the treeherder link was queued
mode, out_build, details = await bus.receive('phabricator')
tip = mock_mc.repo.tip()
assert mode == 'success'
assert out_build == build
assert details['treeherder_url'] == 'https://treeherder.mozilla.org/#/jobs?repo=try&revision={}'.format(tip.node.decode('utf-8'))
task.cancel()
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'Solo PATCH\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-HMBT-deadbeef',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches and config have applied)
assert tip.node != base
assert tip.desc == b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-solo'
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(mock_mc.ssh_key_path)
mock_mc.repo.push.assert_called_with(
dest=b'http://mozilla-central/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
# Check the parent is the solo patch commit
parents = mock_mc.repo.parents(tip.node)
assert len(parents) == 1
parent = parents[0]
assert parent.desc == b'A nice human readable commit message\nDifferential Diff: PHID-DIFF-solo'
# Check the grand parent is the base, not extra
great_parents = mock_mc.repo.parents(parent.node)
assert len(great_parents) == 1
great_parent = great_parents[0]
assert great_parent.node == base
# Extra commit should not appear
assert parent.node != extra
assert great_parent.node != extra
assert 'EXTRA' not in open(os.path.join(repo_dir, 'README.md')).read()