def __init__(
self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
)
def __init__(
self,
pulse_user,
pulse_password,
hooks_configuration,
repositories,
phabricator_api,
cache_root,
publish_phabricator=False,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue
self.mercurial = MercurialWorker(
self.phabricator_api,
publish_phabricator=publish_phabricator,
repositories=repositories,
cache_root=cache_root,
)
# Create web server
self.webserver = WebServer()
async def test_push_to_try(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
with a push to try server
'''
# Get initial tip commit in repo
initial = RepoMock.tip()
# The patched file should not exists at first
repo_dir = RepoMock.root().decode('utf-8')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
)
worker.repo = RepoMock
await worker.handle_diff({
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
# Revision does not exist, will apply on tip
'baseRevision': 'abcdef12345',
})
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'First Line\nSecond Line\n'
# Get tip commit in repo
# It should be different from the initial one (patches have applied)
tip = RepoMock.tip()
assert tip.node != initial.node
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(
worker.ssh_key_path)
RepoMock.push.assert_called_with(
dest=b'ssh://hg.mozilla.org/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
async def test_treeherder_link(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
'''
# Get initial tip commit in repo
initial = RepoMock.tip()
# The patched and config files should not exist at first
repo_dir = RepoMock.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
batch_size=100,
publish_phabricator=True,
)
worker.repo = RepoMock
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
'baseRevision': 'abcdef12345',
}
await worker.push_to_try('PHID-HMBT-somehash', diff)
# Check the treeherder link was published
assert api.mocks.calls[
-1].request.url == 'http://phabricator.test/api/harbormaster.createartifact'
assert api.mocks.calls[-1].response.status_code == 200
# Tip should be updated
tip = RepoMock.tip()
assert tip.node != initial.node
def __init__(
self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
self.http_port = int(os.environ.get('PORT', 9000))
logger.info('HTTP webhook server will listen', port=self.http_port)
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
if mercurial_conf.get('enabled', False):
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
batch_size=mercurial_conf.get('batch_size', 100000),
publish_phabricator=mercurial_conf.get('publish_phabricator',
False),
)
else:
self.mercurial = None
logger.info('Mercurial worker is disabled')
async def test_treeherder_link(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
'''
# Get initial tip commit in repo
initial = mock_mc.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
repositories=[
{
'name': 'mozilla-central',
'ssh_user': '******',
'ssh_key': 'privateSSHkey',
'url': 'http://mozilla-central',
'try_url': 'http://mozilla-central/try',
'batch_size': 100,
}
],
publish_phabricator=True,
cache_root=os.path.dirname(repo_dir),
)
assert len(worker.repositories) == 1
repo = worker.repositories.get('PHID-REPO-mc')
assert repo is not None
repo.repo = mock_mc
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
'baseRevision': 'abcdef12345',
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-HMBT-somehash', diff)
await worker.handle_build(repo, build)
# Check the treeherder link was published
assert api.mocks.calls[-1].request.url == 'http://phabricator.test/api/harbormaster.createartifact'
assert api.mocks.calls[-1].response.status_code == 200
# Tip should be updated
tip = mock_mc.tip()
assert tip.node != initial.node
def __init__(self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
self.http_port = int(os.environ.get('PORT', 9000))
logger.info('HTTP webhook server will listen', port=self.http_port)
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
if mercurial_conf.get('enabled', False):
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
batch_size=mercurial_conf.get('batch_size', 100000),
)
else:
self.mercurial = None
logger.info('Mercurial worker is disabled')
class PulseListener(object):
'''
Listen to pulse messages and trigger new tasks
'''
def __init__(
self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
self.http_port = int(os.environ.get('PORT', 9000))
logger.info('HTTP webhook server will listen', port=self.http_port)
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
if mercurial_conf.get('enabled', False):
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
batch_size=mercurial_conf.get('batch_size', 100000),
publish_phabricator=mercurial_conf.get('publish_phabricator',
False),
)
else:
self.mercurial = None
logger.info('Mercurial worker is disabled')
def run(self):
# Build hooks for each conf
hooks = [self.build_hook(conf) for conf in self.hooks_configuration]
if not hooks:
raise Exception('No hooks created')
# Run hooks pulse listeners together
# but only use hooks with active definitions
def _connect(hook):
out = hook.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
if self.mercurial is not None:
out &= hook.connect_mercurial_queue(self.mercurial.queue)
return out
consumers = [
hook.build_consumer(self.pulse_user, self.pulse_password)
for hook in hooks if _connect(hook)
]
# Add monitoring process
consumers.append(task_monitoring.run())
# Add mercurial process
if self.mercurial is not None:
consumers.append(self.mercurial.run())
# Hooks through web server
consumers.append(self.build_webserver(hooks))
# Run all consumers together
run_consumer(asyncio.gather(*consumers))
def build_hook(self, conf):
'''
Build a new hook instance according to configuration
'''
assert isinstance(conf, dict)
assert 'type' in conf
conf['phabricator_api'] = self.phabricator_api
classes = {
'static-analysis-phabricator': HookPhabricator,
'code-coverage': HookCodeCoverage,
}
hook_class = classes.get(conf['type'])
if hook_class is None:
raise Exception('Unsupported hook {}'.format(conf['type']))
return hook_class(conf)
def build_webserver(self, hooks):
'''
Build an async web server used by hooks
'''
app = web.Application()
# Always add a simple test endpoint
async def ping(request):
return web.Response(text='pong')
app.add_routes([web.get('/ping', ping)])
# Add routes from hooks
for hook in hooks:
app.add_routes(hook.routes)
# Finally build the webserver coroutine
return web._run_app(app, port=self.http_port, print=logger.info)
def add_build(self, build_target_phid):
'''
Fetch a phabricator build and push it in the mercurial queue
'''
assert build_target_phid.startswith('PHID-HMBT-')
if self.mercurial is None:
logger.warn('Skip adding build, mercurial worker is disabled',
build=build_target_phid)
return
# Load the diff from the target
container = self.phabricator_api.find_target_buildable(
build_target_phid)
diffs = self.phabricator_api.search_diffs(
diff_phid=container['fields']['objectPHID'])
loop = asyncio.get_event_loop()
loop.run_until_complete(
self.mercurial.queue.put((build_target_phid, diffs[0])))
logger.info('Pushed build in queue', build=build_target_phid)
class PulseListener(object):
'''
Listen to pulse messages and trigger new tasks
'''
def __init__(self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
if mercurial_conf.get('enabled', False):
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
batch_size=mercurial_conf.get('batch_size', 100000),
publish_phabricator=mercurial_conf.get('publish_phabricator', False),
)
else:
self.mercurial = None
logger.info('Mercurial worker is disabled')
# Create web server
self.webserver = WebServer()
def run(self):
# Build hooks for each conf
hooks = [
self.build_hook(conf)
for conf in self.hooks_configuration
]
if not hooks:
raise Exception('No hooks created')
# Run hooks pulse listeners together
# but only use hooks with active definitions
def _connect(hook):
out = hook.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
out &= hook.connect_queues(
mercurial_queue=self.mercurial.queue if self.mercurial else None,
web_queue=self.webserver.queue,
)
return out
consumers = [
hook.build_consumer(self.pulse_user, self.pulse_password)
for hook in hooks
if _connect(hook)
]
# Add monitoring task
consumers.append(task_monitoring.run())
# Add mercurial task
if self.mercurial is not None:
consumers.append(self.mercurial.run())
# Start the web server in its own process
web_process = self.webserver.start()
# Run all tasks concurrently
run_consumer(asyncio.gather(*consumers))
web_process.join()
def build_hook(self, conf):
'''
Build a new hook instance according to configuration
'''
assert isinstance(conf, dict)
assert 'type' in conf
conf['phabricator_api'] = self.phabricator_api
classes = {
'static-analysis-phabricator': HookPhabricator,
'code-coverage': HookCodeCoverage,
}
hook_class = classes.get(conf['type'])
if hook_class is None:
raise Exception('Unsupported hook {}'.format(conf['type']))
return hook_class(conf)
def add_build(self, build_target_phid):
'''
Fetch a phabricator build and push it in the mercurial queue
'''
assert build_target_phid.startswith('PHID-HMBT-')
if self.mercurial is None:
logger.warn('Skip adding build, mercurial worker is disabled', build=build_target_phid)
return
# Load the diff from the target
container = self.phabricator_api.find_target_buildable(build_target_phid)
diffs = self.phabricator_api.search_diffs(diff_phid=container['fields']['objectPHID'])
loop = asyncio.get_event_loop()
loop.run_until_complete(self.mercurial.queue.put((build_target_phid, diffs[0])))
logger.info('Pushed build in queue', build=build_target_phid)
async def test_push_to_try_existing_rev(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
with a push to try server
but applying on an existing revision
'''
repo_dir = RepoMock.root().decode('utf-8')
def _readme(content):
# Make a commit on README.md in the repo
readme = os.path.join(repo_dir, 'README.md')
with open(readme, 'a') as f:
f.write(content)
_, rev = RepoMock.commit(message=content.encode('utf-8'), user=b'test')
return rev
# Make two commits, the first one is our base
base = _readme('Local base for diffs')
extra = _readme('EXTRA')
# The patched and config files should not exist at first
target = os.path.join(repo_dir, 'solo.txt')
config = os.path.join(repo_dir, 'try_task_config.json')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
batch_size=100,
)
worker.repo = RepoMock
await worker.handle_diff({
'phid': 'PHID-DIFF-solo',
'revisionPHID': 'PHID-DREV-solo',
'id': 9876,
# Revision does not exist, will apply on tip
'baseRevision': base,
})
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'Solo PATCH\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-DIFF-solo',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches and config have applied)
tip = RepoMock.tip()
assert tip.node != base
assert tip.desc == b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-solo'
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(
worker.ssh_key_path)
RepoMock.push.assert_called_with(
dest=b'ssh://hg.mozilla.org/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
# Check the parent is the solo patch commit
parents = RepoMock.parents(tip.node)
assert len(parents) == 1
parent = parents[0]
assert parent.desc == b'A nice human readable commit message\nDifferential Diff: PHID-DIFF-solo'
# Check the grand parent is the base, not extra
great_parents = RepoMock.parents(parent.node)
assert len(great_parents) == 1
great_parent = great_parents[0]
assert great_parent.node == base
# Extra commit should not appear
assert parent.node != extra
assert great_parent.node != extra
assert 'EXTRA' not in open(os.path.join(repo_dir, 'README.md')).read()
async def test_push_to_try(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
with a push to try server
'''
# Get initial tip commit in repo
initial = RepoMock.tip()
# The patched and config files should not exist at first
repo_dir = RepoMock.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
batch_size=100,
)
worker.repo = RepoMock
await worker.handle_diff({
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
# Revision does not exist, will apply on tip
'baseRevision': 'abcdef12345',
})
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'First Line\nSecond Line\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-DIFF-test123',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches + config have applied)
tip = RepoMock.tip()
assert tip.node != initial.node
# Check all commits messages
assert [c.desc for c in RepoMock.log()] == [
b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A second commit message\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A first commit message\nDifferential Diff: PHID-DIFF-xxxx',
b'Readme'
]
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(
worker.ssh_key_path)
RepoMock.push.assert_called_with(
dest=b'ssh://hg.mozilla.org/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
async def test_push_to_try_existing_rev(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
with a push to try server
but applying on an existing revision
'''
repo_dir = RepoMock.root().decode('utf-8')
def _readme(content):
# Make a commit on README.md in the repo
readme = os.path.join(repo_dir, 'README.md')
with open(readme, 'a') as f:
f.write(content)
_, rev = RepoMock.commit(message=content.encode('utf-8'), user=b'test')
return rev
# Make two commits, the first one is our base
base = _readme('Local base for diffs')
extra = _readme('EXTRA')
# The patched and config files should not exist at first
target = os.path.join(repo_dir, 'solo.txt')
config = os.path.join(repo_dir, 'try_task_config.json')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
batch_size=100,
)
worker.repo = RepoMock
await worker.handle_diff({
'phid': 'PHID-DIFF-solo',
'revisionPHID': 'PHID-DREV-solo',
'id': 9876,
# Revision does not exist, will apply on tip
'baseRevision': base,
})
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'Solo PATCH\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-DIFF-solo',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches and config have applied)
tip = RepoMock.tip()
assert tip.node != base
assert tip.desc == b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-solo'
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(worker.ssh_key_path)
RepoMock.push.assert_called_with(
dest=b'ssh://hg.mozilla.org/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
# Check the parent is the solo patch commit
parents = RepoMock.parents(tip.node)
assert len(parents) == 1
parent = parents[0]
assert parent.desc == b'A nice human readable commit message\nDifferential Diff: PHID-DIFF-solo'
# Check the grand parent is the base, not extra
great_parents = RepoMock.parents(parent.node)
assert len(great_parents) == 1
great_parent = great_parents[0]
assert great_parent.node == base
# Extra commit should not appear
assert parent.node != extra
assert great_parent.node != extra
assert 'EXTRA' not in open(os.path.join(repo_dir, 'README.md')).read()
async def test_push_to_try(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
with a push to try server
'''
# Get initial tip commit in repo
initial = RepoMock.tip()
# The patched and config files should not exist at first
repo_dir = RepoMock.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
batch_size=100,
)
worker.repo = RepoMock
await worker.handle_diff({
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
# Revision does not exist, will apply on tip
'baseRevision': 'abcdef12345',
})
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'First Line\nSecond Line\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-DIFF-test123',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches + config have applied)
tip = RepoMock.tip()
assert tip.node != initial.node
# Check all commits messages
assert [c.desc for c in RepoMock.log()] == [
b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A second commit message\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A first commit message\nDifferential Diff: PHID-DIFF-xxxx',
b'Readme'
]
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(worker.ssh_key_path)
RepoMock.push.assert_called_with(
dest=b'ssh://hg.mozilla.org/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
class PulseListener(object):
'''
Listen to pulse messages and trigger new tasks
'''
def __init__(self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
self.http_port = int(os.environ.get('PORT', 9000))
logger.info('HTTP webhook server will listen', port=self.http_port)
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
if mercurial_conf.get('enabled', False):
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
batch_size=mercurial_conf.get('batch_size', 100000),
)
else:
self.mercurial = None
logger.info('Mercurial worker is disabled')
def run(self):
# Build hooks for each conf
hooks = [
self.build_hook(conf)
for conf in self.hooks_configuration
]
if not hooks:
raise Exception('No hooks created')
# Run hooks pulse listeners together
# but only use hooks with active definitions
def _connect(hook):
out = hook.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
if self.mercurial is not None:
out &= hook.connect_mercurial_queue(self.mercurial.queue)
return out
consumers = [
hook.build_consumer(self.pulse_user, self.pulse_password)
for hook in hooks
if _connect(hook)
]
# Add monitoring process
consumers.append(task_monitoring.run())
# Add mercurial process
if self.mercurial is not None:
consumers.append(self.mercurial.run())
# Hooks through web server
consumers.append(self.build_webserver(hooks))
# Run all consumers together
run_consumer(asyncio.gather(*consumers))
def build_hook(self, conf):
'''
Build a new hook instance according to configuration
'''
assert isinstance(conf, dict)
assert 'type' in conf
conf['phabricator_api'] = self.phabricator_api
classes = {
'static-analysis-phabricator': HookPhabricator,
'code-coverage': HookCodeCoverage,
}
hook_class = classes.get(conf['type'])
if hook_class is None:
raise Exception('Unsupported hook {}'.format(conf['type']))
return hook_class(conf)
def build_webserver(self, hooks):
'''
Build an async web server used by hooks
'''
app = web.Application()
# Always add a simple test endpoint
async def ping(request):
return web.Response(text='pong')
app.add_routes([web.get('/ping', ping)])
# Add routes from hooks
for hook in hooks:
app.add_routes(hook.routes)
# Finally build the webserver coroutine
return web._run_app(app, port=self.http_port, print=logger.info)
def add_revision(self, revision):
'''
Fetch a phabricator revision and push it in the mercurial queue
'''
if self.mercurial is None:
logger.warn('Skip adding revision, mercurial worker is disabled', revision=revision)
return
rev = self.phabricator_api.load_revision(rev_id=revision)
logger.info('Found revision', title=rev['fields']['title'])
diffs = self.phabricator_api.search_diffs(diff_phid=rev['fields']['diffPHID'])
assert len(diffs) == 1
loop = asyncio.get_event_loop()
loop.run_until_complete(self.mercurial.queue.put(diffs[0]))
logger.info('Pushed revision in queue', rev=revision)
class PulseListener(object):
'''
Listen to pulse messages and trigger new tasks
'''
def __init__(
self,
pulse_user,
pulse_password,
hooks_configuration,
mercurial_conf,
phabricator_api,
cache_root,
taskcluster_client_id=None,
taskcluster_access_token=None,
):
self.pulse_user = pulse_user
self.pulse_password = pulse_password
self.hooks_configuration = hooks_configuration
self.taskcluster_client_id = taskcluster_client_id
self.taskcluster_access_token = taskcluster_access_token
self.phabricator_api = phabricator_api
task_monitoring.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
)
# Build mercurial worker & queue for mozilla unified
self.mercurial = MercurialWorker(
self.phabricator_api,
ssh_user=mercurial_conf['ssh_user'],
ssh_key=mercurial_conf['ssh_key'],
repo_url=REPO_UNIFIED,
repo_dir=os.path.join(cache_root, 'sa-unified'),
)
def run(self):
# Build hooks for each conf
hooks = [self.build_hook(conf) for conf in self.hooks_configuration]
if not hooks:
raise Exception('No hooks created')
# Run hooks pulse listeners together
# but only use hooks with active definitions
consumers = [
hook.build_consumer(self.pulse_user, self.pulse_password)
for hook in hooks if hook.connect_taskcluster(
self.taskcluster_client_id,
self.taskcluster_access_token,
) and hook.connect_mercurial_queue(self.mercurial.queue)
]
# Add monitoring process
consumers.append(task_monitoring.run())
# Add mercurial process
consumers.append(self.mercurial.run())
# Run all consumers together
run_consumer(asyncio.gather(*consumers))
def build_hook(self, conf):
'''
Build a new hook instance according to configuration
'''
assert isinstance(conf, dict)
assert 'type' in conf
conf['phabricator_api'] = self.phabricator_api
classes = {
'static-analysis-phabricator': HookPhabricator,
'code-coverage': HookCodeCoverage,
}
hook_class = classes.get(conf['type'])
if hook_class is None:
raise Exception('Unsupported hook {}'.format(conf['type']))
return hook_class(conf)
def add_revision(self, revision):
'''
Fetch a phabricator revision and push it in the mercurial queue
'''
rev = self.phabricator_api.load_revision(rev_id=revision)
logger.info('Found revision', title=rev['fields']['title'])
diffs = self.phabricator_api.search_diffs(
diff_phid=rev['fields']['diffPHID'])
assert len(diffs) == 1
loop = asyncio.get_event_loop()
loop.run_until_complete(self.mercurial.queue.put(diffs[0]))
logger.info('Pushed revision in queue', rev=revision)
async def test_failure_general(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
Use a Python common exception to trigger a broken build
'''
# Get initial tip commit in repo
initial = RepoMock.tip()
# The patched and config files should not exist at first
repo_dir = RepoMock.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
batch_size=100,
publish_phabricator=True,
)
worker.repo = RepoMock
diff = {
# Missing revisionPHID will give an assertion error
'phid': 'PHID-DIFF-test123',
'id': 1234,
}
out = await worker.handle_build('PHID-somehash', diff)
assert out is False
# Check the unit result was published
assert api.mocks.calls[
-1].request.url == 'http://phabricator.test/api/harbormaster.sendmessage'
params = json.loads(
urllib.parse.parse_qs(
api.mocks.calls[-1].request.body)['params'][0])
assert params['unit'][0]['duration'] > 0
del params['unit'][0]['duration']
assert params == {
'buildTargetPHID':
'PHID-somehash',
'type':
'fail',
'unit': [{
'name': 'general',
'result': 'broken',
'namespace': 'code-review',
'details':
'WARNING: An error occured in the code review bot.\n\n``````',
'format': 'remarkup',
}],
'lint': [],
'__conduit__': {
'token': 'deadbeef'
}
}
assert api.mocks.calls[-1].response.status_code == 200
# Clone should not be modified
tip = RepoMock.tip()
assert tip.node == initial.node
async def test_push_to_try_nss(PhabricatorMock, mock_nss):
'''
Run mercurial worker on a single diff
with a push to try server, but with NSS support (try syntax)
'''
# Get initial tip commit in repo
initial = mock_nss.tip()
# The patched and config files should not exist at first
repo_dir = mock_nss.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
repositories=[
{
'name': 'nss',
'ssh_user': '******',
'ssh_key': 'privateSSHkey',
'url': 'http://nss',
'try_url': 'http://nss/try',
'try_mode': 'syntax',
'try_syntax': '-a -b XXX -c YYY',
'batch_size': 100,
}
],
publish_phabricator=False,
cache_root=os.path.dirname(repo_dir),
)
assert len(worker.repositories) == 1
repo = worker.repositories.get('PHID-REPO-nss')
assert repo is not None
repo.repo = mock_nss
diff = {
'phid': 'PHID-DIFF-test123',
'revisionPHID': 'PHID-DREV-deadbeef',
'id': 1234,
# Revision does not exist, will apply on tip
'baseRevision': 'abcdef12345',
}
build = MockBuild(1234, 'PHID-REPO-nss', 5678, 'PHID-HMBT-deadbeef', diff)
await worker.handle_build(repo, build)
# Check the treeherder link was NOT published
assert api.mocks.calls[-1].request.url != 'http://phabricator.test/api/harbormaster.createartifact'
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'First Line\nSecond Line\n'
# The config should have content now
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'code-review': {
'phabricator-build-target': 'PHID-HMBT-deadbeef',
}
},
}
# Get tip commit in repo
# It should be different from the initial one (patches + config have applied)
tip = mock_nss.tip()
assert tip.node != initial.node
# Check all commits messages
assert [c.desc for c in mock_nss.log()] == [
b'try: -a -b XXX -c YYY',
b'Bug XXX - A second commit message\nDifferential Diff: PHID-DIFF-test123',
b'Bug XXX - A first commit message\nDifferential Diff: PHID-DIFF-xxxx',
b'Readme'
]
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(repo.ssh_key_path)
mock_nss.push.assert_called_with(
dest=b'http://nss/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
async def test_failure_mercurial(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
and check the treeherder link publication as an artifact
Apply a bad mercurial patch to trigger a mercurial fail
'''
# Get initial tip commit in repo
initial = mock_mc.tip()
# The patched and config files should not exist at first
repo_dir = mock_mc.root().decode('utf-8')
config = os.path.join(repo_dir, 'try_task_config.json')
target = os.path.join(repo_dir, 'test.txt')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
repositories=[
{
'name': 'mozilla-central',
'ssh_user': '******',
'ssh_key': 'privateSSHkey',
'url': 'http://mozilla-central',
'try_url': 'http://mozilla-central/try',
'batch_size': 100,
}
],
publish_phabricator=True,
cache_root=os.path.dirname(repo_dir),
)
assert len(worker.repositories) == 1
repo = worker.repositories.get('PHID-REPO-mc')
assert repo is not None
repo.repo = mock_mc
diff = {
'revisionPHID': 'PHID-DREV-666',
'baseRevision': 'missing',
'phid': 'PHID-DIFF-666',
'id': 666,
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-build-666', diff)
out = await worker.handle_build(repo, build)
assert out is False
# Check the unit result was published
assert api.mocks.calls[-1].request.url == 'http://phabricator.test/api/harbormaster.sendmessage'
params = json.loads(urllib.parse.parse_qs(api.mocks.calls[-1].request.body)['params'][0])
assert params['unit'][0]['duration'] > 0
del params['unit'][0]['duration']
assert params == {
'buildTargetPHID': 'PHID-build-666',
'type': 'fail',
'unit': [
{
'name': 'mercurial',
'result': 'fail',
'namespace': 'code-review',
'details': MERCURIAL_FAILURE,
'format': 'remarkup',
}
],
'lint': [],
'__conduit__': {'token': 'deadbeef'}
}
assert api.mocks.calls[-1].response.status_code == 200
# Clone should not be modified
tip = mock_mc.tip()
assert tip.node == initial.node
async def test_push_to_try_existing_rev(PhabricatorMock, mock_mc):
'''
Run mercurial worker on a single diff
with a push to try server
but applying on an existing revision
'''
repo_dir = mock_mc.root().decode('utf-8')
def _readme(content):
# Make a commit on README.md in the repo
readme = os.path.join(repo_dir, 'README.md')
with open(readme, 'a') as f:
f.write(content)
_, rev = mock_mc.commit(message=content.encode('utf-8'), user=b'test')
return rev
# Make two commits, the first one is our base
base = _readme('Local base for diffs')
extra = _readme('EXTRA')
# The patched and config files should not exist at first
target = os.path.join(repo_dir, 'solo.txt')
config = os.path.join(repo_dir, 'try_task_config.json')
assert not os.path.exists(target)
assert not os.path.exists(config)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
repositories=[
{
'name': 'mozilla-central',
'ssh_user': '******',
'ssh_key': 'privateSSHkey',
'url': 'http://mozilla-central',
'try_url': 'http://mozilla-central/try',
'batch_size': 100,
}
],
publish_phabricator=False,
cache_root=os.path.dirname(repo_dir),
)
assert len(worker.repositories) == 1
repo = worker.repositories.get('PHID-REPO-mc')
assert repo is not None
repo.repo = mock_mc
diff = {
'phid': 'PHID-DIFF-solo',
'revisionPHID': 'PHID-DREV-solo',
'id': 9876,
# Revision does not exist, will apply on tip
'baseRevision': base,
}
build = MockBuild(1234, 'PHID-REPO-mc', 5678, 'PHID-HMBT-deadbeef', diff)
await worker.handle_build(repo, build)
# Check the treeherder link was NOT published
assert api.mocks.calls[-1].request.url != 'http://phabricator.test/api/harbormaster.createartifact'
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'Solo PATCH\n'
# Check the try_task_config file
assert os.path.exists(config)
assert json.load(open(config)) == {
'version': 2,
'parameters': {
'target_tasks_method': 'codereview',
'optimize_target_tasks': True,
'phabricator_diff': 'PHID-HMBT-deadbeef',
}
}
# Get tip commit in repo
# It should be different from the initial one (patches and config have applied)
tip = mock_mc.tip()
assert tip.node != base
assert tip.desc == b'try_task_config for code-review\nDifferential Diff: PHID-DIFF-solo'
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(repo.ssh_key_path)
mock_mc.push.assert_called_with(
dest=b'http://mozilla-central/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
# Check the parent is the solo patch commit
parents = mock_mc.parents(tip.node)
assert len(parents) == 1
parent = parents[0]
assert parent.desc == b'A nice human readable commit message\nDifferential Diff: PHID-DIFF-solo'
# Check the grand parent is the base, not extra
great_parents = mock_mc.parents(parent.node)
assert len(great_parents) == 1
great_parent = great_parents[0]
assert great_parent.node == base
# Extra commit should not appear
assert parent.node != extra
assert great_parent.node != extra
assert 'EXTRA' not in open(os.path.join(repo_dir, 'README.md')).read()
async def test_push_to_try_existing_rev(PhabricatorMock, RepoMock):
'''
Run mercurial worker on a single diff
with a push to try server
but applying on an existing revision
'''
repo_dir = RepoMock.root().decode('utf-8')
def _readme(content):
# Make a commit on README.md in the repo
readme = os.path.join(repo_dir, 'README.md')
with open(readme, 'a') as f:
f.write(content)
_, rev = RepoMock.commit(message=content.encode('utf-8'), user=b'test')
return rev
# Make two commits, the first one is our base
base = _readme('Local base for diffs')
extra = _readme('EXTRA')
print('base', base)
print('extra', extra)
# The patched file should not exists at first
target = os.path.join(repo_dir, 'solo.txt')
assert not os.path.exists(target)
with PhabricatorMock as api:
worker = MercurialWorker(
api,
ssh_user='******',
ssh_key='privateSSHkey',
repo_url='http://mozilla-central',
repo_dir=repo_dir,
)
worker.repo = RepoMock
await worker.handle_diff({
'phid': 'PHID-DIFF-solo',
'revisionPHID': 'PHID-DREV-solo',
'id': 9876,
# Revision does not exist, will apply on tip
'baseRevision': base,
})
# The target should have content now
assert os.path.exists(target)
assert open(target).read() == 'Solo PATCH\n'
# Get tip commit in repo
# It should be different from the initial one (patches have applied)
tip = RepoMock.tip()
assert tip.node != base
# Check the push to try has been called
# with tip commit
ssh_conf = 'ssh -o StrictHostKeyChecking="no" -o User="******" -o IdentityFile="{}"'.format(
worker.ssh_key_path)
RepoMock.push.assert_called_with(
dest=b'ssh://hg.mozilla.org/try',
force=True,
rev=tip.node,
ssh=ssh_conf.encode('utf-8'),
)
# Check the parent is our base (not extra)
parents = RepoMock.parents(tip.node)
assert len(parents) == 1
parent = parents[0]
assert parent.node == base
# Extra commit should not appear
assert parent.node != extra
assert 'EXTRA' not in open(os.path.join(repo_dir, 'README.md')).read()