def connect_taskcluster(self, client_id=None, access_token=None):
'''
Load notification service
'''
self.notify = get_service('notify', client_id, access_token)
self.queue = get_service('queue', client_id, access_token)
self.index = get_service('index', client_id, access_token)
def connect_taskcluster(self, client_id=None, access_token=None):
'''
Load notification service
'''
self.notify = get_service('notify', client_id, access_token)
self.queue = get_service('queue', client_id, access_token)
self.index = get_service('index', client_id, access_token)
def schedule_phase(name, phase):
session = current_app.db.session
try:
phase = session.query(Phase) \
.filter(Release.id == Phase.release_id) \
.filter(Release.name == name) \
.filter(Phase.name == phase).one()
except NoResultFound:
abort(404)
# we must require scope which depends on product
required_permission = f'{SCOPE_PREFIX}/schedule_phase/{phase.release.product}/{phase.name}'
if not current_user.has_permissions(required_permission):
user_permissions = ', '.join(current_user.get_permissions())
abort(401, f'required permission: {required_permission}, user permissions: {user_permissions}')
if phase.submitted:
abort(409, 'Already submitted!')
for signoff in phase.signoffs:
if not signoff.signed:
abort(400, 'Pending signoffs')
task_or_hook = phase.task_json
if 'hook_payload' in task_or_hook:
hooks = get_service('hooks')
client_id = hooks.options['credentials']['clientId'].decode('utf-8')
extra_context = {'clientId': client_id}
result = hooks.triggerHook(
task_or_hook['hook_group_id'],
task_or_hook['hook_id'],
phase.rendered_hook_payload(extra_context=extra_context),
)
phase.task_id = result['status']['taskId']
else:
queue = get_service('queue')
client_id = queue.options['credentials']['clientId'].decode('utf-8')
extra_context = {'clientId': client_id}
queue.createTask(phase.task_id, phase.rendered(extra_context=extra_context))
phase.submitted = True
phase.completed_by = current_user.get_id()
completed = datetime.datetime.utcnow()
phase.completed = completed
if all([ph.submitted for ph in phase.release.phases]):
phase.release.status = 'shipped'
phase.release.completed = completed
session.commit()
notify_via_irc(phase.release.product,
f'Phase {phase.name} was just scheduled '
f'for release {phase.release.product} {phase.release.version} '
f'build{phase.release.build_number} - '
f'(https://tools.taskcluster.net/groups/{phase.task_id})')
return phase.json
def connect_taskcluster(self, client_id=None, access_token=None):
'''
Save hooks and queue services for later use
'''
# Get taskcluster hooks
self.hooks = get_service('hooks', client_id, access_token)
# Get taskcluster queue
self.queue = get_service('queue', client_id, access_token)
return True
def schedule_phase(name, phase):
session = g.db.session
try:
phase = session.query(Phase) \
.filter(Release.id == Phase.release_id) \
.filter(Release.name == name) \
.filter(Phase.name == phase).one()
except NoResultFound:
abort(404)
if phase.submitted:
abort(409, 'Already submitted!')
for signoff in phase.signoffs:
if not signoff.signed:
abort(400, 'Pending signoffs')
task_or_hook = phase.task_json
if 'hook_payload' in task_or_hook:
hooks = get_service('hooks')
client_id = hooks.options['credentials']['clientId'].decode('utf-8')
extra_context = {'clientId': client_id}
result = hooks.triggerHook(
task_or_hook['hook_group_id'],
task_or_hook['hook_id'],
phase.rendered_hook_payload(extra_context=extra_context),
)
phase.task_id = result['status']['taskId']
else:
queue = get_service('queue')
client_id = queue.options['credentials']['clientId'].decode('utf-8')
extra_context = {'clientId': client_id}
queue.createTask(phase.task_id,
phase.rendered(extra_context=extra_context))
phase.submitted = True
phase.completed_by = g.userinfo['email']
completed = datetime.datetime.utcnow()
phase.completed = completed
if all([ph.submitted for ph in phase.release.phases]):
phase.release.status = 'shipped'
phase.release.completed = completed
session.commit()
notify_via_irc(
f'Phase {phase.name} was just scheduled '
f'for release {phase.release.product} {phase.release.version} '
f'build{phase.release.build_number} - '
f'(https://tools.taskcluster.net/groups/{phase.task_id})')
return phase.json
def __init__(self, cache_root, emails, client_id=None, access_token=None):
self.emails = emails
self.cache_root = cache_root
assert os.path.isdir(self.cache_root), \
'Cache root {} is not a dir.'.format(self.cache_root)
# Load TC services & secrets
self.notify = get_service(
'notify',
client_id=client_id,
access_token=access_token,
)
# Clone mozilla-central
self.repo_dir = os.path.join(self.cache_root, 'static-analysis/')
shared_dir = os.path.join(self.cache_root, 'static-analysis-shared')
logger.info('Clone mozilla central', dir=self.repo_dir)
cmd = hglib.util.cmdbuilder('robustcheckout',
REPO_CENTRAL,
self.repo_dir,
purge=True,
sharebase=shared_dir,
branch=b'tip')
cmd.insert(0, hglib.HGPATH)
proc = hglib.util.popen(cmd)
out, err = proc.communicate()
if proc.returncode:
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
# Open new hg client
self.hg = hglib.open(self.repo_dir)
def find_decision_task_id(project, revision):
decision_task_route = '{trust_domain}.v2.{project}.revision.{revision}.taskgraph.decision'.format(
trust_domain=get_trust_domain(project),
project=project,
revision=revision)
index = get_service('index')
return index.findTask(decision_task_route)['taskId']
def __init__(self, repository, revision, task_name_filter, cache_root,
client_id, access_token):
# List of test-suite, sorted alphabetically.
# This way, the index of a suite in the array should be stable enough.
self.suites = [
'web-platform-tests',
]
self.cache_root = cache_root
temp_dir = tempfile.mkdtemp()
self.artifacts_dir = os.path.join(temp_dir, 'ccov-artifacts')
self.ccov_reports_dir = os.path.join(temp_dir, 'code-coverage-reports')
self.client_id = client_id
self.access_token = access_token
self.index_service = get_service('index', client_id, access_token)
self.githubUtils = GitHubUtils(cache_root, client_id, access_token)
if revision is None:
# Retrieve revision of latest codecov build
self.github_revision = uploader.get_latest_codecov()
self.repository = MOZILLA_CENTRAL_REPOSITORY
self.revision = self.githubUtils.git_to_mercurial(
self.github_revision)
self.from_pulse = False
else:
self.github_revision = None
self.repository = repository
self.revision = revision
self.from_pulse = True
self.branch = self.repository[len(HG_BASE):]
assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(
cache_root)
self.repo_dir = os.path.join(cache_root, self.branch)
logger.info('Mercurial revision', revision=self.revision)
task_ids = {}
for platform in [
'linux', 'windows', 'android-test', 'android-emulator'
]:
task = taskcluster.get_task(self.branch, self.revision, platform)
# On try, developers might have requested to run only one platform, and we trust them.
# On mozilla-central, we want to assert that every platform was run (except for android platforms
# as they are unstable).
if task is not None:
task_ids[platform] = task
elif self.repository == MOZILLA_CENTRAL_REPOSITORY and not platform.startswith(
'android'):
raise Exception(
'Code coverage build failed and was not indexed.')
self.artifactsHandler = ArtifactsHandler(task_ids, self.artifacts_dir,
task_name_filter)
def abandon_release(name):
session = current_app.db.session
try:
release = session.query(Release).filter(Release.name == name).one()
# we must require scope which depends on product
required_permission = f"{SCOPE_PREFIX}/abandon_release/{release.product}"
if not current_user.has_permissions(required_permission):
user_permissions = ", ".join(current_user.get_permissions())
abort(
401,
f"required permission: {required_permission}, user permissions: {user_permissions}"
)
# Cancel all submitted task groups first
for phase in filter(lambda x: x.submitted and not x.skipped,
release.phases):
try:
actions = fetch_artifact(phase.task_id, "public/actions.json")
parameters = fetch_artifact(phase.task_id,
"public/parameters.yml")
except ArtifactNotFound:
logger.info("Ignoring not completed action task %s",
phase.task_id)
continue
hook = generate_action_hook(task_group_id=phase.task_id,
action_name="cancel-all",
actions=actions,
parameters=parameters,
input_={})
hooks = get_service("hooks")
client_id = hooks.options["credentials"]["clientId"].decode(
"utf-8")
hook["context"]["clientId"] = client_id
hook_payload_rendered = render_action_hook(
payload=hook["hook_payload"],
context=hook["context"],
delete_params=[
"existing_tasks", "release_history",
"release_partner_config"
])
logger.info("Cancel phase %s by hook %s with payload: %s",
phase.name, hook["hook_id"], hook_payload_rendered)
res = hooks.triggerHook(hook["hook_group_id"], hook["hook_id"],
hook_payload_rendered)
logger.debug("Done: %s", res)
release.status = "aborted"
session.commit()
release_json = release.json
except NoResultFound:
abort(404)
notify_via_irc(
release.product,
f"Release {release.product} {release.version} build{release.build_number} was just canceled."
)
return release_json
def schedule_phase(name, phase):
session = flask.g.db.session
try:
phase = session.query(Phase) \
.filter(Release.id == Phase.release_id) \
.filter(Release.name == name) \
.filter(Phase.name == phase).one()
except NoResultFound:
flask.abort(404)
if phase.submitted:
flask.abort(409, 'Already submitted!')
queue = get_service('queue')
queue.createTask(phase.task_id, phase.rendered)
phase.submitted = True
phase.completed_by = flask.g.userinfo['email']
completed = datetime.datetime.utcnow()
phase.completed = completed
if all([ph.submitted for ph in phase.release.phases]):
phase.release.status = 'shipped'
phase.release.completed = completed
session.commit()
notify_via_irc(
f'Phase {phase.name} was just scheduled '
f'for release {phase.release.product} {phase.release.version} '
f'build{phase.release.build_number} - '
f'(https://tools.taskcluster.net/groups/{phase.task_id})')
return phase.json
def send(self, app_channel):
'''
Build and send report using Taskcluster notification service
'''
# Skip sending when there are no failed merges
if not self.merges:
logger.info('Nothing to report.')
return
def _str(x):
return isinstance(x, bytes) and x.decode('utf-8') or x
def _commit_report(revision, result):
fail_fmt = ' * Merge failed for commit `{}` (parent `{}`)\n\n```\n{}```' # noqa
default_fmt = ' * Merge {} for commit `{}`'
if result.status == 'failed':
return fail_fmt.format(
_str(revision),
_str(result.parent),
result.message,
)
else:
return default_fmt.format(result.status, _str(revision))
# Build markdown output
# Sorting failed merge tests by bugzilla id & branch
subject = '[{}] Uplift bot detected {} merge failures'.format(
app_channel,
len(self.merges),
)
mail = [
'# Failed automated merge test',
''
]
cmp_func = operator.attrgetter('bugzilla_id', 'branch')
merges = sorted(self.merges, key=cmp_func)
merges = itertools.groupby(merges, key=cmp_func)
for keys, failures in merges:
bz_id, branch = keys
mail.append('## Bug [{0}](https://bugzil.la/{0}) - Uplift to {1}\n'.format(bz_id, _str(branch))) # noqa
for merge_test in failures:
mail += [
_commit_report(revision, result)
for revision, result in merge_test.results.items()
]
mail.append('') # newline
mail_md = '\n'.join(mail)
# Send mail report to every mail address
notify = get_service('notify')
for email in self.emails:
notify.email({
'address': email,
'subject': subject,
'content': mail_md,
'template': 'fullscreen',
})
logger.info('Sent report', to=email)
def schedule_phase(name, phase):
session = current_app.db.session
try:
phase = session.query(Phase).filter(
Release.id == Phase.release_id).filter(
Release.name == name).filter(Phase.name == phase).one()
except NoResultFound:
abort(404)
# we must require scope which depends on product
required_permission = f"{SCOPE_PREFIX}/schedule_phase/{phase.release.product}/{phase.name}"
if not current_user.has_permissions(required_permission):
user_permissions = ", ".join(current_user.get_permissions())
abort(
401,
f"required permission: {required_permission}, user permissions: {user_permissions}"
)
if phase.submitted:
abort(409, "Already submitted!")
for signoff in phase.signoffs:
if not signoff.signed:
abort(400, "Pending signoffs")
hook = phase.task_json
if "hook_payload" not in hook:
raise ValueError("Action tasks are not supported")
hooks = get_service("hooks")
client_id = hooks.options["credentials"]["clientId"].decode("utf-8")
extra_context = {"clientId": client_id}
result = hooks.triggerHook(
hook["hook_group_id"], hook["hook_id"],
phase.rendered_hook_payload(extra_context=extra_context))
phase.task_id = result["status"]["taskId"]
phase.submitted = True
phase.completed_by = current_user.get_id()
completed = datetime.datetime.utcnow()
phase.completed = completed
if all([ph.submitted for ph in phase.release.phases]):
phase.release.status = "shipped"
phase.release.completed = completed
session.commit()
root_url = hooks.options["rootUrl"]
url = f"{root_url}/tasks/groups/{phase.task_id}"
# TODO: Remove the condition after migration
if root_url == "https://taskcluster.net":
url = "https://tools.taskcluster.net/groups/{phase.task_id}"
notify_via_irc(
phase.release.product,
f"Phase {phase.name} was just scheduled for release {phase.release.product} {phase.release.version} build{phase.release.build_number} - {url}",
)
return phase.json
def abandon_release(name):
session = current_app.db.session
try:
release = session.query(Release).filter(Release.name == name).one()
# we must require scope which depends on product
required_permission = f'{SCOPE_PREFIX}/abandon_release/{release.product}'
if not current_user.has_permissions(required_permission):
user_permissions = ', '.join(current_user.get_permissions())
abort(
401,
f'required permission: {required_permission}, user permissions: {user_permissions}'
)
# Cancel all submitted task groups first
for phase in filter(lambda x: x.submitted, release.phases):
try:
actions = fetch_actions_json(phase.task_id)
except ActionsJsonNotFound:
logger.info('Ignoring not completed action task %s',
phase.task_id)
continue
hook = generate_action_hook(
task_group_id=phase.task_id,
action_name='cancel-all',
actions=actions,
input_={},
)
hooks = get_service('hooks')
client_id = hooks.options['credentials']['clientId'].decode(
'utf-8')
hook['context']['clientId'] = client_id
hook_payload_rendered = render_action_hook(
payload=hook['hook_payload'],
context=hook['context'],
delete_params=[
'existing_tasks', 'release_history',
'release_partner_config'
],
)
logger.info('Cancel phase %s by hook %s with payload: %s',
phase.name, hook['hook_id'], hook_payload_rendered)
res = hooks.triggerHook(hook['hook_group_id'], hook['hook_id'],
hook_payload_rendered)
logger.debug('Done: %s', res)
release.status = 'aborted'
session.commit()
release_json = release.json
except NoResultFound:
abort(404)
notify_via_irc(
release.product,
f'Release {release.product} {release.version} build{release.build_number} was just canceled.'
)
return release_json
def post_github_status(self, commit_sha):
if self.gecko_dev_user is None or self.gecko_dev_pwd is None:
return
tcGithub = get_service('github', self.client_id, self.access_token)
tcGithub.createStatus('marco-c', 'gecko-dev', commit_sha, {
'state': 'success',
})
def __init__(self, cache_root, client_id, access_token):
self.cache_root = cache_root
self.gecko_dev_user = secrets.get(secrets.GECKO_DEV_USER)
self.gecko_dev_pwd = secrets.get(secrets.GECKO_DEV_PWD)
self.hg_git_mapper = secrets[secrets.HG_GIT_MAPPER] if secrets.HG_GIT_MAPPER in secrets else 'https://mapper.mozilla-releng.net'
self.client_id = client_id
self.access_token = access_token
self.notify_service = get_service('notify', client_id, access_token)
def post_github_status(self, commit_sha):
if self.gecko_dev_user is None or self.gecko_dev_pwd is None:
return
tcGithub = get_service('github', self.client_id, self.access_token)
tcGithub.createStatus('marco-c', 'gecko-dev', commit_sha, {
'state': 'success',
})
def connect_taskcluster(self, client_id=None, access_token=None):
'''
Save hooks and queue services for later use
'''
# Get taskcluster hooks
self.hooks = get_service('hooks', client_id, access_token)
return True
def __init__(self, cache_root, client_id, access_token):
self.cache_root = cache_root
assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(cache_root)
self.client_id = client_id
self.access_token = access_token
self.index_service = get_service('index', client_id, access_token)
def __init__(self, cache_root, client_id, access_token):
self.cache_root = cache_root
assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(cache_root)
self.client_id = client_id
self.access_token = access_token
self.index_service = get_service('index', client_id, access_token)
def connect_taskcluster(self, client_id=None, access_token=None):
'''
Load the hook's task definition through Taskcluster
Save queue service for later use
'''
logger.info('Loading task definition', hook=self.hook_id, group=self.group_id) # noqa
try:
service = get_service('hooks', client_id, access_token)
hook_payload = service.hook(self.group_id, self.hook_id)
self.task_definition = hook_payload['task']
except Exception as e:
logger.warn('Failed to fetch task definition', hook=self.hook_id, group=self.group_id, err=e) # noqa
return False
# Get taskcluster queue
self.queue = get_service('queue', client_id, access_token)
return True
def find_decision_task_id(repo_url, project, revision, product):
trust_domain = get_trust_domain(repo_url, project, product)
if repo_url.startswith("https://github.com"):
# XXX "project" is a gecko-centric term which is translated into a branch in the git world.
branch = project
decision_task_route = f"{trust_domain}.v2.branch.{branch}.revision.{revision}.taskgraph.decision"
else:
decision_task_route = f"{trust_domain}.v2.{project}.revision.{revision}.taskgraph.decision"
index = get_service("index")
return index.findTask(decision_task_route)["taskId"]
def __init__(self, cache_root, client_id, access_token):
self.cache_root = cache_root
self.gecko_dev_user = secrets.get(secrets.GECKO_DEV_USER)
self.gecko_dev_pwd = secrets.get(secrets.GECKO_DEV_PWD)
self.hg_git_mapper = secrets[
secrets.
HG_GIT_MAPPER] if secrets.HG_GIT_MAPPER in secrets else 'https://mapper.mozilla-releng.net'
self.client_id = client_id
self.access_token = access_token
self.notify_service = get_service('notify', client_id, access_token)
def __init__(self,
cache_root,
emails,
app_channel,
mozreview_api_root,
mozreview_enabled=False,
mozreview_publish_success=False,
client_id=None,
access_token=None): # noqa
self.emails = emails
self.app_channel = app_channel
self.mozreview_api_root = mozreview_api_root
self.mozreview_enabled = mozreview_enabled
self.mozreview_publish_success = mozreview_publish_success
self.cache_root = cache_root
assert os.path.isdir(self.cache_root), \
'Cache root {} is not a dir.'.format(self.cache_root)
assert 'MOZCONFIG' in os.environ, \
'Missing MOZCONFIG in environment'
# Save Taskcluster ID for logging
if 'TASK_ID' in os.environ and 'RUN_ID' in os.environ:
self.taskcluster_id = '{} run:{}'.format(
os.environ['TASK_ID'],
os.environ['RUN_ID'],
)
else:
self.taskcluster_id = 'local instance'
# Load TC services & secrets
self.notify = get_service(
'notify',
client_id=client_id,
access_token=access_token,
)
# Clone mozilla-central
self.repo_dir = os.path.join(cache_root, 'central')
shared_dir = os.path.join(cache_root, 'central-shared')
logger.info('Clone mozilla central', dir=self.repo_dir)
cmd = hglib.util.cmdbuilder('robustcheckout',
REPO_CENTRAL,
self.repo_dir,
purge=True,
sharebase=shared_dir,
branch=b'tip')
cmd.insert(0, hglib.HGPATH)
proc = hglib.util.popen(cmd)
out, err = proc.communicate()
if proc.returncode:
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
# Open new hg client
self.hg = hglib.open(self.repo_dir)
def fetch_actions_json(task_id):
try:
queue = get_service('queue')
actions_url = queue.buildUrl('getLatestArtifact', task_id, 'public/actions.json')
q = requests.get(actions_url)
q.raise_for_status()
return q.json()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise ActionsJsonNotFound
raise
def fetch_artifact(task_id, artifact):
try:
queue = get_service('queue')
url = queue.buildUrl('getLatestArtifact', task_id, artifact)
q = requests.get(url)
q.raise_for_status()
return yaml.safe_load(q.text)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise ArtifactNotFound
raise
def fetch_actions_json(task_id):
try:
queue = get_service('queue')
actions_url = queue.buildUrl('getLatestArtifact', task_id,
'public/actions.json')
q = requests.get(actions_url)
q.raise_for_status()
return q.json()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise ActionsJsonNotFound
raise
def __init__(self, configuration, client_id, access_token):
self.emails, = self.requires(configuration, 'emails')
assert len(self.emails) > 0, 'Missing emails data'
# Load TC services & secrets
self.notify = get_service(
'notify',
client_id=client_id,
access_token=access_token,
)
logger.info('Mail report enabled', emails=self.emails)
def __init__(self, repository, revision, cache_root, client_id, access_token):
# List of test-suite, sorted alphabetically.
# This way, the index of a suite in the array should be stable enough.
self.suites = [
'web-platform-tests',
]
self.cache_root = cache_root
temp_dir = tempfile.mkdtemp()
self.artifacts_dir = os.path.join(temp_dir, 'ccov-artifacts')
self.ccov_reports_dir = os.path.join(temp_dir, 'code-coverage-reports')
self.client_id = client_id
self.access_token = access_token
self.index_service = get_service('index', client_id, access_token)
self.githubUtils = GitHubUtils(cache_root, client_id, access_token)
if revision is None:
# Retrieve revision of latest codecov build
self.github_revision = uploader.get_latest_codecov()
self.repository = MOZILLA_CENTRAL_REPOSITORY
self.revision = self.githubUtils.git_to_mercurial(self.github_revision)
self.from_pulse = False
else:
self.github_revision = None
self.repository = repository
self.revision = revision
self.from_pulse = True
branch = self.repository[len(HG_BASE):]
assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(cache_root)
self.repo_dir = os.path.join(cache_root, branch)
logger.info('Mercurial revision', revision=self.revision)
task_ids = {}
for platform in ['linux', 'windows', 'android-test', 'android-emulator']:
task = taskcluster.get_task(branch, self.revision, platform)
# On try, developers might have requested to run only one platform, and we trust them.
# On mozilla-central, we want to assert that every platform was run (except for android platforms
# as they are unstable).
if task is not None:
task_ids[platform] = task
elif self.repository == MOZILLA_CENTRAL_REPOSITORY and not platform.startswith('android'):
raise Exception('Code coverage build failed and was not indexed.')
self.artifactsHandler = ArtifactsHandler(task_ids, self.artifacts_dir)
def __init__(self, revision, cache_root, client_id, access_token):
# List of test-suite, sorted alphabetically.
# This way, the index of a suite in the array should be stable enough.
self.suites = [
'web-platform-tests',
]
self.cache_root = cache_root
assert os.path.isdir(cache_root), 'Cache root {} is not a dir.'.format(
cache_root)
self.repo_dir = os.path.join(cache_root, 'mozilla-central')
temp_dir = tempfile.mkdtemp()
self.artifacts_dir = os.path.join(temp_dir, 'ccov-artifacts')
self.ccov_reports_dir = os.path.join(temp_dir, 'code-coverage-reports')
self.client_id = client_id
self.access_token = access_token
self.index_service = get_service('index', client_id, access_token)
self.githubUtils = GitHubUtils(cache_root, client_id, access_token)
if revision is None:
# Retrieve revision of latest codecov build
self.github_revision = uploader.get_latest_codecov()
self.revision = self.githubUtils.git_to_mercurial(
self.github_revision)
self.from_pulse = False
else:
self.github_revision = None
self.revision = revision
self.from_pulse = True
self.notifier = Notifier(self.repo_dir, revision, client_id,
access_token)
logger.info('Mercurial revision', revision=self.revision)
task_ids = {
'linux':
taskcluster.get_task('mozilla-central', self.revision, 'linux'),
'windows':
taskcluster.get_task('mozilla-central', self.revision, 'win'),
'android-test':
taskcluster.get_task('mozilla-central', self.revision,
'android-test'),
'android-emulator':
taskcluster.get_task('mozilla-central', self.revision,
'android-emulator'),
}
self.artifactsHandler = ArtifactsHandler(task_ids, self.artifacts_dir)
def abandon_release(name):
session = g.db.session
try:
r = session.query(Release).filter(Release.name == name).one()
# Cancel all submitted task groups first
for phase in filter(lambda x: x.submitted, r.phases):
try:
actions = fetch_actions_json(phase.task_id)
except ActionsJsonNotFound:
logger.info('Ignoring not completed action task %s',
phase.task_id)
continue
hook = generate_action_hook(
task_group_id=phase.task_id,
action_name='cancel-all',
actions=actions,
input_={},
)
hooks = get_service('hooks')
client_id = hooks.options['credentials']['clientId'].decode(
'utf-8')
hook['context']['clientId'] = client_id
hook_payload_rendered = render_action_hook(
payload=hook['hook_payload'],
context=hook['context'],
delete_params=[
'existing_tasks', 'release_history',
'release_partner_config'
],
)
logger.info('Cancel phase %s by hook %s with payload: %s',
phase.name, hook['hook_id'], hook_payload_rendered)
res = hooks.triggerHook(hook['hook_group_id'], hook['hook_id'],
hook_payload_rendered)
logger.debug('Done: %s', res)
r.status = 'aborted'
session.commit()
release = r.json
except NoResultFound:
abort(404)
notify_via_irc(
f'Release {r.product} {r.version} build{r.build_number} was just canceled.'
)
return release
def trigger_task(task_group_id, commit):
'''
Trigger a code coverage task to build covdir at a specified revision
'''
assert 'mercurial' in commit
name = 'covdir {} - {} - {}'.format(secrets[secrets.APP_CHANNEL],
commit['timestamp'],
commit['mercurial'])
hooks = taskcluster.get_service('hooks')
payload = {
'REPOSITORY': MC_REPO,
'REVISION': commit['mercurial'],
'taskGroupId': task_group_id,
'taskName': name,
}
hook_id = HOOK_ID.format(app_channel=secrets[secrets.APP_CHANNEL])
return hooks.triggerHook(HOOK_GROUP, hook_id, payload)
def main():
# CLI args
parser = argparse.ArgumentParser()
parser.add_argument('--nb-tasks',
type=int,
default=5,
help='NB of tasks to create')
parser.add_argument('--unique',
choices=('day', 'week'),
help='Trigger only one task per day or week')
parser.add_argument('--group',
type=str,
default=slugId(),
help='Task group to create/update')
parser.add_argument('--dry-run',
action='store_true',
default=False,
help='List actions without triggering any new task')
args = parser.parse_args()
# List existing tags & commits
print('Group', args.group)
queue = taskcluster.get_service('queue')
try:
group = queue.listTaskGroup(args.group)
commits = [
task['task']['payload']['env']['REVISION']
for task in group['tasks']
if task['status']['state'] not in ('failed', 'exception')
]
print('Found {} commits processed in task group {}'.format(
len(commits), args.group))
except Exception as e:
print('Invalid task group : {}'.format(e))
commits = []
# Trigger a task for each commit
for commit in list_commits(args.nb_tasks, args.unique, commits):
print(
'Triggering commit {mercurial} from {timestamp}'.format(**commit))
if args.dry_run:
print('>>> No trigger on dry run')
else:
out = trigger_task(args.group, commit)
print('>>>', out['status']['taskId'])
def __init__(self,
cache_root,
emails,
mozreview,
mozreview_enabled=False,
client_id=None,
access_token=None): # noqa
self.emails = emails
self.mozreview = mozreview
self.mozreview_enabled = mozreview_enabled
self.cache_root = cache_root
assert os.path.isdir(self.cache_root), \
'Cache root {} is not a dir.'.format(self.cache_root)
assert 'MOZCONFIG' in os.environ, \
'Missing MOZCONFIG in environment'
# Load TC services & secrets
self.notify = get_service(
'notify',
client_id=client_id,
access_token=access_token,
)
# Clone mozilla-central
self.repo_dir = os.path.join(cache_root, 'central')
shared_dir = os.path.join(cache_root, 'central-shared')
logger.info('Clone mozilla central', dir=self.repo_dir)
cmd = hglib.util.cmdbuilder('robustcheckout',
REPO_CENTRAL,
self.repo_dir,
purge=True,
sharebase=shared_dir,
branch=b'tip')
cmd.insert(0, hglib.HGPATH)
proc = hglib.util.popen(cmd)
out, err = proc.communicate()
if proc.returncode:
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
# Open new hg client
self.hg = hglib.open(self.repo_dir)
# Setup clang
self.clang = ClangTidy(self.repo_dir, settings.target)
def abandon_release(name):
session = g.db.session
try:
r = session.query(Release).filter(Release.name == name).one()
# Cancel all submitted task groups first
for phase in filter(lambda x: x.submitted, r.phases):
try:
actions = fetch_actions_json(phase.task_id)
except ActionsJsonNotFound:
logger.info('Ignoring not completed action task %s',
phase.task_id)
continue
hook = generate_action_hook(
decision_task_id=phase.task_id,
action_name='cancel-all',
actions=actions,
)
# some parameters contain a lot of entries, so we hit the payload
# size limit. We don't use this parameter in any case, safe to
# remove
for long_param in ('existing_tasks', 'release_history',
'release_partner_config'):
del hook['context']['parameters'][long_param]
logger.info('Cancel phase %s by hook %s', phase.name, hook)
hooks = get_service('hooks')
res = hooks.triggerHook(hook['hook_group_id'], hook['hook_id'],
hook['hook_payload'])
logger.debug('Done: %s', res)
r.status = 'aborted'
session.commit()
release = r.json
except NoResultFound:
abort(404)
notify_via_irc(
f'Release {r.product} {r.version} build{r.build_number} was just canceled.'
)
return release
def do_schedule_phase(session, phase):
if phase.submitted:
abort(409, "Already submitted!")
for signoff in phase.signoffs:
if not signoff.signed:
abort(400, "Pending signoffs")
hook = phase.task_json
hooks = get_service("hooks")
client_id = hooks.options["credentials"]["clientId"].decode("utf-8")
extra_context = {"clientId": client_id}
try:
result = hooks.triggerHook(
hook["hook_group_id"], hook["hook_id"],
phase.rendered_hook_payload(extra_context=extra_context))
phase.task_id = result["status"]["taskId"]
except TaskclusterRestFailure as e:
abort(400, str(e))
phase.submitted = True
completed = datetime.datetime.utcnow()
phase.completed_by = current_user.get_id()
phase.completed = completed
# If the previous phases are not submitted, mark them as submitted and they
# will be calculated as skipped because they don't have taskId associated
for ph in phase.release.phases:
if ph.name == phase.name:
break
if not ph.submitted:
ph.submitted = True
ph.completed_by = current_user.get_id()
ph.completed = completed
session.commit()
return phase
def __init__(self, repo_dir, revision, client_id, access_token):
self.repo_dir = repo_dir
self.revision = revision
self.notify_service = get_service('notify', client_id, access_token)
def find_decision_task_id(project, revision):
decision_task_route = f'{get_trust_domain(project)}.v2.{project}.revision.{revision}.taskgraph.decision'
index = get_service('index')
return index.findTask(decision_task_route)['taskId']
def __init__(self, repo_dir, revision, client_id, access_token):
self.repo_dir = repo_dir
self.revision = revision
self.notify_service = get_service('notify', client_id, access_token)
def find_decision_task_id(project, revision):
decision_task_route = f'{get_trust_domain(project)}.v2.{project}.revision.{revision}.taskgraph.decision'
index = get_service('index')
return index.findTask(decision_task_route)['taskId']