def manual_backfill(revision, buildername, max_revisions, dry_run=False):
"""
This function is used to trigger jobs for a range of revisions
when a user clicks the backfill icon for a job on Treeherder.
It backfills to the last known job on Treeherder.
"""
repo_url = query_repo_url_from_buildername(buildername)
# We want to use data from treeherder for manual backfilling for long term.
set_query_source("treeherder")
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=revision,
before=max_revisions,
after=-1, # We don't want the current job in the revision to be included.
return_revision_list=True)
filtered_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=False)
trigger_range(
buildername=buildername,
revisions=filtered_revlist,
times=1,
dry_run=dry_run,
extra_properties={
'mozci_request': {
'type': 'manual_backfill',
'builders': [buildername]}
}
)
def manual_backfill(revision, buildername, max_pushes=None, dry_run=False):
"""
This function is used to trigger jobs for a range of revisions
when a user clicks the backfill icon for a job on Treeherder.
It backfills to the last known job on Treeherder.
"""
max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername)
repo_url = query_repo_url_from_buildername(buildername)
# We want to use data from treeherder for manual backfilling for long term.
set_query_source("treeherder")
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=revision,
before=max_pushes,
after=-1, # We don't want the current job in the revision to be included.
return_revision_list=True)
filtered_revlist = revlist
# Talos jobs are generally always green and we want to fill in all holes in a range
if 'talos' not in buildername:
filtered_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=False)
trigger_range(
buildername=buildername,
revisions=filtered_revlist,
times=1,
dry_run=dry_run,
extra_properties={
'mozci_request': {
'type': 'manual_backfill',
'builders': [buildername]}
}
)
def manual_backfill(revision, buildername, dry_run=False):
"""
This function is used to trigger jobs for a range of revisions
when a user clicks the backfill icon for a job on Treeherder.
It backfills to the last known job on Treeherder.
"""
factor = 1.5
seta_skip = get_max_pushes(buildername)
# Use SETA's skip pushes times factor
max_pushes = seta_skip * factor
repo_url = query_repo_url_from_buildername(buildername)
# We want to use data from treeherder for manual backfilling for long term.
set_query_source("treeherder")
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=revision,
before=max_pushes,
after=-1, # We don't want the current job in the revision to be included.
return_revision_list=True)
LOG.info("We're *aiming* to backfill; note that we ignore the revision that you request "
"to backfill from ({}) up to {} pushes (seta skip: {}; factor: {}) "
"and we backfill up to the last green if found.".format(
revision[:12], max_pushes, seta_skip, factor))
LOG.info("https://treeherder.mozilla.org/#/jobs?repo={}&filter-searchStr={}"
"&tochange={}&fromchange={}".format(
repo_url.split('/')[-1],
buildername,
revlist[-1],
revlist[0],
))
filtered_revlist = revlist
# Talos jobs are generally always green and we want to fill in all holes in a range
if 'talos' not in buildername:
filtered_revlist = _filter_backfill_revlist(buildername, list(reversed(revlist)),
only_successful=True)
if len(filtered_revlist) == 0:
LOG.info("We don't have a revision list to work with.")
return
if len(revlist) != len(filtered_revlist):
LOG.info("NOTICE: We were aiming for a revlist of {}, however, we only "
"need to backfill {} revisions".format(len(revlist), len(filtered_revlist)))
trigger_range(
buildername=buildername,
revisions=filtered_revlist,
times=1,
dry_run=dry_run,
extra_properties={
'mozci_request': {
'type': 'manual_backfill',
'builders': [buildername]}
}
)
def manual_backfill(revision, buildername, dry_run=False):
"""
This function is used to trigger jobs for a range of revisions
when a user clicks the backfill icon for a job on Treeherder.
It backfills to the last known job on Treeherder.
"""
factor = 1.5
seta_skip = get_max_pushes(buildername)
# Use SETA's skip pushes times factor
max_pushes = seta_skip * factor
repo_url = query_repo_url_from_buildername(buildername)
# We want to use data from treeherder for manual backfilling for long term.
set_query_source("treeherder")
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=revision,
before=max_pushes,
after=-1, # We don't want the current job in the revision to be included.
return_revision_list=True)
LOG.info("We're *aiming* to backfill; note that we ignore the revision that you request "
"to backfill from ({}) up to {} pushes (seta skip: {}; factor: {}) "
"and we backfill up to the last green if found.".format(
revision[:12], max_pushes, seta_skip, factor))
LOG.info("https://treeherder.mozilla.org/#/jobs?repo={}&filter-searchStr={}"
"&tochange={}&fromchange={}".format(
repo_url.split('/')[-1],
buildername,
revlist[-1],
revlist[0],
))
filtered_revlist = revlist
# Talos jobs are generally always green and we want to fill in all holes in a range
if 'talos' not in buildername:
filtered_revlist = _filter_backfill_revlist(buildername, list(reversed(revlist)),
only_successful=True)
if len(filtered_revlist) == 0:
LOG.info("We don't have a revision list to work with.")
return
if len(revlist) != len(filtered_revlist):
LOG.info("NOTICE: We were aiming for a revlist of {}, however, we only "
"need to backfill {} revisions".format(len(revlist), len(filtered_revlist)))
trigger_range(
buildername=buildername,
revisions=filtered_revlist,
times=1,
dry_run=dry_run,
extra_properties={
'mozci_request': {
'type': 'manual_backfill',
'builders': [buildername]}
}
)
def test_query_pushes_by_specified_revision_range_return_changesets(self, get,
query_push_by_revision):
changesets = query_pushes_by_specified_revision_range(repo_url=self.repo_url,
revision=self.revision,
before=1, after=1,
return_revision_list=True)
assert len(changesets) == 3
assert changesets == ['eb15e3f893453d6a4472f8905271aba33f8b68d5',
'1c5b4332e2f1b73fe03977b69371e9a08503bff3',
'724f0a71d62171da1357e6c1f93453359e54206b']
def find_backfill_revlist(buildername, revision, max_pushes=None):
"""Determine which revisions we need to trigger in order to backfill.
This function is generally called by automatic backfilling on pulse_actions.
We need to take into consideration that a job might not be run for many revisions
due to SETA. We also might have a permanent failure appear after a reconfiguration
(a new job is added).
When a permanent failure appears, we keep on adding load unnecessarily
by triggering coalesced jobs in between pushes.
Long lived failing job (it could be hidden):
* push N -> failed job
* push N-1 -> failed/coalesced job
* push N-2 -> failed/coalesced job
...
* push N-max_pushes-1 -> failed/coalesced job
If the list of revision we need to trigger is larger than max_pushes
it means that we either have not had that job scheduled beyond max_pushes
or it has been failing forever.
"""
max_pushes = max_pushes if max_pushes is not None else get_max_pushes(
buildername)
# XXX: There is a chance that a green job has run in a newer push (the priority was higher),
# however, this is unlikely.
# XXX: We might need to consider when a backout has already landed and stop backfilling
LOG.info("BACKFILL-START:%s_%s begins." % (revision[0:8], buildername))
revlist = query_pushes_by_specified_revision_range(
repo_url=query_repo_url_from_buildername(buildername),
revision=revision,
before=max_pushes - 1,
after=0,
return_revision_list=True)
new_revlist = _filter_backfill_revlist(buildername,
revlist,
only_successful=True)
if len(new_revlist) >= max_pushes:
# It is likely that we are facing a long lived permanent failure
LOG.debug(
"We're not going to backfill %s since it is likely to be a permanent "
"failure." % buildername)
LOG.info("BACKFILL-END:%s_%s will not backfill." %
(revision[0:8], buildername))
return []
else:
LOG.info("BACKFILL-END:%s_%s will backfill %s." %
(revision[0:8], buildername, new_revlist))
return new_revlist
def test_query_pushes_by_specified_revision_range_return_changesets(
self, get, query_push_by_revision):
changesets = query_pushes_by_specified_revision_range(
repo_url=self.repo_url,
revision=self.revision,
before=1,
after=1,
return_revision_list=True)
assert len(changesets) == 3
assert changesets == [
'eb15e3f893453d6a4472f8905271aba33f8b68d5',
'1c5b4332e2f1b73fe03977b69371e9a08503bff3',
'724f0a71d62171da1357e6c1f93453359e54206b'
]
def determine_revlist(repo_url, buildername, rev, back_revisions,
delta, from_rev, backfill, skips, max_revisions):
"""Determine which revisions we need to trigger."""
if back_revisions:
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=rev,
before=back_revisions,
after=0,
return_revision_list=True)
elif delta:
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=rev,
before=delta,
after=delta,
return_revision_list=True)
elif from_rev:
revlist = query_pushes_by_revision_range(
repo_url=repo_url,
to_revision=rev,
from_revision=from_rev,
return_revision_list=True)
elif backfill:
revlist = find_backfill_revlist(
buildername=buildername,
revision=rev,
max_pushes=max_revisions,
)
else:
revlist = [rev]
if skips:
revlist = revlist[::skips]
return revlist
def determine_revlist(repo_url, buildername, rev, back_revisions,
delta, from_rev, backfill, skips, max_revisions):
"""Determine which revisions we need to trigger."""
if back_revisions:
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=rev,
before=back_revisions,
after=0,
return_revision_list=True)
elif delta:
revlist = query_pushes_by_specified_revision_range(
repo_url=repo_url,
revision=rev,
before=delta,
after=delta,
return_revision_list=True)
elif from_rev:
revlist = query_pushes_by_revision_range(
repo_url=repo_url,
to_revision=rev,
from_revision=from_rev,
return_revision_list=True)
elif backfill:
revlist = find_backfill_revlist(
buildername=buildername,
revision=rev,
max_pushes=max_revisions,
)
else:
revlist = [rev]
if skips:
revlist = revlist[::skips]
return revlist
def find_backfill_revlist(buildername, revision, max_pushes=None):
"""Determine which revisions we need to trigger in order to backfill.
This function is generally called by automatic backfilling on pulse_actions.
We need to take into consideration that a job might not be run for many revisions
due to SETA. We also might have a permanent failure appear after a reconfiguration
(a new job is added).
When a permanent failure appears, we keep on adding load unnecessarily
by triggering coalesced jobs in between pushes.
Long lived failing job (it could be hidden):
* push N -> failed job
* push N-1 -> failed/coalesced job
* push N-2 -> failed/coalesced job
...
* push N-max_pushes-1 -> failed/coalesced job
If the list of revision we need to trigger is larger than max_pushes
it means that we either have not had that job scheduled beyond max_pushes
or it has been failing forever.
"""
max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername)
# XXX: There is a chance that a green job has run in a newer push (the priority was higher),
# however, this is unlikely.
# XXX: We might need to consider when a backout has already landed and stop backfilling
LOG.info("BACKFILL-START:%s_%s begins." % (revision[0:8], buildername))
revlist = query_pushes_by_specified_revision_range(
repo_url=query_repo_url_from_buildername(buildername),
revision=revision,
before=max_pushes - 1,
after=0,
return_revision_list=True
)
new_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=True)
if len(new_revlist) >= max_pushes:
# It is likely that we are facing a long lived permanent failure
LOG.debug("We're not going to backfill %s since it is likely to be a permanent "
"failure." % buildername)
LOG.info("BACKFILL-END:%s_%s will not backfill." % (revision[0:8], buildername))
return []
else:
LOG.info("BACKFILL-END:%s_%s will backfill %s." %
(revision[0:8], buildername, new_revlist))
return new_revlist
def test_query_pushes_by_specified_revision_range(self, get, query_push_by_revision):
pushes = query_pushes_by_specified_revision_range(repo_url=self.repo_url,
revision=self.revision,
before=1,
after=1)
# This part is totally duplicate with the test_query_pushes_by_pushid_range,
# because we are calling query_pushes_by_pushid_range inside this function.
assert len(pushes) == 3
push_id_list = []
changeset_list = []
for push in pushes:
push_id_list.append(push.id)
changeset_list.append(push.changesets[0].node)
assert push_id_list == ['53348', '53349', '53350']
assert changeset_list == ['eb15e3f893453d6a4472f8905271aba33f8b68d5',
'1c5b4332e2f1b73fe03977b69371e9a08503bff3',
'724f0a71d62171da1357e6c1f93453359e54206b']
def test_query_pushes_by_specified_revision_range(self, get,
query_push_by_revision):
pushes = query_pushes_by_specified_revision_range(
repo_url=self.repo_url, revision=self.revision, before=1, after=1)
# This part is totally duplicate with the test_query_pushes_by_pushid_range,
# because we are calling query_pushes_by_pushid_range inside this function.
assert len(pushes) == 3
push_id_list = []
changeset_list = []
for push in pushes:
push_id_list.append(push.id)
changeset_list.append(push.changesets[0].node)
assert push_id_list == ['53348', '53349', '53350']
assert changeset_list == [
'eb15e3f893453d6a4472f8905271aba33f8b68d5',
'1c5b4332e2f1b73fe03977b69371e9a08503bff3',
'724f0a71d62171da1357e6c1f93453359e54206b'
]
