def manual_backfill(revision, buildername, max_revisions, dry_run=False): """ This function is used to trigger jobs for a range of revisions when a user clicks the backfill icon for a job on Treeherder. It backfills to the last known job on Treeherder. """ repo_url = query_repo_url_from_buildername(buildername) # We want to use data from treeherder for manual backfilling for long term. set_query_source("treeherder") revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=revision, before=max_revisions, after=-1, # We don't want the current job in the revision to be included. return_revision_list=True) filtered_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=False) trigger_range( buildername=buildername, revisions=filtered_revlist, times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'manual_backfill', 'builders': [buildername]} } )
def manual_backfill(revision, buildername, max_pushes=None, dry_run=False): """ This function is used to trigger jobs for a range of revisions when a user clicks the backfill icon for a job on Treeherder. It backfills to the last known job on Treeherder. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername) repo_url = query_repo_url_from_buildername(buildername) # We want to use data from treeherder for manual backfilling for long term. set_query_source("treeherder") revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=revision, before=max_pushes, after=-1, # We don't want the current job in the revision to be included. return_revision_list=True) filtered_revlist = revlist # Talos jobs are generally always green and we want to fill in all holes in a range if 'talos' not in buildername: filtered_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=False) trigger_range( buildername=buildername, revisions=filtered_revlist, times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'manual_backfill', 'builders': [buildername]} } )
def manual_backfill(revision, buildername, dry_run=False): """ This function is used to trigger jobs for a range of revisions when a user clicks the backfill icon for a job on Treeherder. It backfills to the last known job on Treeherder. """ factor = 1.5 seta_skip = get_max_pushes(buildername) # Use SETA's skip pushes times factor max_pushes = seta_skip * factor repo_url = query_repo_url_from_buildername(buildername) # We want to use data from treeherder for manual backfilling for long term. set_query_source("treeherder") revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=revision, before=max_pushes, after=-1, # We don't want the current job in the revision to be included. return_revision_list=True) LOG.info("We're *aiming* to backfill; note that we ignore the revision that you request " "to backfill from ({}) up to {} pushes (seta skip: {}; factor: {}) " "and we backfill up to the last green if found.".format( revision[:12], max_pushes, seta_skip, factor)) LOG.info("https://treeherder.mozilla.org/#/jobs?repo={}&filter-searchStr={}" "&tochange={}&fromchange={}".format( repo_url.split('/')[-1], buildername, revlist[-1], revlist[0], )) filtered_revlist = revlist # Talos jobs are generally always green and we want to fill in all holes in a range if 'talos' not in buildername: filtered_revlist = _filter_backfill_revlist(buildername, list(reversed(revlist)), only_successful=True) if len(filtered_revlist) == 0: LOG.info("We don't have a revision list to work with.") return if len(revlist) != len(filtered_revlist): LOG.info("NOTICE: We were aiming for a revlist of {}, however, we only " "need to backfill {} revisions".format(len(revlist), len(filtered_revlist))) trigger_range( buildername=buildername, revisions=filtered_revlist, times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'manual_backfill', 'builders': [buildername]} } )
def test_query_pushes_by_specified_revision_range_return_changesets(self, get, query_push_by_revision): changesets = query_pushes_by_specified_revision_range(repo_url=self.repo_url, revision=self.revision, before=1, after=1, return_revision_list=True) assert len(changesets) == 3 assert changesets == ['eb15e3f893453d6a4472f8905271aba33f8b68d5', '1c5b4332e2f1b73fe03977b69371e9a08503bff3', '724f0a71d62171da1357e6c1f93453359e54206b']
def find_backfill_revlist(buildername, revision, max_pushes=None): """Determine which revisions we need to trigger in order to backfill. This function is generally called by automatic backfilling on pulse_actions. We need to take into consideration that a job might not be run for many revisions due to SETA. We also might have a permanent failure appear after a reconfiguration (a new job is added). When a permanent failure appears, we keep on adding load unnecessarily by triggering coalesced jobs in between pushes. Long lived failing job (it could be hidden): * push N -> failed job * push N-1 -> failed/coalesced job * push N-2 -> failed/coalesced job ... * push N-max_pushes-1 -> failed/coalesced job If the list of revision we need to trigger is larger than max_pushes it means that we either have not had that job scheduled beyond max_pushes or it has been failing forever. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes( buildername) # XXX: There is a chance that a green job has run in a newer push (the priority was higher), # however, this is unlikely. # XXX: We might need to consider when a backout has already landed and stop backfilling LOG.info("BACKFILL-START:%s_%s begins." % (revision[0:8], buildername)) revlist = query_pushes_by_specified_revision_range( repo_url=query_repo_url_from_buildername(buildername), revision=revision, before=max_pushes - 1, after=0, return_revision_list=True) new_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=True) if len(new_revlist) >= max_pushes: # It is likely that we are facing a long lived permanent failure LOG.debug( "We're not going to backfill %s since it is likely to be a permanent " "failure." % buildername) LOG.info("BACKFILL-END:%s_%s will not backfill." % (revision[0:8], buildername)) return [] else: LOG.info("BACKFILL-END:%s_%s will backfill %s." % (revision[0:8], buildername, new_revlist)) return new_revlist
def test_query_pushes_by_specified_revision_range_return_changesets( self, get, query_push_by_revision): changesets = query_pushes_by_specified_revision_range( repo_url=self.repo_url, revision=self.revision, before=1, after=1, return_revision_list=True) assert len(changesets) == 3 assert changesets == [ 'eb15e3f893453d6a4472f8905271aba33f8b68d5', '1c5b4332e2f1b73fe03977b69371e9a08503bff3', '724f0a71d62171da1357e6c1f93453359e54206b' ]
def determine_revlist(repo_url, buildername, rev, back_revisions, delta, from_rev, backfill, skips, max_revisions): """Determine which revisions we need to trigger.""" if back_revisions: revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=rev, before=back_revisions, after=0, return_revision_list=True) elif delta: revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=rev, before=delta, after=delta, return_revision_list=True) elif from_rev: revlist = query_pushes_by_revision_range( repo_url=repo_url, to_revision=rev, from_revision=from_rev, return_revision_list=True) elif backfill: revlist = find_backfill_revlist( buildername=buildername, revision=rev, max_pushes=max_revisions, ) else: revlist = [rev] if skips: revlist = revlist[::skips] return revlist
def find_backfill_revlist(buildername, revision, max_pushes=None): """Determine which revisions we need to trigger in order to backfill. This function is generally called by automatic backfilling on pulse_actions. We need to take into consideration that a job might not be run for many revisions due to SETA. We also might have a permanent failure appear after a reconfiguration (a new job is added). When a permanent failure appears, we keep on adding load unnecessarily by triggering coalesced jobs in between pushes. Long lived failing job (it could be hidden): * push N -> failed job * push N-1 -> failed/coalesced job * push N-2 -> failed/coalesced job ... * push N-max_pushes-1 -> failed/coalesced job If the list of revision we need to trigger is larger than max_pushes it means that we either have not had that job scheduled beyond max_pushes or it has been failing forever. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername) # XXX: There is a chance that a green job has run in a newer push (the priority was higher), # however, this is unlikely. # XXX: We might need to consider when a backout has already landed and stop backfilling LOG.info("BACKFILL-START:%s_%s begins." % (revision[0:8], buildername)) revlist = query_pushes_by_specified_revision_range( repo_url=query_repo_url_from_buildername(buildername), revision=revision, before=max_pushes - 1, after=0, return_revision_list=True ) new_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=True) if len(new_revlist) >= max_pushes: # It is likely that we are facing a long lived permanent failure LOG.debug("We're not going to backfill %s since it is likely to be a permanent " "failure." % buildername) LOG.info("BACKFILL-END:%s_%s will not backfill." % (revision[0:8], buildername)) return [] else: LOG.info("BACKFILL-END:%s_%s will backfill %s." % (revision[0:8], buildername, new_revlist)) return new_revlist
def test_query_pushes_by_specified_revision_range(self, get, query_push_by_revision): pushes = query_pushes_by_specified_revision_range(repo_url=self.repo_url, revision=self.revision, before=1, after=1) # This part is totally duplicate with the test_query_pushes_by_pushid_range, # because we are calling query_pushes_by_pushid_range inside this function. assert len(pushes) == 3 push_id_list = [] changeset_list = [] for push in pushes: push_id_list.append(push.id) changeset_list.append(push.changesets[0].node) assert push_id_list == ['53348', '53349', '53350'] assert changeset_list == ['eb15e3f893453d6a4472f8905271aba33f8b68d5', '1c5b4332e2f1b73fe03977b69371e9a08503bff3', '724f0a71d62171da1357e6c1f93453359e54206b']
def test_query_pushes_by_specified_revision_range(self, get, query_push_by_revision): pushes = query_pushes_by_specified_revision_range( repo_url=self.repo_url, revision=self.revision, before=1, after=1) # This part is totally duplicate with the test_query_pushes_by_pushid_range, # because we are calling query_pushes_by_pushid_range inside this function. assert len(pushes) == 3 push_id_list = [] changeset_list = [] for push in pushes: push_id_list.append(push.id) changeset_list.append(push.changesets[0].node) assert push_id_list == ['53348', '53349', '53350'] assert changeset_list == [ 'eb15e3f893453d6a4472f8905271aba33f8b68d5', '1c5b4332e2f1b73fe03977b69371e9a08503bff3', '724f0a71d62171da1357e6c1f93453359e54206b' ]