def manual_backfill(revision, buildername, max_pushes=None, dry_run=False): """ This function is used to trigger jobs for a range of revisions when a user clicks the backfill icon for a job on Treeherder. It backfills to the last known job on Treeherder. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername) repo_url = query_repo_url_from_buildername(buildername) # We want to use data from treeherder for manual backfilling for long term. set_query_source("treeherder") revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=revision, before=max_pushes, after=-1, # We don't want the current job in the revision to be included. return_revision_list=True) filtered_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=False) trigger_range( buildername=buildername, revisions=filtered_revlist, times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'manual_backfill', 'builders': [buildername]} } )
def manual_backfill(revision, buildername, max_pushes=None, dry_run=False): """ This function is used to trigger jobs for a range of revisions when a user clicks the backfill icon for a job on Treeherder. It backfills to the last known job on Treeherder. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername) repo_url = query_repo_url_from_buildername(buildername) # We want to use data from treeherder for manual backfilling for long term. set_query_source("treeherder") revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=revision, before=max_pushes, after=-1, # We don't want the current job in the revision to be included. return_revision_list=True) filtered_revlist = revlist # Talos jobs are generally always green and we want to fill in all holes in a range if 'talos' not in buildername: filtered_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=False) trigger_range( buildername=buildername, revisions=filtered_revlist, times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'manual_backfill', 'builders': [buildername]} } )
def manual_backfill(revision, buildername, dry_run=False): """ This function is used to trigger jobs for a range of revisions when a user clicks the backfill icon for a job on Treeherder. It backfills to the last known job on Treeherder. """ factor = 1.5 seta_skip = get_max_pushes(buildername) # Use SETA's skip pushes times factor max_pushes = seta_skip * factor repo_url = query_repo_url_from_buildername(buildername) # We want to use data from treeherder for manual backfilling for long term. set_query_source("treeherder") revlist = query_pushes_by_specified_revision_range( repo_url=repo_url, revision=revision, before=max_pushes, after=-1, # We don't want the current job in the revision to be included. return_revision_list=True) LOG.info("We're *aiming* to backfill; note that we ignore the revision that you request " "to backfill from ({}) up to {} pushes (seta skip: {}; factor: {}) " "and we backfill up to the last green if found.".format( revision[:12], max_pushes, seta_skip, factor)) LOG.info("https://treeherder.mozilla.org/#/jobs?repo={}&filter-searchStr={}" "&tochange={}&fromchange={}".format( repo_url.split('/')[-1], buildername, revlist[-1], revlist[0], )) filtered_revlist = revlist # Talos jobs are generally always green and we want to fill in all holes in a range if 'talos' not in buildername: filtered_revlist = _filter_backfill_revlist(buildername, list(reversed(revlist)), only_successful=True) if len(filtered_revlist) == 0: LOG.info("We don't have a revision list to work with.") return if len(revlist) != len(filtered_revlist): LOG.info("NOTICE: We were aiming for a revlist of {}, however, we only " "need to backfill {} revisions".format(len(revlist), len(filtered_revlist))) trigger_range( buildername=buildername, revisions=filtered_revlist, times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'manual_backfill', 'builders': [buildername]} } )
def find_backfill_revlist(buildername, revision, max_pushes=None): """Determine which revisions we need to trigger in order to backfill. This function is generally called by automatic backfilling on pulse_actions. We need to take into consideration that a job might not be run for many revisions due to SETA. We also might have a permanent failure appear after a reconfiguration (a new job is added). When a permanent failure appears, we keep on adding load unnecessarily by triggering coalesced jobs in between pushes. Long lived failing job (it could be hidden): * push N -> failed job * push N-1 -> failed/coalesced job * push N-2 -> failed/coalesced job ... * push N-max_pushes-1 -> failed/coalesced job If the list of revision we need to trigger is larger than max_pushes it means that we either have not had that job scheduled beyond max_pushes or it has been failing forever. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes( buildername) # XXX: There is a chance that a green job has run in a newer push (the priority was higher), # however, this is unlikely. # XXX: We might need to consider when a backout has already landed and stop backfilling LOG.info("BACKFILL-START:%s_%s begins." % (revision[0:8], buildername)) revlist = query_pushes_by_specified_revision_range( repo_url=query_repo_url_from_buildername(buildername), revision=revision, before=max_pushes - 1, after=0, return_revision_list=True) new_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=True) if len(new_revlist) >= max_pushes: # It is likely that we are facing a long lived permanent failure LOG.debug( "We're not going to backfill %s since it is likely to be a permanent " "failure." % buildername) LOG.info("BACKFILL-END:%s_%s will not backfill." % (revision[0:8], buildername)) return [] else: LOG.info("BACKFILL-END:%s_%s will backfill %s." % (revision[0:8], buildername, new_revlist)) return new_revlist
def find_backfill_revlist(buildername, revision, max_pushes=None): """Determine which revisions we need to trigger in order to backfill. This function is generally called by automatic backfilling on pulse_actions. We need to take into consideration that a job might not be run for many revisions due to SETA. We also might have a permanent failure appear after a reconfiguration (a new job is added). When a permanent failure appears, we keep on adding load unnecessarily by triggering coalesced jobs in between pushes. Long lived failing job (it could be hidden): * push N -> failed job * push N-1 -> failed/coalesced job * push N-2 -> failed/coalesced job ... * push N-max_pushes-1 -> failed/coalesced job If the list of revision we need to trigger is larger than max_pushes it means that we either have not had that job scheduled beyond max_pushes or it has been failing forever. """ max_pushes = max_pushes if max_pushes is not None else get_max_pushes(buildername) # XXX: There is a chance that a green job has run in a newer push (the priority was higher), # however, this is unlikely. # XXX: We might need to consider when a backout has already landed and stop backfilling LOG.info("BACKFILL-START:%s_%s begins." % (revision[0:8], buildername)) revlist = query_pushes_by_specified_revision_range( repo_url=query_repo_url_from_buildername(buildername), revision=revision, before=max_pushes - 1, after=0, return_revision_list=True ) new_revlist = _filter_backfill_revlist(buildername, revlist, only_successful=True) if len(new_revlist) >= max_pushes: # It is likely that we are facing a long lived permanent failure LOG.debug("We're not going to backfill %s since it is likely to be a permanent " "failure." % buildername) LOG.info("BACKFILL-END:%s_%s will not backfill." % (revision[0:8], buildername)) return [] else: LOG.info("BACKFILL-END:%s_%s will backfill %s." % (revision[0:8], buildername, new_revlist)) return new_revlist
def test_get_max_pushes_with_no_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert get_max_pushes( "Platform2 mozilla-beta talos tp5o-e10s") == MAX_PUSHES
def test_get_max_pushes_with_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert get_max_pushes( "Windows 7 VM 32-bit mozilla-inbound opt test mochitest-2") == 8
def test_get_max_pushes_with_no_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert get_max_pushes("Platform2 mozilla-beta talos tp5o") == MAX_PUSHES
def test_get_max_pushes_with_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert get_max_pushes("Windows 7 VM 32-bit mozilla-inbound opt test mochitest-2") == 8
def test_get_max_pushes_with_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert get_max_pushes( "Ubuntu VM 12.04 x64 fx-team opt test mochitest-2") == 7
def test_get_max_pushes_with_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = MOCK_ALLTHETHINGS self.assertEquals( get_max_pushes("Rev4 MacOSX Snow Leopard 10.6 fx-team debug test cppunit"), 7)
def test_get_max_pushes_with_seta(self, fetch_allthethings_data): """get_max_pushes should return the number of pushes associated to the SETA scheduler.""" fetch_allthethings_data.return_value = ALLTHETHINGS assert get_max_pushes("Ubuntu VM 12.04 x64 fx-team opt test mochitest-2") == 7