def main(): options = parse_args() if not valid_credentials(): sys.exit(-1) if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) if options.rev == 'tip': repo_url = query_repo_url(options.repo) options.rev = query_repo_tip(repo_url) LOG.info("The tip of %s is %s", options.repo, options.rev) filters_in = options.includes.split(',') + [options.repo] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames( buildernames=query_builders(repo_name=options.repo), include=filters_in, exclude=filters_out) if len(buildernames) == 0: LOG.info("0 jobs match these filters, please try again.") return cont = raw_input( "%i jobs will be triggered, do you wish to continue? y/n/d (d=show details) " % len(buildernames)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(buildernames)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[options.rev], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode( { 'repo': query_repo_name_from_buildername(buildername), 'fromchange': options.rev, 'tochange': options.rev, 'filter-searchStr': buildername }))
def trigger_range(self, buildername, repo_name, revisions, times, dry_run, files, trigger_build_if_missing): trigger_range(buildername=buildername, revisions=revisions, times=times, dry_run=dry_run, files=files, trigger_build_if_missing=trigger_build_if_missing)
def main(): options = parse_args() repo_url = query_repo_url(options.repo) if not valid_credentials(): sys.exit(-1) if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) if options.rev == 'tip': revision = query_repo_tip(repo_url) LOG.info("The tip of %s is %s", options.repo, revision) else: revision = query_full_revision_info(repo_url, options.rev) filters_in = options.includes.split(',') + [options.repo] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames( buildernames=query_builders(repo_name=options.repo), include=filters_in, exclude=filters_out ) if len(buildernames) == 0: LOG.info("0 jobs match these filters, please try again.") return cont = raw_input("%i jobs will be triggered, do you wish to continue? y/n/d (d=show details) " % len(buildernames)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(buildernames)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[revision], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': query_repo_name_from_buildername(buildername), 'fromchange': revision, 'tochange': revision, 'filter-searchStr': buildername}))
def trigger_range(self, buildername, repo_name, revisions, times, dry_run, files, trigger_build_if_missing): trigger_range( buildername=buildername, revisions=revisions, times=times, dry_run=dry_run, files=files, trigger_build_if_missing=trigger_build_if_missing, )
def on_event(data, message, dry_run): """Automatically backfill failed jobs.""" # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] status = payload["status"] buildername = payload["buildername"] # Backfill a failed job if status in [FAILURE, WARNING]: buildername = filter_invalid_builders(buildername) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildername is None: if not dry_run: # We need to ack the message to remove it from our queue message.ack() return revision = payload["revision"] LOG.info("**") # visual separator LOG.info("Failed job found at revision %s. Buildername: %s", revision, buildername) try: # We want to ensure 1 appearance of the job on every revision revlist = find_backfill_revlist( revision=revision, max_revisions=get_maxRevisions(buildername), buildername=buildername) trigger_range( buildername=buildername, revisions=revlist[1:], times=1, dry_run=dry_run, trigger_build_if_missing=False ) if not dry_run: # We need to ack the message to remove it from our queue message.ack() except ConnectionError: # The message has not been acked so we will try again LOG.warning("Connection error. Trying again") except PushlogError, e: # Unable to retrieve pushlog data. Please check repo_url and revision specified. LOG.warning(str(e)) except Exception, e: # The message has not been acked so we will try again LOG.warning(str(e)) raise
def on_event(data, message, dry_run, acknowledge): """Automatically backfill failed jobs.""" # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] status = payload["status"] buildername = payload["buildername"] # Backfill a failed job if status in [FAILURE, WARNING]: buildername = filter_invalid_builders(buildername) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildername is None: if acknowledge: # We need to ack the message to remove it from our queue message.ack() return revision = payload["revision"] LOG.info("**") # visual separator LOG.info("Failed job found at revision %s. Buildername: %s", revision, buildername) try: # We want to ensure 1 appearance of the job on every revision revlist = find_backfill_revlist(buildername=buildername, revision=revision) trigger_range(buildername=buildername, revisions=revlist[1:], times=1, dry_run=dry_run, trigger_build_if_missing=False) if acknowledge: # We need to ack the message to remove it from our queue message.ack() except ConnectionError: # The message has not been acked so we will try again LOG.warning("Connection error. Trying again") except PushlogError, e: # Unable to retrieve pushlog data. Please check repo_url and revision specified. LOG.warning(str(e)) except Exception, e: # The message has not been acked so we will try again LOG.warning(str(e)) raise
def trigger_missing_jobs_for_revision(self, repo_name, revision, dry_run=False, trigger_build_if_missing=True): """ Trigger missing jobs for a given revision. Jobs containing 'b2g' or 'pgo' in their buildername will not be triggered. """ builders_for_repo = list_builders(repo_name=repo_name) for buildername in builders_for_repo: trigger_range( buildername=buildername, revisions=[revision], times=1, dry_run=dry_run, extra_properties={"mozci_request": {"type": "trigger_missing_jobs_for_revision"}}, trigger_build_if_missing=trigger_build_if_missing, )
def main(): options = parse_args() if options.debug: LOG.setLevel(logging.DEBUG) logging.getLogger("requests").setLevel(logging.DEBUG) LOG.info("Setting DEBUG level") else: LOG.setLevel(logging.INFO) # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) filters_in = options.includes.split(' ') + [options.repo] buildernames = query_builders() for word in filters_in: buildernames = filter(lambda x: word in x, buildernames) if options.exclude: filters_out = options.exclude.split(' ') for word in filters_out: buildernames = filter(lambda x: word not in x, buildernames) if len(buildernames) > options.lim: LOG.info('There %i matching buildernames, the limit is %i. If you really want' 'to trigger everything, try again with --limit %i.' % (len(buildernames), options.lim, options.lim)) exit(1) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[options.rev], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': query_repo_name_from_buildername(buildername), 'fromchange': options.rev, 'tochange': options.rev, 'filter-searchStr': buildername}))
def main(): options = parse_args() if options.debug: LOG.setLevel(logging.DEBUG) logging.getLogger("requests").setLevel(logging.DEBUG) LOG.info("Setting DEBUG level") else: LOG.setLevel(logging.INFO) # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) filters_in = options.includes.split(',') + [options.repo] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames(filters_in, filters_out, query_builders()) cont = raw_input("%i jobs will be triggered, do you wish to continue? y/n/d (d=show details) " % len(buildernames)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(buildernames)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[options.rev], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': query_repo_name_from_buildername(buildername), 'fromchange': options.rev, 'tochange': options.rev, 'filter-searchStr': buildername}))
def trigger_missing_jobs_for_revision(self, repo_name, revision, dry_run=False, trigger_build_if_missing=True): """ Trigger missing jobs for a given revision. Jobs containing 'b2g' or 'pgo' in their buildername will not be triggered. """ builders_for_repo = list_builders(repo_name=repo_name) for buildername in builders_for_repo: trigger_range(buildername=buildername, revisions=[revision], times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'trigger_missing_jobs_for_revision' } }, trigger_build_if_missing=trigger_build_if_missing)
def on_event(data, message, dry_run): """Automatically backfill failed jobs.""" # We need to ack the message to remove it from our queue message.ack() # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] status = payload["status"] buildername = payload["buildername"] # Backfill a failed job if status in [FAILURE, WARNING]: revision = payload["revision"] LOG.info("**") # visual separator LOG.info("Failed job found at revision %s. Buildername: %s", revision, buildername) # We want to assure 1 appearance of the job on each of the revisions repo_url = query_repo_url_from_buildername(buildername) revlist = find_backfill_revlist( repo_url=repo_url, revision=revision, max_revisions=MAX_REVISIONS, buildername=buildername) trigger_range( buildername=buildername, revisions=revlist[1:], times=1, dry_run=dry_run, trigger_build_if_missing=False ) else: # TODO: change this to debug after a testing period LOG.info("'%s' with status %i. Nothing to be done.", buildername, status)
revlist = backfill_revlist( options.buildername, revlist, options.times, options.dry_run ) else: revlist = [options.rev] if options.skips: revlist = revlist[::options.skips] try: trigger_range( buildername=options.buildername, revisions=revlist, times=options.times, dry_run=options.dry_run ) except Exception, e: LOG.exception(e) exit(1) if revlist: LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': query_repo_name_from_buildername(options.buildername), 'fromchange': revlist[-1], 'tochange': revlist[0], 'filter-searchStr': options.buildername}))
def main(): options = parse_args() validate_options(options) repo_url = query_repo_url(options.repo_name) if not valid_credentials(): sys.exit(-1) if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) if options.buildernames: options.buildernames = sanitize_buildernames(options.buildernames) repo_url = query_repo_url_from_buildername(options.buildernames[0]) if not options.repo_name: options.repo_name = query_repo_name_from_buildername( options.buildernames[0]) if options.rev == 'tip': revision = query_repo_tip(repo_url) LOG.info("The tip of %s is %s", options.repo_name, revision) else: revision = query_full_revision_info(repo_url, options.rev) # Mode 1: Trigger coalesced jobs if options.coalesced: query_api = BuildApi() request_ids = query_api.find_all_jobs_by_status( options.repo_name, revision, COALESCED) if len(request_ids) == 0: LOG.info('We did not find any coalesced job') for request_id in request_ids: make_retrigger_request(repo_name=options.repo_name, request_id=request_id, dry_run=options.dry_run) return # Mode #2: Fill-in a revision if options.fill_revision: trigger_missing_jobs_for_revision(repo_name=options.repo_name, revision=revision, dry_run=options.dry_run) return # Mode #3: Trigger jobs based on revision list modifiers for buildername in options.buildernames: revlist = determine_revlist(repo_url=repo_url, buildername=buildername, rev=revision, back_revisions=options.back_revisions, delta=options.delta, from_rev=options.from_rev, backfill=options.backfill, skips=options.skips, max_revisions=options.max_revisions) try: trigger_range( buildername=buildername, revisions=revlist, times=options.times, dry_run=options.dry_run, files=options.files, trigger_build_if_missing=options.trigger_build_if_missing) except Exception, e: LOG.exception(e) exit(1) if revlist: LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({ 'repo': options.repo_name, 'fromchange': revlist[-1], 'tochange': revlist[0], 'filter-searchStr': buildername }))
def main(): alerts = getAlerts() for alert in alerts: # new alert LOG.info("Running alert for: [%s, %s, %s]" % (alert['test'], alert['buildername'], alert['revision'])) if alert['stage'] == 0: LOG.info("We are in stage 0.") if checkMerge(alert['revision'], alert['buildername']) or 'pgo' in alert['buildername']: LOG.info("We are ignoring alert: %s since it is either a merge or a pgo job." % alert['test']) alert['stage'] = -1 # We need to have manual inspection in this case. alert['user'] = '******' updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) else: alert['stage'] = 1 # trigger jobs for backfill if alert['stage'] == 1: LOG.info("We are in stage 1, and going to backfill jobs.") revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2) # Setting Treeherder as the source for querying. set_query_source("treeherder") trigger_range(alert['buildername'], revisionList, times=6, dry_run=DRY_RUN) alert['stage'] = 2 # We want some time interval between stage 1 and 2, so we exit. updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue # verify jobs for backfill if alert['stage'] == 2: LOG.info("We are in stage 2, and going to verify if jobs are backfilled.") revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2) for revision in revisionList: dataPoints = getSuccessfulJobs(revision, alert['buildername']) # If dataPoints are less than 6, it means that builds/jobs are still running. if dataPoints < 6: print "data points <6 for revision: %s" % revision # We wait for 6 hours for all triggered tests to complete, # And if they don't then we mark them for manual intervention/ alert['loop'] += 1 if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME) / CYCLE_TIME: LOG.info("The jobs did not complete backfilling in time, assigning for human inspection.") alert['stage'] = -1 alert['user'] = '******' else: LOG.info("The jobs have not completed backfilling. Looping back to stage 1.") alert['stage'] = 1 break if alert['stage'] != 2: print "updating alert and then continue, not stage 2" updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue badRevisions = [] # Reset the loop for upcoming stages alert['loop'] = 0 for i in range(1, len(revisionList)): print "getting results for revision number: %s" % i results = compare(alert['test'], alert['buildername'], revisionList[i], revisionList[i-1]) print "compare returned: %s" % results if results < -2.0: print "appending bad revision to list: %s"% revisionList[i] badRevisions.append(revisionList[i]) if len(badRevisions) != 1: LOG.info("There are too many bad revisions: %s for alert %s on buildername %s, " "assigning for human inspection." % (badRevisions, alert['test'], alert['buildername'])) alert['stage'] = -1 # too noisy, something bad happened alert['user'] = '******' print "too many bad revisions, update alert to human" updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue if checkMerge(badRevisions[0], alert['buildername']): LOG.info("The bad revision %s identified for alert %s on buildername %s is a merge, " "assigning for human inspection" % (badRevisions[0], alert['test'], alert['buildername'])) alert['stage'] = -1 # A merge revision is a bad revision, manually inspect alert['user'] = '******' if alert['revision'] != badRevisions[0]: LOG.info("Alert_Manager misreported the bad revision. The actual bad revision is %s " "for alert %s on %s buildername." % (badRevisions[0], alert['test'], alert['buildername'])) alert['revision'] = badRevisions[0] # we misreported initially, change the actual regression revision print "setting stage = 3!" alert['stage'] = 3 # Trigger all talos stage if alert['stage'] == 3: LOG.info("We are in stage 3, and going to trigger all_talos jobs.") repo_name = query_repo_name_from_buildername(alert['buildername']) # Setting Treeherder as the source for querying. set_query_source("treeherder") trigger_all_talos_jobs(repo_name, alert['revision'], times=6, dry_run=DRY_RUN) previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0] trigger_all_talos_jobs(repo_name, previousRevision, times=6, dry_run=DRY_RUN) alert['stage'] = 4 updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue # Verify All talos stage is completed if alert['stage'] == 4: LOG.info("We are in stage 4, and going to verify if all_talos ran successfully.") previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0] repo_name = query_repo_name_from_buildername(alert['buildername']) all_buildernames = build_talos_buildernames_for_repo(repo_name) for revision in [alert['revision'], previousRevision]: for buildername in all_buildernames: dataPoints = getSuccessfulJobs(revision, buildername) if dataPoints < 6: # We wait for 8 hours for all talos tests to complete, # And if they don't then we mark them for manual intervention alert['loop'] += 1 if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME + TIME_TO_WAIT) / CYCLE_TIME: LOG.info("The all talos jobs for alert %s on %s revision did not complete in time, " " assigning for human inspection." % (alert['test'], alert['revision'])) alert['stage'] = -1 alert['user'] = '******' else: alert['stage'] = 3 break if alert['stage'] != 4: break if alert['stage'] != 4: updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue alert['stage'] = 5 # final stage, sheriff will check for this. alert['user'] = '******' LOG.info("All automated parts are complete.") updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user'])
def main(): alerts = getAlerts() for alert in alerts: # new alert if alert['stage'] == 0: if checkMerge(alert['revision'], alert['buildername']) or 'pgo' in alert['buildername']: LOG.info("We are ignoring this alert since it is either a merge or a pgo job.") alert['stage'] = -1 # We need to have manual inspection in this case. alert['user'] = '******' updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) else: alert['stage'] = 1 # trigger jobs for backfill if alert['stage'] == 1: LOG.info("We are in stage 1, and going to backfill jobs.") revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2) trigger_range(alert['buildername'], revisionList, times=6, dry_run=DRY_RUN) alert['stage'] = 2 # We want some time interval between stage 1 and 2, so we exit. updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue # verify jobs for backfill if alert['stage'] == 2: LOG.info("We are in stage 2, and going to verify if jobs are backfilled.") revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2) for revision in revisionList: dataPoints = getSuccessfulJobs(revision, alert['buildername']) # If dataPoints are less than 6, it means that builds/jobs are still running. if dataPoints < 6: # We wait for 6 hours for all triggered tests to complete, # And if they don't then we mark them for manual intervention/ alert['loop'] += 1 if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME) / CYCLE_TIME: alert['stage'] = -1 alert['user'] = '******' else: alert['stage'] = 1 break if alert['stage'] != 2: updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue badRevisions = [] # Reset the loop for upcoming stages alert['loop'] = 0 for i in range(1, len(revisionList)): results = compare(alert['test'], alert['buildername'], revisionList[i], revisionList[i-1]) if results < -2.0: badRevisions.append(revisionList[i]) if len(badRevisions) != 1: alert['stage'] = -1 # too noisy, something bad happened alert['user'] = '******' updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue if checkMerge(badRevisions[0], alert['buildername']): alert['stage'] = -1 # A merge revision is a bad revision, manually inspect alert['user'] = '******' if alert['revision'] != badRevisions[0]: alert['revision'] = badRevisions[0] # we misreported initially, change the actual regression revision alert['stage'] = 3 # Trigger all talos stage if alert['stage'] == 3: LOG.info("We are in stage 3, and going to trigger all_talos jobs.") repo_name = query_repo_name_from_buildername(alert['buildername']) trigger_all_talos_jobs(repo_name, alert['revision'], times=6, dry_run=DRY_RUN) previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0] trigger_all_talos_jobs(repo_name, previousRevision, times=6, dry_run=DRY_RUN) alert['stage'] = 4 updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue # Verify All talos stage is completed if alert['stage'] == 4: LOG.info("We are in stage 4, and going to verify if all_talos ran successfully.") previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0] repo_name = query_repo_name_from_buildername(alert['buildername']) all_buildernames = build_talos_buildernames_for_repo(repo_name) for revision in [alert['revision'], previousRevision]: for buildername in all_buildernames: dataPoints = getSuccessfulJobs(revision, buildername) if dataPoints < 6: # We wait for 8 hours for all talos tests to complete, # And if they don't then we mark them for manual intervention alert['loop'] += 1 if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME + TIME_TO_WAIT) / CYCLE_TIME: alert['stage'] = -1 alert['user'] = '******' else: alert['stage'] = 3 break if alert['stage'] != 4: break if alert['stage'] != 4: updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue alert['stage'] = 5 # final stage, sheriff will check for this. updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user'])
def main(): options = parse_args() validate_options(options) if options.debug: LOG.setLevel(logging.DEBUG) logging.getLogger("requests").setLevel(logging.DEBUG) LOG.info("Setting DEBUG level") else: LOG.setLevel(logging.INFO) # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) options.buildername = sanitize_buildername(options.buildername) repo_url = query_repo_url_from_buildername(options.buildername) if options.back_revisions: push_info = query_revision_info(repo_url, options.rev) end_id = int(push_info["pushid"]) # newest revision start_id = end_id - options.back_revisions revlist = query_pushid_range(repo_url=repo_url, start_id=start_id, end_id=end_id) elif options.delta: revlist = query_revisions_range_from_revision_and_delta( repo_url, options.rev, options.delta) elif options.from_rev: revlist = query_revisions_range( repo_url, to_revision=options.rev, from_revision=options.from_rev) elif options.backfill: push_info = query_revision_info(repo_url, options.rev) # A known bad revision end_id = int(push_info["pushid"]) # newest revision # The furthest we will go to find the last good job # We might find a good job before that start_id = end_id - options.max_revisions + 1 revlist = query_pushid_range(repo_url=repo_url, start_id=start_id, end_id=end_id) revlist = backfill_revlist( options.buildername, revlist, options.times, options.dry_run ) else: revlist = [options.rev] if options.skips: revlist = revlist[::options.skips] try: trigger_range( buildername=options.buildername, revisions=revlist, times=options.times, dry_run=options.dry_run, files=options.files ) except Exception, e: LOG.exception(e) exit(1)
def main(): options = parse_args() validate_options(options) valid_credentials() if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) if options.buildernames: options.buildernames = sanitize_buildernames(options.buildernames) repo_url = query_repo_url_from_buildername(options.buildernames[0]) if not options.repo_name: options.repo_name = query_repo_name_from_buildername(options.buildernames[0]) if options.rev == 'tip': repo_url = query_repo_url(options.repo_name) options.rev = query_repo_tip(repo_url) LOG.info("The tip of %s is %s", options.repo_name, options.rev) if options.coalesced: query_api = BuildApi() request_ids = query_api.find_all_jobs_by_status(options.repo_name, options.rev, COALESCED) if len(request_ids) == 0: LOG.info('We did not find any coalesced job') for request_id in request_ids: make_retrigger_request(repo_name=options.repo_name, request_id=request_id, dry_run=options.dry_run) return for buildername in options.buildernames: revlist = determine_revlist( repo_url=repo_url, buildername=buildername, rev=options.rev, back_revisions=options.back_revisions, delta=options.delta, from_rev=options.from_rev, backfill=options.backfill, skips=options.skips, max_revisions=options.max_revisions) try: trigger_range( buildername=buildername, revisions=revlist, times=options.times, dry_run=options.dry_run, files=options.files, trigger_build_if_missing=options.trigger_build_if_missing ) except Exception, e: LOG.exception(e) exit(1) if revlist: LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': options.repo_name, 'fromchange': revlist[-1], 'tochange': revlist[0], 'filter-searchStr': buildername}))
def main(): alerts = getAlerts() for alert in alerts: # new alert if alert['stage'] == 0: if checkMerge( alert['revision'], alert['buildername']) or 'pgo' in alert['buildername']: LOG.info( "We are ignoring this alert since it is either a merge or a pgo job." ) alert[ 'stage'] = -1 # We need to have manual inspection in this case. alert['user'] = '******' updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) else: alert['stage'] = 1 # trigger jobs for backfill if alert['stage'] == 1: LOG.info("We are in stage 1, and going to backfill jobs.") revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2) trigger_range(alert['buildername'], revisionList, times=6, dry_run=DRY_RUN) alert['stage'] = 2 # We want some time interval between stage 1 and 2, so we exit. updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue # verify jobs for backfill if alert['stage'] == 2: LOG.info( "We are in stage 2, and going to verify if jobs are backfilled." ) revisionList = getRevisions(alert['revision'], alert['buildername'], start=-2, end=2) for revision in revisionList: dataPoints = getSuccessfulJobs(revision, alert['buildername']) # If dataPoints are less than 6, it means that builds/jobs are still running. if dataPoints < 6: # We wait for 6 hours for all triggered tests to complete, # And if they don't then we mark them for manual intervention/ alert['loop'] += 1 if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME) / CYCLE_TIME: alert['stage'] = -1 alert['user'] = '******' else: alert['stage'] = 1 break if alert['stage'] != 2: updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue badRevisions = [] # Reset the loop for upcoming stages alert['loop'] = 0 for i in range(1, len(revisionList)): results = compare(alert['test'], alert['buildername'], revisionList[i], revisionList[i - 1]) if results < -2.0: badRevisions.append(revisionList[i]) if len(badRevisions) != 1: alert['stage'] = -1 # too noisy, something bad happened alert['user'] = '******' updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue if checkMerge(badRevisions[0], alert['buildername']): alert[ 'stage'] = -1 # A merge revision is a bad revision, manually inspect alert['user'] = '******' if alert['revision'] != badRevisions[0]: alert['revision'] = badRevisions[ 0] # we misreported initially, change the actual regression revision alert['stage'] = 3 # Trigger all talos stage if alert['stage'] == 3: LOG.info("We are in stage 3, and going to trigger all_talos jobs.") repo_name = query_repo_name_from_buildername(alert['buildername']) trigger_all_talos_jobs(repo_name, alert['revision'], times=6, dry_run=DRY_RUN) previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0] trigger_all_talos_jobs(repo_name, previousRevision, times=6, dry_run=DRY_RUN) alert['stage'] = 4 updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue # Verify All talos stage is completed if alert['stage'] == 4: LOG.info( "We are in stage 4, and going to verify if all_talos ran successfully." ) previousRevision = getRevisions(alert['revision'], alert['buildername'], start=-1, end=-1)[0] repo_name = query_repo_name_from_buildername(alert['buildername']) all_buildernames = build_talos_buildernames_for_repo(repo_name) for revision in [alert['revision'], previousRevision]: for buildername in all_buildernames: dataPoints = getSuccessfulJobs(revision, buildername) if dataPoints < 6: # We wait for 8 hours for all talos tests to complete, # And if they don't then we mark them for manual intervention alert['loop'] += 1 if alert['loop'] > (TIME_TO_BUILD + TIME_TO_TEST + PENDING_TIME + TIME_TO_WAIT) / CYCLE_TIME: alert['stage'] = -1 alert['user'] = '******' else: alert['stage'] = 3 break if alert['stage'] != 4: break if alert['stage'] != 4: updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user']) continue alert['stage'] = 5 # final stage, sheriff will check for this. updateAlert(alert['id'], alert['revision'], alert['buildername'], alert['test'], alert['stage'], alert['loop'], alert['user'])