def main(): options = parse_args() if options.debug: LOG.setLevel(logging.DEBUG) logging.getLogger("requests").setLevel(logging.DEBUG) LOG.info("Setting DEBUG level") else: LOG.setLevel(logging.INFO) # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) pgo = False if options.repo_name in PGO_ONLY_BRANCHES or options.pgo: pgo = True buildernames = build_talos_buildernames_for_repo(options.repo_name, pgo) filters_in, filters_out = [], [] if options.includes: filters_in = options.includes.split(',') if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames(filters_in, filters_out, buildernames) for buildername in buildernames: trigger_job(revision=options.revision, buildername=buildername, times=options.times, dry_run=options.dry_run)
def main(): options = parse_args() if not valid_credentials(): sys.exit(-1) if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) if options.rev == 'tip': repo_url = query_repo_url(options.repo) options.rev = query_repo_tip(repo_url) LOG.info("The tip of %s is %s", options.repo, options.rev) filters_in = options.includes.split(',') + [options.repo] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames( buildernames=query_builders(repo_name=options.repo), include=filters_in, exclude=filters_out) if len(buildernames) == 0: LOG.info("0 jobs match these filters, please try again.") return cont = raw_input( "%i jobs will be triggered, do you wish to continue? y/n/d (d=show details) " % len(buildernames)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(buildernames)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[options.rev], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode( { 'repo': query_repo_name_from_buildername(buildername), 'fromchange': options.rev, 'tochange': options.rev, 'filter-searchStr': buildername }))
def get_upstream_buildernames(repo_name): """Return every upstream buildername in a repo.""" buildernames = filter_buildernames([repo_name], ["hg bundle", "pgo"], list_builders()) upstream_jobs = [] for buildername in buildernames: if not is_downstream(buildername): upstream_jobs.append(buildername) return upstream_jobs
def test_include_exclude(self): """filter_buildernames should return a list matching the criteria.""" buildernames = MOCK_ALLTHETHINGS['builders'].keys() self.assertEquals( filter_buildernames(include=['repo', 'mochitest-1'], exclude=['debug'], buildernames=buildernames), ['Platform1 repo opt test mochitest-1'])
def main(): options = parse_args() repo_url = query_repo_url(options.repo) if not valid_credentials(): sys.exit(-1) if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) if options.rev == 'tip': revision = query_repo_tip(repo_url) LOG.info("The tip of %s is %s", options.repo, revision) else: revision = query_full_revision_info(repo_url, options.rev) filters_in = options.includes.split(',') + [options.repo] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames( buildernames=query_builders(repo_name=options.repo), include=filters_in, exclude=filters_out ) if len(buildernames) == 0: LOG.info("0 jobs match these filters, please try again.") return cont = raw_input("%i jobs will be triggered, do you wish to continue? y/n/d (d=show details) " % len(buildernames)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(buildernames)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[revision], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': query_repo_name_from_buildername(buildername), 'fromchange': revision, 'tochange': revision, 'filter-searchStr': buildername}))
def test_include_exclude(self): """filter_buildernames should return a list matching the criteria.""" buildernames = MOCK_ALLTHETHINGS['builders'].keys() self.assertEquals( filter_buildernames( include=['repo', 'mochitest-1'], exclude=['debug'], buildernames=buildernames ), ['Platform1 repo opt test mochitest-1'] )
def test_include_exclude(self): """filter_buildernames should return a list matching the criteria.""" buildernames = ALLTHETHINGS['builders'].keys() obtained = sorted( filter_buildernames(include=['try', 'mochitest-1', 'Windows'], exclude=['debug', 'pgo'], buildernames=buildernames)) expected = [ u'Windows 7 VM 32-bit try opt test mochitest-1', u'Windows 8 64-bit try opt test mochitest-1', u'Windows XP 32-bit try opt test mochitest-1', ] assert obtained == expected
def test_include_exclude(self): """filter_buildernames should return a list matching the criteria.""" buildernames = ALLTHETHINGS['builders'].keys() obtained = sorted(filter_buildernames( include=['try', 'mochitest-1', 'Windows'], exclude=['debug', 'pgo'], buildernames=buildernames )) expected = [ u'Windows 10 64-bit try opt test mochitest-1', u'Windows 7 VM 32-bit try opt test mochitest-1', u'Windows 8 64-bit try opt test mochitest-1', u'Windows XP 32-bit try opt test mochitest-1', ] assert obtained == expected
def trigger_missing_jobs_for_revision(repo_name, revision, dry_run=False): """ Trigger missing jobs for a given revision. Jobs have any of ('hg bundle', 'b2g', 'pgo') in their buildername will not be triggered. """ all_buildernames = filter_buildernames([repo_name], ['hg bundle', 'b2g', 'pgo'], allthethings.list_builders()) for buildername in all_buildernames: trigger_range(buildername=buildername, revisions=[revision], times=1, dry_run=dry_run, extra_properties={'mozci_request': { 'type': 'trigger_missing_jobs_for_revision'} })
def trigger_missing_jobs_for_revision(repo_name, revision, dry_run=False): """ Trigger missing jobs for a given revision. Jobs have any of ('hg bundle', 'b2g', 'pgo') in their buildername will not be triggered. """ all_buildernames = filter_buildernames([repo_name], ['hg bundle', 'b2g', 'pgo'], allthethings.list_builders()) for buildername in all_buildernames: trigger_range(buildername=buildername, revisions=[revision], times=1, dry_run=dry_run, extra_properties={ 'mozci_request': { 'type': 'trigger_missing_jobs_for_revision' } })
def main(): options = parse_args() if options.debug: LOG.setLevel(logging.DEBUG) logging.getLogger("requests").setLevel(logging.DEBUG) LOG.info("Setting DEBUG level") else: LOG.setLevel(logging.INFO) # requests is too noisy and adds no value logging.getLogger("requests").setLevel(logging.WARNING) filters_in = options.includes.split(',') + [options.repo] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') buildernames = filter_buildernames(filters_in, filters_out, query_builders()) cont = raw_input("%i jobs will be triggered, do you wish to continue? y/n/d (d=show details) " % len(buildernames)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(buildernames)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) for buildername in buildernames: trigger_range( buildername=buildername, revisions=[options.rev], times=options.times, dry_run=options.dry_run, ) LOG.info('https://treeherder.mozilla.org/#/jobs?%s' % urllib.urlencode({'repo': query_repo_name_from_buildername(buildername), 'fromchange': options.rev, 'tochange': options.rev, 'filter-searchStr': buildername}))
def _includes_excludes(options, repo_name): filters_in = options.includes.split(',') + [repo_name] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') job_names = filter_buildernames( buildernames=query_builders(repo_name=repo_name), include=filters_in, exclude=filters_out) if len(job_names) == 0: LOG.info("0 jobs match these filters. please try again.") return if options.existing_only: # We query all successful jobs for a given revision and filter # them by include/exclude filters. trigger_build_if_missing = False successful_jobs = TreeherderApi().find_all_jobs_by_status( repo_name=repo_name, revision=revision, status=SUCCESS) # We will filter out all the existing job from those successful job we have. job_names = [ buildername for buildername in successful_jobs if buildername in job_names ] cont = raw_input( "The ones which have existing builds out of %i jobs will be triggered,\ do you wish to continue? y/n/d (d=show details) " % len(job_names)) else: cont = raw_input("%i jobs will be triggered, do you wish to continue? \ y/n/d (d=show details) " % len(job_names)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1)
def _includes_excludes(options, repo_name): filters_in = options.includes.split(',') + [repo_name] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') job_names = filter_buildernames( buildernames=query_builders(repo_name=repo_name), include=filters_in, exclude=filters_out ) if len(job_names) == 0: LOG.info("0 jobs match these filters. please try again.") return if options.existing_only: # We query all successful jobs for a given revision and filter # them by include/exclude filters. trigger_build_if_missing = False successful_jobs = TreeherderApi().find_all_jobs_by_status( repo_name=repo_name, revision=revision, status=SUCCESS) # We will filter out all the existing job from those successful job we have. job_names = [buildername for buildername in successful_jobs if buildername in job_names] cont = raw_input("The ones which have existing builds out of %i jobs will be triggered,\ do you wish to continue? y/n/d (d=show details) " % len(job_names)) else: cont = raw_input("%i jobs will be triggered, do you wish to continue? \ y/n/d (d=show details) " % len(job_names)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1)
def main(): options = parse_args() if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) validate_options(options) if not valid_credentials(): sys.exit(-1) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) if options.buildernames: options.buildernames = sanitize_buildernames(options.buildernames) repo_url = query_repo_url_from_buildername(options.buildernames[0]) if not options.repo_name: repo_name = query_repo_name_from_buildername(options.buildernames[0]) else: repo_name = options.repo_name repo_url = query_repo_url(repo_name) if options.rev == 'tip': revision = query_repo_tip(repo_url).changesets[0].node LOG.info("The tip of %s is %s", repo_name, revision) else: revision = query_push_by_revision(repo_url, options.rev, return_revision_list=True) # Schedule jobs through TaskCluster if --taskcluster option has been set to true if options.taskcluster: mgr = TaskClusterBuildbotManager() else: mgr = BuildAPIManager() trigger_build_if_missing = options.trigger_build_if_missing if repo_name == 'try': trigger_build_if_missing = False # Mode 1: Trigger coalesced jobs if options.coalesced: query_api = BuildApi() request_ids = query_api.find_all_jobs_by_status(repo_name, revision, COALESCED) if len(request_ids) == 0: LOG.info('We did not find any coalesced job') for request_id in request_ids: make_retrigger_request(repo_name=repo_name, request_id=request_id, auth=get_credentials(), dry_run=options.dry_run) return # Mode #2: Fill-in a revision or trigger_test_jobs_only if options.fill_revision or options.trigger_tests_only: mgr.trigger_missing_jobs_for_revision( repo_name=repo_name, revision=revision, dry_run=options.dry_run, trigger_build_if_missing=not options.trigger_tests_only ) return # Mode #3: Trigger jobs based on revision list modifiers if not (options.includes or options.exclude or options.failed_jobs): job_names = options.buildernames # Mode 4 - Schedule every builder matching --includes and does not match --exclude. elif options.includes or options.exclude: filters_in = options.includes.split(',') + [repo_name] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') job_names = filter_buildernames( buildernames=query_builders(repo_name=repo_name), include=filters_in, exclude=filters_out ) if len(job_names) == 0: LOG.info("0 jobs match these filters. please try again.") return if options.existing_only: # We query all succesful jobs for a given revision and filter # them by include/exclude filters. trigger_build_if_missing = False successful_jobs = TreeherderApi().find_all_jobs_by_status( repo_name=repo_name, revision=revision, status=SUCCESS) # We will filter out all the existing job from those successful job we have. job_names = [buildername for buildername in successful_jobs if buildername in job_names] cont = raw_input("The ones which have existing builds out of %i jobs will be triggered,\ do you wish to continue? y/n/d (d=show details) " % len(job_names)) else: cont = raw_input("%i jobs will be triggered, do you wish to continue? \ y/n/d (d=show details) " % len(job_names)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) # Mode 5: Use --failed-jobs to trigger jobs for particular revision elif options.failed_jobs: job_names = TreeherderApi().find_all_jobs_by_status( repo_name=repo_name, revision=revision, status=WARNING) for buildername in job_names: revlist = determine_revlist( repo_url=repo_url, buildername=buildername, rev=revision, back_revisions=options.back_revisions, delta=options.delta, from_rev=options.from_rev, backfill=options.backfill, skips=options.skips, max_revisions=options.max_revisions) _print_treeherder_link( revlist=revlist, repo_name=repo_name, buildername=buildername, revision=revision, log=LOG, includes=options.includes, exclude=options.exclude) try: mgr.trigger_range( buildername=buildername, repo_name=repo_name, revisions=revlist, times=options.times, dry_run=options.dry_run, files=options.files, trigger_build_if_missing=trigger_build_if_missing ) except Exception, e: LOG.exception(e) exit(1)
def main(): options = parse_args() if options.debug: LOG = setup_logging(logging.DEBUG) else: LOG = setup_logging(logging.INFO) validate_options(options) if not options.dry_run and not valid_credentials(): sys.exit(-1) # Setting the QUERY_SOURCE global variable in mozci.py set_query_source(options.query_source) if options.buildernames: options.buildernames = sanitize_buildernames(options.buildernames) repo_url = query_repo_url_from_buildername(options.buildernames[0]) if not options.repo_name: repo_name = query_repo_name_from_buildername(options.buildernames[0]) else: repo_name = options.repo_name repo_url = query_repo_url(repo_name) if options.rev == 'tip': revision = query_repo_tip(repo_url).changesets[0].node LOG.info("The tip of %s is %s", repo_name, revision) else: revision = query_push_by_revision(repo_url, options.rev, return_revision_list=True) # Schedule jobs through TaskCluster if --taskcluster option has been set to true if options.taskcluster: mgr = TaskClusterBuildbotManager() else: mgr = BuildAPIManager() trigger_build_if_missing = options.trigger_build_if_missing if repo_name == 'try': trigger_build_if_missing = False # Mode 0: Backfill if options.backfill: manual_backfill(revision, options.buildernames[0], dry_run=options.dry_run) return # Mode 1: Trigger coalesced jobs if options.coalesced: query_api = BuildApi() request_ids = query_api.find_all_jobs_by_status(repo_name, revision, COALESCED) if len(request_ids) == 0: LOG.info('We did not find any coalesced job') for request_id in request_ids: make_retrigger_request(repo_name=repo_name, request_id=request_id, auth=get_credentials(), dry_run=options.dry_run) return # Mode #2: Fill-in a revision or trigger_test_jobs_only if options.fill_revision or options.trigger_tests_only: mgr.trigger_missing_jobs_for_revision( repo_name=repo_name, revision=revision, dry_run=options.dry_run, trigger_build_if_missing=not options.trigger_tests_only ) return # Mode #3: Trigger jobs based on revision list modifiers if not (options.includes or options.exclude or options.failed_jobs): job_names = options.buildernames # Mode 4 - Schedule every builder matching --includes and does not match --exclude. elif options.includes or options.exclude: filters_in = options.includes.split(',') + [repo_name] filters_out = [] if options.exclude: filters_out = options.exclude.split(',') job_names = filter_buildernames( buildernames=query_builders(repo_name=repo_name), include=filters_in, exclude=filters_out ) if len(job_names) == 0: LOG.info("0 jobs match these filters. please try again.") return if options.existing_only: # We query all successful jobs for a given revision and filter # them by include/exclude filters. trigger_build_if_missing = False successful_jobs = TreeherderApi().find_all_jobs_by_status( repo_name=repo_name, revision=revision, status=SUCCESS) # We will filter out all the existing job from those successful job we have. job_names = [buildername for buildername in successful_jobs if buildername in job_names] cont = raw_input("The ones which have existing builds out of %i jobs will be triggered,\ do you wish to continue? y/n/d (d=show details) " % len(job_names)) else: cont = raw_input("%i jobs will be triggered, do you wish to continue? \ y/n/d (d=show details) " % len(job_names)) if cont.lower() == 'd': LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names)) cont = raw_input("Do you wish to continue? y/n ") if cont.lower() != 'y': exit(1) # Mode 5: Use --failed-jobs to trigger jobs for particular revision elif options.failed_jobs: job_names = TreeherderApi().find_all_jobs_by_status( repo_name=repo_name, revision=revision, status=WARNING) for buildername in job_names: revlist = determine_revlist( repo_url=repo_url, buildername=buildername, rev=revision, back_revisions=options.back_revisions, delta=options.delta, from_rev=options.from_rev, backfill=options.backfill, skips=options.skips, max_revisions=options.max_revisions) _print_treeherder_link( revlist=revlist, repo_name=repo_name, buildername=buildername, revision=revision, log=LOG, includes=options.includes, exclude=options.exclude) try: mgr.trigger_range( buildername=buildername, repo_name=repo_name, revisions=revlist, times=options.times, dry_run=options.dry_run, files=options.files, trigger_build_if_missing=trigger_build_if_missing ) except Exception, e: LOG.exception(e) exit(1)