def main():
    parser = ArgumentParser()
    parser.add_argument("--debug",
                        action="store_true",
                        dest="debug",
                        help="set debug for logging.")

    parser.add_argument("--dry-run",
                        action="store_true",
                        dest="dry_run",
                        help="Dry run. No real actions are taken.")

    parser.add_argument('task_ids',
                        metavar='task_id',
                        type=str,
                        nargs='+',
                        help='Task IDs to work with.')

    options = parser.parse_args()

    if options.debug:
        LOG = setup_logging(logging.DEBUG)
    else:
        LOG = setup_logging()

    sch = TaskClusterBuildbotManager()
    for t_id in options.task_ids:
        ret_code = sch.retrigger(uuid=t_id, dry_run=options.dry_run)
        if ret_code < 0:
            LOG.warning("We could not retrigger task %s" % t_id)
def main():
    parser = ArgumentParser()
    parser.add_argument("--debug",
                        action="store_true",
                        dest="debug",
                        help="set debug for logging.")

    parser.add_argument("--dry-run",
                        action="store_true",
                        dest="dry_run",
                        help="Dry run. No real actions are taken.")

    parser.add_argument('task_ids',
                        metavar='task_id',
                        type=str,
                        nargs='+',
                        help='Task IDs to work with.')

    options = parser.parse_args()

    if options.debug:
        LOG = setup_logging(logging.DEBUG)
    else:
        LOG = setup_logging()

    sch = TaskClusterBuildbotManager()
    for t_id in options.task_ids:
        ret_code = sch.retrigger(uuid=t_id, dry_run=options.dry_run)
        if ret_code < 0:
            LOG.warning("We could not retrigger task %s" % t_id)
示例#3
0
def main():
    parser = ArgumentParser()
    parser.add_argument("--debug",
                        action="store_true",
                        dest="debug",
                        help="set debug for logging.")

    parser.add_argument("--dry-run",
                        action="store_true",
                        dest="dry_run",
                        help="Dry run. No real actions are taken.")

    parser.add_argument("--repo-name",
                        action="store",
                        dest="repo_name",
                        type=str,
                        help="Repository name, e.g. mozilla-inbound.")

    parser.add_argument("--revision",
                        action="store",
                        dest="revision",
                        type=str,
                        help="12-char representing a push.")

    parser.add_argument('builders_graph',
                        metavar='builders_graph',
                        help='Graph of builders in the form of: '
                             'dict(builder: [dep_builders].')

    options = parser.parse_args()

    if options.debug:
        setup_logging(logging.DEBUG)
    else:
        setup_logging()

    mgr = TaskClusterBuildbotManager()
    # XXX: test what happens when we have a bad graph
    mgr.schedule_graph(
        repo_name=options.repo_name,
        revision=options.revision,
        builders_graph=ast.literal_eval(options.builders_graph),
        #  dry_run=options.dry_run
    )
def main():
    parser = ArgumentParser()
    parser.add_argument("--debug",
                        action="store_true",
                        dest="debug",
                        help="set debug for logging.")

    parser.add_argument("--dry-run",
                        action="store_true",
                        dest="dry_run",
                        help="Dry run. No real actions are taken.")

    parser.add_argument("--repo-name",
                        action="store",
                        dest="repo_name",
                        type=str,
                        help="Repository name, e.g. mozilla-inbound.")

    parser.add_argument("--revision",
                        action="store",
                        dest="revision",
                        type=str,
                        help="12-char representing a push.")

    parser.add_argument("--trigger-from-task-id",
                        action="store",
                        dest="trigger_from_task_id",
                        type=str,
                        help="Trigger builders based on build task (use with "
                        "--builders).")

    parser.add_argument("--builders",
                        action="store",
                        dest="builders",
                        type=str,
                        help="Use this if you want to pass a list of builders "
                        "(e.g. \"['builder 1']\".")

    parser.add_argument(
        "--children-of",
        action="store",
        dest="children_of",
        type=str,
        help="This allows you to request a list of all the associated "
        "test jobs to a build job.")

    parser.add_argument("-g",
                        "--graph",
                        action="store",
                        dest="builders_graph",
                        help='Graph of builders in the form of: '
                        'dict(builder: [dep_builders].')

    options = parser.parse_args()

    if options.debug:
        setup_logging(logging.DEBUG)
    else:
        setup_logging()

    assert options.repo_name and options.revision, \
        "Make sure you specify --repo-name and --revion"

    if not options.dry_run and not credentials_available():
        sys.exit(1)

    mgr = TaskClusterBuildbotManager()
    builders = None
    if options.builders:
        builders = ast.literal_eval(options.builders)
    else:
        builders = get_downstream_jobs(options.children_of)

    if options.trigger_from_task_id and builders:
        trigger_builders_based_on_task_id(repo_name=options.repo_name,
                                          revision=options.revision,
                                          task_id=options.trigger_from_task_id,
                                          builders=builders,
                                          dry_run=options.dry_run)
    elif builders and len(builders) == 1:
        mgr.schedule_arbitrary_job(repo_name=options.repo_name,
                                   revision=options.revision,
                                   uuid=builders[0],
                                   dry_run=options.dry_run)
    elif options.builders_graph:
        mgr.schedule_graph(repo_name=options.repo_name,
                           revision=options.revision,
                           builders_graph=ast.literal_eval(
                               options.builders_graph),
                           dry_run=options.dry_run)
    else:
        print "Please read the help menu to know what options are available to you."
示例#5
0
def on_runnable_job_event(data, message, dry_run, treeherder_host,
                          acknowledge):
    # Cleaning mozci caches
    buildjson.BUILDS_CACHE = {}
    query_jobs.JOBS_CACHE = {}

    treeherder_client = TreeherderClient(host='treeherder.allizom.org')

    # XXX:
    # Grabbing data received over pulse
    repo_name = data["project"]
    requester = data["requester"]
    resultset_id = data["resultset_id"]
    buildernames = data["buildernames"]

    resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]
    revision = resultset["revision"]
    author = resultset["author"]
    status = None

    treeherder_link = TREEHERDER % {
        'host': treeherder_host,
        'repo': repo_name,
        'revision': resultset['revision']
    }

    message_sender = MessageHandler()
    if not (requester.endswith('@mozilla.com') or author == requester
            or whitelisted_users(requester)):
        # We want to see this in the alerts
        LOG.error("Notice that we're letting %s schedule jobs for %s." %
                  (requester, treeherder_link))
    '''
    # Everyone can press the button, but only authorized users can trigger jobs
    # TODO: remove this when proper LDAP identication is set up on TH
    if not (requester.endswith('@mozilla.com') or author == requester or
            whitelisted_users(requester)):

        if acknowledge:
            # Remove message from pulse queue
            message.ack()

        # We publish a message saying we will not trigger the job
        pulse_message = {
            'resultset_id': resultset_id,
            'requester': requester,
            'status': "Could not determine if the user is authorized, nothing was triggered."}
        routing_key = '{}.{}'.format(repo_name, 'runnable')
        try:
            message_sender.publish_message(pulse_message, routing_key)
        except:
            LOG.warning("Failed to publish message over pulse stream.")

        LOG.error("Requester %s is not allowed to trigger jobs on %s." %
                  (requester, treeherder_link))
        return  # Raising an exception adds too much noise
    '''

    LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link))
    LOG.info("List of builders:")
    for b in buildernames:
        LOG.info("- %s" % b)

    buildernames = filter_invalid_builders(buildernames)

    # Treeherder can send us invalid builder names
    # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038
    if buildernames is None:
        if acknowledge:
            # We need to ack the message to remove it from our queue
            message.ack()
        return

    builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder(
        builders=buildernames,
        revision=revision,
        complete=False  # XXX: This can be removed when BBB is in use
    )

    if builders_graph != {}:
        mgr = TaskClusterBuildbotManager()
        mgr.schedule_graph(
            repo_name=repo_name,
            revision=revision,
            metadata={
                'name':
                'pulse_actions_graph',
                'description':
                'Adding new jobs to push via pulse_actions/treeherder for %s.'
                % requester,
                'owner':
                requester,
                'source':
                treeherder_link,
            },
            builders_graph=builders_graph,
            dry_run=dry_run)
    else:
        LOG.info("We don't have anything to schedule through TaskCluster")

    if other_builders_to_schedule:
        # XXX: We should be able to replace this once all Buildbot jobs run through BBB
        # XXX: There might be a work around with
        #      https://github.com/mozilla/mozilla_ci_tools/issues/424
        LOG.info("We're going to schedule these builders via Buildapi: %s" %
                 str(other_builders_to_schedule))
        # This is used for test jobs which need an existing Buildbot job to be scheduled
        for buildername in other_builders_to_schedule:
            trigger_job(revision, buildername, dry_run=dry_run)
    else:
        LOG.info("We don't have anything to schedule through Buildapi")

    # Send a pulse message showing what we did
    message_sender = MessageHandler()
    pulse_message = {
        'resultset_id': resultset_id,
        'graph': builders_graph,
        'requester': requester,
        'status': status
    }
    routing_key = '{}.{}'.format(repo_name, 'runnable')
    try:
        message_sender.publish_message(pulse_message, routing_key)
    except:
        LOG.warning("Failed to publish message over pulse stream.")

    if acknowledge:
        # We need to ack the message to remove it from our queue
        message.ack()
示例#6
0
def main():
    options = parse_args()
    if options.debug:
        LOG = setup_logging(logging.DEBUG)
    else:
        LOG = setup_logging(logging.INFO)

    validate_options(options)
    if not valid_credentials():
        sys.exit(-1)

    # Setting the QUERY_SOURCE global variable in mozci.py
    set_query_source(options.query_source)

    if options.buildernames:
        options.buildernames = sanitize_buildernames(options.buildernames)
        repo_url = query_repo_url_from_buildername(options.buildernames[0])

    if not options.repo_name:
        repo_name = query_repo_name_from_buildername(options.buildernames[0])
    else:
        repo_name = options.repo_name

    repo_url = query_repo_url(repo_name)
    if options.rev == 'tip':
        revision = query_repo_tip(repo_url).changesets[0].node
        LOG.info("The tip of %s is %s", repo_name, revision)

    else:
        revision = query_push_by_revision(repo_url, options.rev,
                                          return_revision_list=True)

    # Schedule jobs through TaskCluster if --taskcluster option has been set to true
    if options.taskcluster:
        mgr = TaskClusterBuildbotManager()
    else:
        mgr = BuildAPIManager()

    trigger_build_if_missing = options.trigger_build_if_missing
    if repo_name == 'try':
        trigger_build_if_missing = False
    # Mode 1: Trigger coalesced jobs
    if options.coalesced:
        query_api = BuildApi()
        request_ids = query_api.find_all_jobs_by_status(repo_name,
                                                        revision, COALESCED)
        if len(request_ids) == 0:
            LOG.info('We did not find any coalesced job')
        for request_id in request_ids:
            make_retrigger_request(repo_name=repo_name,
                                   request_id=request_id,
                                   auth=get_credentials(),
                                   dry_run=options.dry_run)

        return

    # Mode #2: Fill-in a revision or trigger_test_jobs_only
    if options.fill_revision or options.trigger_tests_only:
        mgr.trigger_missing_jobs_for_revision(
            repo_name=repo_name,
            revision=revision,
            dry_run=options.dry_run,
            trigger_build_if_missing=not options.trigger_tests_only
        )
        return

    # Mode #3: Trigger jobs based on revision list modifiers
    if not (options.includes or options.exclude or options.failed_jobs):
        job_names = options.buildernames

    # Mode 4 - Schedule every builder matching --includes and does not match --exclude.
    elif options.includes or options.exclude:
        filters_in = options.includes.split(',') + [repo_name]
        filters_out = []

        if options.exclude:
            filters_out = options.exclude.split(',')

        job_names = filter_buildernames(
            buildernames=query_builders(repo_name=repo_name),
            include=filters_in,
            exclude=filters_out
        )
        if len(job_names) == 0:
            LOG.info("0 jobs match these filters. please try again.")
            return

        if options.existing_only:
            # We query all succesful jobs for a given revision and filter
            # them by include/exclude filters.
            trigger_build_if_missing = False
            successful_jobs = TreeherderApi().find_all_jobs_by_status(
                repo_name=repo_name,
                revision=revision,
                status=SUCCESS)
            # We will filter out all the existing job from those successful job we have.
            job_names = [buildername for buildername in successful_jobs
                         if buildername in job_names]
            cont = raw_input("The ones which have existing builds out of %i jobs will be triggered,\
                             do you wish to continue? y/n/d (d=show details) " % len(job_names))
        else:
            cont = raw_input("%i jobs will be triggered, do you wish to continue? \
                              y/n/d (d=show details) " % len(job_names))

        if cont.lower() == 'd':
            LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names))
            cont = raw_input("Do you wish to continue? y/n ")

        if cont.lower() != 'y':
            exit(1)

    # Mode 5: Use --failed-jobs to trigger jobs for particular revision
    elif options.failed_jobs:
        job_names = TreeherderApi().find_all_jobs_by_status(
            repo_name=repo_name,
            revision=revision,
            status=WARNING)

    for buildername in job_names:
        revlist = determine_revlist(
            repo_url=repo_url,
            buildername=buildername,
            rev=revision,
            back_revisions=options.back_revisions,
            delta=options.delta,
            from_rev=options.from_rev,
            backfill=options.backfill,
            skips=options.skips,
            max_revisions=options.max_revisions)

        _print_treeherder_link(
            revlist=revlist,
            repo_name=repo_name,
            buildername=buildername,
            revision=revision,
            log=LOG,
            includes=options.includes,
            exclude=options.exclude)

        try:
            mgr.trigger_range(
                buildername=buildername,
                repo_name=repo_name,
                revisions=revlist,
                times=options.times,
                dry_run=options.dry_run,
                files=options.files,
                trigger_build_if_missing=trigger_build_if_missing
            )
        except Exception, e:
            LOG.exception(e)
            exit(1)
def on_runnable_job_event(data, message, dry_run, stage):
    # Cleaning mozci caches
    buildjson.BUILDS_CACHE = {}
    query_jobs.JOBS_CACHE = {}

    if stage:
        treeherder_client = TreeherderClient(host='treeherder.allizom.org')
    else:
        treeherder_client = TreeherderClient()

    # Grabbing data received over pulse
    repo_name = data["project"]
    requester = data["requester"]
    resultset_id = data["resultset_id"]
    buildernames = data["buildernames"]

    resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]
    revision = resultset["revision"]
    author = resultset["author"]
    status = None

    treeherder_link = TREEHERDER % {'repo': repo_name, 'revision': resultset['revision']}

    message_sender = MessageHandler()
    # Everyone can press the button, but only authorized users can trigger jobs
    # TODO: remove this when proper LDAP identication is set up on TH
    if not (requester.endswith('@mozilla.com') or author == requester or
            whitelisted_users(requester)):

        if not dry_run:
            # Remove message from pulse queue
            message.ack()

        # We publish a message saying we will not trigger the job
        pulse_message = {
            'resultset_id': resultset_id,
            'requester': requester,
            'status': "Could not determine if the user is authorized, nothing was triggered."}
        routing_key = '{}.{}'.format(repo_name, 'runnable')
        try:
            message_sender.publish_message(pulse_message, routing_key)
        except:
            LOG.warning("Failed to publish message over pulse stream.")

        LOG.error("Requester %s is not allowed to trigger jobs on %s." %
                  (requester, treeherder_link))
        return  # Raising an exception adds too much noise

    LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link))
    LOG.info("List of builders:")
    for b in buildernames:
        LOG.info("- %s" % b)

    buildernames = filter_invalid_builders(buildernames)

    # Treeherder can send us invalid builder names
    # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038
    if buildernames is None:
        if not dry_run:
            # We need to ack the message to remove it from our queue
            message.ack()
        return

    builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder(
        builders=buildernames,
        revision=revision,
        complete=False  # XXX: This can be removed when BBB is in use
    )

    if builders_graph != {}:
        mgr = TaskClusterBuildbotManager()
        mgr.schedule_graph(
            repo_name=repo_name,
            revision=revision,
            metadata={
                'name': 'pulse_actions_graph',
                'description':
                    'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester,
                'owner': requester,
                'source': treeherder_link,
            },
            builders_graph=builders_graph,
            dry_run=dry_run)
    else:
        LOG.info("We don't have anything to schedule through TaskCluster")

    if other_builders_to_schedule:
        # XXX: We should be able to replace this once all Buildbot jobs run through BBB
        # XXX: There might be a work around with
        #      https://github.com/mozilla/mozilla_ci_tools/issues/424
        LOG.info("We're going to schedule these builders via Buildapi.")
        # This is used for test jobs which need an existing Buildbot job to be scheduled
        for buildername in other_builders_to_schedule:
            trigger_job(revision, buildername, dry_run=dry_run)
    else:
        LOG.info("We don't have anything to schedule through Buildapi")

    # Send a pulse message showing what we did
    message_sender = MessageHandler()
    pulse_message = {
        'resultset_id': resultset_id,
        'graph': builders_graph,
        'requester': requester,
        'status': status}
    routing_key = '{}.{}'.format(repo_name, 'runnable')
    try:
        message_sender.publish_message(pulse_message, routing_key)
    except:
        LOG.warning("Failed to publish message over pulse stream.")

    if not dry_run:
        # We need to ack the message to remove it from our queue
        message.ack()
def main():
    parser = ArgumentParser()
    parser.add_argument("--debug",
                        action="store_true",
                        dest="debug",
                        help="set debug for logging.")

    parser.add_argument("--dry-run",
                        action="store_true",
                        dest="dry_run",
                        help="Dry run. No real actions are taken.")

    parser.add_argument("--repo-name",
                        action="store",
                        dest="repo_name",
                        type=str,
                        help="Repository name, e.g. mozilla-inbound.")

    parser.add_argument("--revision",
                        action="store",
                        dest="revision",
                        type=str,
                        help="12-char representing a push.")

    parser.add_argument("--trigger-from-task-id",
                        action="store",
                        dest="trigger_from_task_id",
                        type=str,
                        help="Trigger builders based on build task (use with "
                        "--builders).")

    parser.add_argument("--builders",
                        action="store",
                        dest="builders",
                        type=str,
                        help="Use this if you want to pass a list of builders "
                        "(e.g. \"['builder 1']\".")

    parser.add_argument("--children-of",
                        action="store",
                        dest="children_of",
                        type=str,
                        help="This allows you to request a list of all the associated "
                        "test jobs to a build job.")

    parser.add_argument("-g", "--graph",
                        action="store",
                        dest="builders_graph",
                        help='Graph of builders in the form of: '
                             'dict(builder: [dep_builders].')

    options = parser.parse_args()

    if options.debug:
        setup_logging(logging.DEBUG)
    else:
        setup_logging()

    assert options.repo_name and options.revision, \
        "Make sure you specify --repo-name and --revision"

    if not options.dry_run and not credentials_available():
        sys.exit(1)

    mgr = TaskClusterBuildbotManager()
    builders = None
    if options.builders:
        builders = ast.literal_eval(options.builders)
    else:
        builders = get_downstream_jobs(options.children_of)

    if options.trigger_from_task_id and builders:
        trigger_builders_based_on_task_id(
            repo_name=options.repo_name,
            revision=options.revision,
            task_id=options.trigger_from_task_id,
            builders=builders,
            dry_run=options.dry_run
        )
    elif builders and len(builders) == 1:
        mgr.schedule_arbitrary_job(
            repo_name=options.repo_name,
            revision=options.revision,
            uuid=builders[0],
            dry_run=options.dry_run
        )
    elif options.builders_graph:
        mgr.schedule_graph(
            repo_name=options.repo_name,
            revision=options.revision,
            builders_graph=ast.literal_eval(options.builders_graph),
            dry_run=options.dry_run
        )
    else:
        print "Please read the help menu to know what options are available to you."