Ejemplo n.º 1
0
def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
                                  beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
                                  beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
                                  build_tools_repo_path, checksums_enabled, dashboard_check, en_US_config,
                                  extra_balrog_submitter_params, final_verify_channels,
                                  final_verify_platforms, uptake_monitoring_platforms,
                                  funsize_balrog_api_root, l10n_config,
                                  l10n_changesets, mozharness_changeset, next_version,
                                  partial_updates, partner_repacks_platforms,
                                  postrelease_bouncer_aliases_enabled, uptake_monitoring_enabled,
                                  postrelease_version_bump_enabled,
                                  product, public_key, push_to_candidates_enabled,
                                  push_to_releases_automatic, push_to_releases_enabled, release_channels,
                                  repo_path, revision, signing_class, signing_pvt_key, source_enabled,
                                  tuxedo_server_url, update_verify_enabled, updates_builder_enabled,
                                  updates_enabled, verifyConfigs, version, publish_to_balrog_channels):
    """simple wrapper that sanitizes whatever calls make_task_graph uses universally known kwargs"""

    kwargs = dict(
        appVersion=appVersion,
        balrog_api_root=balrog_api_root,
        balrog_password=balrog_password,
        balrog_username=balrog_username,
        beetmover_aws_access_key_id=beetmover_aws_access_key_id,
        beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
        beetmover_candidates_bucket=beetmover_candidates_bucket,
        bouncer_enabled=bouncer_enabled,
        branch=branch,
        buildNumber=buildNumber,
        build_tools_repo_path=build_tools_repo_path,
        checksums_enabled=checksums_enabled,
        dashboard_check=dashboard_check,
        en_US_config=en_US_config,
        final_verify_channels=final_verify_channels,
        final_verify_platforms=final_verify_platforms,
        uptake_monitoring_platforms=uptake_monitoring_platforms,
        funsize_balrog_api_root=funsize_balrog_api_root,
        l10n_changesets=l10n_changesets,
        l10n_config=l10n_config,
        mozharness_changeset=mozharness_changeset,
        next_version=next_version,
        partial_updates=partial_updates,
        partner_repacks_platforms=partner_repacks_platforms,
        postrelease_bouncer_aliases_enabled=postrelease_bouncer_aliases_enabled,
        uptake_monitoring_enabled=uptake_monitoring_enabled,
        postrelease_version_bump_enabled=postrelease_version_bump_enabled,
        product=product,
        public_key=public_key,
        push_to_candidates_enabled=push_to_candidates_enabled,
        push_to_releases_automatic=push_to_releases_automatic,
        push_to_releases_enabled=push_to_releases_enabled,
        release_channels=release_channels,
        repo_path=repo_path,
        revision=revision,
        signing_class=signing_class,
        signing_pvt_key=signing_pvt_key,
        source_enabled=source_enabled,
        tuxedo_server_url=tuxedo_server_url,
        update_verify_enabled=update_verify_enabled,
        updates_builder_enabled=updates_builder_enabled,
        updates_enabled=updates_enabled,
        verifyConfigs=verifyConfigs,
        version=version,
        publish_to_balrog_channels=publish_to_balrog_channels,
    )
    if extra_balrog_submitter_params:
        kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params

    # don't import releasetasks until required within function impl to avoid global failures
    # during nosetests
    from releasetasks import make_task_graph
    return make_task_graph(**kwargs)
Ejemplo n.º 2
0
def main(options):
    log.info('Loading config from %s' % options.config)
    config = load_config(options.config)

    if config.getboolean('release-runner', 'verbose'):
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                        level=log_level)
    # Suppress logging of retry(), see bug 925321 for the details
    logging.getLogger("util.retry").setLevel(logging.WARN)

    # Shorthand
    api_root = config.get('api', 'api_root')
    username = config.get('api', 'username')
    password = config.get('api', 'password')
    buildbot_configs = config.get('release-runner', 'buildbot_configs')
    buildbot_configs_branch = config.get('release-runner',
                                         'buildbot_configs_branch')
    sleeptime = config.getint('release-runner', 'sleeptime')
    notify_from = get_config(config, 'release-runner', 'notify_from', None)
    notify_to = get_config(config, 'release-runner', 'notify_to', None)
    docker_worker_key = get_config(config, 'release-runner',
                                   'docker_worker_key', None)
    signing_pvt_key = get_config(config, 'signing', 'pvt_key', None)
    if isinstance(notify_to, basestring):
        notify_to = [x.strip() for x in notify_to.split(',')]
    smtp_server = get_config(config, 'release-runner', 'smtp_server',
                             'localhost')
    tc_config = {
        "credentials": {
            "clientId": get_config(config, "taskcluster", "client_id", None),
            "accessToken": get_config(config, "taskcluster", "access_token",
                                      None),
        }
    }
    configs_workdir = 'buildbot-configs'
    balrog_username = get_config(config, "balrog", "username", None)
    balrog_password = get_config(config, "balrog", "password", None)
    extra_balrog_submitter_params = get_config(
        config, "balrog", "extra_balrog_submitter_params", None)
    beetmover_aws_access_key_id = get_config(config, "beetmover",
                                             "aws_access_key_id", None)
    beetmover_aws_secret_access_key = get_config(config, "beetmover",
                                                 "aws_secret_access_key", None)
    gpg_key_path = get_config(config, "signing", "gpg_key_path", None)

    # TODO: replace release sanity with direct checks of en-US and l10n revisions (and other things if needed)

    rr = ReleaseRunner(api_root=api_root, username=username, password=password)
    scheduler = Scheduler(tc_config)
    index = Index(tc_config)
    queue = Queue(tc_config)

    # Main loop waits for new releases, processes them and exits.
    while True:
        try:
            log.debug('Fetching release requests')
            rr.get_release_requests()
            if rr.new_releases:
                for release in rr.new_releases:
                    log.info('Got a new release request: %s' % release)
                break
            else:
                log.debug('Sleeping for %d seconds before polling again' %
                          sleeptime)
                time.sleep(sleeptime)
        except:
            log.error("Caught exception when polling:", exc_info=True)
            sys.exit(5)

    retry(mercurial,
          args=(buildbot_configs, configs_workdir),
          kwargs=dict(branch=buildbot_configs_branch))

    if 'symlinks' in config.sections():
        format_dict = dict(buildbot_configs=configs_workdir)
        for target in config.options('symlinks'):
            symlink = config.get('symlinks', target).format(**format_dict)
            if path.exists(symlink):
                log.warning("Skipping %s -> %s symlink" % (symlink, target))
            else:
                log.info("Adding %s -> %s symlink" % (symlink, target))
                os.symlink(target, symlink)

    # TODO: this won't work for Thunderbird...do we care?
    branch = release["branch"].split("/")[-1]
    branchConfig = readBranchConfig(path.join(configs_workdir, "mozilla"),
                                    branch=branch)

    release_channels = update_channels(
        release["version"], branchConfig["release_channel_mappings"])
    # candidate releases are split in two graphs and release-runner only handles the first
    # graph of tasks. so parts like postrelease, push_to_releases/mirrors, and mirror dependant
    # channels are handled in the second generated graph outside of release-runner.
    # This is not elegant but it should do the job for now
    candidate_release = is_candidate_release(release_channels)
    if candidate_release:
        postrelease_enabled = False
        final_verify_channels = [
            c for c in release_channels
            if c not in branchConfig.get('mirror_requiring_channels', [])
        ]
        # TODO - use publish_to_balrog_channels once releasetasks publishes to balrog
        publish_to_balrog_channels = [
            c for c in release_channels
            if c not in branchConfig.get('mirror_requiring_channels', [])
        ]
        push_to_releases_enabled = False
    else:
        postrelease_enabled = branchConfig['postrelease_version_bump_enabled']
        final_verify_channels = release_channels
        publish_to_balrog_channels = release_channels
        push_to_releases_enabled = True

    rc = 0
    for release in rr.new_releases:
        graph_id = slugId()
        try:
            rr.update_status(release, 'Generating task graph')
            l10n_changesets = parsePlainL10nChangesets(
                rr.get_release_l10n(release["name"]))

            kwargs = {
                "public_key":
                docker_worker_key,
                "version":
                release["version"],
                "next_version":
                bump_version(release["version"]),
                "appVersion":
                getAppVersion(release["version"]),
                "buildNumber":
                release["buildNumber"],
                "source_enabled":
                True,
                "checksums_enabled":
                True,
                "repo_path":
                release["branch"],
                "revision":
                release["mozillaRevision"],
                "product":
                release["product"],
                # if mozharness_revision is not passed, use 'revision'
                "mozharness_changeset":
                release.get('mh_changeset') or release['mozillaRevision'],
                "partial_updates":
                getPartials(release),
                "branch":
                branch,
                "updates_enabled":
                bool(release["partials"]),
                "l10n_config":
                get_l10n_config(release, branchConfig, branch, l10n_changesets,
                                index),
                "en_US_config":
                get_en_US_config(release, branchConfig, branch, index),
                "verifyConfigs": {},
                "balrog_api_root":
                branchConfig["balrog_api_root"],
                "funsize_balrog_api_root":
                branchConfig["funsize_balrog_api_root"],
                "balrog_username":
                balrog_username,
                "balrog_password":
                balrog_password,
                "beetmover_aws_access_key_id":
                beetmover_aws_access_key_id,
                "beetmover_aws_secret_access_key":
                beetmover_aws_secret_access_key,
                # TODO: stagin specific, make them configurable
                "signing_class":
                "release-signing",
                "bouncer_enabled":
                branchConfig["bouncer_enabled"],
                "release_channels":
                release_channels,
                "final_verify_channels":
                final_verify_channels,
                "signing_pvt_key":
                signing_pvt_key,
                "build_tools_repo_path":
                branchConfig['build_tools_repo_path'],
                "push_to_candidates_enabled":
                branchConfig['push_to_candidates_enabled'],
                "postrelease_bouncer_aliases_enabled":
                branchConfig['postrelease_bouncer_aliases_enabled'],
                "tuxedo_server_url":
                branchConfig['tuxedoServerUrl'],
                "postrelease_version_bump_enabled":
                postrelease_enabled,
                "push_to_releases_enabled":
                push_to_releases_enabled,
                "push_to_releases_automatic":
                branchConfig['push_to_releases_automatic'],
                "beetmover_candidates_bucket":
                branchConfig["beetmover_buckets"][release["product"]],
            }
            if extra_balrog_submitter_params:
                kwargs[
                    "extra_balrog_submitter_params"] = extra_balrog_submitter_params

            validate_graph_kwargs(queue, gpg_key_path, **kwargs)
            graph = make_task_graph(**kwargs)
            rr.update_status(release, "Submitting task graph")
            log.info("Task graph generated!")
            import pprint
            log.debug(pprint.pformat(graph, indent=4, width=160))
            print scheduler.createTaskGraph(graph_id, graph)

            rr.mark_as_completed(release)
            email_release_drivers(smtp_server=smtp_server,
                                  from_=notify_from,
                                  to=notify_to,
                                  release=release,
                                  graph_id=graph_id)
        except:
            # We explicitly do not raise an error here because there's no
            # reason not to start other releases if creating the Task Graph
            # fails for another one. We _do_ need to set this in order to exit
            # with the right code, though.
            rc = 2
            rr.mark_as_failed(
                release,
                'Failed to start release promotion (graph ID: %s)' % graph_id)
            log.exception("Failed to start release promotion for graph %s %s",
                          graph_id, release)

    if rc != 0:
        sys.exit(rc)
Ejemplo n.º 3
0
def main(options):
    log.info('Loading config from %s' % options.config)
    config = load_config(options.config)

    if config.getboolean('release-runner', 'verbose'):
        log_level = logging.DEBUG
    else:
        log_level = logging.INFO
    logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
                        level=log_level)
    # Suppress logging of retry(), see bug 925321 for the details
    logging.getLogger("util.retry").setLevel(logging.WARN)

    # Shorthand
    api_root = config.get('api', 'api_root')
    username = config.get('api', 'username')
    password = config.get('api', 'password')
    buildbot_configs = config.get('release-runner', 'buildbot_configs')
    buildbot_configs_branch = config.get('release-runner',
                                         'buildbot_configs_branch')
    sleeptime = config.getint('release-runner', 'sleeptime')
    notify_from = get_config(config, 'release-runner', 'notify_from', None)
    notify_to = get_config(config, 'release-runner', 'notify_to', None)
    docker_worker_key = get_config(config, 'release-runner',
                                   'docker_worker_key', None)
    if isinstance(notify_to, basestring):
        notify_to = [x.strip() for x in notify_to.split(',')]
    smtp_server = get_config(config, 'release-runner', 'smtp_server',
                             'localhost')
    tc_config = {
        "credentials": {
            "clientId": get_config(config, "taskcluster", "client_id", None),
            "accessToken": get_config(config, "taskcluster", "access_token", None),
        }
    }
    configs_workdir = 'buildbot-configs'
    balrog_username = get_config(config, "balrog", "username", None)
    balrog_password = get_config(config, "balrog", "password", None)

    # TODO: replace release sanity with direct checks of en-US and l10n revisions (and other things if needed)

    rr = ReleaseRunner(api_root=api_root, username=username, password=password)
    scheduler = Scheduler(tc_config)
    index = Index(tc_config)

    # Main loop waits for new releases, processes them and exits.
    while True:
        try:
            log.debug('Fetching release requests')
            rr.get_release_requests()
            if rr.new_releases:
                for release in rr.new_releases:
                    log.info('Got a new release request: %s' % release)
                break
            else:
                log.debug('Sleeping for %d seconds before polling again' %
                          sleeptime)
                time.sleep(sleeptime)
        except:
            log.error("Caught exception when polling:", exc_info=True)
            sys.exit(5)

    retry(mercurial, args=(buildbot_configs, configs_workdir), kwargs=dict(branch=buildbot_configs_branch))

    if 'symlinks' in config.sections():
        format_dict = dict(buildbot_configs=configs_workdir)
        for target in config.options('symlinks'):
            symlink = config.get('symlinks', target).format(**format_dict)
            if path.exists(symlink):
                log.warning("Skipping %s -> %s symlink" % (symlink, target))
            else:
                log.info("Adding %s -> %s symlink" % (symlink, target))
                os.symlink(target, symlink)

    # TODO: this won't work for Thunderbird...do we care?
    branch = release["branch"].split("/")[-1]
    branchConfig = readBranchConfig(path.join(configs_workdir, "mozilla"), branch=branch)

    rc = 0
    for release in rr.new_releases:
        try:
            rr.update_status(release, 'Generating task graph')
            l10n_changesets = parsePlainL10nChangesets(rr.get_release_l10n(release["name"]))

            kwargs = {
                "public_key": docker_worker_key,
                "version": release["version"],
                "appVersion": getAppVersion(release["version"]),
                "buildNumber": release["buildNumber"],
                "source_enabled": True,
                "repo_path": release["branch"],
                "revision": release["mozillaRevision"],
                "product": release["product"],
                "partial_updates": getPartials(release),
                "branch": branch,
                "updates_enabled": bool(release["partials"]),
                "enUS_platforms": branchConfig["release_platforms"],
                "l10n_config": get_l10n_config(release, branchConfig, branch, l10n_changesets, index),
                "en_US_config": get_en_US_config(release, branchConfig, branch, index),
                "verifyConfigs": {},
                "balrog_api_root": branchConfig["balrog_api_root"],
                "balrog_username": balrog_username,
                "balrog_password": balrog_password,
                # TODO: stagin specific, make them configurable
                "signing_class": "dep-signing",
            }
            verifyConfigTemplate = "{branch}-{product}-{plat}.cfg"
            for plat in branchConfig["release_platforms"]:
                kwargs["verifyConfigs"][plat] = verifyConfigTemplate.format(
                    branch=kwargs['branch'],
                    product=kwargs['product'],
                    plat=plat,
                )

            validate_graph_kwargs(**kwargs)

            graph_id = slugId()
            graph = make_task_graph(**kwargs)

            rr.update_status(release, "Submitting task graph")

            log.info("Task graph generated!")
            import pprint
            log.debug(pprint.pformat(graph, indent=4, width=160))
            print scheduler.createTaskGraph(graph_id, graph)

            rr.mark_as_completed(release)
        except:
            # We explicitly do not raise an error here because there's no
            # reason not to start other releases if creating the Task Graph
            # fails for another one. We _do_ need to set this in order to exit
            # with the right code, though.
            rc = 2
            rr.update_status(release, 'Failed to start release promotion')
            log.exception("Failed to start release promotion for {}: ".format(release))

    if rc != 0:
        sys.exit(rc)
Ejemplo n.º 4
0
def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
                                  beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
                                  beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
                                  build_tools_repo_path, candidates_fennec_enabled,
                                  checksums_enabled, binary_transparency_enabled, en_US_config,
                                  root_home_dir, extra_balrog_submitter_params, final_verify_channels,
                                  final_verify_platforms, uptake_monitoring_platforms,
                                  funsize_balrog_api_root, l10n_config, signing_cert,
                                  l10n_changesets, mozharness_changeset, next_version,
                                  partial_updates, partner_repacks_platforms,
                                  eme_free_repacks_platforms, sha1_repacks_platforms,
                                  postrelease_bouncer_aliases_enabled, uptake_monitoring_enabled,
                                  postrelease_version_bump_enabled, moz_disable_mar_cert_verification,
                                  postrelease_mark_as_shipped_enabled, accepted_mar_channel_id, public_key,
                                  product, stage_product, funsize_product, push_to_candidates_enabled,
                                  push_to_releases_automatic, push_to_releases_enabled, release_channels,
                                  repo_path, revision, signing_class, signing_pvt_key, source_enabled,
                                  tuxedo_server_url, update_verify_enabled, updates_builder_enabled,
                                  updates_enabled, verifyConfigs, version, publish_to_balrog_channels,
                                  snap_enabled, update_verify_channel, update_verify_requires_cdn_push,
                                  release_eta):
    """simple wrapper that sanitizes whatever calls make_task_graph uses universally known kwargs"""

    kwargs = dict(
        appVersion=appVersion,
        balrog_api_root=balrog_api_root,
        balrog_password=balrog_password,
        balrog_username=balrog_username,
        beetmover_aws_access_key_id=beetmover_aws_access_key_id,
        beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
        beetmover_candidates_bucket=beetmover_candidates_bucket,
        bouncer_enabled=bouncer_enabled,
        branch=branch,
        buildNumber=buildNumber,
        build_tools_repo_path=build_tools_repo_path,
        candidates_fennec_enabled=candidates_fennec_enabled,
        checksums_enabled=checksums_enabled,
        binary_transparency_enabled=binary_transparency_enabled,
        en_US_config=en_US_config,
        final_verify_channels=final_verify_channels,
        final_verify_platforms=final_verify_platforms,
        uptake_monitoring_platforms=uptake_monitoring_platforms,
        funsize_balrog_api_root=funsize_balrog_api_root,
        l10n_changesets=l10n_changesets,
        l10n_config=l10n_config,
        mozharness_changeset=mozharness_changeset,
        next_version=next_version,
        partial_updates=partial_updates,
        partner_repacks_platforms=partner_repacks_platforms,
        eme_free_repacks_platforms=eme_free_repacks_platforms,
        sha1_repacks_platforms=sha1_repacks_platforms,
        postrelease_bouncer_aliases_enabled=postrelease_bouncer_aliases_enabled,
        uptake_monitoring_enabled=uptake_monitoring_enabled,
        postrelease_version_bump_enabled=postrelease_version_bump_enabled,
        postrelease_mark_as_shipped_enabled=postrelease_mark_as_shipped_enabled,
        product=product,
        public_key=public_key,
        push_to_candidates_enabled=push_to_candidates_enabled,
        push_to_releases_automatic=push_to_releases_automatic,
        push_to_releases_enabled=push_to_releases_enabled,
        release_channels=release_channels,
        repo_path=repo_path,
        revision=revision,
        signing_class=signing_class,
        accepted_mar_channel_id=accepted_mar_channel_id,
        signing_cert=signing_cert,
        moz_disable_mar_cert_verification=moz_disable_mar_cert_verification,
        root_home_dir=root_home_dir,
        signing_pvt_key=signing_pvt_key,
        source_enabled=source_enabled,
        stage_product=stage_product,
        tuxedo_server_url=tuxedo_server_url,
        update_verify_enabled=update_verify_enabled,
        updates_builder_enabled=updates_builder_enabled,
        updates_enabled=updates_enabled,
        verifyConfigs=verifyConfigs,
        version=version,
        publish_to_balrog_channels=publish_to_balrog_channels,
        snap_enabled=snap_enabled,
        update_verify_channel=update_verify_channel,
        update_verify_requires_cdn_push=update_verify_requires_cdn_push,
        funsize_product=funsize_product,
        release_eta=release_eta,
    )
    if extra_balrog_submitter_params:
        kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params

    # don't import releasetasks until required within function impl to avoid global failures
    # during nosetests
    from releasetasks import make_task_graph
    return make_task_graph(**kwargs)