def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
build_tools_repo_path, checksums_enabled, dashboard_check, en_US_config,
extra_balrog_submitter_params, final_verify_channels,
final_verify_platforms, uptake_monitoring_platforms,
funsize_balrog_api_root, l10n_config,
l10n_changesets, mozharness_changeset, next_version,
partial_updates, partner_repacks_platforms,
postrelease_bouncer_aliases_enabled, uptake_monitoring_enabled,
postrelease_version_bump_enabled,
product, public_key, push_to_candidates_enabled,
push_to_releases_automatic, push_to_releases_enabled, release_channels,
repo_path, revision, signing_class, signing_pvt_key, source_enabled,
tuxedo_server_url, update_verify_enabled, updates_builder_enabled,
updates_enabled, verifyConfigs, version, publish_to_balrog_channels):
"""simple wrapper that sanitizes whatever calls make_task_graph uses universally known kwargs"""
kwargs = dict(
appVersion=appVersion,
balrog_api_root=balrog_api_root,
balrog_password=balrog_password,
balrog_username=balrog_username,
beetmover_aws_access_key_id=beetmover_aws_access_key_id,
beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
beetmover_candidates_bucket=beetmover_candidates_bucket,
bouncer_enabled=bouncer_enabled,
branch=branch,
buildNumber=buildNumber,
build_tools_repo_path=build_tools_repo_path,
checksums_enabled=checksums_enabled,
dashboard_check=dashboard_check,
en_US_config=en_US_config,
final_verify_channels=final_verify_channels,
final_verify_platforms=final_verify_platforms,
uptake_monitoring_platforms=uptake_monitoring_platforms,
funsize_balrog_api_root=funsize_balrog_api_root,
l10n_changesets=l10n_changesets,
l10n_config=l10n_config,
mozharness_changeset=mozharness_changeset,
next_version=next_version,
partial_updates=partial_updates,
partner_repacks_platforms=partner_repacks_platforms,
postrelease_bouncer_aliases_enabled=postrelease_bouncer_aliases_enabled,
uptake_monitoring_enabled=uptake_monitoring_enabled,
postrelease_version_bump_enabled=postrelease_version_bump_enabled,
product=product,
public_key=public_key,
push_to_candidates_enabled=push_to_candidates_enabled,
push_to_releases_automatic=push_to_releases_automatic,
push_to_releases_enabled=push_to_releases_enabled,
release_channels=release_channels,
repo_path=repo_path,
revision=revision,
signing_class=signing_class,
signing_pvt_key=signing_pvt_key,
source_enabled=source_enabled,
tuxedo_server_url=tuxedo_server_url,
update_verify_enabled=update_verify_enabled,
updates_builder_enabled=updates_builder_enabled,
updates_enabled=updates_enabled,
verifyConfigs=verifyConfigs,
version=version,
publish_to_balrog_channels=publish_to_balrog_channels,
)
if extra_balrog_submitter_params:
kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
# don't import releasetasks until required within function impl to avoid global failures
# during nosetests
from releasetasks import make_task_graph
return make_task_graph(**kwargs)
def main(options):
log.info('Loading config from %s' % options.config)
config = load_config(options.config)
if config.getboolean('release-runner', 'verbose'):
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=log_level)
# Suppress logging of retry(), see bug 925321 for the details
logging.getLogger("util.retry").setLevel(logging.WARN)
# Shorthand
api_root = config.get('api', 'api_root')
username = config.get('api', 'username')
password = config.get('api', 'password')
buildbot_configs = config.get('release-runner', 'buildbot_configs')
buildbot_configs_branch = config.get('release-runner',
'buildbot_configs_branch')
sleeptime = config.getint('release-runner', 'sleeptime')
notify_from = get_config(config, 'release-runner', 'notify_from', None)
notify_to = get_config(config, 'release-runner', 'notify_to', None)
docker_worker_key = get_config(config, 'release-runner',
'docker_worker_key', None)
signing_pvt_key = get_config(config, 'signing', 'pvt_key', None)
if isinstance(notify_to, basestring):
notify_to = [x.strip() for x in notify_to.split(',')]
smtp_server = get_config(config, 'release-runner', 'smtp_server',
'localhost')
tc_config = {
"credentials": {
"clientId": get_config(config, "taskcluster", "client_id", None),
"accessToken": get_config(config, "taskcluster", "access_token",
None),
}
}
configs_workdir = 'buildbot-configs'
balrog_username = get_config(config, "balrog", "username", None)
balrog_password = get_config(config, "balrog", "password", None)
extra_balrog_submitter_params = get_config(
config, "balrog", "extra_balrog_submitter_params", None)
beetmover_aws_access_key_id = get_config(config, "beetmover",
"aws_access_key_id", None)
beetmover_aws_secret_access_key = get_config(config, "beetmover",
"aws_secret_access_key", None)
gpg_key_path = get_config(config, "signing", "gpg_key_path", None)
# TODO: replace release sanity with direct checks of en-US and l10n revisions (and other things if needed)
rr = ReleaseRunner(api_root=api_root, username=username, password=password)
scheduler = Scheduler(tc_config)
index = Index(tc_config)
queue = Queue(tc_config)
# Main loop waits for new releases, processes them and exits.
while True:
try:
log.debug('Fetching release requests')
rr.get_release_requests()
if rr.new_releases:
for release in rr.new_releases:
log.info('Got a new release request: %s' % release)
break
else:
log.debug('Sleeping for %d seconds before polling again' %
sleeptime)
time.sleep(sleeptime)
except:
log.error("Caught exception when polling:", exc_info=True)
sys.exit(5)
retry(mercurial,
args=(buildbot_configs, configs_workdir),
kwargs=dict(branch=buildbot_configs_branch))
if 'symlinks' in config.sections():
format_dict = dict(buildbot_configs=configs_workdir)
for target in config.options('symlinks'):
symlink = config.get('symlinks', target).format(**format_dict)
if path.exists(symlink):
log.warning("Skipping %s -> %s symlink" % (symlink, target))
else:
log.info("Adding %s -> %s symlink" % (symlink, target))
os.symlink(target, symlink)
# TODO: this won't work for Thunderbird...do we care?
branch = release["branch"].split("/")[-1]
branchConfig = readBranchConfig(path.join(configs_workdir, "mozilla"),
branch=branch)
release_channels = update_channels(
release["version"], branchConfig["release_channel_mappings"])
# candidate releases are split in two graphs and release-runner only handles the first
# graph of tasks. so parts like postrelease, push_to_releases/mirrors, and mirror dependant
# channels are handled in the second generated graph outside of release-runner.
# This is not elegant but it should do the job for now
candidate_release = is_candidate_release(release_channels)
if candidate_release:
postrelease_enabled = False
final_verify_channels = [
c for c in release_channels
if c not in branchConfig.get('mirror_requiring_channels', [])
]
# TODO - use publish_to_balrog_channels once releasetasks publishes to balrog
publish_to_balrog_channels = [
c for c in release_channels
if c not in branchConfig.get('mirror_requiring_channels', [])
]
push_to_releases_enabled = False
else:
postrelease_enabled = branchConfig['postrelease_version_bump_enabled']
final_verify_channels = release_channels
publish_to_balrog_channels = release_channels
push_to_releases_enabled = True
rc = 0
for release in rr.new_releases:
graph_id = slugId()
try:
rr.update_status(release, 'Generating task graph')
l10n_changesets = parsePlainL10nChangesets(
rr.get_release_l10n(release["name"]))
kwargs = {
"public_key":
docker_worker_key,
"version":
release["version"],
"next_version":
bump_version(release["version"]),
"appVersion":
getAppVersion(release["version"]),
"buildNumber":
release["buildNumber"],
"source_enabled":
True,
"checksums_enabled":
True,
"repo_path":
release["branch"],
"revision":
release["mozillaRevision"],
"product":
release["product"],
# if mozharness_revision is not passed, use 'revision'
"mozharness_changeset":
release.get('mh_changeset') or release['mozillaRevision'],
"partial_updates":
getPartials(release),
"branch":
branch,
"updates_enabled":
bool(release["partials"]),
"l10n_config":
get_l10n_config(release, branchConfig, branch, l10n_changesets,
index),
"en_US_config":
get_en_US_config(release, branchConfig, branch, index),
"verifyConfigs": {},
"balrog_api_root":
branchConfig["balrog_api_root"],
"funsize_balrog_api_root":
branchConfig["funsize_balrog_api_root"],
"balrog_username":
balrog_username,
"balrog_password":
balrog_password,
"beetmover_aws_access_key_id":
beetmover_aws_access_key_id,
"beetmover_aws_secret_access_key":
beetmover_aws_secret_access_key,
# TODO: stagin specific, make them configurable
"signing_class":
"release-signing",
"bouncer_enabled":
branchConfig["bouncer_enabled"],
"release_channels":
release_channels,
"final_verify_channels":
final_verify_channels,
"signing_pvt_key":
signing_pvt_key,
"build_tools_repo_path":
branchConfig['build_tools_repo_path'],
"push_to_candidates_enabled":
branchConfig['push_to_candidates_enabled'],
"postrelease_bouncer_aliases_enabled":
branchConfig['postrelease_bouncer_aliases_enabled'],
"tuxedo_server_url":
branchConfig['tuxedoServerUrl'],
"postrelease_version_bump_enabled":
postrelease_enabled,
"push_to_releases_enabled":
push_to_releases_enabled,
"push_to_releases_automatic":
branchConfig['push_to_releases_automatic'],
"beetmover_candidates_bucket":
branchConfig["beetmover_buckets"][release["product"]],
}
if extra_balrog_submitter_params:
kwargs[
"extra_balrog_submitter_params"] = extra_balrog_submitter_params
validate_graph_kwargs(queue, gpg_key_path, **kwargs)
graph = make_task_graph(**kwargs)
rr.update_status(release, "Submitting task graph")
log.info("Task graph generated!")
import pprint
log.debug(pprint.pformat(graph, indent=4, width=160))
print scheduler.createTaskGraph(graph_id, graph)
rr.mark_as_completed(release)
email_release_drivers(smtp_server=smtp_server,
from_=notify_from,
to=notify_to,
release=release,
graph_id=graph_id)
except:
# We explicitly do not raise an error here because there's no
# reason not to start other releases if creating the Task Graph
# fails for another one. We _do_ need to set this in order to exit
# with the right code, though.
rc = 2
rr.mark_as_failed(
release,
'Failed to start release promotion (graph ID: %s)' % graph_id)
log.exception("Failed to start release promotion for graph %s %s",
graph_id, release)
if rc != 0:
sys.exit(rc)
def main(options):
log.info('Loading config from %s' % options.config)
config = load_config(options.config)
if config.getboolean('release-runner', 'verbose'):
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=log_level)
# Suppress logging of retry(), see bug 925321 for the details
logging.getLogger("util.retry").setLevel(logging.WARN)
# Shorthand
api_root = config.get('api', 'api_root')
username = config.get('api', 'username')
password = config.get('api', 'password')
buildbot_configs = config.get('release-runner', 'buildbot_configs')
buildbot_configs_branch = config.get('release-runner',
'buildbot_configs_branch')
sleeptime = config.getint('release-runner', 'sleeptime')
notify_from = get_config(config, 'release-runner', 'notify_from', None)
notify_to = get_config(config, 'release-runner', 'notify_to', None)
docker_worker_key = get_config(config, 'release-runner',
'docker_worker_key', None)
if isinstance(notify_to, basestring):
notify_to = [x.strip() for x in notify_to.split(',')]
smtp_server = get_config(config, 'release-runner', 'smtp_server',
'localhost')
tc_config = {
"credentials": {
"clientId": get_config(config, "taskcluster", "client_id", None),
"accessToken": get_config(config, "taskcluster", "access_token", None),
}
}
configs_workdir = 'buildbot-configs'
balrog_username = get_config(config, "balrog", "username", None)
balrog_password = get_config(config, "balrog", "password", None)
# TODO: replace release sanity with direct checks of en-US and l10n revisions (and other things if needed)
rr = ReleaseRunner(api_root=api_root, username=username, password=password)
scheduler = Scheduler(tc_config)
index = Index(tc_config)
# Main loop waits for new releases, processes them and exits.
while True:
try:
log.debug('Fetching release requests')
rr.get_release_requests()
if rr.new_releases:
for release in rr.new_releases:
log.info('Got a new release request: %s' % release)
break
else:
log.debug('Sleeping for %d seconds before polling again' %
sleeptime)
time.sleep(sleeptime)
except:
log.error("Caught exception when polling:", exc_info=True)
sys.exit(5)
retry(mercurial, args=(buildbot_configs, configs_workdir), kwargs=dict(branch=buildbot_configs_branch))
if 'symlinks' in config.sections():
format_dict = dict(buildbot_configs=configs_workdir)
for target in config.options('symlinks'):
symlink = config.get('symlinks', target).format(**format_dict)
if path.exists(symlink):
log.warning("Skipping %s -> %s symlink" % (symlink, target))
else:
log.info("Adding %s -> %s symlink" % (symlink, target))
os.symlink(target, symlink)
# TODO: this won't work for Thunderbird...do we care?
branch = release["branch"].split("/")[-1]
branchConfig = readBranchConfig(path.join(configs_workdir, "mozilla"), branch=branch)
rc = 0
for release in rr.new_releases:
try:
rr.update_status(release, 'Generating task graph')
l10n_changesets = parsePlainL10nChangesets(rr.get_release_l10n(release["name"]))
kwargs = {
"public_key": docker_worker_key,
"version": release["version"],
"appVersion": getAppVersion(release["version"]),
"buildNumber": release["buildNumber"],
"source_enabled": True,
"repo_path": release["branch"],
"revision": release["mozillaRevision"],
"product": release["product"],
"partial_updates": getPartials(release),
"branch": branch,
"updates_enabled": bool(release["partials"]),
"enUS_platforms": branchConfig["release_platforms"],
"l10n_config": get_l10n_config(release, branchConfig, branch, l10n_changesets, index),
"en_US_config": get_en_US_config(release, branchConfig, branch, index),
"verifyConfigs": {},
"balrog_api_root": branchConfig["balrog_api_root"],
"balrog_username": balrog_username,
"balrog_password": balrog_password,
# TODO: stagin specific, make them configurable
"signing_class": "dep-signing",
}
verifyConfigTemplate = "{branch}-{product}-{plat}.cfg"
for plat in branchConfig["release_platforms"]:
kwargs["verifyConfigs"][plat] = verifyConfigTemplate.format(
branch=kwargs['branch'],
product=kwargs['product'],
plat=plat,
)
validate_graph_kwargs(**kwargs)
graph_id = slugId()
graph = make_task_graph(**kwargs)
rr.update_status(release, "Submitting task graph")
log.info("Task graph generated!")
import pprint
log.debug(pprint.pformat(graph, indent=4, width=160))
print scheduler.createTaskGraph(graph_id, graph)
rr.mark_as_completed(release)
except:
# We explicitly do not raise an error here because there's no
# reason not to start other releases if creating the Task Graph
# fails for another one. We _do_ need to set this in order to exit
# with the right code, though.
rc = 2
rr.update_status(release, 'Failed to start release promotion')
log.exception("Failed to start release promotion for {}: ".format(release))
if rc != 0:
sys.exit(rc)
def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
build_tools_repo_path, candidates_fennec_enabled,
checksums_enabled, binary_transparency_enabled, en_US_config,
root_home_dir, extra_balrog_submitter_params, final_verify_channels,
final_verify_platforms, uptake_monitoring_platforms,
funsize_balrog_api_root, l10n_config, signing_cert,
l10n_changesets, mozharness_changeset, next_version,
partial_updates, partner_repacks_platforms,
eme_free_repacks_platforms, sha1_repacks_platforms,
postrelease_bouncer_aliases_enabled, uptake_monitoring_enabled,
postrelease_version_bump_enabled, moz_disable_mar_cert_verification,
postrelease_mark_as_shipped_enabled, accepted_mar_channel_id, public_key,
product, stage_product, funsize_product, push_to_candidates_enabled,
push_to_releases_automatic, push_to_releases_enabled, release_channels,
repo_path, revision, signing_class, signing_pvt_key, source_enabled,
tuxedo_server_url, update_verify_enabled, updates_builder_enabled,
updates_enabled, verifyConfigs, version, publish_to_balrog_channels,
snap_enabled, update_verify_channel, update_verify_requires_cdn_push,
release_eta):
"""simple wrapper that sanitizes whatever calls make_task_graph uses universally known kwargs"""
kwargs = dict(
appVersion=appVersion,
balrog_api_root=balrog_api_root,
balrog_password=balrog_password,
balrog_username=balrog_username,
beetmover_aws_access_key_id=beetmover_aws_access_key_id,
beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
beetmover_candidates_bucket=beetmover_candidates_bucket,
bouncer_enabled=bouncer_enabled,
branch=branch,
buildNumber=buildNumber,
build_tools_repo_path=build_tools_repo_path,
candidates_fennec_enabled=candidates_fennec_enabled,
checksums_enabled=checksums_enabled,
binary_transparency_enabled=binary_transparency_enabled,
en_US_config=en_US_config,
final_verify_channels=final_verify_channels,
final_verify_platforms=final_verify_platforms,
uptake_monitoring_platforms=uptake_monitoring_platforms,
funsize_balrog_api_root=funsize_balrog_api_root,
l10n_changesets=l10n_changesets,
l10n_config=l10n_config,
mozharness_changeset=mozharness_changeset,
next_version=next_version,
partial_updates=partial_updates,
partner_repacks_platforms=partner_repacks_platforms,
eme_free_repacks_platforms=eme_free_repacks_platforms,
sha1_repacks_platforms=sha1_repacks_platforms,
postrelease_bouncer_aliases_enabled=postrelease_bouncer_aliases_enabled,
uptake_monitoring_enabled=uptake_monitoring_enabled,
postrelease_version_bump_enabled=postrelease_version_bump_enabled,
postrelease_mark_as_shipped_enabled=postrelease_mark_as_shipped_enabled,
product=product,
public_key=public_key,
push_to_candidates_enabled=push_to_candidates_enabled,
push_to_releases_automatic=push_to_releases_automatic,
push_to_releases_enabled=push_to_releases_enabled,
release_channels=release_channels,
repo_path=repo_path,
revision=revision,
signing_class=signing_class,
accepted_mar_channel_id=accepted_mar_channel_id,
signing_cert=signing_cert,
moz_disable_mar_cert_verification=moz_disable_mar_cert_verification,
root_home_dir=root_home_dir,
signing_pvt_key=signing_pvt_key,
source_enabled=source_enabled,
stage_product=stage_product,
tuxedo_server_url=tuxedo_server_url,
update_verify_enabled=update_verify_enabled,
updates_builder_enabled=updates_builder_enabled,
updates_enabled=updates_enabled,
verifyConfigs=verifyConfigs,
version=version,
publish_to_balrog_channels=publish_to_balrog_channels,
snap_enabled=snap_enabled,
update_verify_channel=update_verify_channel,
update_verify_requires_cdn_push=update_verify_requires_cdn_push,
funsize_product=funsize_product,
release_eta=release_eta,
)
if extra_balrog_submitter_params:
kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
# don't import releasetasks until required within function impl to avoid global failures
# during nosetests
from releasetasks import make_task_graph
return make_task_graph(**kwargs)