def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition", required=True,
type=argparse.FileType('r'))
parser.add_argument("--output-filename", required=True)
parser.add_argument("-q", "--quiet", dest="log_level",
action="store_const", const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.log_level)
task = json.load(args.task_definition)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
manifest = []
for e in task["extra"]["funsize"]["completes"]:
to_mar = e["to_mar"]
locale = e["locale"]
output_filename = args.output_filename.format(locale=locale)
verify_allowed_url(to_mar)
work_env = WorkEnv()
work_env.setup()
complete_mars = {}
dest = os.path.join(work_env.workdir, "to.mar")
unpack_dir = os.path.join(work_env.workdir, "to")
download(to_mar, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, signing_certs)
# Changing the compression strips the signature
change_mar_compression(work_env, dest)
complete_mars["hash"] = get_hash(dest)
unpack(work_env, dest, unpack_dir)
log.info("AV-scanning %s ...", unpack_dir)
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
mar_data = {
"file_to_sign": output_filename,
"hash": get_hash(dest),
}
shutil.copy(dest, os.path.join(args.artifacts_dir, output_filename))
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--allow-staging-prefixes",
action="store_true",
default=strtobool(
os.environ.get('FUNSIZE_ALLOW_STAGING_PREFIXES',
"false")),
help="Allow files from staging buckets.")
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
parser.add_argument('--arch',
type=str,
required=True,
choices=BCJ_OPTIONS.keys(),
help='The archtecture you are building.')
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert (get_keysize(signing_certs['sha1']) == 2048)
assert (get_keysize(signing_certs['sha384']) == 4096)
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_certs))
loop.close()
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
def get_key_from_cmdline(parser, args):
"""Return the signing key and signing algoritm from the commandline."""
if args.keyfiles:
signing_key = open(args.keyfiles[0], 'rb').read()
bits = get_keysize(signing_key)
if bits == 2048:
signing_algorithm = 'sha1'
elif bits == 4096:
signing_algorithm = 'sha384'
else:
parser.error("Unsupported key size {} from key {}".format(bits, args.keyfiles[0]))
print("Using {} to sign using algorithm {!s}".format(args.keyfiles[0], signing_algorithm))
else:
signing_key = None
signing_algorithm = None
return signing_key, signing_algorithm
def main():
args = process_arguments()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_cert = args.signing_cert.read()
assert get_keysize(signing_cert) == 4096
artifacts_dir = Path(args.artifacts_dir)
if not artifacts_dir.exists():
artifacts_dir.mkdir()
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_cert))
loop.close()
manifest_file = artifacts_dir / "manifest.json"
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition", required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam", action="store_true", default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q", "--quiet", dest="log_level",
action="store_const", const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(os.environ.get('DATADOG_API_SECRET')).get('key')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
# For use in shell scripts.
write_dogrc(dd_api_key)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
_timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_certs))
loop.close()
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(manifest[0]['branch']),
"platform:{}".format(manifest[0]['platform']),
]
ddstats.timing('task_duration', time.time() - start,
start, tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
if dd_api_key:
time.sleep(10)
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam",
action="store_true",
default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert (get_keysize(signing_certs['sha1']) == 2048)
assert (get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(
os.environ.get('DATADOG_API_SECRET')).get('key')
# Create this even when not sending metrics, so the context manager
# statements work.
ddstats = ThreadStats(namespace='releng.releases.partials')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
use_old_format = False
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
with ddstats.timer('mar.download.time'):
download(f, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, signing_certs)
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
complete_mars["%s_hash" % mar_type] = get_hash(dest)
with ddstats.timer('mar.unpack.time'):
unpack(work_env, dest, unpack_dir)
if mar_type == 'from':
version = get_option(unpack_dir,
filename="application.ini",
section="App",
option="Version")
major = int(version.split(".")[0])
# The updater for versions less than 56.0 requires BZ2
# compressed MAR files
if major < 56:
use_old_format = True
log.info("Forcing BZ2 compression for %s", f)
log.info("AV-scanning %s ...", unpack_dir)
metric_tags = [
"platform:{}".format(e['platform']),
]
with ddstats.timer('mar.clamscan.time', tags=metric_tags):
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
path = os.path.join(work_env.workdir, "to")
from_path = os.path.join(work_env.workdir, "from")
mar_data = {
"ACCEPTED_MAR_CHANNEL_IDS":
get_option(path,
filename="update-settings.ini",
section="Settings",
option="ACCEPTED_MAR_CHANNEL_IDS"),
"version":
get_option(path,
filename="application.ini",
section="App",
option="Version"),
"to_buildid":
get_option(path,
filename="application.ini",
section="App",
option="BuildID"),
"from_buildid":
get_option(from_path,
filename="application.ini",
section="App",
option="BuildID"),
"appName":
get_option(from_path,
filename="application.ini",
section="App",
option="Name"),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceRepository"),
"revision":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceStamp"),
"from_mar":
e["from_mar"],
"to_mar":
e["to_mar"],
"platform":
e["platform"],
"locale":
e["locale"],
}
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ[
"ACCEPTED_MAR_CHANNEL_IDS"]
for field in ("update_number", "previousVersion",
"previousBuildNumber", "toVersion", "toBuildNumber"):
if field in e:
mar_data[field] = e[field]
mar_data.update(complete_mars)
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
if 'dest_mar' in e:
mar_name = e['dest_mar']
else:
# default to formatted name if not specified
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once
work_env.download_buildsystem_bits(repo=mar_data["repo"],
revision=mar_data["revision"])
metric_tags = [
"branch:{}".format(mar_data['branch']),
"platform:{}".format(mar_data['platform']),
# If required. Shouldn't add much useful info, but increases
# cardinality of metrics substantially, so avoided.
# "locale:{}".format(mar_data['locale']),
]
with ddstats.timer('generate_partial.time', tags=metric_tags):
generate_partial(work_env, from_path, path, dest_mar,
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
mar_data["version"], use_old_format)
mar_data["size"] = os.path.getsize(dest_mar)
metric_tags.append("unit:bytes")
# Allows us to find out how many releases there were between the two,
# making buckets of the file sizes easier.
metric_tags.append("update_number:{}".format(
mar_data.get('update_number', 0)))
ddstats.gauge('partial_mar_size', mar_data['size'], tags=metric_tags)
mar_data["hash"] = get_hash(dest_mar)
shutil.copy(dest_mar, args.artifacts_dir)
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(mar_data['branch']),
]
ddstats.timing('task_duration',
time.time() - start,
start,
tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
time.sleep(10)
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition", required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam", action="store_true", default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q", "--quiet", dest="log_level",
action="store_const", const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(os.environ.get('DATADOG_API_SECRET')).get('key')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
else:
log.info("No metric collection")
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam("--stdout", "--verbose",
_timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_certs))
loop.close()
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(manifest[0]['branch']),
]
ddstats.timing('task_duration', time.time() - start,
start, tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
if dd_api_key:
time.sleep(10)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-url-prefix", required=True,
help="URL prefix for MAR")
parser.add_argument("--manifest", required=True)
parser.add_argument("-a", "--api-root", required=True,
help="Balrog API root")
parser.add_argument("-d", "--dummy", action="store_true",
help="Add '-dummy' suffix to branch name")
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("-v", "--verbose", action="store_const",
dest="loglevel", const=logging.DEBUG,
default=logging.INFO)
parser.add_argument("--product", help="Override product name from application.ini")
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.loglevel)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("boto").setLevel(logging.WARNING)
balrog_username = os.environ.get("BALROG_USERNAME")
balrog_password = os.environ.get("BALROG_PASSWORD")
if not balrog_username and not balrog_password:
raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
"variables should be set")
# blob suffix used for releases only
suffix = os.environ.get("BALROG_BLOB_SUFFIX")
s3_bucket = os.environ.get("S3_BUCKET")
aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
if not (s3_bucket and aws_access_key_id and aws_secret_access_key):
log.warn("Skipping S3 uploads...")
uploads_enabled = False
else:
uploads_enabled = True
manifest = json.load(open(args.manifest))
auth = (balrog_username, balrog_password)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
for e in manifest:
complete_info = [{
"hash": e["to_hash"],
"size": e["to_size"],
}]
partial_info = [{
"hash": e["hash"],
"size": e["size"],
}]
if "previousVersion" in e and "previousBuildNumber" in e:
log.info("Release style balrog submission")
partial_info[0]["previousVersion"] = e["previousVersion"]
partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
dummy=args.dummy, suffix=suffix)
productName = args.product or e["appName"]
if suffix:
log.warning("Not submitting complete info")
complete_info = None
retry(lambda: submitter.run(
platform=e["platform"], productName=productName,
version=e["toVersion"],
build_number=e["toBuildNumber"],
appVersion=e["version"], extVersion=e["version"],
buildID=e["to_buildid"], locale=e["locale"],
hashFunction='sha512',
partialInfo=partial_info, completeInfo=complete_info),
attempts=30, sleeptime=10, max_sleeptime=60, jitter=3,
)
elif "from_buildid" in e and uploads_enabled:
log.info("Nightly style balrog submission")
partial_mar_url = "{}/{}".format(args.artifacts_url_prefix,
e["mar"])
complete_mar_url = e["to_mar"]
dest_prefix = "{branch}/{buildid}".format(
branch=e["branch"], buildid=e["to_buildid"])
partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"])
complete_mar_filename = "{appName}-{branch}-{version}-" \
"{platform}-{locale}.complete.mar"
complete_mar_filename = complete_mar_filename.format(
appName=e["appName"], branch=e["branch"],
version=e["version"], platform=e["platform"],
locale=e["locale"]
)
complete_mar_dest = "{}/{}".format(dest_prefix,
complete_mar_filename)
partial_info[0]["url"] = verify_copy_to_s3(
s3_bucket, aws_access_key_id, aws_secret_access_key,
partial_mar_url, partial_mar_dest, signing_certs)
complete_info[0]["url"] = verify_copy_to_s3(
s3_bucket, aws_access_key_id, aws_secret_access_key,
complete_mar_url, complete_mar_dest, signing_certs)
partial_info[0]["from_buildid"] = e["from_buildid"]
submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth,
dummy=args.dummy)
productName = args.product or e["appName"]
retry(lambda: submitter.run(
platform=e["platform"], buildID=e["to_buildid"],
productName=productName, branch=e["branch"],
appVersion=e["version"], locale=e["locale"],
hashFunction='sha512', extVersion=e["version"],
partialInfo=partial_info, completeInfo=complete_info),
attempts=30, sleeptime=10, max_sleeptime=60, jitter=3,
)
else:
raise RuntimeError("Cannot determine Balrog submission style")
def main():
start = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--allow-staging-prefixes",
action="store_true",
default=strtobool(
os.environ.get('FUNSIZE_ALLOW_STAGING_PREFIXES',
"false")),
help="Allow files from staging buckets.")
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
parser.add_argument('--arch',
type=str,
required=True,
choices=BCJ_OPTIONS.keys(),
help='The archtecture you are building.')
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log.setLevel(args.log_level)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert (get_keysize(signing_certs['sha1']) == 2048)
assert (get_keysize(signing_certs['sha384']) == 4096)
# Intended for local testing.
dd_api_key = os.environ.get('DATADOG_API_KEY')
# Intended for Taskcluster.
if not dd_api_key and os.environ.get('DATADOG_API_SECRET'):
dd_api_key = get_secret(
os.environ.get('DATADOG_API_SECRET')).get('key')
if dd_api_key:
dd_options = {
'api_key': dd_api_key,
}
log.info("Starting metric collection")
initialize(**dd_options)
ddstats.start(flush_interval=1)
# For use in shell scripts.
write_dogrc(dd_api_key)
else:
log.info("No metric collection")
loop = asyncio.get_event_loop()
manifest = loop.run_until_complete(async_main(args, signing_certs))
loop.close()
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
log.debug("{}".format(json.dumps(manifest, indent=2, sort_keys=True)))
# Warning: Assumption that one partials task will always be for one branch.
metric_tags = [
"branch:{}".format(manifest[0]['branch']),
"platform:{}".format(manifest[0]['platform']),
]
ddstats.timing('task_duration',
time.time() - start,
start,
tags=metric_tags)
# Wait for all the metrics to flush. If the program ends before
# they've been sent, they'll be dropped.
# Should be more than the flush_interval for the ThreadStats object
if dd_api_key:
time.sleep(10)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-url-prefix", required=True,
help="URL prefix for MAR")
parser.add_argument("--manifest", required=True)
parser.add_argument("-a", "--api-root", required=True,
help="Balrog API root")
parser.add_argument("-d", "--dummy", action="store_true",
help="Add '-dummy' suffix to branch name")
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("-v", "--verbose", action="store_const",
dest="loglevel", const=logging.DEBUG,
default=logging.INFO)
parser.add_argument("--product", help="Override product name from application.ini")
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.loglevel)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("boto").setLevel(logging.WARNING)
balrog_username = os.environ.get("BALROG_USERNAME")
balrog_password = os.environ.get("BALROG_PASSWORD")
if not balrog_username and not balrog_password:
raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
"variables should be set")
s3_bucket = os.environ.get("S3_BUCKET")
aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
if not (s3_bucket and aws_access_key_id and aws_secret_access_key):
log.warn("Skipping S3 uploads...")
uploads_enabled = False
else:
uploads_enabled = True
manifest = json.load(open(args.manifest))
auth = (balrog_username, balrog_password)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
for e in manifest:
complete_info = [{
"hash": e["to_hash"],
"size": e["to_size"],
}]
partial_info = [{
"hash": e["hash"],
"size": e["size"],
}]
if "previousVersion" in e and "previousBuildNumber" in e:
log.info("Release style balrog submission")
partial_info[0]["previousVersion"] = e["previousVersion"]
partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
dummy=args.dummy)
productName = args.product or e["appName"]
retry(lambda: submitter.run(
platform=e["platform"], productName=productName,
version=e["toVersion"],
build_number=e["toBuildNumber"],
appVersion=e["version"], extVersion=e["version"],
buildID=e["to_buildid"], locale=e["locale"],
hashFunction='sha512',
partialInfo=partial_info, completeInfo=complete_info,
))
elif "from_buildid" in e and uploads_enabled:
log.info("Nightly style balrog submission")
partial_mar_url = "{}/{}".format(args.artifacts_url_prefix,
e["mar"])
complete_mar_url = e["to_mar"]
dest_prefix = "{branch}/{buildid}".format(
branch=e["branch"], buildid=e["to_buildid"])
partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"])
complete_mar_filename = "{appName}-{branch}-{version}-" \
"{platform}-{locale}.complete.mar"
complete_mar_filename = complete_mar_filename.format(
appName=e["appName"], branch=e["branch"],
version=e["version"], platform=e["platform"],
locale=e["locale"]
)
complete_mar_dest = "{}/{}".format(dest_prefix,
complete_mar_filename)
partial_info[0]["url"] = verify_copy_to_s3(
s3_bucket, aws_access_key_id, aws_secret_access_key,
partial_mar_url, partial_mar_dest, signing_certs)
complete_info[0]["url"] = verify_copy_to_s3(
s3_bucket, aws_access_key_id, aws_secret_access_key,
complete_mar_url, complete_mar_dest, signing_certs)
partial_info[0]["from_buildid"] = e["from_buildid"]
submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth,
dummy=args.dummy)
productName = args.product or e["appName"]
retry(lambda: submitter.run(
platform=e["platform"], buildID=e["to_buildid"],
productName=productName, branch=e["branch"],
appVersion=e["version"], locale=e["locale"],
hashFunction='sha512', extVersion=e["version"],
partialInfo=partial_info, completeInfo=complete_info),
attempts=30, sleeptime=10, max_sleeptime=60,
)
else:
raise RuntimeError("Cannot determine Balrog submission style")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-dir", required=True)
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("--task-definition",
required=True,
type=argparse.FileType('r'))
parser.add_argument("--filename-template",
default=DEFAULT_FILENAME_TEMPLATE)
parser.add_argument("--no-freshclam",
action="store_true",
default=False,
help="Do not refresh ClamAV DB")
parser.add_argument("-q",
"--quiet",
dest="log_level",
action="store_const",
const=logging.WARNING,
default=logging.DEBUG)
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.log_level)
task = json.load(args.task_definition)
# TODO: verify task["extra"]["funsize"]["partials"] with jsonschema
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert (get_keysize(signing_certs['sha1']) == 2048)
assert (get_keysize(signing_certs['sha384']) == 4096)
if args.no_freshclam:
log.info("Skipping freshclam")
else:
log.info("Refreshing clamav db...")
try:
redo.retry(lambda: sh.freshclam(
"--stdout", "--verbose", _timeout=300, _err_to_out=True))
log.info("Done.")
except sh.ErrorReturnCode:
log.warning("Freshclam failed, skipping DB update")
manifest = []
for e in task["extra"]["funsize"]["partials"]:
for mar in (e["from_mar"], e["to_mar"]):
verify_allowed_url(mar)
work_env = WorkEnv()
# TODO: run setup once
work_env.setup()
complete_mars = {}
use_old_format = False
for mar_type, f in (("from", e["from_mar"]), ("to", e["to_mar"])):
dest = os.path.join(work_env.workdir, "{}.mar".format(mar_type))
unpack_dir = os.path.join(work_env.workdir, mar_type)
download(f, dest)
if not os.getenv("MOZ_DISABLE_MAR_CERT_VERIFICATION"):
verify_signature(dest, signing_certs)
complete_mars["%s_size" % mar_type] = os.path.getsize(dest)
complete_mars["%s_hash" % mar_type] = get_hash(dest)
if mar_type == 'to' and not is_lzma_compressed_mar(dest):
use_old_format = True
unpack(work_env, dest, unpack_dir)
log.info("AV-scanning %s ...", unpack_dir)
sh.clamscan("-r", unpack_dir, _timeout=600, _err_to_out=True)
log.info("Done.")
path = os.path.join(work_env.workdir, "to")
from_path = os.path.join(work_env.workdir, "from")
mar_data = {
"ACCEPTED_MAR_CHANNEL_IDS":
get_option(path,
filename="update-settings.ini",
section="Settings",
option="ACCEPTED_MAR_CHANNEL_IDS"),
"version":
get_option(path,
filename="application.ini",
section="App",
option="Version"),
"to_buildid":
get_option(path,
filename="application.ini",
section="App",
option="BuildID"),
"from_buildid":
get_option(from_path,
filename="application.ini",
section="App",
option="BuildID"),
"appName":
get_option(from_path,
filename="application.ini",
section="App",
option="Name"),
# Use Gecko repo and rev from platform.ini, not application.ini
"repo":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceRepository"),
"revision":
get_option(path,
filename="platform.ini",
section="Build",
option="SourceStamp"),
"from_mar":
e["from_mar"],
"to_mar":
e["to_mar"],
"platform":
e["platform"],
"locale":
e["locale"],
}
# Override ACCEPTED_MAR_CHANNEL_IDS if needed
if "ACCEPTED_MAR_CHANNEL_IDS" in os.environ:
mar_data["ACCEPTED_MAR_CHANNEL_IDS"] = os.environ[
"ACCEPTED_MAR_CHANNEL_IDS"]
for field in ("update_number", "previousVersion",
"previousBuildNumber", "toVersion", "toBuildNumber"):
if field in e:
mar_data[field] = e[field]
mar_data.update(complete_mars)
# if branch not set explicitly use repo-name
mar_data["branch"] = e.get("branch",
mar_data["repo"].rstrip("/").split("/")[-1])
mar_name = args.filename_template.format(**mar_data)
mar_data["mar"] = mar_name
dest_mar = os.path.join(work_env.workdir, mar_name)
# TODO: download these once
work_env.download_buildsystem_bits(repo=mar_data["repo"],
revision=mar_data["revision"])
generate_partial(work_env, from_path, path, dest_mar,
mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
mar_data["version"], use_old_format)
mar_data["size"] = os.path.getsize(dest_mar)
mar_data["hash"] = get_hash(dest_mar)
shutil.copy(dest_mar, args.artifacts_dir)
work_env.cleanup()
manifest.append(mar_data)
manifest_file = os.path.join(args.artifacts_dir, "manifest.json")
with open(manifest_file, "w") as fp:
json.dump(manifest, fp, indent=2, sort_keys=True)
def main(argv=None):
"""Main CLI entry point."""
parser = build_argparser()
args = parser.parse_args(argv)
logging.basicConfig(level=args.loglevel, format="%(message)s")
# Make sure only one action has been specified
if len([
a for a in [
args.create, args.extract, args.verify, args.list,
args.list_detailed
] if a is not None
]) != 1:
parser.error(
"Must specify something to do (one of -c, -x, -t, -T, -v)")
if args.create and not args.files:
parser.error("Must specify at least one file to add to marfile")
if args.verify and not args.keyfiles:
parser.error("Must specify a key file when verifying")
if args.extract:
if args.compression not in (None, 'bz2', 'xz', 'auto'):
parser.error('Unsupported compression type')
marfile = os.path.abspath(args.extract)
if args.chdir:
os.chdir(args.chdir)
do_extract(marfile, os.getcwd(), args.compression)
elif args.verify:
if do_verify(args.verify, args.keyfiles):
print("Verification OK")
return
else:
print("Verification failed")
sys.exit(1)
elif args.list:
print("\n".join(do_list(args.list)))
elif args.list_detailed:
print("\n".join(do_list(args.list_detailed, detailed=True)))
elif args.create:
if args.compression not in (None, 'bz2', 'xz'):
parser.error('Unsupported compression type')
marfile = os.path.abspath(args.create)
if args.keyfiles:
signing_key = open(args.keyfiles[0], 'rb').read()
bits = get_keysize(signing_key)
if bits == 2048:
signing_algorithm = 'sha1'
elif bits == 4096:
signing_algorithm = 'sha384'
else:
parser.error("Unsupported key size {} from key {}".format(
bits, args.keyfiles[0]))
print("Using {} to sign using algorithm {!s}".format(
args.keyfiles[0], signing_algorithm))
else:
signing_key = None
signing_algorithm = None
if args.chdir:
os.chdir(args.chdir)
do_create(marfile,
args.files,
args.compression,
productversion=args.productversion,
channel=args.channel,
signing_key=signing_key,
signing_algorithm=signing_algorithm)
# sanity check; should never happen
else: # pragma: no cover
parser.error("Unsupported action")
