def test_same_latest_data(self, get_data, update_build):
partials = [{
"from": "pr1-b1-nightly-b0",
"filesize": 1,
"hashValue": "p_hash1",
"fileUrl": "p_url1"
}]
completes = [{
"from": "*",
"filesize": 2,
"hashValue": "c_hash1",
"fileUrl": "c_url1"
}]
partial_info = [{
"url": "p_url1",
"hash": "p_hash1",
"size": 1,
"from_buildid": "b0"
}]
complete_info = [{
"url": "c_url1",
"hash": "c_hash1",
"size": 2,
"from": "*"
}]
data = {
"buildID": "b1",
"appVersion": "a1",
"displayVersion": "a1",
"platformVersion": "v1",
"partials": partials,
"completes": completes
}
get_data.side_effect = [
# first call, the dated blob, assume it contains the same data
(data, 1),
# second call, get the "latest" blob's data version, data itself is
# not important and discarded
(data, 100),
# Third call, get data from the dated blob
(data, 1)
]
submitter = NightlySubmitterV4("api_root", auth=None)
submitter.run(platform="linux64",
buildID="b1",
productName="pr1",
branch="b1",
appVersion="a1",
locale="l1",
hashFunction='sha512',
extVersion="v1",
partialInfo=partial_info,
completeInfo=complete_info)
self.assertEqual(update_build.call_count, 0)
def test_new_data(self, get_data, update_build):
"""SingleLocale.update_build() should be called twice when new data
submitted"""
get_data.side_effect = [
# first call, the dated blob, assume there is no data yet
({}, None),
# second call, get the "latest" blob's data
({}, 100),
# Third call, get data from the dated blob
({
"buildID":
"b1",
"appVersion":
"a1",
"displayVersion":
"a1",
"partials": [{
"fileUrl": "p_url1",
"from": "pr1-b1-nightly-b0",
"hashValue": "p_hash1",
"filesize": 1
}],
"platformVersion":
"v1",
"completes": [{
"fileUrl": "c_url1",
"hashValue": "c_hash1",
"from": "*",
"filesize": 2
}]
}, 1)
]
partial_info = [{
"url": "p_url1",
"hash": "p_hash1",
"size": 1,
"from_buildid": "b0"
}]
complete_info = [{
"url": "c_url1",
"hash": "c_hash1",
"size": 2,
}]
submitter = NightlySubmitterV4("api_root", auth=None)
submitter.run(platform="linux64",
buildID="b1",
productName="pr1",
branch="b1",
appVersion="a1",
locale="l1",
hashFunction='sha512',
extVersion="v1",
partialInfo=partial_info,
completeInfo=complete_info)
self.assertEqual(update_build.call_count, 2)
def create_submitter(e, balrog_auth, config):
from balrog.submitter.cli import NightlySubmitterV4, ReleaseSubmitterV4 # noqa: E402
auth = balrog_auth
if "previousVersion" in e and "previousBuildNumber" in e:
# TODO: to rewrite this only for CMAR. partials are handled externally
# by funsize
log.info("Release style balrog submission")
complete_info = [{
"hash": e["to_hash"],
"size": e["to_size"],
}]
partial_info = [{
"hash": e["hash"],
"size": e["size"],
}]
partial_info[0]["previousVersion"] = e["previousVersion"]
partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
submitter = ReleaseSubmitterV4(api_root=config['api_root'],
auth=auth,
dummy=config['dummy'])
return submitter, {'platform': e["platform"],
'productName': e["appName"],
'version': e["toVersion"],
'build_number': e["toBuildNumber"],
'appVersion': e["version"],
'extVersion': e["version"],
'buildID': e["to_buildid"],
'locale': e["locale"],
'hashFunction': 'sha512',
'partialInfo': partial_info,
'completeInfo': complete_info}
elif "tc_nightly" in e:
log.info("Taskcluster Nightly Fennec style Balrog submission")
complete_info = e['completeInfo']
submitter = NightlySubmitterV4(api_root=config['api_root'], auth=auth,
dummy=config['dummy'],
url_replacements=e.get('url_replacements', []))
return submitter, {'platform': e["platform"],
'buildID': e["buildid"],
'productName': e["appName"],
'branch': e["branch"],
'appVersion': e["appVersion"],
'locale': e["locale"],
'hashFunction': e['hashType'],
'extVersion': e["extVersion"],
'completeInfo': complete_info}
else:
raise RuntimeError("Unknown Balrog submission style. Check manifest.json")
def create_locale_submitter(e, balrog_auth, config):
from balrog.submitter.cli import NightlySubmitterV4, ReleaseSubmitterV4 # noqa: E402
auth = balrog_auth
if "tc_release" in e:
log.info("Taskcluster Release style Balrog submission")
complete_info = e['completeInfo']
partial_info = e.get('partialInfo')
submitter = ReleaseSubmitterV4(api_root=config['api_root'], auth=auth,
dummy=config['dummy'])
data = {
'platform': e['platform'],
'productName': e['appName'],
'appVersion': e['appVersion'],
'version': e['version'],
'build_number': e['build_number'],
'locale': e['locale'],
'hashFunction': e['hashType'],
'extVersion': e['extVersion'],
'buildID': e['buildid'],
'completeInfo': complete_info
}
if partial_info:
data['partialInfo'] = partial_info
return submitter, data
elif "tc_nightly" in e:
log.info("Taskcluster Nightly style Balrog submission")
complete_info = e['completeInfo']
partial_info = e.get('partialInfo')
submitter = NightlySubmitterV4(api_root=config['api_root'], auth=auth,
dummy=config['dummy'],
url_replacements=e.get('url_replacements', []))
data = {
'platform': e["platform"],
'buildID': e["buildid"],
'productName': e["appName"],
'branch': e["branch"],
'appVersion': e["appVersion"],
'locale': e["locale"],
'hashFunction': e['hashType'],
'extVersion': e["extVersion"],
'completeInfo': complete_info
}
if partial_info:
data['partialInfo'] = partial_info
return submitter, data
else:
raise RuntimeError("Unknown Balrog submission style. Check manifest.json")
def test_no_canonical_ur_replacement(self):
submitter = NightlySubmitterV4(api_root=None,
auth=None,
url_replacements=None)
completeInfo = [{
'size': 123,
'hash': 'abcd',
'url': 'http://ftp.mozilla.org/url'
}]
data = submitter._get_update_data("prod", "brnch", completeInfo)
self.assertDictEqual(
data, {
'completes': [{
'fileUrl': 'http://ftp.mozilla.org/url',
'filesize': 123,
'from': '*',
'hashValue': 'abcd'
}]
})
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--artifacts-url-prefix", required=True,
help="URL prefix for MAR")
parser.add_argument("--manifest", required=True)
parser.add_argument("-a", "--api-root", required=True,
help="Balrog API root")
parser.add_argument("-d", "--dummy", action="store_true",
help="Add '-dummy' suffix to branch name")
parser.add_argument("--sha1-signing-cert", required=True)
parser.add_argument("--sha384-signing-cert", required=True)
parser.add_argument("-v", "--verbose", action="store_const",
dest="loglevel", const=logging.DEBUG,
default=logging.INFO)
parser.add_argument("--product", help="Override product name from application.ini")
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.loglevel)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("boto").setLevel(logging.WARNING)
balrog_username = os.environ.get("BALROG_USERNAME")
balrog_password = os.environ.get("BALROG_PASSWORD")
if not balrog_username and not balrog_password:
raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
"variables should be set")
s3_bucket = os.environ.get("S3_BUCKET")
aws_access_key_id = os.environ.get("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
if not (s3_bucket and aws_access_key_id and aws_secret_access_key):
log.warn("Skipping S3 uploads...")
uploads_enabled = False
else:
uploads_enabled = True
manifest = json.load(open(args.manifest))
auth = (balrog_username, balrog_password)
signing_certs = {
'sha1': open(args.sha1_signing_cert, 'rb').read(),
'sha384': open(args.sha384_signing_cert, 'rb').read(),
}
assert(get_keysize(signing_certs['sha1']) == 2048)
assert(get_keysize(signing_certs['sha384']) == 4096)
for e in manifest:
complete_info = [{
"hash": e["to_hash"],
"size": e["to_size"],
}]
partial_info = [{
"hash": e["hash"],
"size": e["size"],
}]
if "previousVersion" in e and "previousBuildNumber" in e:
log.info("Release style balrog submission")
partial_info[0]["previousVersion"] = e["previousVersion"]
partial_info[0]["previousBuildNumber"] = e["previousBuildNumber"]
submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
dummy=args.dummy)
productName = args.product or e["appName"]
retry(lambda: submitter.run(
platform=e["platform"], productName=productName,
version=e["toVersion"],
build_number=e["toBuildNumber"],
appVersion=e["version"], extVersion=e["version"],
buildID=e["to_buildid"], locale=e["locale"],
hashFunction='sha512',
partialInfo=partial_info, completeInfo=complete_info,
))
elif "from_buildid" in e and uploads_enabled:
log.info("Nightly style balrog submission")
partial_mar_url = "{}/{}".format(args.artifacts_url_prefix,
e["mar"])
complete_mar_url = e["to_mar"]
dest_prefix = "{branch}/{buildid}".format(
branch=e["branch"], buildid=e["to_buildid"])
partial_mar_dest = "{}/{}".format(dest_prefix, e["mar"])
complete_mar_filename = "{appName}-{branch}-{version}-" \
"{platform}-{locale}.complete.mar"
complete_mar_filename = complete_mar_filename.format(
appName=e["appName"], branch=e["branch"],
version=e["version"], platform=e["platform"],
locale=e["locale"]
)
complete_mar_dest = "{}/{}".format(dest_prefix,
complete_mar_filename)
partial_info[0]["url"] = verify_copy_to_s3(
s3_bucket, aws_access_key_id, aws_secret_access_key,
partial_mar_url, partial_mar_dest, signing_certs)
complete_info[0]["url"] = verify_copy_to_s3(
s3_bucket, aws_access_key_id, aws_secret_access_key,
complete_mar_url, complete_mar_dest, signing_certs)
partial_info[0]["from_buildid"] = e["from_buildid"]
submitter = NightlySubmitterV4(api_root=args.api_root, auth=auth,
dummy=args.dummy)
productName = args.product or e["appName"]
retry(lambda: submitter.run(
platform=e["platform"], buildID=e["to_buildid"],
productName=productName, branch=e["branch"],
appVersion=e["version"], locale=e["locale"],
hashFunction='sha512', extVersion=e["version"],
partialInfo=partial_info, completeInfo=complete_info),
attempts=30, sleeptime=10, max_sleeptime=60,
)
else:
raise RuntimeError("Cannot determine Balrog submission style")
if options.url_replacements:
for replacement in options.url_replacements:
from_, to = replacement.split(",")
url_replacements.append([from_, to])
if options.type_ == "nightly":
isOSUpdate = props.get('isOSUpdate', None)
updateKwargs = {}
if options.schema_version == 3:
submitter = NightlySubmitterV3(options.api_root,
auth,
options.dummy,
url_replacements=url_replacements)
else:
submitter = NightlySubmitterV4(options.api_root,
auth,
options.dummy,
url_replacements=url_replacements)
updateKwargs["completeInfo"] = [{
'size': props['completeMarSize'],
'hash': props['completeMarHash'],
'url': props['completeMarUrl'],
}]
if "partialInfo" in props:
updateKwargs["partialInfo"] = props["partialInfo"]
submitter.run(props['platform'],
props['buildid'],
props['appName'],
props['branch'],
props['appVersion'],