def testMultipleFiles(self):
expected = {'a': {'size': 1, 'hashes': {'sha512': 'd'}},
'b': {'size': 2, 'hashes': {'sha512': 'e'}}}
got = parseChecksumsFile("""\
d sha512 1 a
e sha512 2 b
""")
self.assertEquals(got, expected)
def testMultipleHashes(self):
expected = {'yyy': {'size': 5, 'hashes': {'sha512': 'aaa',
'md5': 'bbb'}}}
got = parseChecksumsFile("""\
aaa sha512 5 yyy
bbb md5 5 yyy
""")
self.assertEquals(got, expected)
def getChecksum(platform, locale, checksumsFile):
if checksums.get(platform, {}).get(locale):
log.debug(
"Using in-memory checksums for %s %s" % (platform, locale))
else:
try:
cacheChecksums(platform, locale,
parseChecksumsFile(open(checksumsFile).read()))
log.debug(
"Using on-disk checksums for %s %s" % (platform, locale))
except (IOError, ValueError):
contents = requests.get(
checksumsUrl, config={'danger_mode': True}).content
log.debug("Using newly downloaded checksums for %s %s" %
(platform, locale))
# Cache the sums in-memory and on-disk.
cacheChecksums(platform, locale, parseChecksumsFile(contents))
with open(checksumsFile, 'w') as f:
f.write(contents)
return checksums[platform][locale]
def testMultipleHashes(self):
expected = {
'yyy': {
'size': 5,
'hashes': {
'sha512': 'aaa',
'md5': 'bbb'
}
}
}
got = parseChecksumsFile("""\
aaa sha512 5 yyy
bbb md5 5 yyy
""")
self.assertEquals(got, expected)
def testMultipleFiles(self):
expected = {
'a': {
'size': 1,
'hashes': {
'sha512': 'd'
}
},
'b': {
'size': 2,
'hashes': {
'sha512': 'e'
}
}
}
got = parseChecksumsFile("""\
d sha512 1 a
e sha512 2 b
""")
self.assertEquals(got, expected)
def createRepacks(sourceRepo,
revision,
l10nRepoDir,
l10nBaseRepo,
mozconfigPath,
srcMozconfigPath,
objdir,
makeDirs,
appName,
locales,
product,
version,
buildNumber,
stageServer,
stageUsername,
stageSshKey,
ftpServer,
compareLocalesRepo,
merge,
platform,
brand,
appVersion,
generatePartials=False,
partialUpdates=None,
usePymake=False,
tooltoolManifest=None,
tooltool_script=None,
tooltool_urls=None,
balrog_submitter=None,
balrog_hash="sha512",
mozillaDir=None,
mozillaSrcDir=None,
bucket_prefix=None):
buildid = retry(getBuildID,
args=(platform, product, version, buildNumber,
'candidates', ftpServer))
log.info('Got buildid: %s' % buildid)
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"DOWNLOAD_HOST": ftpServer,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
"MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
bucket_prefix=bucket_prefix,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H",
1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
mercurial(sourceRepo, sourceRepoName)
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, srcMozconfigPath,
l10nRepoDir, makeDirs, env, tooltoolManifest,
tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(ftpServer, product, brand, version, buildNumber,
platform),
kwargs={
'signed': signed,
'usePymake': usePymake
})
env.update(input_env)
if product == "thunderbird" and platform == "macosx64":
# TODO: FIXME: HACK: KILLME:
# Terrible, terrible, terrible hack to work around bug 1234935 and make
# the build system happier
absMozillaSrcDir = path.abspath(
path.join(sourceRepoName, mozillaSrcDir))
run_cmd(['ln', '-sf', '../obj-l10n', absMozillaSrcDir])
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(ftpServer, product, oldVersion, oldBuildNumber,
platform, l))
checksums_file = repackLocale(
locale=l,
l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo,
revision=revision,
localeSrcDir=localeSrcDir,
l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo,
env=env,
absObjdir=absObjdir,
merge=merge,
productName=product,
platform=platform,
version=version,
partialUpdates=partialUpdates,
buildNumber=buildNumber,
stageServer=ftpServer,
mozillaDir=mozillaDir,
mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size":
info["size"],
"hash":
info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion":
previousVersion,
"previousBuildNumber":
partialUpdates[previousVersion]['buildNumber'],
"size":
info["size"],
"hash":
info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
})
except Exception, e:
print_exc()
failed.append((l, format_exc()))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
locales, product, version, buildNumber,
stageServer, stageUsername, stageSshKey, ftpServer,
compareLocalesRepo, merge, platform, brand, appVersion,
generatePartials=False, partialUpdates=None,
usePymake=False, tooltoolManifest=None,
tooltool_script=None, tooltool_urls=None,
balrog_submitter=None, balrog_hash="sha512",
mozillaDir=None, mozillaSrcDir=None, bucket_prefix=None):
buildid = retry(getBuildID, args=(platform, product, version,
buildNumber, 'candidates', ftpServer))
log.info('Got buildid: %s' % buildid)
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"DOWNLOAD_HOST": ftpServer,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
"MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
bucket_prefix=bucket_prefix,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
mercurial(sourceRepo, sourceRepoName)
update(sourceRepoName, revision=revision)
l10nRepackPrep(
sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir,
makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(ftpServer, product, brand, version, buildNumber,
platform),
kwargs={'signed': signed,
'usePymake': usePymake})
env.update(input_env)
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(ftpServer, product, oldVersion, oldBuildNumber,
platform, l)
)
checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo, revision=revision,
localeSrcDir=localeSrcDir, l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo, env=env,
absObjdir=absObjdir, merge=merge,
productName=product, platform=platform,
version=version, partialUpdates=partialUpdates,
buildNumber=buildNumber, stageServer=ftpServer,
mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion": previousVersion,
"previousBuildNumber": partialUpdates[previousVersion]['buildNumber'],
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
}
)
except Exception, e:
print_exc()
failed.append((l, format_exc()))
def testSimple(self):
expected = {'xxxxx': {'size': 1, 'hashes': {'sha512': 'aaa'}}}
got = parseChecksumsFile("aaa sha512 1 xxxxx")
self.assertEquals(got, expected)
def testSpacesInFilename(self):
expected = {'b d': {'size': 2, 'hashes': {'sha512': 'g'}}}
got = parseChecksumsFile("""\
g sha512 2 b d
""")
self.assertEquals(got, expected)
def createRepacks(sourceRepo,
revision,
l10nRepoDir,
l10nBaseRepo,
mozconfigPath,
srcMozconfigPath,
objdir,
makeDirs,
appName,
locales,
product,
version,
buildNumber,
stageServer,
stageUsername,
stageSshKey,
compareLocalesRepo,
merge,
platform,
brand,
appVersion,
generatePartials=False,
partialUpdates=None,
usePymake=False,
tooltoolManifest=None,
tooltool_script=None,
tooltool_urls=None,
balrog_submitter=None,
balrog_hash="sha512",
buildid=None):
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", "")
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\') + \
" -H " + \
os.environ['MOZ_SIGN_CMD'].split(' ')[-1]
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
retry(mercurial, args=(sourceRepo, sourceRepoName))
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, srcMozconfigPath,
l10nRepoDir, makeDirs, env, tooltoolManifest,
tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(stageServer, product, brand, version, buildNumber,
platform),
kwargs={
'signed': signed,
'usePymake': usePymake
})
env.update(input_env)
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(stageServer, product, oldVersion, oldBuildNumber,
platform, l))
checksums_file = repackLocale(
locale=l,
l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo,
revision=revision,
localeSrcDir=localeSrcDir,
l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo,
env=env,
absObjdir=absObjdir,
merge=merge,
productName=product,
platform=platform,
version=version,
partialUpdates=partialUpdates,
buildNumber=buildNumber,
stageServer=stageServer)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
marInfo = defaultdict(dict)
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
marInfo['complete']['hash'] = info['hashes'][
balrog_hash]
marInfo['complete']['size'] = info['size']
if not marInfo['complete']:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeMarSize': marInfo['complete']['size'],
'completeMarHash': marInfo['complete']['hash'],
})
except Exception, e:
print_exc()
failed.append((l, format_exc()))
def testSimple(self):
expected = {'xxxxx': {'size': 1, 'hashes': {'sha512': 'aaa'}}}
got = parseChecksumsFile("aaa sha512 1 xxxxx")
self.assertEquals(got, expected)
def testSpacesInFilename(self):
expected = {'b d': {'size': 2, 'hashes': {'sha512': 'g'}}}
got = parseChecksumsFile("""\
g sha512 2 b d
""")
self.assertEquals(got, expected)
