def fileKey(f):
info = fileInfo(f, product)
locale = info['locale']
leading_path = info['leading_path']
if locale == firstLocale and not leading_path:
localeVal = 0
else:
localeVal = 1
if info['format'] == 'exe':
exeVal = 0
else:
exeVal = 1
return (localeVal, leading_path, locale, exeVal, f)
def fileKey(f):
info = fileInfo(f, product)
locale = info['locale']
leading_path = info['leading_path']
if locale == firstLocale and not leading_path:
localeVal = 0
else:
localeVal = 1
if info['format'] == 'exe':
exeVal = 0
else:
exeVal = 1
return (localeVal, leading_path, locale, exeVal, f)
def filterFiles(files, product):
""" Filter out files that we can't sign. Right now this means that
anything that isn't a win32 .exe or .mar file gets filtered out"""
for f in files[:]:
skip = False
try:
info = fileInfo(f, product)
if info['platform'] != 'win32':
skip = True
if info['contents'] not in ('complete', 'installer'):
skip = True
except ValueError:
skip = True
if skip:
files.remove(f)
if 'win32' in f:
if 'xpi' not in f:
log.info("Skipping %s", f)
return files
def filterFiles(files, product):
""" Filter out files that we can't sign. Right now this means that
anything that isn't a win32 .exe or .mar file gets filtered out"""
for f in files[:]:
skip = False
try:
info = fileInfo(f, product)
if info['platform'] != 'win32':
skip = True
if info['contents'] not in ('complete', 'installer'):
skip = True
except ValueError:
skip = True
if skip:
files.remove(f)
if 'win32' in f:
if 'xpi' not in f:
log.info("Skipping %s", f)
return files
def createRepacks(sourceRepo,
revision,
l10nRepoDir,
l10nBaseRepo,
mozconfigPath,
srcMozconfigPath,
objdir,
makeDirs,
appName,
locales,
product,
version,
buildNumber,
stageServer,
stageUsername,
stageSshKey,
ftpServer,
compareLocalesRepo,
merge,
platform,
brand,
appVersion,
generatePartials=False,
partialUpdates=None,
usePymake=False,
tooltoolManifest=None,
tooltool_script=None,
tooltool_urls=None,
balrog_submitter=None,
balrog_hash="sha512",
mozillaDir=None,
mozillaSrcDir=None,
bucket_prefix=None):
buildid = retry(getBuildID,
args=(platform, product, version, buildNumber,
'candidates', ftpServer))
log.info('Got buildid: %s' % buildid)
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"DOWNLOAD_HOST": ftpServer,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
"MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
bucket_prefix=bucket_prefix,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H",
1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
mercurial(sourceRepo, sourceRepoName)
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, srcMozconfigPath,
l10nRepoDir, makeDirs, env, tooltoolManifest,
tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(ftpServer, product, brand, version, buildNumber,
platform),
kwargs={
'signed': signed,
'usePymake': usePymake
})
env.update(input_env)
if product == "thunderbird" and platform == "macosx64":
# TODO: FIXME: HACK: KILLME:
# Terrible, terrible, terrible hack to work around bug 1234935 and make
# the build system happier
absMozillaSrcDir = path.abspath(
path.join(sourceRepoName, mozillaSrcDir))
run_cmd(['ln', '-sf', '../obj-l10n', absMozillaSrcDir])
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(ftpServer, product, oldVersion, oldBuildNumber,
platform, l))
checksums_file = repackLocale(
locale=l,
l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo,
revision=revision,
localeSrcDir=localeSrcDir,
l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo,
env=env,
absObjdir=absObjdir,
merge=merge,
productName=product,
platform=platform,
version=version,
partialUpdates=partialUpdates,
buildNumber=buildNumber,
stageServer=ftpServer,
mozillaDir=mozillaDir,
mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size":
info["size"],
"hash":
info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion":
previousVersion,
"previousBuildNumber":
partialUpdates[previousVersion]['buildNumber'],
"size":
info["size"],
"hash":
info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
})
except Exception, e:
print_exc()
failed.append((l, format_exc()))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
locales, product, version, buildNumber,
stageServer, stageUsername, stageSshKey, ftpServer,
compareLocalesRepo, merge, platform, brand, appVersion,
generatePartials=False, partialUpdates=None,
usePymake=False, tooltoolManifest=None,
tooltool_script=None, tooltool_urls=None,
balrog_submitter=None, balrog_hash="sha512",
mozillaDir=None, mozillaSrcDir=None, bucket_prefix=None):
buildid = retry(getBuildID, args=(platform, product, version,
buildNumber, 'candidates', ftpServer))
log.info('Got buildid: %s' % buildid)
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"DOWNLOAD_HOST": ftpServer,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
"MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
bucket_prefix=bucket_prefix,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
mercurial(sourceRepo, sourceRepoName)
update(sourceRepoName, revision=revision)
l10nRepackPrep(
sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir,
makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(ftpServer, product, brand, version, buildNumber,
platform),
kwargs={'signed': signed,
'usePymake': usePymake})
env.update(input_env)
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(ftpServer, product, oldVersion, oldBuildNumber,
platform, l)
)
checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo, revision=revision,
localeSrcDir=localeSrcDir, l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo, env=env,
absObjdir=absObjdir, merge=merge,
productName=product, platform=platform,
version=version, partialUpdates=partialUpdates,
buildNumber=buildNumber, stageServer=ftpServer,
mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion": previousVersion,
"previousBuildNumber": partialUpdates[previousVersion]['buildNumber'],
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
}
)
except Exception, e:
print_exc()
failed.append((l, format_exc()))
def _doFileInfoTest(self, path, product, info):
self.assertEqual(fileInfo(path, product), info)
def _doFileInfoTest(self, path, product, info):
self.assertEqual(fileInfo(path, product), info)
