def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
relbranchChangesets = len(tags)
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
revision = get_revision(reponame)
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def cleanup_wrapper():
# use an attribute full_clean in order to keep track of
# whether or not a full cleanup is required.
# This is done since cleanup_wrapper's scope doesn't let us
# access process_patchset globals, given the way it is used.
if not hasattr(cleanup_wrapper, 'full_clean'):
cleanup_wrapper.full_clean = False
if not hasattr(cleanup_wrapper, 'branch'):
cleanup_wrapper.branch = data['branch']
if cleanup_wrapper.branch != data['branch']:
cleanup_wrapper.full_clean = False
cleanup_wrapper.branch = data['branch']
# only wipe the repositories every second cleanup
if cleanup_wrapper.full_clean:
clear_branch(data['branch'])
log_msg('Wiped repositories for: %s' % data['branch'])
else:
active_repo = os.path.join('active', data['branch'])
update(active_repo)
log_msg('Update -C on active repo for: %s' % data['branch'])
cleanup_wrapper.full_clean = not cleanup_wrapper.full_clean
clone_revision = clone_branch(data['branch'], data['branch_url'])
if clone_revision == None:
# TODO: Handle clone error
log_msg('[HgPusher] Clone error...')
return
def process_configs(repo, attempt):
"""Helper method that encapsulates all of the things necessary
to run release runner for all releases."""
log.info("Bumping %s, attempt #%s" % (repo, attempt))
for release in rr.new_releases:
rr.update_status(release, 'Writing configs')
l10nContents = rr.get_release_l10n(release['name'])
tags.extend(getTags(
getBaseTag(release['product'], release['version']),
release['buildNumber'])
)
update(configs_workdir, revision='default')
cfgFile = getReleaseConfigName(
release['product'], path.basename(release['branch']),
release['version'], staging)
bump_configs(release=release, cfgFile=cfgFile,
l10nContents=l10nContents, workdir=configs_workdir,
hg_username=hg_username,
productionBranch=buildbot_configs_branch)
rr.update_status(release, 'Running release sanity')
rs_args = get_release_sanity_args(configs_workdir, release,
cfgFile, masters_json,
buildbot_configs_branch)
release_sanity_script = "%s/buildbot-helpers/release_sanity.py" % tools_workdir
run_cmd(['python', release_sanity_script] + rs_args +
['--dry-run'])
rr.update_status(
release, 'Waiting for other releases to run release sanity'
)
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
l10nIni, compareLocalesRepo, env, merge=True,
prevMar=None, productName=None, platform=None,
version=None, oldVersion=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
retry(mercurial, args=(repo, localeDir))
update(localeDir, revision=revision)
compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
l10nIni, revision=revision, merge=merge)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env)
UPLOAD_EXTRA_FILES = []
if prevMar:
nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir,
'../../dist')))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version)
partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
run_cmd(['rm', '-rf', previous, current])
run_cmd(['mkdir', previous, current])
run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
'../../../../%s' % prevMar],
cwd=path.join(nativeDistDir, 'previous'), env=env)
run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
current_mar], cwd=path.join(nativeDistDir, 'current'), env=env)
run_cmd(
['bash', '../../tools/update-packaging/make_incremental_update.sh',
partial_mar, previous, current], cwd=nativeDistDir, env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd(['bash', '-c',
'%s -f gpg -f mar "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)],
env=env)
UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name))
retry(run_cmd,
args=(["make", "upload", "AB_CD=%s" % locale,
'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],),
kwargs={'cwd': localeSrcDir, 'env': env})
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts, defaultBranch='default'):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, locales, ftpProduct,
appName, version, appVersion, buildNumber, stageServer,
stageUsername, stageSshKey, compareLocalesRepo, merge,
platform, stage_platform, brand, mobileDirName):
sourceRepoName = path.split(sourceRepo)[-1]
nightlyDir = "candidates"
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_PKG_VERSION": version,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
# Android signing
"JARSIGNER": os.path.join(os.getcwd(), "scripts", "release",
"signing", "mozpass.py")
}
build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName)
retry(mercurial, args=(sourceRepo, sourceRepoName))
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath,
l10nRepoDir, makeDirs, localeSrcDir, env)
fullCandidatesDir = makeCandidatesDir(appName, version, buildNumber,
protocol='http', server=stageServer,
nightlyDir=nightlyDir)
input_env = retry(downloadReleaseBuilds,
args=(stageServer, ftpProduct, brand, version,
buildNumber, stage_platform, fullCandidatesDir))
env.update(input_env)
print "env pre-locale: %s" % str(env)
failed = []
for l in locales:
try:
# adding locale into builddir
env["POST_UPLOAD_CMD"] = postUploadCmdPrefix(
to_mobile_candidates=True,
product=appName,
version=version,
builddir='%s/%s' % (stage_platform, l),
buildNumber=buildNumber,
nightly_dir=nightlyDir,)
print "env post-locale: %s" % str(env)
repackLocale(str(l), l10nRepoDir, l10nBaseRepo, revision,
localeSrcDir, l10nIni, compareLocalesRepo, env, merge)
except Exception, e:
failed.append((l, format_exc()))
def soft_clean(self):
"""
Only does an update -C on the active repository
to get rid of any applied, not committed patches.
"""
active_repo = os.path.join('active', self.branch)
# get rid of any imported and qpush-ed patches
log.debug('qpop -a and rm -rf .hg/patches')
(success, err, ret) = run_hg(['qpop', '-a', '-R', active_repo])
run_cmd(['rm', '-rf', os.path.join(active_repo, '.hg/patches')])
log.debug('Update -C on active repo for: %s' % (self.branch))
update(active_repo)
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni,
revision="default", merge=True):
retry(mercurial, args=(repo, "compare-locales"))
update("compare-locales", revision=revision)
mergeDir = path.join(localeSrcDir, "merged")
if path.exists(mergeDir):
log.info("Deleting %s" % mergeDir)
shutil.rmtree(mergeDir)
run_cmd(["python", path.join("compare-locales", "scripts",
"compare-locales"),
"-m", mergeDir,
l10nIni,
l10nRepoDir, locale],
env={"PYTHONPATH": path.join("compare-locales", "lib")})
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config[
'hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
def bump_configs(release, cfgFile, l10nContents, workdir,
hg_username, productionBranch, defaultBranch='default'):
# Update the production branch first, because that's where we want to read
# the templates from
update(workdir, productionBranch)
cfgDir = path.join(workdir, 'mozilla')
templateFile = path.join(cfgDir, '%s.template' % cfgFile)
tags = set(getTags(getBaseTag(release['product'], release['version']),
release['buildNumber']))
cfgFile = path.join(cfgDir, cfgFile)
l10nChangesetsFile = path.join(
cfgDir,
readReleaseConfig(cfgFile)['l10nRevisionFile']
)
subs = release.copy()
if 'partials' in release:
subs['partials'] = getPartials(release)
# This is true 99% of the time. It's exceedingly rare that we ship a point
# release that we first push to the beta channel. If we need to, the
# expectation is that this will be ignored by hardcoding True in the
# template.
if isFinalRelease(release["version"]):
subs["betaChannelEnabled"] = True
else:
subs["betaChannelEnabled"] = False
with open(templateFile) as f:
template = f.read()
releaseConfig = substituteReleaseConfig(template, **subs)
# Write out the new configs on the production branch...
with open(cfgFile, 'w') as f:
f.write(releaseConfig)
with open(l10nChangesetsFile, 'w') as f:
f.write(l10nContents)
prodRev = commit(workdir, 'Update release config for %s' % release['name'],
user=hg_username)
# We always force tagging, because it makes it easier to retrigger a
# release that fails for infrastructure reasons.
tag(workdir, tags, rev=prodRev, force=True, user=hg_username)
# And then write the same files to the default branch
update(workdir, defaultBranch)
with open(cfgFile, 'w') as f:
f.write(releaseConfig)
with open(l10nChangesetsFile, 'w') as f:
f.write(l10nContents)
commit(workdir, 'Update release config for %s' % release['name'],
user=hg_username)
def bump_configs(release, cfgFile, l10nContents, workdir,
hg_username, productionBranch, defaultBranch='default'):
# Update the production branch first, because that's where we want to read
# the templates from
update(workdir, productionBranch)
cfgDir = path.join(workdir, 'mozilla')
templateFile = path.join(cfgDir, '%s.template' % cfgFile)
tags = getTags(getBaseTag(release['product'], release['version']),
release['buildNumber'])
cfgFile = path.join(cfgDir, cfgFile)
l10nChangesetsFile = path.join(
cfgDir,
readReleaseConfig(cfgFile)['l10nRevisionFile']
)
subs = release.copy()
if 'partials' in release:
subs['partials'] = getPartials(release)
with open(templateFile) as f:
template = f.read()
releaseConfig = substituteReleaseConfig(template, **subs)
# Write out the new configs on the production branch...
with open(cfgFile, 'w') as f:
f.write(releaseConfig)
with open(l10nChangesetsFile, 'w') as f:
f.write(l10nContents)
prodRev = commit(workdir, 'Update release config for %s' % release['name'],
user=hg_username)
# We always force tagging, because it makes it easier to retrigger a
# release that fails for infrastructure reasons.
tag(workdir, tags, rev=prodRev, force=True, user=hg_username)
# And then write the same files to the default branch
update(workdir, defaultBranch)
with open(cfgFile, 'w') as f:
f.write(releaseConfig)
with open(l10nChangesetsFile, 'w') as f:
f.write(l10nContents)
commit(workdir, 'Update release config for %s' % release['name'],
user=hg_username)
check_buildbot()
if not options.dryrun and not options.skip_reconfig:
from util.fabric.common import check_fabric, FabricHelper
check_fabric()
if options.configs_dir:
configs_dir = options.configs_dir
cleanup_configs = False
else:
cleanup_configs = True
configs_dir = mkdtemp()
remote = make_hg_url(HG, options.configs_repo_url)
retry(mercurial, args=(remote, configs_dir),
kwargs={'branch': options.configs_branch})
update(configs_dir, options.configs_branch)
# https://bugzilla.mozilla.org/show_bug.cgi?id=678103#c5
# This goes through the list of config files in reverse order, which is a
# hacky way of making sure that the config file that's listed first is the
# one that's loaded in releaseConfig for the sendchange.
for releaseConfigFile in list(reversed(options.releaseConfigFiles)):
abs_release_config_file = path.join(configs_dir, 'mozilla',
releaseConfigFile)
releaseConfig = readReleaseConfig(abs_release_config_file)
products.append(releaseConfig['productName'])
if not options.buildNumber:
log.warn("No buildNumber specified, using buildNumber in"
" release_config, which may be out of date!")
options.buildNumber = releaseConfig['buildNumber']
"--buildbot-configs",
help="buildbot-configs mercurial repo URL",
default=os.environ.get("BUILDBOT_CONFIGS",
DEFAULT_BUILDBOT_CONFIGS_REPO))
parser.add_argument("-t", "--release-tag", required=True)
parser.add_argument("--product", help="Override for stage_product")
parser.add_argument("--ssh-user", required=True)
parser.add_argument("--ssh-key", required=True)
parser.add_argument("--overwrite", default=False, action="store_true")
parser.add_argument("--extra-excludes", action="append")
parser.add_argument("actions", nargs="+", help="Script actions")
args = parser.parse_args()
actions = args.actions
mercurial(args.buildbot_configs, "buildbot-configs")
update("buildbot-configs", revision=args.release_tag)
releaseConfigFile = path.join("buildbot-configs", args.release_config)
releaseConfig = readReleaseConfig(releaseConfigFile,
required=REQUIRED_RELEASE_CONFIG)
productName = args.product or releaseConfig['stage_product']
version = releaseConfig['version']
buildNumber = releaseConfig['buildNumber']
stageServer = releaseConfig['stagingServer']
stageUsername = args.ssh_user
stageSshKey = args.ssh_key
stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey)
createIndexFiles = releaseConfig.get('makeIndexFiles', False) \
and productName != 'xulrunner'
syncPartnerBundles = releaseConfig.get('syncPartnerBundles', False) \
dest="tooltool_script",
default=[],
action="append")
parser.add_option("--tooltool-url", dest="tooltool_urls", action="append")
parser.add_option("--use-pymake",
dest="use_pymake",
action="store_true",
default=False)
# todo: maybe read these from branch/release config? is that even possible for credentials file?
parser.add_option("--balrog-api-root", dest="balrog_api_root")
parser.add_option("--credentials-file", dest="credentials_file")
parser.add_option("--balrog-username", dest="balrog_username")
options, args = parser.parse_args()
mercurial(options.buildbotConfigs, "buildbot-configs")
update("buildbot-configs", revision=options.releaseTag)
sys.path.append(os.getcwd())
branchConfig, releaseConfig = validate(options, args)
sourceRepoInfo = releaseConfig["sourceRepositories"][
options.source_repo_key]
try:
brandName = releaseConfig["brandName"]
except KeyError:
brandName = releaseConfig["productName"].title()
platform = options.platform
if platform == "linux":
platform = "linux32"
mozconfig = path.join(get_repo_dirname(sourceRepoInfo["path"]),
releaseConfig["appName"], "config", "mozconfigs",
def repackLocale(locale,
l10nRepoDir,
l10nBaseRepo,
revision,
localeSrcDir,
l10nIni,
compareLocalesRepo,
env,
absObjdir,
merge=True,
productName=None,
platform=None,
version=None,
partialUpdates=None,
buildNumber=None,
stageServer=None,
mozillaDir=None,
mozillaSrcDir=None,
marSignatureFormat='mar'):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
mercurial(repo, localeDir)
update(localeDir, revision=revision)
# It's a bad assumption to make, but the source dir is currently always
# one level above the objdir.
absSourceRepoPath = path.split(absObjdir)[0]
use_pymake = env.get("USE_PYMAKE", False)
make = getMakeCommand(use_pymake, absSourceRepoPath)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
if use_pymake:
env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"])
else:
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
UPLOAD_EXTRA_FILES = []
if mozillaDir:
nativeDistDir = path.normpath(
path.abspath(path.join(localeSrcDir,
'../../%s/dist' % mozillaDir)))
else:
nativeDistDir = path.normpath(
path.abspath(path.join(localeSrcDir, '../../dist')))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
if platform.startswith('win'):
mar += ".exe"
mbsdiff += ".exe"
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName,
version)
unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl'
make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh'
prevMarDir = '../../../../'
if mozillaSrcDir:
# Compensate for having the objdir or not.
additionalParent = ''
if mozillaDir:
additionalParent = '../'
unwrap_full_update = '../../../%s%s/tools/update-packaging/unwrap_full_update.pl' % (
additionalParent, mozillaSrcDir)
make_incremental_update = '../../%s%s/tools/update-packaging/make_incremental_update.sh' % (
additionalParent, mozillaSrcDir)
prevMarDir = '../../../../%s' % additionalParent
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
log.info("Download mar tools")
if stageServer:
candidates_dir = makeCandidatesDir(productName,
version,
buildNumber,
protocol="http",
server=stageServer)
if not path.isfile(msys2windows(mar)):
marUrl = '/'.join([
p.strip('/') for p in
[candidates_dir, 'mar-tools', platform,
path.basename(mar)]
])
run_cmd(['mkdir', '-p', path.dirname(mar)])
log.info("Downloading %s to %s", marUrl, mar)
urlretrieve(marUrl, msys2windows(mar))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mar])
if not path.isfile(msys2windows(mbsdiff)):
mbsdiffUrl = '/'.join([
p.strip('/') for p in [
candidates_dir, 'mar-tools', platform,
path.basename(mbsdiff)
]
])
run_cmd(['mkdir', '-p', path.dirname(mbsdiff)])
log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff)
urlretrieve(mbsdiffUrl, msys2windows(mbsdiff))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mbsdiff])
else:
log.warning('stageServer not set. mar tools will *not* be downloaded.')
compareLocales(compareLocalesRepo,
locale,
l10nRepoDir,
localeSrcDir,
l10nIni,
revision=revision,
merge=merge)
make_installers_env = env.copy()
make_installers_env['MOZ_OBJDIR'] = absObjdir
run_cmd(make + ["installers-%s" % locale],
cwd=localeSrcDir,
env=make_installers_env)
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(current)])
run_cmd(['mkdir', current])
run_cmd(['perl', unwrap_full_update, current_mar],
cwd=path.join(nativeDistDir, 'current'),
env=env)
for oldVersion in partialUpdates:
prevMar = partialUpdates[oldVersion]['mar']
if prevMar:
partial_mar_name = '%s-%s-%s.partial.mar' % (productName,
oldVersion, version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(previous)])
run_cmd(['mkdir', previous])
run_cmd(
['perl', unwrap_full_update,
'%s/%s' % (prevMarDir, prevMar)],
cwd=path.join(nativeDistDir, 'previous'),
env=env)
run_cmd([
'bash', make_incremental_update, partial_mar, previous, current
],
cwd=nativeDistDir,
env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd([
'bash', '-c',
'%s -f %s "%s"' % (os.environ['MOZ_SIGN_CMD'],
marSignatureFormat, partial_mar)
],
env=env)
UPLOAD_EXTRA_FILES.append('%s/%s.asc' %
(updateDir, partial_mar_name))
else:
log.warning("Skipping partial MAR creation for %s %s" %
(oldVersion, locale))
env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES)
retry(run_cmd,
args=(make + ["upload", "AB_CD=%s" % locale], ),
kwargs={
'cwd': localeSrcDir,
'env': env
})
# return the location of the checksums file, because consumers may want
# some information about the files that were generated.
# Some versions of make that we use (at least pymake) imply --print-directory
# We need to turn it off to avoid getting extra output that mess up our
# parsing of the checksum file path.
curdir = os.getcwd()
try:
os.chdir(localeSrcDir)
relative_checksums = get_output(make + [
"--no-print-directory", "echo-variable-CHECKSUM_FILE",
"AB_CD=%s" % locale
],
env=env,
cwd=localeSrcDir).strip("\"'\n\r")
return path.normpath(path.join(localeSrcDir, relative_checksums))
finally:
os.chdir(curdir)
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
l10nIni, compareLocalesRepo, env, absObjdir, merge=True,
productName=None, platform=None,
version=None, partialUpdates=None,
buildNumber=None, stageServer=None,
mozillaDir=None, mozillaSrcDir=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
mercurial(repo, localeDir)
update(localeDir, revision=revision)
# It's a bad assumption to make, but the source dir is currently always
# one level above the objdir.
absSourceRepoPath = path.split(absObjdir)[0]
use_pymake = env.get("USE_PYMAKE", False)
make = getMakeCommand(use_pymake, absSourceRepoPath)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
if use_pymake:
env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"])
else:
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
UPLOAD_EXTRA_FILES = []
if mozillaDir:
nativeDistDir = path.normpath(path.abspath(
path.join(localeSrcDir, '../../%s/dist' % mozillaDir)))
else:
nativeDistDir = path.normpath(path.abspath(
path.join(localeSrcDir, '../../dist')))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
if platform.startswith('win'):
mar += ".exe"
mbsdiff += ".exe"
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (
updateAbsDir, productName, version)
unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl'
make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh'
prevMarDir = '../../../../'
if mozillaSrcDir:
# Compensate for having the objdir or not.
additionalParent = ''
if mozillaDir:
additionalParent = '../'
unwrap_full_update = '../../../%s%s/tools/update-packaging/unwrap_full_update.pl' % (additionalParent, mozillaSrcDir)
make_incremental_update = '../../%s%s/tools/update-packaging/make_incremental_update.sh' % (additionalParent, mozillaSrcDir)
prevMarDir = '../../../../%s' % additionalParent
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
log.info("Download mar tools")
if stageServer:
candidates_dir = makeCandidatesDir(productName, version, buildNumber,
protocol="http", server=stageServer)
if not path.isfile(msys2windows(mar)):
marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \
dict(c_dir=candidates_dir, platform=platform,
mar=path.basename(mar))
run_cmd(['mkdir', '-p', path.dirname(mar)])
log.info("Downloading %s to %s", marUrl, mar)
urlretrieve(marUrl, msys2windows(mar))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mar])
if not path.isfile(msys2windows(mbsdiff)):
mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \
dict(c_dir=candidates_dir, platform=platform,
mbsdiff=path.basename(mbsdiff))
run_cmd(['mkdir', '-p', path.dirname(mbsdiff)])
log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff)
urlretrieve(mbsdiffUrl, msys2windows(mbsdiff))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mbsdiff])
else:
log.warning('stageServer not set. mar tools will *not* be downloaded.')
compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
l10nIni, revision=revision, merge=merge)
run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env)
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(current)])
run_cmd(['mkdir', current])
run_cmd(['perl', unwrap_full_update, current_mar],
cwd=path.join(nativeDistDir, 'current'), env=env)
for oldVersion in partialUpdates:
prevMar = partialUpdates[oldVersion]['mar']
if prevMar:
partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
# Our Windows-native rm from bug 727551 requires Windows-style paths
run_cmd(['rm', '-rf', msys2windows(previous)])
run_cmd(['mkdir', previous])
run_cmd(
['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)],
cwd=path.join(nativeDistDir, 'previous'), env=env)
run_cmd(['bash', make_incremental_update, partial_mar, previous,
current], cwd=nativeDistDir, env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd(['bash', '-c',
'%s -f mar -f gpg "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)],
env=env)
UPLOAD_EXTRA_FILES.append(
'%s/%s.asc' % (updateDir, partial_mar_name))
else:
log.warning(
"Skipping partial MAR creation for %s %s" % (oldVersion,
locale))
env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES)
retry(run_cmd,
args=(make + ["upload", "AB_CD=%s" % locale], ),
kwargs={'cwd': localeSrcDir, 'env': env})
# return the location of the checksums file, because consumers may want
# some information about the files that were generated.
# Some versions of make that we use (at least pymake) imply --print-directory
# We need to turn it off to avoid getting extra output that mess up our
# parsing of the checksum file path.
curdir = os.getcwd()
try:
os.chdir(localeSrcDir)
relative_checksums = get_output(make +
["--no-print-directory", "echo-variable-CHECKSUM_FILE", "AB_CD=%s" % locale],
env=env).strip("\"'\n")
return path.normpath(path.join(localeSrcDir, relative_checksums))
finally:
os.chdir(curdir)
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
l10nIni, compareLocalesRepo, env, merge=True,
productName=None, platform=None,
version=None, partialUpdates=None,
buildNumber=None, stageServer=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
retry(mercurial, args=(repo, localeDir))
update(localeDir, revision=revision)
mozillaDir = ''
if 'thunderbird' in productName:
mozillaDir = 'mozilla/'
# split on \\ since we care about the absSourceRepoPath for pymake, which
# is windows.
absSourceRepoPath = os.path.join(os.getcwd(), localeSrcDir.split("\\")[0])
use_pymake = env.get("USE_PYMAKE", False)
make = getMakeCommand(use_pymake, absSourceRepoPath)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
if use_pymake:
env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"])
else:
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
UPLOAD_EXTRA_FILES = []
nativeDistDir = path.normpath(path.abspath(
path.join(localeSrcDir, '../../%sdist' % mozillaDir)))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
if platform.startswith('win'):
mar += ".exe"
mbsdiff += ".exe"
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (
updateAbsDir, productName, version)
unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl'
make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh'
prevMarDir = '../../../../'
if mozillaDir:
unwrap_full_update = '../../../../%stools/update-packaging/unwrap_full_update.pl' % mozillaDir
make_incremental_update = '../../../%stools/update-packaging/make_incremental_update.sh' % mozillaDir
prevMarDir = '../../../../../'
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
log.info("Download mar tools")
if stageServer:
candidates_dir = makeCandidatesDir(productName, version, buildNumber,
protocol="http", server=stageServer)
if not path.isfile(msys2windows(mar)):
marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \
dict(c_dir=candidates_dir, platform=platform,
mar=path.basename(mar))
run_cmd(['mkdir', '-p', path.dirname(mar)])
log.info("Downloading %s to %s", marUrl, mar)
urlretrieve(marUrl, msys2windows(mar))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mar])
if not path.isfile(msys2windows(mbsdiff)):
mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \
dict(c_dir=candidates_dir, platform=platform,
mbsdiff=path.basename(mbsdiff))
run_cmd(['mkdir', '-p', path.dirname(mbsdiff)])
log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff)
urlretrieve(mbsdiffUrl, msys2windows(mbsdiff))
if not sys.platform.startswith('win'):
run_cmd(['chmod', '755', mbsdiff])
else:
log.warning('stageServer not set. mar tools will *not* be downloaded.')
compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
l10nIni, revision=revision, merge=merge)
run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env)
run_cmd(['rm', '-rf', current])
run_cmd(['mkdir', current])
run_cmd(['perl', unwrap_full_update, current_mar],
cwd=path.join(nativeDistDir, 'current'), env=env)
for oldVersion in partialUpdates:
prevMar = partialUpdates[oldVersion]['mar']
if prevMar:
partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
run_cmd(['rm', '-rf', previous])
run_cmd(['mkdir', previous])
run_cmd(
['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)],
cwd=path.join(nativeDistDir, 'previous'), env=env)
run_cmd(['bash', make_incremental_update, partial_mar, previous,
current], cwd=nativeDistDir, env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd(['bash', '-c',
'%s -f mar -f gpg "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)],
env=env)
UPLOAD_EXTRA_FILES.append(
'%s/%s.asc' % (updateDir, partial_mar_name))
else:
log.warning(
"Skipping partial MAR creation for %s %s" % (oldVersion,
locale))
env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES)
retry(run_cmd,
args=(make + ["upload", "AB_CD=%s" % locale], ),
kwargs={'cwd': localeSrcDir, 'env': env})
def main():
logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument("--hg-user", default="ffxbld <*****@*****.**>",
help="Mercurial username to be passed to hg -u")
args = parser.parse_args()
hg_user = args.hg_user
# prep the repos
for d, repo in ((mc_dir, mc_repo), (ma_dir, ma_repo), (mb_dir, mb_repo)):
mercurial(repo, d)
log.info("Cleaning up %s...", d)
strip_outgoing(d)
update(d, branch="default")
curr_mc_version = get_major_version(mc_dir)
curr_ma_version = get_major_version(ma_dir)
curr_mb_version = get_major_version(mb_dir)
next_mc_version = str(int(curr_mc_version) + 1)
next_ma_version = str(int(curr_ma_version) + 1)
next_mb_version = str(int(curr_mb_version) + 1)
# mozilla-central
mc_revision = get_revision(mc_dir)
mc_tag = "FIREFOX_AURORA_%s_BASE" % curr_mc_version
tag(mc_dir, tags=[mc_tag], rev=mc_revision, user=hg_user,
msg="Added %s tag for changeset %s. IGNORE BROKEN CHANGESETS DONTBUILD CLOSED TREE NO BUG a=release" %
(mc_tag, mc_revision))
new_mc_revision = get_revision(mc_dir)
bump_version(mc_dir, curr_mc_version, next_mc_version, "a1", "a1",
bump_major=True)
raw_input("Hit 'return' to display diffs onscreen")
run_cmd(["hg", "diff"], cwd=mc_dir)
raw_input("If the diff looks good hit return to commit those changes")
commit(mc_dir, user=hg_user,
msg="Version bump. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release")
raw_input("Go ahead and push mozilla-central...and continue to "
"mozilla-aurora to mozilla-beta uplift ")
# mozilla-aurora
ma_revision = get_revision(ma_dir)
ma_tag = "FIREFOX_BETA_%s_BASE" % curr_ma_version
ma_end_tag = "FIREFOX_AURORA_%s_END" % curr_ma_version
# pull must use revision not tag
pull(mc_dir, dest=ma_dir, revision=new_mc_revision)
merge_via_debugsetparents(
ma_dir, old_head=ma_revision, new_head=new_mc_revision,
user=hg_user, msg="Merge old head via |hg debugsetparents %s %s|. "
"CLOSED TREE DONTBUILD a=release" % (new_mc_revision, ma_revision))
tag(ma_dir, tags=[ma_tag, ma_end_tag], rev=ma_revision, user=hg_user,
msg="Added %s %s tags for changeset %s. IGNORE BROKEN CHANGESETS DONTBUILD CLOSED TREE NO BUG a=release" %
(ma_tag, ma_end_tag, ma_revision))
log.info("Reverting locales")
for f in locale_files:
run_cmd(["hg", "revert", "-r", ma_end_tag, f], cwd=ma_dir)
bump_version(ma_dir, next_ma_version, next_ma_version, "a1", "a2")
raw_input("Hit 'return' to display diffs onscreen")
run_cmd(["hg", "diff"], cwd=ma_dir)
raw_input("If the diff looks good hit return to commit those changes")
commit(ma_dir, user=hg_user, msg="Version bump. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release")
replace(path.join(ma_dir, "browser/confvars.sh"),
"MOZ_BRANDING_DIRECTORY=browser/branding/nightly",
"MOZ_BRANDING_DIRECTORY=browser/branding/aurora")
replace(path.join(ma_dir, "browser/confvars.sh"),
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-central",
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-aurora")
replace(path.join(ma_dir, "browser/confvars.sh"),
"MAR_CHANNEL_ID=firefox-mozilla-central",
"MAR_CHANNEL_ID=firefox-mozilla-aurora")
for d in branding_dirs:
for f in branding_files:
replace(path.join(ma_dir, d, f),
"ac_add_options --with-branding=mobile/android/branding/nightly",
"ac_add_options --with-branding=mobile/android/branding/aurora")
if f == "l10n-nightly":
replace(path.join(ma_dir, d, f),
"ac_add_options --with-l10n-base=../../l10n-central",
"ac_add_options --with-l10n-base=..")
for f in profiling_files:
replace(path.join(ma_dir, f), "ac_add_options --enable-profiling", "")
for f in elf_hack_files:
replace(path.join(ma_dir, f),
"ac_add_options --disable-elf-hack # --enable-elf-hack conflicts with --enable-profiling", "")
raw_input("Hit 'return' to display diffs onscreen")
run_cmd(["hg", "diff"], cwd=ma_dir)
raw_input("If the diff looks good hit return to commit those changes")
commit(ma_dir, user=hg_user,
msg="Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release")
raw_input("Go ahead and push mozilla-aurora changes.")
# mozilla-beta
mb_revision = get_revision(mb_dir)
mb_tag = "FIREFOX_BETA_%s_END" % curr_mb_version
# pull must use revision not tag
pull(ma_dir, dest=mb_dir, revision=ma_revision)
merge_via_debugsetparents(
mb_dir, old_head=mb_revision, new_head=ma_revision,
user=hg_user, msg="Merge old head via |hg debugsetparents %s %s|. "
"CLOSED TREE DONTBUILD a=release" % (ma_revision, mb_revision))
tag(mb_dir, tags=[mb_tag], rev=mb_revision, user=hg_user,
msg="Added %s tag for changeset %s. IGNORE BROKEN CHANGESETS DONTBUILD CLOSED TREE NO BUG a=release" %
(mb_tag, mb_revision))
bump_version(mb_dir, next_mb_version, next_mb_version, "a2", "")
raw_input("Hit 'return' to display diffs onscreen")
run_cmd(["hg", "diff"], cwd=mb_dir)
raw_input("If the diff looks good hit return to commit those changes")
commit(mb_dir, user=hg_user, msg="Version bump. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release")
replace(path.join(mb_dir, "browser/confvars.sh"),
"MOZ_BRANDING_DIRECTORY=browser/branding/aurora",
"MOZ_BRANDING_DIRECTORY=browser/branding/nightly")
replace(path.join(mb_dir, "browser/confvars.sh"),
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-aurora",
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release")
replace(path.join(mb_dir, "browser/confvars.sh"),
"MAR_CHANNEL_ID=firefox-mozilla-aurora",
"MAR_CHANNEL_ID=firefox-mozilla-beta")
for d in branding_dirs:
for f in branding_files:
replace(path.join(mb_dir, d, f),
"ac_add_options --with-branding=mobile/android/branding/aurora",
"ac_add_options --with-branding=mobile/android/branding/beta")
raw_input("Hit 'return' to display diffs onscreen")
run_cmd(["hg", "diff"], cwd=mb_dir)
raw_input("If the diff looks good hit return to commit those changes")
commit(mb_dir, user=hg_user,
msg="Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release")
raw_input("Go ahead and push mozilla-beta changes.")
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
locales, product, version, buildNumber,
stageServer, stageUsername, stageSshKey,
compareLocalesRepo, merge, platform, brand,
generatePartials=False, partialUpdates=None,
appVersion=None, usePymake=False, tooltoolManifest=None,
tooltool_script=None, tooltool_urls=None):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", "")
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\') + \
" -H " + \
os.environ['MOZ_SIGN_CMD'].split(' ')[-1]
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
retry(mercurial, args=(sourceRepo, sourceRepoName))
update(sourceRepoName, revision=revision)
l10nRepackPrep(
sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir,
makeDirs, localeSrcDir, env, tooltoolManifest, tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(stageServer, product, brand, version, buildNumber,
platform),
kwargs={'signed': signed,
'usePymake': usePymake})
env.update(input_env)
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(stageServer, product, oldVersion, oldBuildNumber,
platform, l)
)
repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo, revision=revision,
localeSrcDir=localeSrcDir, l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo, env=env,
merge=merge,
productName=product, platform=platform,
version=version, partialUpdates=partialUpdates,
buildNumber=buildNumber, stageServer=stageServer)
except Exception, e:
failed.append((l, format_exc()))
objdir="obj-l10n"
)
parser.add_option("-c", "--configfile", dest="configfile")
parser.add_option("-B", "--branch", dest="branch")
parser.add_option("-b", "--buildbot-configs", dest="buildbotConfigs")
parser.add_option("-p", "--platform", dest="platform")
parser.add_option("-s", "--stage-platform", dest="stage_platform",
default=None)
parser.add_option("-o", "--objdir", dest="objdir")
parser.add_option("-l", "--locale", dest="locales", action="append")
parser.add_option("--chunks", dest="chunks", type="int")
parser.add_option("--this-chunk", dest="thisChunk", type="int")
options, args = parser.parse_args()
mercurial(options.buildbotConfigs, "buildbot-configs")
update("buildbot-configs", revision="default")
branchConfig = validate(options, args)
platformConfig = branchConfig["platforms"][options.platform]
mobileDirName = platformConfig.get('mobile_dir', 'mobile')
if options.chunks:
locales = getNightlyLocalesForChunk(mobileDirName,
options.branch, options.platform,
options.chunks, options.thisChunk)
else:
locales = options.locales
mozconfig = path.join("buildbot-configs", "mozilla2",
platformConfig["mozconfig"], "l10n-mozconfig")
ftpProduct = "mobile"
except Exception, e:
# Rather than catching individual problems and giving very specific
# status updates to the kickoff application, we use this catch-all.
# Because nearly all problems require looking at the release runner
# logs and manual intervention, it's not worth the pain and ugliness
# to do more than this.
for release in rr.new_releases:
rr.mark_as_failed(release, 'Failed: %s' % repr(e))
raise
rc = 0
for release in rr.new_releases:
try:
rr.update_status(release, 'Running sendchange command')
staging = config.getboolean('release-runner', 'staging')
update(configs_workdir, revision='default')
cfgFile = path.join(
configs_workdir, 'mozilla',
getReleaseConfigName(release['product'],
path.basename(release['branch']),
release['version'], staging))
enUSPlatforms = readReleaseConfig(cfgFile)['enUSPlatforms']
rr.start_release_automation(release, sendchange_master,
enUSPlatforms)
except:
# We explicitly do not raise an error here because there's no
# reason not to start other releases if the sendchange fails for
# another one. We _do_ need to set this in order to exit
# with the right code, though.
rc = 2
rr.update_status(release, 'Sendchange failed')
def testUpdateRevision(self):
rev = clone(self.repodir, self.wc, update_dest=False)
self.assertEquals(rev, None)
rev = update(self.wc, revision=self.revisions[1])
self.assertEquals(rev, self.revisions[1])
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, appName, locales, product,
version, buildNumber, stageServer, stageUsername, stageSshKey,
compareLocalesRepo, merge, platform, brand,
generatePartials=False, oldVersion=None,
oldBuildNumber=None, appVersion=None):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
)
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
retry(mercurial, args=(sourceRepo, sourceRepoName))
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, l10nRepoDir, makeDirs,
localeSrcDir, env)
input_env = retry(downloadReleaseBuilds,
args=(stageServer, product, brand, version, buildNumber,
platform),
kwargs={'signed': signed})
env.update(input_env)
failed = []
for l in locales:
try:
prevMar = None
if generatePartials:
prevMar = retry(
downloadUpdateIgnore404,
args=(stageServer, product, oldVersion, oldBuildNumber,
platform, l)
)
repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo, revision=revision,
localeSrcDir=localeSrcDir, l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo, env=env,
merge=merge, prevMar=prevMar,
productName=product, platform=platform,
version=version, oldVersion=oldVersion)
except Exception, e:
failed.append((l, format_exc()))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
locales, product, version, buildNumber,
stageServer, stageUsername, stageSshKey, ftpServer,
compareLocalesRepo, merge, platform, brand, appVersion,
generatePartials=False, partialUpdates=None,
usePymake=False, tooltoolManifest=None,
tooltool_script=None, tooltool_urls=None,
balrog_submitter=None, balrog_hash="sha512",
mozillaDir=None, mozillaSrcDir=None, bucket_prefix=None):
buildid = retry(getBuildID, args=(platform, product, version,
buildNumber, 'candidates', ftpServer))
log.info('Got buildid: %s' % buildid)
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"DOWNLOAD_HOST": ftpServer,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
"MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
bucket_prefix=bucket_prefix,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
mercurial(sourceRepo, sourceRepoName)
update(sourceRepoName, revision=revision)
l10nRepackPrep(
sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir,
makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(ftpServer, product, brand, version, buildNumber,
platform),
kwargs={'signed': signed,
'usePymake': usePymake})
env.update(input_env)
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(ftpServer, product, oldVersion, oldBuildNumber,
platform, l)
)
checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo, revision=revision,
localeSrcDir=localeSrcDir, l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo, env=env,
absObjdir=absObjdir, merge=merge,
productName=product, platform=platform,
version=version, partialUpdates=partialUpdates,
buildNumber=buildNumber, stageServer=ftpServer,
mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion": previousVersion,
"previousBuildNumber": partialUpdates[previousVersion]['buildNumber'],
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
}
)
except Exception, e:
print_exc()
failed.append((l, format_exc()))
def tagRepo(config,
repo,
reponame,
revision,
tags,
bumpFiles,
relbranch,
pushAttempts,
defaultBranch='default'):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output([
'hg', 'commit', '-u', config['hgUsername'], '-m',
getBumpCommitMessage(config['productName'],
config['version'])
],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output([
'hg', 'commit', '-u', config['hgUsername'], '-m',
getBumpCommitMessage(config['productName'],
config['version'])
],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
for opt in required_options:
if not options_dict[opt]:
parser.error("Required option %s not present" % opt)
if options.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
update_platform = buildbot2updatePlatforms(options.platform)[0]
ftp_platform = buildbot2ftp(options.platform)
full_check_locales = options.full_check_locales
# Variables from release config
retry(mercurial, args=(options.buildbot_configs, 'buildbot-configs'))
update('buildbot-configs', revision=options.release_tag)
release_config = validate(options)
product_name = release_config['productName']
staging_server = FTP_SERVER_TEMPLATE % release_config['stagingServer']
aus_server_url = release_config['ausServerUrl']
build_number = release_config['buildNumber']
previous_releases_staging_server = FTP_SERVER_TEMPLATE % \
release_config.get('previousReleasesStagingServer',
release_config['stagingServer'])
# Current version data
pc = PatcherConfig(open(options.config).read())
app_name = pc['appName']
to_version = pc['current-update']['to']
to_ = makeReleaseRepackUrls(
product_name, app_name, to_version, options.platform,
for opt in required_options:
if not options_dict[opt]:
parser.error("Required option %s not present" % opt)
if options.verbose:
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
update_platform = buildbot2updatePlatforms(options.platform)[0]
ftp_platform = buildbot2ftp(options.platform)
full_check_locales = options.full_check_locales
# Variables from release config
retry(mercurial, args=(options.buildbot_configs, 'buildbot-configs'))
update('buildbot-configs', revision=options.release_tag)
release_config = validate(options)
product_name = release_config['productName']
staging_server = FTP_SERVER_TEMPLATE % release_config['stagingServer']
aus_server_url = release_config['ausServerUrl']
build_number = release_config['buildNumber']
previous_releases_staging_server = FTP_SERVER_TEMPLATE % \
release_config.get('previousReleasesStagingServer',
release_config['stagingServer'])
# Current version data
pc = PatcherConfig(open(options.config).read())
app_name = pc['appName']
to_version = pc['current-update']['to']
to_ = makeReleaseRepackUrls(product_name,
app_name,
def tag_and_push(repo, attempt):
update(workdir, branch)
tag(workdir, tags, rev=branch, force=True, user=hg_username)
log.info("Tagged %s, attempt #%s" % (repo, attempt))
except Exception, e:
# Rather than catching individual problems and giving very specific
# status updates to the kickoff application, we use this catch-all.
# Because nearly all problems require looking at the release runner
# logs and manual intervention, it's not worth the pain and ugliness
# to do more than this.
for release in rr.new_releases:
rr.mark_as_failed(release, 'Failed: %s' % repr(e))
raise
rc = 0
for release in rr.new_releases:
try:
rr.update_status(release, 'Running sendchange command')
staging = config.getboolean('release-runner', 'staging')
update(configs_workdir, revision='default')
cfgFile = path.join(configs_workdir,
'mozilla',
getReleaseConfigName(release['product'],
path.basename(release['branch']),
release['version'], staging))
enUSPlatforms = readReleaseConfig(cfgFile)['enUSPlatforms']
rr.start_release_automation(release, sendchange_master, enUSPlatforms)
except:
# We explicitly do not raise an error here because there's no
# reason not to start other releases if the sendchange fails for
# another one. We _do_ need to set this in order to exit
# with the right code, though.
rc = 2
rr.update_status(release, 'Sendchange failed')
log.error('Sendchange failed for %s: ' % release, exc_info=True)
parser.add_argument("-b", "--buildbot-configs",
help="buildbot-configs mercurial repo URL",
default=os.environ.get("BUILDBOT_CONFIGS",
DEFAULT_BUILDBOT_CONFIGS_REPO))
parser.add_argument("-t", "--release-tag", required=True)
parser.add_argument("--product", help="Override for stage_product")
parser.add_argument("--ssh-user", required=True)
parser.add_argument("--ssh-key", required=True)
parser.add_argument("--overwrite", default=False, action="store_true")
parser.add_argument("--extra-excludes", action="append")
parser.add_argument("actions", nargs="+", help="Script actions")
args = parser.parse_args()
actions = args.actions
mercurial(args.buildbot_configs, "buildbot-configs")
update("buildbot-configs", revision=args.release_tag)
releaseConfigFile = path.join("buildbot-configs", args.release_config)
releaseConfig = readReleaseConfig(
releaseConfigFile, required=REQUIRED_RELEASE_CONFIG)
productName = args.product or releaseConfig['stage_product']
version = releaseConfig['version']
buildNumber = releaseConfig['buildNumber']
stageServer = releaseConfig['stagingServer']
stageUsername = args.ssh_user
stageSshKey = args.ssh_key
stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey)
createIndexFiles = releaseConfig.get('makeIndexFiles', False) \
and productName != 'xulrunner'
syncPartnerBundles = releaseConfig.get('syncPartnerBundles', False) \
def tag_and_push(repo, attempt):
update(workdir, branch)
tag(workdir, tags, rev=branch, force=True, user=hg_username)
log.info("Tagged %s, attempt #%s" % (repo, attempt))
parser.add_option("-o", "--objdir", dest="objdir")
parser.add_option("-l", "--locale", dest="locales", action="append")
parser.add_option("--source-repo-key", dest="source_repo_key")
parser.add_option("--chunks", dest="chunks", type="int")
parser.add_option("--this-chunk", dest="thisChunk", type="int")
parser.add_option("--generate-partials", dest="generatePartials",
action='store_true', default=False)
options, args = parser.parse_args()
if options.generatePartials:
makeDirs.extend([
path.join("modules", "libbz2"),
path.join("other-licenses", "bsdiff")
])
retry(mercurial, args=(options.buildbotConfigs, "buildbot-configs"))
update("buildbot-configs", revision=options.releaseTag)
sys.path.append(os.getcwd())
branchConfig, releaseConfig = validate(options, args)
sourceRepoInfo = releaseConfig["sourceRepositories"][options.source_repo_key]
try:
brandName = releaseConfig["brandName"]
except KeyError:
brandName = releaseConfig["productName"].title()
mozconfig = path.join("buildbot-configs", "mozilla2", options.platform,
sourceRepoInfo['name'], "release", "l10n-mozconfig")
if options.chunks:
locales = retry(getReleaseLocalesForChunk,
args=(releaseConfig["productName"], releaseConfig["appName"],
releaseConfig["version"], int(releaseConfig["buildNumber"]),
sourceRepoInfo["path"], options.platform,
def main():
logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
parser = argparse.ArgumentParser()
parser.add_argument("--from-dir", default="mozilla-beta",
help="Working directory of repo to be merged from")
parser.add_argument("--from-repo",
default="ssh://hg.mozilla.org/releases/mozilla-beta",
help="Repo to be merged from")
parser.add_argument("--to-dir", default="mozilla-release",
help="Working directory of repo to be merged to")
parser.add_argument(
"--to-repo", default="ssh://hg.mozilla.org/releases/mozilla-release",
help="Repo to be merged to")
parser.add_argument("--hg-user", default="ffxbld <*****@*****.**>",
help="Mercurial username to be passed to hg -u")
parser.add_argument("--remove-locale", dest="remove_locales", action="append",
required=True,
help="Locales to be removed from release shipped-locales")
args = parser.parse_args()
from_dir = args.from_dir
to_dir = args.to_dir
from_repo = args.from_repo
to_repo = args.to_repo
hg_user = args.hg_user
with retrying(mercurial) as clone:
for (d, repo) in ((from_dir, from_repo), (to_dir, to_repo)):
clone(repo, d)
log.info("Cleaning up %s...", d)
strip_outgoing(d)
update(d, branch="default")
beta_rev = get_revision(from_dir)
release_rev = get_revision(to_dir)
now = datetime.datetime.now()
date = now.strftime("%Y%m%d")
# TODO: make this tag consistent with other branches
release_base_tag = "RELEASE_BASE_" + date
log.info("Tagging %s beta with %s", beta_rev, release_base_tag)
tag(from_dir, tags=[release_base_tag], rev=beta_rev, user=hg_user,
msg="Added %s tag for changeset %s. DONTBUILD CLOSED TREE a=release" %
(release_base_tag, beta_rev))
new_beta_rev = get_revision(from_dir)
raw_input("Push mozilla-beta and hit Return")
pull(from_dir, dest=to_dir)
merge_via_debugsetparents(
to_dir, old_head=release_rev, new_head=new_beta_rev, user=hg_user,
msg="Merge old head via |hg debugsetparents %s %s|. "
"CLOSED TREE DONTBUILD a=release" % (new_beta_rev, release_rev))
replace(
path.join(to_dir, "browser/confvars.sh"),
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release",
"ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release")
replace(path.join(to_dir, "browser/confvars.sh"),
"MAR_CHANNEL_ID=firefox-mozilla-beta",
"MAR_CHANNEL_ID=firefox-mozilla-release")
for d in branding_dirs:
for f in branding_files:
replace(
path.join(to_dir, d, f),
"ac_add_options --with-branding=mobile/android/branding/beta",
"ac_add_options --with-branding=mobile/android/branding/official")
if args.remove_locales:
log.info("Removing locales: %s", args.remove_locales)
remove_locales(path.join(to_dir, "browser/locales/shipped-locales"),
args.remove_locales)
log.warn("Apply any manual changes, such as disabling features.")
raw_input("Hit 'return' to display channel, branding, and feature diffs onscreen")
run_cmd(["hg", "diff"], cwd=to_dir)
raw_input("If the diff looks good hit return to commit those changes")
commit(to_dir, user=hg_user,
msg="Update configs. CLOSED TREE a=release ba=release")
raw_input("Go ahead and push mozilla-release changes.")
def testUpdateRevision(self):
rev = clone(self.repodir, self.wc, update_dest=False)
self.assertEquals(rev, None)
rev = update(self.wc, revision=self.revisions[1])
self.assertEquals(rev, self.revisions[1])
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName,
locales, product, version, buildNumber,
stageServer, stageUsername, stageSshKey,
compareLocalesRepo, merge, platform, brand, appVersion,
generatePartials=False, partialUpdates=None,
usePymake=False, tooltoolManifest=None,
tooltool_script=None, tooltool_urls=None,
balrog_submitter=None, balrog_hash="sha512", buildid=None,
mozillaDir=None, mozillaSrcDir=None):
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", "")
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
retry(mercurial, args=(sourceRepo, sourceRepoName))
update(sourceRepoName, revision=revision)
l10nRepackPrep(
sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir,
makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(stageServer, product, brand, version, buildNumber,
platform),
kwargs={'signed': signed,
'usePymake': usePymake})
env.update(input_env)
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(stageServer, product, oldVersion, oldBuildNumber,
platform, l)
)
checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo, revision=revision,
localeSrcDir=localeSrcDir, l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo, env=env,
absObjdir=absObjdir, merge=merge,
productName=product, platform=platform,
version=version, partialUpdates=partialUpdates,
buildNumber=buildNumber, stageServer=stageServer,
mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion": previousVersion,
"previousBuildNumber": partialUpdates[previousVersion]['buildNumber'],
"size": info["size"],
"hash": info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
}
)
except Exception, e:
print_exc()
failed.append((l, format_exc()))
