def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): relbranchChangesets = len(tags) defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 revision = get_revision(reponame) except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise
def cleanup_wrapper(): # use an attribute full_clean in order to keep track of # whether or not a full cleanup is required. # This is done since cleanup_wrapper's scope doesn't let us # access process_patchset globals, given the way it is used. if not hasattr(cleanup_wrapper, 'full_clean'): cleanup_wrapper.full_clean = False if not hasattr(cleanup_wrapper, 'branch'): cleanup_wrapper.branch = data['branch'] if cleanup_wrapper.branch != data['branch']: cleanup_wrapper.full_clean = False cleanup_wrapper.branch = data['branch'] # only wipe the repositories every second cleanup if cleanup_wrapper.full_clean: clear_branch(data['branch']) log_msg('Wiped repositories for: %s' % data['branch']) else: active_repo = os.path.join('active', data['branch']) update(active_repo) log_msg('Update -C on active repo for: %s' % data['branch']) cleanup_wrapper.full_clean = not cleanup_wrapper.full_clean clone_revision = clone_branch(data['branch'], data['branch_url']) if clone_revision == None: # TODO: Handle clone error log_msg('[HgPusher] Clone error...') return
def process_configs(repo, attempt): """Helper method that encapsulates all of the things necessary to run release runner for all releases.""" log.info("Bumping %s, attempt #%s" % (repo, attempt)) for release in rr.new_releases: rr.update_status(release, 'Writing configs') l10nContents = rr.get_release_l10n(release['name']) tags.extend(getTags( getBaseTag(release['product'], release['version']), release['buildNumber']) ) update(configs_workdir, revision='default') cfgFile = getReleaseConfigName( release['product'], path.basename(release['branch']), release['version'], staging) bump_configs(release=release, cfgFile=cfgFile, l10nContents=l10nContents, workdir=configs_workdir, hg_username=hg_username, productionBranch=buildbot_configs_branch) rr.update_status(release, 'Running release sanity') rs_args = get_release_sanity_args(configs_workdir, release, cfgFile, masters_json, buildbot_configs_branch) release_sanity_script = "%s/buildbot-helpers/release_sanity.py" % tools_workdir run_cmd(['python', release_sanity_script] + rs_args + ['--dry-run']) rr.update_status( release, 'Waiting for other releases to run release sanity' )
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge=True, prevMar=None, productName=None, platform=None, version=None, oldVersion=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) retry(mercurial, args=(repo, localeDir)) update(localeDir, revision=revision) compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env) UPLOAD_EXTRA_FILES = [] if prevMar: nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir, '../../dist'))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version) partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) env['MAR'] = mar env['MBSDIFF'] = mbsdiff run_cmd(['rm', '-rf', previous, current]) run_cmd(['mkdir', previous, current]) run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl', '../../../../%s' % prevMar], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl', current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) run_cmd( ['bash', '../../tools/update-packaging/make_incremental_update.sh', partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f gpg -f mar "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name)) retry(run_cmd, args=(["make", "upload", "AB_CD=%s" % locale, 'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],), kwargs={'cwd': localeSrcDir, 'env': env})
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts, defaultBranch='default'): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): # set relbranchChangesets=1 because tag() generates exactly 1 commit relbranchChangesets = 1 defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # We always want our tags pointing at the tip of the relbranch # so we need to grab the current revision after we've switched # branches and bumped versions. revision = get_revision(reponame) # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # This is the bump of the version on the default branch # We do it after the other one in order to get the tip of the # repository back on default, thus avoiding confusion. if len(bumpFiles) > 0: update(reponame, revision=defaultBranch) bump(reponame, bumpFiles, 'nextVersion') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) defaultBranchChangesets += 1 except subprocess.CalledProcessError, e: if e.returncode != 1 or "nothing changed" not in e.output: raise
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, locales, ftpProduct, appName, version, appVersion, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, stage_platform, brand, mobileDirName): sourceRepoName = path.split(sourceRepo)[-1] nightlyDir = "candidates" localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"]) env = { "MOZ_OBJDIR": objdir, "MOZ_PKG_VERSION": version, "UPLOAD_HOST": stageServer, "UPLOAD_USER": stageUsername, "UPLOAD_SSH_KEY": stageSshKey, "UPLOAD_TO_TEMP": "1", # Android signing "JARSIGNER": os.path.join(os.getcwd(), "scripts", "release", "signing", "mozpass.py") } build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName) retry(mercurial, args=(sourceRepo, sourceRepoName)) update(sourceRepoName, revision=revision) l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, l10nRepoDir, makeDirs, localeSrcDir, env) fullCandidatesDir = makeCandidatesDir(appName, version, buildNumber, protocol='http', server=stageServer, nightlyDir=nightlyDir) input_env = retry(downloadReleaseBuilds, args=(stageServer, ftpProduct, brand, version, buildNumber, stage_platform, fullCandidatesDir)) env.update(input_env) print "env pre-locale: %s" % str(env) failed = [] for l in locales: try: # adding locale into builddir env["POST_UPLOAD_CMD"] = postUploadCmdPrefix( to_mobile_candidates=True, product=appName, version=version, builddir='%s/%s' % (stage_platform, l), buildNumber=buildNumber, nightly_dir=nightlyDir,) print "env post-locale: %s" % str(env) repackLocale(str(l), l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge) except Exception, e: failed.append((l, format_exc()))
def soft_clean(self): """ Only does an update -C on the active repository to get rid of any applied, not committed patches. """ active_repo = os.path.join('active', self.branch) # get rid of any imported and qpush-ed patches log.debug('qpop -a and rm -rf .hg/patches') (success, err, ret) = run_hg(['qpop', '-a', '-R', active_repo]) run_cmd(['rm', '-rf', os.path.join(active_repo, '.hg/patches')]) log.debug('Update -C on active repo for: %s' % (self.branch)) update(active_repo)
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision="default", merge=True): retry(mercurial, args=(repo, "compare-locales")) update("compare-locales", revision=revision) mergeDir = path.join(localeSrcDir, "merged") if path.exists(mergeDir): log.info("Deleting %s" % mergeDir) shutil.rmtree(mergeDir) run_cmd(["python", path.join("compare-locales", "scripts", "compare-locales"), "-m", mergeDir, l10nIni, l10nRepoDir, locale], env={"PYTHONPATH": path.join("compare-locales", "lib")})
def tagRepo(repo, attempt, config, revision, tags): # set totalChangesets=1 because tag() generates exactly 1 commit totalChangesets = 1 # update to the desired revision first, then to the tip of revision's # branch to avoid new head creation update(repo, revision=revision) update(repo) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config[ 'hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions")
def bump_configs(release, cfgFile, l10nContents, workdir, hg_username, productionBranch, defaultBranch='default'): # Update the production branch first, because that's where we want to read # the templates from update(workdir, productionBranch) cfgDir = path.join(workdir, 'mozilla') templateFile = path.join(cfgDir, '%s.template' % cfgFile) tags = set(getTags(getBaseTag(release['product'], release['version']), release['buildNumber'])) cfgFile = path.join(cfgDir, cfgFile) l10nChangesetsFile = path.join( cfgDir, readReleaseConfig(cfgFile)['l10nRevisionFile'] ) subs = release.copy() if 'partials' in release: subs['partials'] = getPartials(release) # This is true 99% of the time. It's exceedingly rare that we ship a point # release that we first push to the beta channel. If we need to, the # expectation is that this will be ignored by hardcoding True in the # template. if isFinalRelease(release["version"]): subs["betaChannelEnabled"] = True else: subs["betaChannelEnabled"] = False with open(templateFile) as f: template = f.read() releaseConfig = substituteReleaseConfig(template, **subs) # Write out the new configs on the production branch... with open(cfgFile, 'w') as f: f.write(releaseConfig) with open(l10nChangesetsFile, 'w') as f: f.write(l10nContents) prodRev = commit(workdir, 'Update release config for %s' % release['name'], user=hg_username) # We always force tagging, because it makes it easier to retrigger a # release that fails for infrastructure reasons. tag(workdir, tags, rev=prodRev, force=True, user=hg_username) # And then write the same files to the default branch update(workdir, defaultBranch) with open(cfgFile, 'w') as f: f.write(releaseConfig) with open(l10nChangesetsFile, 'w') as f: f.write(l10nContents) commit(workdir, 'Update release config for %s' % release['name'], user=hg_username)
def bump_configs(release, cfgFile, l10nContents, workdir, hg_username, productionBranch, defaultBranch='default'): # Update the production branch first, because that's where we want to read # the templates from update(workdir, productionBranch) cfgDir = path.join(workdir, 'mozilla') templateFile = path.join(cfgDir, '%s.template' % cfgFile) tags = getTags(getBaseTag(release['product'], release['version']), release['buildNumber']) cfgFile = path.join(cfgDir, cfgFile) l10nChangesetsFile = path.join( cfgDir, readReleaseConfig(cfgFile)['l10nRevisionFile'] ) subs = release.copy() if 'partials' in release: subs['partials'] = getPartials(release) with open(templateFile) as f: template = f.read() releaseConfig = substituteReleaseConfig(template, **subs) # Write out the new configs on the production branch... with open(cfgFile, 'w') as f: f.write(releaseConfig) with open(l10nChangesetsFile, 'w') as f: f.write(l10nContents) prodRev = commit(workdir, 'Update release config for %s' % release['name'], user=hg_username) # We always force tagging, because it makes it easier to retrigger a # release that fails for infrastructure reasons. tag(workdir, tags, rev=prodRev, force=True, user=hg_username) # And then write the same files to the default branch update(workdir, defaultBranch) with open(cfgFile, 'w') as f: f.write(releaseConfig) with open(l10nChangesetsFile, 'w') as f: f.write(l10nContents) commit(workdir, 'Update release config for %s' % release['name'], user=hg_username)
check_buildbot() if not options.dryrun and not options.skip_reconfig: from util.fabric.common import check_fabric, FabricHelper check_fabric() if options.configs_dir: configs_dir = options.configs_dir cleanup_configs = False else: cleanup_configs = True configs_dir = mkdtemp() remote = make_hg_url(HG, options.configs_repo_url) retry(mercurial, args=(remote, configs_dir), kwargs={'branch': options.configs_branch}) update(configs_dir, options.configs_branch) # https://bugzilla.mozilla.org/show_bug.cgi?id=678103#c5 # This goes through the list of config files in reverse order, which is a # hacky way of making sure that the config file that's listed first is the # one that's loaded in releaseConfig for the sendchange. for releaseConfigFile in list(reversed(options.releaseConfigFiles)): abs_release_config_file = path.join(configs_dir, 'mozilla', releaseConfigFile) releaseConfig = readReleaseConfig(abs_release_config_file) products.append(releaseConfig['productName']) if not options.buildNumber: log.warn("No buildNumber specified, using buildNumber in" " release_config, which may be out of date!") options.buildNumber = releaseConfig['buildNumber']
"--buildbot-configs", help="buildbot-configs mercurial repo URL", default=os.environ.get("BUILDBOT_CONFIGS", DEFAULT_BUILDBOT_CONFIGS_REPO)) parser.add_argument("-t", "--release-tag", required=True) parser.add_argument("--product", help="Override for stage_product") parser.add_argument("--ssh-user", required=True) parser.add_argument("--ssh-key", required=True) parser.add_argument("--overwrite", default=False, action="store_true") parser.add_argument("--extra-excludes", action="append") parser.add_argument("actions", nargs="+", help="Script actions") args = parser.parse_args() actions = args.actions mercurial(args.buildbot_configs, "buildbot-configs") update("buildbot-configs", revision=args.release_tag) releaseConfigFile = path.join("buildbot-configs", args.release_config) releaseConfig = readReleaseConfig(releaseConfigFile, required=REQUIRED_RELEASE_CONFIG) productName = args.product or releaseConfig['stage_product'] version = releaseConfig['version'] buildNumber = releaseConfig['buildNumber'] stageServer = releaseConfig['stagingServer'] stageUsername = args.ssh_user stageSshKey = args.ssh_key stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey) createIndexFiles = releaseConfig.get('makeIndexFiles', False) \ and productName != 'xulrunner' syncPartnerBundles = releaseConfig.get('syncPartnerBundles', False) \
dest="tooltool_script", default=[], action="append") parser.add_option("--tooltool-url", dest="tooltool_urls", action="append") parser.add_option("--use-pymake", dest="use_pymake", action="store_true", default=False) # todo: maybe read these from branch/release config? is that even possible for credentials file? parser.add_option("--balrog-api-root", dest="balrog_api_root") parser.add_option("--credentials-file", dest="credentials_file") parser.add_option("--balrog-username", dest="balrog_username") options, args = parser.parse_args() mercurial(options.buildbotConfigs, "buildbot-configs") update("buildbot-configs", revision=options.releaseTag) sys.path.append(os.getcwd()) branchConfig, releaseConfig = validate(options, args) sourceRepoInfo = releaseConfig["sourceRepositories"][ options.source_repo_key] try: brandName = releaseConfig["brandName"] except KeyError: brandName = releaseConfig["productName"].title() platform = options.platform if platform == "linux": platform = "linux32" mozconfig = path.join(get_repo_dirname(sourceRepoInfo["path"]), releaseConfig["appName"], "config", "mozconfigs",
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, absObjdir, merge=True, productName=None, platform=None, version=None, partialUpdates=None, buildNumber=None, stageServer=None, mozillaDir=None, mozillaSrcDir=None, marSignatureFormat='mar'): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) mercurial(repo, localeDir) update(localeDir, revision=revision) # It's a bad assumption to make, but the source dir is currently always # one level above the objdir. absSourceRepoPath = path.split(absObjdir)[0] use_pymake = env.get("USE_PYMAKE", False) make = getMakeCommand(use_pymake, absSourceRepoPath) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): if use_pymake: env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"]) else: env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" UPLOAD_EXTRA_FILES = [] if mozillaDir: nativeDistDir = path.normpath( path.abspath(path.join(localeSrcDir, '../../%s/dist' % mozillaDir))) else: nativeDistDir = path.normpath( path.abspath(path.join(localeSrcDir, '../../dist'))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir if platform.startswith('win'): mar += ".exe" mbsdiff += ".exe" current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version) unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl' make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh' prevMarDir = '../../../../' if mozillaSrcDir: # Compensate for having the objdir or not. additionalParent = '' if mozillaDir: additionalParent = '../' unwrap_full_update = '../../../%s%s/tools/update-packaging/unwrap_full_update.pl' % ( additionalParent, mozillaSrcDir) make_incremental_update = '../../%s%s/tools/update-packaging/make_incremental_update.sh' % ( additionalParent, mozillaSrcDir) prevMarDir = '../../../../%s' % additionalParent env['MAR'] = mar env['MBSDIFF'] = mbsdiff log.info("Download mar tools") if stageServer: candidates_dir = makeCandidatesDir(productName, version, buildNumber, protocol="http", server=stageServer) if not path.isfile(msys2windows(mar)): marUrl = '/'.join([ p.strip('/') for p in [candidates_dir, 'mar-tools', platform, path.basename(mar)] ]) run_cmd(['mkdir', '-p', path.dirname(mar)]) log.info("Downloading %s to %s", marUrl, mar) urlretrieve(marUrl, msys2windows(mar)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mar]) if not path.isfile(msys2windows(mbsdiff)): mbsdiffUrl = '/'.join([ p.strip('/') for p in [ candidates_dir, 'mar-tools', platform, path.basename(mbsdiff) ] ]) run_cmd(['mkdir', '-p', path.dirname(mbsdiff)]) log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff) urlretrieve(mbsdiffUrl, msys2windows(mbsdiff)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mbsdiff]) else: log.warning('stageServer not set. mar tools will *not* be downloaded.') compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) make_installers_env = env.copy() make_installers_env['MOZ_OBJDIR'] = absObjdir run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=make_installers_env) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(current)]) run_cmd(['mkdir', current]) run_cmd(['perl', unwrap_full_update, current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) for oldVersion in partialUpdates: prevMar = partialUpdates[oldVersion]['mar'] if prevMar: partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(previous)]) run_cmd(['mkdir', previous]) run_cmd( ['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd([ 'bash', make_incremental_update, partial_mar, previous, current ], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd([ 'bash', '-c', '%s -f %s "%s"' % (os.environ['MOZ_SIGN_CMD'], marSignatureFormat, partial_mar) ], env=env) UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name)) else: log.warning("Skipping partial MAR creation for %s %s" % (oldVersion, locale)) env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES) retry(run_cmd, args=(make + ["upload", "AB_CD=%s" % locale], ), kwargs={ 'cwd': localeSrcDir, 'env': env }) # return the location of the checksums file, because consumers may want # some information about the files that were generated. # Some versions of make that we use (at least pymake) imply --print-directory # We need to turn it off to avoid getting extra output that mess up our # parsing of the checksum file path. curdir = os.getcwd() try: os.chdir(localeSrcDir) relative_checksums = get_output(make + [ "--no-print-directory", "echo-variable-CHECKSUM_FILE", "AB_CD=%s" % locale ], env=env, cwd=localeSrcDir).strip("\"'\n\r") return path.normpath(path.join(localeSrcDir, relative_checksums)) finally: os.chdir(curdir)
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, absObjdir, merge=True, productName=None, platform=None, version=None, partialUpdates=None, buildNumber=None, stageServer=None, mozillaDir=None, mozillaSrcDir=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) mercurial(repo, localeDir) update(localeDir, revision=revision) # It's a bad assumption to make, but the source dir is currently always # one level above the objdir. absSourceRepoPath = path.split(absObjdir)[0] use_pymake = env.get("USE_PYMAKE", False) make = getMakeCommand(use_pymake, absSourceRepoPath) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): if use_pymake: env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"]) else: env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" UPLOAD_EXTRA_FILES = [] if mozillaDir: nativeDistDir = path.normpath(path.abspath( path.join(localeSrcDir, '../../%s/dist' % mozillaDir))) else: nativeDistDir = path.normpath(path.abspath( path.join(localeSrcDir, '../../dist'))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir if platform.startswith('win'): mar += ".exe" mbsdiff += ".exe" current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % ( updateAbsDir, productName, version) unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl' make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh' prevMarDir = '../../../../' if mozillaSrcDir: # Compensate for having the objdir or not. additionalParent = '' if mozillaDir: additionalParent = '../' unwrap_full_update = '../../../%s%s/tools/update-packaging/unwrap_full_update.pl' % (additionalParent, mozillaSrcDir) make_incremental_update = '../../%s%s/tools/update-packaging/make_incremental_update.sh' % (additionalParent, mozillaSrcDir) prevMarDir = '../../../../%s' % additionalParent env['MAR'] = mar env['MBSDIFF'] = mbsdiff log.info("Download mar tools") if stageServer: candidates_dir = makeCandidatesDir(productName, version, buildNumber, protocol="http", server=stageServer) if not path.isfile(msys2windows(mar)): marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \ dict(c_dir=candidates_dir, platform=platform, mar=path.basename(mar)) run_cmd(['mkdir', '-p', path.dirname(mar)]) log.info("Downloading %s to %s", marUrl, mar) urlretrieve(marUrl, msys2windows(mar)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mar]) if not path.isfile(msys2windows(mbsdiff)): mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \ dict(c_dir=candidates_dir, platform=platform, mbsdiff=path.basename(mbsdiff)) run_cmd(['mkdir', '-p', path.dirname(mbsdiff)]) log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff) urlretrieve(mbsdiffUrl, msys2windows(mbsdiff)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mbsdiff]) else: log.warning('stageServer not set. mar tools will *not* be downloaded.') compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(current)]) run_cmd(['mkdir', current]) run_cmd(['perl', unwrap_full_update, current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) for oldVersion in partialUpdates: prevMar = partialUpdates[oldVersion]['mar'] if prevMar: partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) # Our Windows-native rm from bug 727551 requires Windows-style paths run_cmd(['rm', '-rf', msys2windows(previous)]) run_cmd(['mkdir', previous]) run_cmd( ['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['bash', make_incremental_update, partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f mar -f gpg "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append( '%s/%s.asc' % (updateDir, partial_mar_name)) else: log.warning( "Skipping partial MAR creation for %s %s" % (oldVersion, locale)) env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES) retry(run_cmd, args=(make + ["upload", "AB_CD=%s" % locale], ), kwargs={'cwd': localeSrcDir, 'env': env}) # return the location of the checksums file, because consumers may want # some information about the files that were generated. # Some versions of make that we use (at least pymake) imply --print-directory # We need to turn it off to avoid getting extra output that mess up our # parsing of the checksum file path. curdir = os.getcwd() try: os.chdir(localeSrcDir) relative_checksums = get_output(make + ["--no-print-directory", "echo-variable-CHECKSUM_FILE", "AB_CD=%s" % locale], env=env).strip("\"'\n") return path.normpath(path.join(localeSrcDir, relative_checksums)) finally: os.chdir(curdir)
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge=True, productName=None, platform=None, version=None, partialUpdates=None, buildNumber=None, stageServer=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) retry(mercurial, args=(repo, localeDir)) update(localeDir, revision=revision) mozillaDir = '' if 'thunderbird' in productName: mozillaDir = 'mozilla/' # split on \\ since we care about the absSourceRepoPath for pymake, which # is windows. absSourceRepoPath = os.path.join(os.getcwd(), localeSrcDir.split("\\")[0]) use_pymake = env.get("USE_PYMAKE", False) make = getMakeCommand(use_pymake, absSourceRepoPath) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): if use_pymake: env["LOCALE_MERGEDIR"] = msys2windows(env["LOCALE_MERGEDIR"]) else: env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" UPLOAD_EXTRA_FILES = [] nativeDistDir = path.normpath(path.abspath( path.join(localeSrcDir, '../../%sdist' % mozillaDir))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir if platform.startswith('win'): mar += ".exe" mbsdiff += ".exe" current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % ( updateAbsDir, productName, version) unwrap_full_update = '../../../tools/update-packaging/unwrap_full_update.pl' make_incremental_update = '../../tools/update-packaging/make_incremental_update.sh' prevMarDir = '../../../../' if mozillaDir: unwrap_full_update = '../../../../%stools/update-packaging/unwrap_full_update.pl' % mozillaDir make_incremental_update = '../../../%stools/update-packaging/make_incremental_update.sh' % mozillaDir prevMarDir = '../../../../../' env['MAR'] = mar env['MBSDIFF'] = mbsdiff log.info("Download mar tools") if stageServer: candidates_dir = makeCandidatesDir(productName, version, buildNumber, protocol="http", server=stageServer) if not path.isfile(msys2windows(mar)): marUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mar)s" % \ dict(c_dir=candidates_dir, platform=platform, mar=path.basename(mar)) run_cmd(['mkdir', '-p', path.dirname(mar)]) log.info("Downloading %s to %s", marUrl, mar) urlretrieve(marUrl, msys2windows(mar)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mar]) if not path.isfile(msys2windows(mbsdiff)): mbsdiffUrl = "%(c_dir)s/mar-tools/%(platform)s/%(mbsdiff)s" % \ dict(c_dir=candidates_dir, platform=platform, mbsdiff=path.basename(mbsdiff)) run_cmd(['mkdir', '-p', path.dirname(mbsdiff)]) log.info("Downloading %s to %s", mbsdiffUrl, mbsdiff) urlretrieve(mbsdiffUrl, msys2windows(mbsdiff)) if not sys.platform.startswith('win'): run_cmd(['chmod', '755', mbsdiff]) else: log.warning('stageServer not set. mar tools will *not* be downloaded.') compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) run_cmd(make + ["installers-%s" % locale], cwd=localeSrcDir, env=env) run_cmd(['rm', '-rf', current]) run_cmd(['mkdir', current]) run_cmd(['perl', unwrap_full_update, current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) for oldVersion in partialUpdates: prevMar = partialUpdates[oldVersion]['mar'] if prevMar: partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) run_cmd(['rm', '-rf', previous]) run_cmd(['mkdir', previous]) run_cmd( ['perl', unwrap_full_update, '%s/%s' % (prevMarDir, prevMar)], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['bash', make_incremental_update, partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f mar -f gpg "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append( '%s/%s.asc' % (updateDir, partial_mar_name)) else: log.warning( "Skipping partial MAR creation for %s %s" % (oldVersion, locale)) env['UPLOAD_EXTRA_FILES'] = ' '.join(UPLOAD_EXTRA_FILES) retry(run_cmd, args=(make + ["upload", "AB_CD=%s" % locale], ), kwargs={'cwd': localeSrcDir, 'env': env})
def main(): logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument("--hg-user", default="ffxbld <*****@*****.**>", help="Mercurial username to be passed to hg -u") args = parser.parse_args() hg_user = args.hg_user # prep the repos for d, repo in ((mc_dir, mc_repo), (ma_dir, ma_repo), (mb_dir, mb_repo)): mercurial(repo, d) log.info("Cleaning up %s...", d) strip_outgoing(d) update(d, branch="default") curr_mc_version = get_major_version(mc_dir) curr_ma_version = get_major_version(ma_dir) curr_mb_version = get_major_version(mb_dir) next_mc_version = str(int(curr_mc_version) + 1) next_ma_version = str(int(curr_ma_version) + 1) next_mb_version = str(int(curr_mb_version) + 1) # mozilla-central mc_revision = get_revision(mc_dir) mc_tag = "FIREFOX_AURORA_%s_BASE" % curr_mc_version tag(mc_dir, tags=[mc_tag], rev=mc_revision, user=hg_user, msg="Added %s tag for changeset %s. IGNORE BROKEN CHANGESETS DONTBUILD CLOSED TREE NO BUG a=release" % (mc_tag, mc_revision)) new_mc_revision = get_revision(mc_dir) bump_version(mc_dir, curr_mc_version, next_mc_version, "a1", "a1", bump_major=True) raw_input("Hit 'return' to display diffs onscreen") run_cmd(["hg", "diff"], cwd=mc_dir) raw_input("If the diff looks good hit return to commit those changes") commit(mc_dir, user=hg_user, msg="Version bump. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release") raw_input("Go ahead and push mozilla-central...and continue to " "mozilla-aurora to mozilla-beta uplift ") # mozilla-aurora ma_revision = get_revision(ma_dir) ma_tag = "FIREFOX_BETA_%s_BASE" % curr_ma_version ma_end_tag = "FIREFOX_AURORA_%s_END" % curr_ma_version # pull must use revision not tag pull(mc_dir, dest=ma_dir, revision=new_mc_revision) merge_via_debugsetparents( ma_dir, old_head=ma_revision, new_head=new_mc_revision, user=hg_user, msg="Merge old head via |hg debugsetparents %s %s|. " "CLOSED TREE DONTBUILD a=release" % (new_mc_revision, ma_revision)) tag(ma_dir, tags=[ma_tag, ma_end_tag], rev=ma_revision, user=hg_user, msg="Added %s %s tags for changeset %s. IGNORE BROKEN CHANGESETS DONTBUILD CLOSED TREE NO BUG a=release" % (ma_tag, ma_end_tag, ma_revision)) log.info("Reverting locales") for f in locale_files: run_cmd(["hg", "revert", "-r", ma_end_tag, f], cwd=ma_dir) bump_version(ma_dir, next_ma_version, next_ma_version, "a1", "a2") raw_input("Hit 'return' to display diffs onscreen") run_cmd(["hg", "diff"], cwd=ma_dir) raw_input("If the diff looks good hit return to commit those changes") commit(ma_dir, user=hg_user, msg="Version bump. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release") replace(path.join(ma_dir, "browser/confvars.sh"), "MOZ_BRANDING_DIRECTORY=browser/branding/nightly", "MOZ_BRANDING_DIRECTORY=browser/branding/aurora") replace(path.join(ma_dir, "browser/confvars.sh"), "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-central", "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-aurora") replace(path.join(ma_dir, "browser/confvars.sh"), "MAR_CHANNEL_ID=firefox-mozilla-central", "MAR_CHANNEL_ID=firefox-mozilla-aurora") for d in branding_dirs: for f in branding_files: replace(path.join(ma_dir, d, f), "ac_add_options --with-branding=mobile/android/branding/nightly", "ac_add_options --with-branding=mobile/android/branding/aurora") if f == "l10n-nightly": replace(path.join(ma_dir, d, f), "ac_add_options --with-l10n-base=../../l10n-central", "ac_add_options --with-l10n-base=..") for f in profiling_files: replace(path.join(ma_dir, f), "ac_add_options --enable-profiling", "") for f in elf_hack_files: replace(path.join(ma_dir, f), "ac_add_options --disable-elf-hack # --enable-elf-hack conflicts with --enable-profiling", "") raw_input("Hit 'return' to display diffs onscreen") run_cmd(["hg", "diff"], cwd=ma_dir) raw_input("If the diff looks good hit return to commit those changes") commit(ma_dir, user=hg_user, msg="Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release") raw_input("Go ahead and push mozilla-aurora changes.") # mozilla-beta mb_revision = get_revision(mb_dir) mb_tag = "FIREFOX_BETA_%s_END" % curr_mb_version # pull must use revision not tag pull(ma_dir, dest=mb_dir, revision=ma_revision) merge_via_debugsetparents( mb_dir, old_head=mb_revision, new_head=ma_revision, user=hg_user, msg="Merge old head via |hg debugsetparents %s %s|. " "CLOSED TREE DONTBUILD a=release" % (ma_revision, mb_revision)) tag(mb_dir, tags=[mb_tag], rev=mb_revision, user=hg_user, msg="Added %s tag for changeset %s. IGNORE BROKEN CHANGESETS DONTBUILD CLOSED TREE NO BUG a=release" % (mb_tag, mb_revision)) bump_version(mb_dir, next_mb_version, next_mb_version, "a2", "") raw_input("Hit 'return' to display diffs onscreen") run_cmd(["hg", "diff"], cwd=mb_dir) raw_input("If the diff looks good hit return to commit those changes") commit(mb_dir, user=hg_user, msg="Version bump. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release") replace(path.join(mb_dir, "browser/confvars.sh"), "MOZ_BRANDING_DIRECTORY=browser/branding/aurora", "MOZ_BRANDING_DIRECTORY=browser/branding/nightly") replace(path.join(mb_dir, "browser/confvars.sh"), "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-aurora", "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release") replace(path.join(mb_dir, "browser/confvars.sh"), "MAR_CHANNEL_ID=firefox-mozilla-aurora", "MAR_CHANNEL_ID=firefox-mozilla-beta") for d in branding_dirs: for f in branding_files: replace(path.join(mb_dir, d, f), "ac_add_options --with-branding=mobile/android/branding/aurora", "ac_add_options --with-branding=mobile/android/branding/beta") raw_input("Hit 'return' to display diffs onscreen") run_cmd(["hg", "diff"], cwd=mb_dir) raw_input("If the diff looks good hit return to commit those changes") commit(mb_dir, user=hg_user, msg="Update configs. IGNORE BROKEN CHANGESETS CLOSED TREE NO BUG a=release ba=release") raw_input("Go ahead and push mozilla-beta changes.")
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName, locales, product, version, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, brand, generatePartials=False, partialUpdates=None, appVersion=None, usePymake=False, tooltoolManifest=None, tooltool_script=None, tooltool_urls=None): sourceRepoName = path.split(sourceRepo)[-1] localeSrcDir = path.join(sourceRepoName, objdir, appName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"]) env = { "MOZ_OBJDIR": objdir, "MOZ_MAKE_COMPLETE_MAR": "1", "UPLOAD_HOST": stageServer, "UPLOAD_USER": stageUsername, "UPLOAD_SSH_KEY": stageSshKey, "UPLOAD_TO_TEMP": "1", "MOZ_PKG_PRETTYNAMES": "1", "MOZILLA_REV": os.getenv('MOZILLA_REV', ''), "COMM_REV": os.getenv('COMM_REV', ''), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", "") } if appVersion is None or version != appVersion: env["MOZ_PKG_VERSION"] = version signed = False if os.environ.get('MOZ_SIGN_CMD'): env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD'] signed = True env['POST_UPLOAD_CMD'] = postUploadCmdPrefix( to_candidates=True, product=product, version=version, buildNumber=buildNumber, signed=signed, ) if usePymake: env['USE_PYMAKE'] = "1" env['MOZILLA_OFFICIAL'] = "1" env["MOZ_SIGN_CMD"] = "python " + \ path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \ " --cachedir " + \ path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \ " -t " + \ path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \ " -n " + \ path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \ " -c " + \ path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\') + \ " -H " + \ os.environ['MOZ_SIGN_CMD'].split(' ')[-1] build.misc.cleanupObjdir(sourceRepoName, objdir, appName) retry(mercurial, args=(sourceRepo, sourceRepoName)) update(sourceRepoName, revision=revision) l10nRepackPrep( sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir, makeDirs, localeSrcDir, env, tooltoolManifest, tooltool_script, tooltool_urls) input_env = retry(downloadReleaseBuilds, args=(stageServer, product, brand, version, buildNumber, platform), kwargs={'signed': signed, 'usePymake': usePymake}) env.update(input_env) failed = [] for l in locales: try: if generatePartials: for oldVersion in partialUpdates: oldBuildNumber = partialUpdates[oldVersion]['buildNumber'] partialUpdates[oldVersion]['mar'] = retry( downloadUpdateIgnore404, args=(stageServer, product, oldVersion, oldBuildNumber, platform, l) ) repackLocale(locale=l, l10nRepoDir=l10nRepoDir, l10nBaseRepo=l10nBaseRepo, revision=revision, localeSrcDir=localeSrcDir, l10nIni=l10nIni, compareLocalesRepo=compareLocalesRepo, env=env, merge=merge, productName=product, platform=platform, version=version, partialUpdates=partialUpdates, buildNumber=buildNumber, stageServer=stageServer) except Exception, e: failed.append((l, format_exc()))
objdir="obj-l10n" ) parser.add_option("-c", "--configfile", dest="configfile") parser.add_option("-B", "--branch", dest="branch") parser.add_option("-b", "--buildbot-configs", dest="buildbotConfigs") parser.add_option("-p", "--platform", dest="platform") parser.add_option("-s", "--stage-platform", dest="stage_platform", default=None) parser.add_option("-o", "--objdir", dest="objdir") parser.add_option("-l", "--locale", dest="locales", action="append") parser.add_option("--chunks", dest="chunks", type="int") parser.add_option("--this-chunk", dest="thisChunk", type="int") options, args = parser.parse_args() mercurial(options.buildbotConfigs, "buildbot-configs") update("buildbot-configs", revision="default") branchConfig = validate(options, args) platformConfig = branchConfig["platforms"][options.platform] mobileDirName = platformConfig.get('mobile_dir', 'mobile') if options.chunks: locales = getNightlyLocalesForChunk(mobileDirName, options.branch, options.platform, options.chunks, options.thisChunk) else: locales = options.locales mozconfig = path.join("buildbot-configs", "mozilla2", platformConfig["mozconfig"], "l10n-mozconfig") ftpProduct = "mobile"
except Exception, e: # Rather than catching individual problems and giving very specific # status updates to the kickoff application, we use this catch-all. # Because nearly all problems require looking at the release runner # logs and manual intervention, it's not worth the pain and ugliness # to do more than this. for release in rr.new_releases: rr.mark_as_failed(release, 'Failed: %s' % repr(e)) raise rc = 0 for release in rr.new_releases: try: rr.update_status(release, 'Running sendchange command') staging = config.getboolean('release-runner', 'staging') update(configs_workdir, revision='default') cfgFile = path.join( configs_workdir, 'mozilla', getReleaseConfigName(release['product'], path.basename(release['branch']), release['version'], staging)) enUSPlatforms = readReleaseConfig(cfgFile)['enUSPlatforms'] rr.start_release_automation(release, sendchange_master, enUSPlatforms) except: # We explicitly do not raise an error here because there's no # reason not to start other releases if the sendchange fails for # another one. We _do_ need to set this in order to exit # with the right code, though. rc = 2 rr.update_status(release, 'Sendchange failed')
def testUpdateRevision(self): rev = clone(self.repodir, self.wc, update_dest=False) self.assertEquals(rev, None) rev = update(self.wc, revision=self.revisions[1]) self.assertEquals(rev, self.revisions[1])
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, appName, locales, product, version, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, brand, generatePartials=False, oldVersion=None, oldBuildNumber=None, appVersion=None): sourceRepoName = path.split(sourceRepo)[-1] localeSrcDir = path.join(sourceRepoName, objdir, appName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"]) env = { "MOZ_OBJDIR": objdir, "MOZ_MAKE_COMPLETE_MAR": "1", "UPLOAD_HOST": stageServer, "UPLOAD_USER": stageUsername, "UPLOAD_SSH_KEY": stageSshKey, "UPLOAD_TO_TEMP": "1", "MOZ_PKG_PRETTYNAMES": "1", } if appVersion is None or version != appVersion: env["MOZ_PKG_VERSION"] = version signed = False if os.environ.get('MOZ_SIGN_CMD'): env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD'] signed = True env['POST_UPLOAD_CMD'] = postUploadCmdPrefix( to_candidates=True, product=product, version=version, buildNumber=buildNumber, signed=signed, ) build.misc.cleanupObjdir(sourceRepoName, objdir, appName) retry(mercurial, args=(sourceRepo, sourceRepoName)) update(sourceRepoName, revision=revision) l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, l10nRepoDir, makeDirs, localeSrcDir, env) input_env = retry(downloadReleaseBuilds, args=(stageServer, product, brand, version, buildNumber, platform), kwargs={'signed': signed}) env.update(input_env) failed = [] for l in locales: try: prevMar = None if generatePartials: prevMar = retry( downloadUpdateIgnore404, args=(stageServer, product, oldVersion, oldBuildNumber, platform, l) ) repackLocale(locale=l, l10nRepoDir=l10nRepoDir, l10nBaseRepo=l10nBaseRepo, revision=revision, localeSrcDir=localeSrcDir, l10nIni=l10nIni, compareLocalesRepo=compareLocalesRepo, env=env, merge=merge, prevMar=prevMar, productName=product, platform=platform, version=version, oldVersion=oldVersion) except Exception, e: failed.append((l, format_exc()))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName, locales, product, version, buildNumber, stageServer, stageUsername, stageSshKey, ftpServer, compareLocalesRepo, merge, platform, brand, appVersion, generatePartials=False, partialUpdates=None, usePymake=False, tooltoolManifest=None, tooltool_script=None, tooltool_urls=None, balrog_submitter=None, balrog_hash="sha512", mozillaDir=None, mozillaSrcDir=None, bucket_prefix=None): buildid = retry(getBuildID, args=(platform, product, version, buildNumber, 'candidates', ftpServer)) log.info('Got buildid: %s' % buildid) sourceRepoName = path.split(sourceRepo)[-1] absObjdir = path.abspath(path.join(sourceRepoName, objdir)) localeSrcDir = path.join(absObjdir, appName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"]) env = { "MOZ_OBJDIR": objdir, "MOZ_MAKE_COMPLETE_MAR": "1", "DOWNLOAD_HOST": ftpServer, "UPLOAD_HOST": stageServer, "UPLOAD_USER": stageUsername, "UPLOAD_SSH_KEY": stageSshKey, "UPLOAD_TO_TEMP": "1", "MOZ_PKG_PRETTYNAMES": "1", "MOZILLA_REV": os.getenv('MOZILLA_REV', ''), "COMM_REV": os.getenv('COMM_REV', ''), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""), "MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""), } if appVersion is None or version != appVersion: env["MOZ_PKG_VERSION"] = version signed = False if os.environ.get('MOZ_SIGN_CMD'): env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD'] signed = True env['POST_UPLOAD_CMD'] = postUploadCmdPrefix( to_candidates=True, product=product, version=version, buildNumber=buildNumber, signed=signed, bucket_prefix=bucket_prefix, ) if usePymake: env['USE_PYMAKE'] = "1" env['MOZILLA_OFFICIAL'] = "1" env["MOZ_SIGN_CMD"] = "python " + \ path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \ " --cachedir " + \ path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \ " -t " + \ path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \ " -n " + \ path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \ " -c " + \ path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\') signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H") for s in signingServers: env["MOZ_SIGN_CMD"] += " -H %s" % s.strip() build.misc.cleanupObjdir(sourceRepoName, objdir, appName) mercurial(sourceRepo, sourceRepoName) update(sourceRepoName, revision=revision) l10nRepackPrep( sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir, makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls) input_env = retry(downloadReleaseBuilds, args=(ftpServer, product, brand, version, buildNumber, platform), kwargs={'signed': signed, 'usePymake': usePymake}) env.update(input_env) failed = [] for l in locales: try: if generatePartials: for oldVersion in partialUpdates: oldBuildNumber = partialUpdates[oldVersion]['buildNumber'] partialUpdates[oldVersion]['mar'] = retry( downloadUpdateIgnore404, args=(ftpServer, product, oldVersion, oldBuildNumber, platform, l) ) checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir, l10nBaseRepo=l10nBaseRepo, revision=revision, localeSrcDir=localeSrcDir, l10nIni=l10nIni, compareLocalesRepo=compareLocalesRepo, env=env, absObjdir=absObjdir, merge=merge, productName=product, platform=platform, version=version, partialUpdates=partialUpdates, buildNumber=buildNumber, stageServer=ftpServer, mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir) if balrog_submitter: # TODO: partials, after bug 797033 is fixed checksums = parseChecksumsFile(open(checksums_file).read()) completeInfo = [] partialInfo = [] for f, info in checksums.iteritems(): if f.endswith('.complete.mar'): completeInfo.append({ "size": info["size"], "hash": info["hashes"][balrog_hash], }) if f.endswith('.partial.mar'): pathInfo = fileInfo(f, product.lower()) previousVersion = pathInfo["previousVersion"] partialInfo.append({ "previousVersion": previousVersion, "previousBuildNumber": partialUpdates[previousVersion]['buildNumber'], "size": info["size"], "hash": info["hashes"][balrog_hash], }) if not completeInfo: raise Exception("Couldn't find complete mar info") retry(balrog_submitter.run, kwargs={ 'platform': platform, 'productName': product.capitalize(), 'appVersion': appVersion, 'version': version, 'build_number': buildNumber, 'locale': l, 'hashFunction': balrog_hash, 'extVersion': appVersion, 'buildID': buildid, 'completeInfo': completeInfo, 'partialInfo': partialInfo, } ) except Exception, e: print_exc() failed.append((l, format_exc()))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts, defaultBranch='default'): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): # set relbranchChangesets=1 because tag() generates exactly 1 commit relbranchChangesets = 1 defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output([ 'hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version']) ], cwd=reponame) relbranchChangesets += 1 except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # We always want our tags pointing at the tip of the relbranch # so we need to grab the current revision after we've switched # branches and bumped versions. revision = get_revision(reponame) # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # This is the bump of the version on the default branch # We do it after the other one in order to get the tip of the # repository back on default, thus avoiding confusion. if len(bumpFiles) > 0: update(reponame, revision=defaultBranch) bump(reponame, bumpFiles, 'nextVersion') run_cmd(['hg', 'diff'], cwd=repo) try: get_output([ 'hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version']) ], cwd=reponame) defaultBranchChangesets += 1 except subprocess.CalledProcessError, e: if e.returncode != 1 or "nothing changed" not in e.output: raise
for opt in required_options: if not options_dict[opt]: parser.error("Required option %s not present" % opt) if options.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) update_platform = buildbot2updatePlatforms(options.platform)[0] ftp_platform = buildbot2ftp(options.platform) full_check_locales = options.full_check_locales # Variables from release config retry(mercurial, args=(options.buildbot_configs, 'buildbot-configs')) update('buildbot-configs', revision=options.release_tag) release_config = validate(options) product_name = release_config['productName'] staging_server = FTP_SERVER_TEMPLATE % release_config['stagingServer'] aus_server_url = release_config['ausServerUrl'] build_number = release_config['buildNumber'] previous_releases_staging_server = FTP_SERVER_TEMPLATE % \ release_config.get('previousReleasesStagingServer', release_config['stagingServer']) # Current version data pc = PatcherConfig(open(options.config).read()) app_name = pc['appName'] to_version = pc['current-update']['to'] to_ = makeReleaseRepackUrls( product_name, app_name, to_version, options.platform,
for opt in required_options: if not options_dict[opt]: parser.error("Required option %s not present" % opt) if options.verbose: log.setLevel(logging.DEBUG) else: log.setLevel(logging.INFO) update_platform = buildbot2updatePlatforms(options.platform)[0] ftp_platform = buildbot2ftp(options.platform) full_check_locales = options.full_check_locales # Variables from release config retry(mercurial, args=(options.buildbot_configs, 'buildbot-configs')) update('buildbot-configs', revision=options.release_tag) release_config = validate(options) product_name = release_config['productName'] staging_server = FTP_SERVER_TEMPLATE % release_config['stagingServer'] aus_server_url = release_config['ausServerUrl'] build_number = release_config['buildNumber'] previous_releases_staging_server = FTP_SERVER_TEMPLATE % \ release_config.get('previousReleasesStagingServer', release_config['stagingServer']) # Current version data pc = PatcherConfig(open(options.config).read()) app_name = pc['appName'] to_version = pc['current-update']['to'] to_ = makeReleaseRepackUrls(product_name, app_name,
def tag_and_push(repo, attempt): update(workdir, branch) tag(workdir, tags, rev=branch, force=True, user=hg_username) log.info("Tagged %s, attempt #%s" % (repo, attempt))
except Exception, e: # Rather than catching individual problems and giving very specific # status updates to the kickoff application, we use this catch-all. # Because nearly all problems require looking at the release runner # logs and manual intervention, it's not worth the pain and ugliness # to do more than this. for release in rr.new_releases: rr.mark_as_failed(release, 'Failed: %s' % repr(e)) raise rc = 0 for release in rr.new_releases: try: rr.update_status(release, 'Running sendchange command') staging = config.getboolean('release-runner', 'staging') update(configs_workdir, revision='default') cfgFile = path.join(configs_workdir, 'mozilla', getReleaseConfigName(release['product'], path.basename(release['branch']), release['version'], staging)) enUSPlatforms = readReleaseConfig(cfgFile)['enUSPlatforms'] rr.start_release_automation(release, sendchange_master, enUSPlatforms) except: # We explicitly do not raise an error here because there's no # reason not to start other releases if the sendchange fails for # another one. We _do_ need to set this in order to exit # with the right code, though. rc = 2 rr.update_status(release, 'Sendchange failed') log.error('Sendchange failed for %s: ' % release, exc_info=True)
parser.add_argument("-b", "--buildbot-configs", help="buildbot-configs mercurial repo URL", default=os.environ.get("BUILDBOT_CONFIGS", DEFAULT_BUILDBOT_CONFIGS_REPO)) parser.add_argument("-t", "--release-tag", required=True) parser.add_argument("--product", help="Override for stage_product") parser.add_argument("--ssh-user", required=True) parser.add_argument("--ssh-key", required=True) parser.add_argument("--overwrite", default=False, action="store_true") parser.add_argument("--extra-excludes", action="append") parser.add_argument("actions", nargs="+", help="Script actions") args = parser.parse_args() actions = args.actions mercurial(args.buildbot_configs, "buildbot-configs") update("buildbot-configs", revision=args.release_tag) releaseConfigFile = path.join("buildbot-configs", args.release_config) releaseConfig = readReleaseConfig( releaseConfigFile, required=REQUIRED_RELEASE_CONFIG) productName = args.product or releaseConfig['stage_product'] version = releaseConfig['version'] buildNumber = releaseConfig['buildNumber'] stageServer = releaseConfig['stagingServer'] stageUsername = args.ssh_user stageSshKey = args.ssh_key stageSshKey = path.join(os.path.expanduser("~"), ".ssh", stageSshKey) createIndexFiles = releaseConfig.get('makeIndexFiles', False) \ and productName != 'xulrunner' syncPartnerBundles = releaseConfig.get('syncPartnerBundles', False) \
parser.add_option("-o", "--objdir", dest="objdir") parser.add_option("-l", "--locale", dest="locales", action="append") parser.add_option("--source-repo-key", dest="source_repo_key") parser.add_option("--chunks", dest="chunks", type="int") parser.add_option("--this-chunk", dest="thisChunk", type="int") parser.add_option("--generate-partials", dest="generatePartials", action='store_true', default=False) options, args = parser.parse_args() if options.generatePartials: makeDirs.extend([ path.join("modules", "libbz2"), path.join("other-licenses", "bsdiff") ]) retry(mercurial, args=(options.buildbotConfigs, "buildbot-configs")) update("buildbot-configs", revision=options.releaseTag) sys.path.append(os.getcwd()) branchConfig, releaseConfig = validate(options, args) sourceRepoInfo = releaseConfig["sourceRepositories"][options.source_repo_key] try: brandName = releaseConfig["brandName"] except KeyError: brandName = releaseConfig["productName"].title() mozconfig = path.join("buildbot-configs", "mozilla2", options.platform, sourceRepoInfo['name'], "release", "l10n-mozconfig") if options.chunks: locales = retry(getReleaseLocalesForChunk, args=(releaseConfig["productName"], releaseConfig["appName"], releaseConfig["version"], int(releaseConfig["buildNumber"]), sourceRepoInfo["path"], options.platform,
def main(): logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO) parser = argparse.ArgumentParser() parser.add_argument("--from-dir", default="mozilla-beta", help="Working directory of repo to be merged from") parser.add_argument("--from-repo", default="ssh://hg.mozilla.org/releases/mozilla-beta", help="Repo to be merged from") parser.add_argument("--to-dir", default="mozilla-release", help="Working directory of repo to be merged to") parser.add_argument( "--to-repo", default="ssh://hg.mozilla.org/releases/mozilla-release", help="Repo to be merged to") parser.add_argument("--hg-user", default="ffxbld <*****@*****.**>", help="Mercurial username to be passed to hg -u") parser.add_argument("--remove-locale", dest="remove_locales", action="append", required=True, help="Locales to be removed from release shipped-locales") args = parser.parse_args() from_dir = args.from_dir to_dir = args.to_dir from_repo = args.from_repo to_repo = args.to_repo hg_user = args.hg_user with retrying(mercurial) as clone: for (d, repo) in ((from_dir, from_repo), (to_dir, to_repo)): clone(repo, d) log.info("Cleaning up %s...", d) strip_outgoing(d) update(d, branch="default") beta_rev = get_revision(from_dir) release_rev = get_revision(to_dir) now = datetime.datetime.now() date = now.strftime("%Y%m%d") # TODO: make this tag consistent with other branches release_base_tag = "RELEASE_BASE_" + date log.info("Tagging %s beta with %s", beta_rev, release_base_tag) tag(from_dir, tags=[release_base_tag], rev=beta_rev, user=hg_user, msg="Added %s tag for changeset %s. DONTBUILD CLOSED TREE a=release" % (release_base_tag, beta_rev)) new_beta_rev = get_revision(from_dir) raw_input("Push mozilla-beta and hit Return") pull(from_dir, dest=to_dir) merge_via_debugsetparents( to_dir, old_head=release_rev, new_head=new_beta_rev, user=hg_user, msg="Merge old head via |hg debugsetparents %s %s|. " "CLOSED TREE DONTBUILD a=release" % (new_beta_rev, release_rev)) replace( path.join(to_dir, "browser/confvars.sh"), "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-beta,firefox-mozilla-release", "ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-release") replace(path.join(to_dir, "browser/confvars.sh"), "MAR_CHANNEL_ID=firefox-mozilla-beta", "MAR_CHANNEL_ID=firefox-mozilla-release") for d in branding_dirs: for f in branding_files: replace( path.join(to_dir, d, f), "ac_add_options --with-branding=mobile/android/branding/beta", "ac_add_options --with-branding=mobile/android/branding/official") if args.remove_locales: log.info("Removing locales: %s", args.remove_locales) remove_locales(path.join(to_dir, "browser/locales/shipped-locales"), args.remove_locales) log.warn("Apply any manual changes, such as disabling features.") raw_input("Hit 'return' to display channel, branding, and feature diffs onscreen") run_cmd(["hg", "diff"], cwd=to_dir) raw_input("If the diff looks good hit return to commit those changes") commit(to_dir, user=hg_user, msg="Update configs. CLOSED TREE a=release ba=release") raw_input("Go ahead and push mozilla-release changes.")
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, srcMozconfigPath, objdir, makeDirs, appName, locales, product, version, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, brand, appVersion, generatePartials=False, partialUpdates=None, usePymake=False, tooltoolManifest=None, tooltool_script=None, tooltool_urls=None, balrog_submitter=None, balrog_hash="sha512", buildid=None, mozillaDir=None, mozillaSrcDir=None): sourceRepoName = path.split(sourceRepo)[-1] absObjdir = path.abspath(path.join(sourceRepoName, objdir)) localeSrcDir = path.join(absObjdir, appName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"]) env = { "MOZ_OBJDIR": objdir, "MOZ_MAKE_COMPLETE_MAR": "1", "UPLOAD_HOST": stageServer, "UPLOAD_USER": stageUsername, "UPLOAD_SSH_KEY": stageSshKey, "UPLOAD_TO_TEMP": "1", "MOZ_PKG_PRETTYNAMES": "1", "MOZILLA_REV": os.getenv('MOZILLA_REV', ''), "COMM_REV": os.getenv('COMM_REV', ''), "LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", "") } if appVersion is None or version != appVersion: env["MOZ_PKG_VERSION"] = version signed = False if os.environ.get('MOZ_SIGN_CMD'): env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD'] signed = True env['POST_UPLOAD_CMD'] = postUploadCmdPrefix( to_candidates=True, product=product, version=version, buildNumber=buildNumber, signed=signed, ) if usePymake: env['USE_PYMAKE'] = "1" env['MOZILLA_OFFICIAL'] = "1" env["MOZ_SIGN_CMD"] = "python " + \ path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \ " --cachedir " + \ path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \ " -t " + \ path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \ " -n " + \ path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \ " -c " + \ path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\') signingServers = os.environ["MOZ_SIGN_CMD"].split("-H", 1)[1].split("-H") for s in signingServers: env["MOZ_SIGN_CMD"] += " -H %s" % s.strip() build.misc.cleanupObjdir(sourceRepoName, objdir, appName) retry(mercurial, args=(sourceRepo, sourceRepoName)) update(sourceRepoName, revision=revision) l10nRepackPrep( sourceRepoName, objdir, mozconfigPath, srcMozconfigPath, l10nRepoDir, makeDirs, env, tooltoolManifest, tooltool_script, tooltool_urls) input_env = retry(downloadReleaseBuilds, args=(stageServer, product, brand, version, buildNumber, platform), kwargs={'signed': signed, 'usePymake': usePymake}) env.update(input_env) failed = [] for l in locales: try: if generatePartials: for oldVersion in partialUpdates: oldBuildNumber = partialUpdates[oldVersion]['buildNumber'] partialUpdates[oldVersion]['mar'] = retry( downloadUpdateIgnore404, args=(stageServer, product, oldVersion, oldBuildNumber, platform, l) ) checksums_file = repackLocale(locale=l, l10nRepoDir=l10nRepoDir, l10nBaseRepo=l10nBaseRepo, revision=revision, localeSrcDir=localeSrcDir, l10nIni=l10nIni, compareLocalesRepo=compareLocalesRepo, env=env, absObjdir=absObjdir, merge=merge, productName=product, platform=platform, version=version, partialUpdates=partialUpdates, buildNumber=buildNumber, stageServer=stageServer, mozillaDir=mozillaDir, mozillaSrcDir=mozillaSrcDir) if balrog_submitter: # TODO: partials, after bug 797033 is fixed checksums = parseChecksumsFile(open(checksums_file).read()) completeInfo = [] partialInfo = [] for f, info in checksums.iteritems(): if f.endswith('.complete.mar'): completeInfo.append({ "size": info["size"], "hash": info["hashes"][balrog_hash], }) if f.endswith('.partial.mar'): pathInfo = fileInfo(f, product.lower()) previousVersion = pathInfo["previousVersion"] partialInfo.append({ "previousVersion": previousVersion, "previousBuildNumber": partialUpdates[previousVersion]['buildNumber'], "size": info["size"], "hash": info["hashes"][balrog_hash], }) if not completeInfo: raise Exception("Couldn't find complete mar info") retry(balrog_submitter.run, kwargs={ 'platform': platform, 'productName': product.capitalize(), 'appVersion': appVersion, 'version': version, 'build_number': buildNumber, 'locale': l, 'hashFunction': balrog_hash, 'extVersion': appVersion, 'buildID': buildid, 'completeInfo': completeInfo, 'partialInfo': partialInfo, } ) except Exception, e: print_exc() failed.append((l, format_exc()))