def updateProductDetails(productName, version, productDetailsRepo, mozillaComRepo, svnSshKey, dryRun=False): """ Add a new version to the product details """ os.environ["SVN_SSH"] = "ssh -i %s" % svnSshKey cwd = os.getcwd() pdDir = path.join(cwd, "product-details.svn") mcDir = path.join(cwd, "mozilla.com.svn") retry(checkoutSVN, args=(pdDir, productDetailsRepo), attempts=3) retry(checkoutSVN, args=(mcDir, mozillaComRepo), attempts=3) # Update the PHP files updateProductDetailFiles(pdDir, productName, version) # Export to json exportJSON(pdDir) # Commit to svn commitSVN(pdDir, productName, version, dryRun) # Get the svn revision of the p-d repository svnRev = getSVNrev(pdDir) # Update Mozilla.com updateRev(mcDir, svnRev) # commit Mozilla.com commitSVN(pdDir, productName, version, dryRun)
def make_generic_head_request(page_url): """Make generic HEAD request to check page existence""" def _get(): req = requests.head(page_url, timeout=60) req.raise_for_status() retry(_get, attempts=5, sleeptime=1)
def main(name=None, config_path=None): if name not in (None, '__main__'): return config = setup_config(config_path) setup_logging(config['verbose']) task = get_task(config) validate_task_schema(config, task) server = get_task_server(task, config) balrog_auth, config = update_config(config, server) config['upstream_artifacts'] = get_upstream_artifacts(task) # hacking the tools repo dependency by first reading its location from # the config file and only then loading the module from subdfolder sys.path.insert(0, os.path.join(config['tools_location'], 'lib/python')) # Until we get rid of our tools dep, this import(s) will break flake8 E402 from util.retry import retry # noqa: E402 # Read the manifest from disk manifest = get_manifest(config) for e in manifest: # Get release metadata from manifest submitter, release = create_submitter(e, balrog_auth, config) # Connect to balrog and submit the metadata retry(lambda: submitter.run(**release))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def tagRepo(repo, attempt, config, revision, tags): # set totalChangesets=1 because tag() generates exactly 1 commit totalChangesets = 1 # update to the desired revision first, then to the tip of revision's # branch to avoid new head creation update(repo, revision=revision) update(repo) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions") pushRepo = make_hg_url(HG, repo, protocol='ssh') def tag_wrapper(r, n): tagRepo(r, n, config, revision, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'], config['hgSshKey']) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, tag_wrapper, pushAttempts), kwargs=dict(ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts): remote = make_hg_url(HG, repo) mercurial(remote, reponame) def tagRepo(repo, attempt, config, revision, tags): # set totalChangesets=1 because tag() generates exactly 1 commit totalChangesets = 1 # update to the desired revision first, then to the tip of revision's # branch to avoid new head creation update(repo, revision=revision) update(repo) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config[ 'hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions") pushRepo = make_hg_url(HG, repo, protocol='ssh') def tag_wrapper(r, n): tagRepo(r, n, config, revision, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'], config['hgSshKey']) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, tag_wrapper, pushAttempts), kwargs=dict(ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']))
def main(): parser = argparse.ArgumentParser() parser.add_argument("--manifest", required=True) parser.add_argument("-a", "--api-root", required=True, help="Balrog API root") parser.add_argument("-v", "--verbose", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) parser.add_argument("--product", help="Override product name from application.ini") args = parser.parse_args() logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=args.loglevel) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("boto").setLevel(logging.WARNING) balrog_username = os.environ.get("BALROG_USERNAME") balrog_password = os.environ.get("BALROG_PASSWORD") suffix = os.environ.get("BALROG_BLOB_SUFFIX") if not balrog_username and not balrog_password: raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " "variables should be set") if not suffix: raise RuntimeError( "BALROG_BLOB_SUFFIX environment variable should be set") manifest = json.load(open(args.manifest)) auth = (balrog_username, balrog_password) for e in manifest: complete_info = [{ "hash": e["hash"], "size": e["size"], }] submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, suffix=suffix) productName = args.product or e["appName"] retry( lambda: submitter.run(platform=e["platform"], productName=productName, version=e["toVersion"], build_number=e["toBuildNumber"], appVersion=e["version"], extVersion=e["version"], buildID=e["to_buildid"], locale=e["locale"], hashFunction='sha512', completeInfo=complete_info), attempts=30, sleeptime=10, max_sleeptime=60, jitter=3, )
def do_reboot(slaveapi, slave): url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add("reboot") retry(requests.post, args=(str(url),)) # Because SlaveAPI fully escalates reboots (all the way to IT bug filing), # there's no reason for us to watch for it to complete. log.info("%s - Reboot queued", slave) return
def do_reboot(slaveapi, slave): url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add("reboot") retry(requests.post, args=(str(url), )) # Because SlaveAPI fully escalates reboots (all the way to IT bug filing), # there's no reason for us to watch for it to complete. log.info("%s - Reboot queued", slave) return
def action_update(master): print "sleeping 30 seconds to make sure that hg.m.o syncs NFS... ", time.sleep(30) print OK with show('running'): retry(run, args=('source bin/activate && make update',), kwargs={'workdir': master['basedir']}, sleeptime=10, retry_exceptions=(SystemExit,)) print OK, "updated %(hostname)s:%(basedir)s" % master
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge=True, prevMar=None, productName=None, platform=None, version=None, oldVersion=None): repo = "/".join([l10nBaseRepo, locale]) localeDir = path.join(l10nRepoDir, locale) retry(mercurial, args=(repo, localeDir)) update(localeDir, revision=revision) compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision=revision, merge=merge) env["AB_CD"] = locale env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged")) if sys.platform.startswith('win'): env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"]) if sys.platform.startswith('darwin'): env["MOZ_PKG_PLATFORM"] = "mac" run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env) UPLOAD_EXTRA_FILES = [] if prevMar: nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir, '../../dist'))) posixDistDir = windows2msys(nativeDistDir) mar = '%s/host/bin/mar' % posixDistDir mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir current = '%s/current' % posixDistDir previous = '%s/previous' % posixDistDir updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale) updateAbsDir = '%s/%s' % (posixDistDir, updateDir) current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version) partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion, version) partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name) UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name)) env['MAR'] = mar env['MBSDIFF'] = mbsdiff run_cmd(['rm', '-rf', previous, current]) run_cmd(['mkdir', previous, current]) run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl', '../../../../%s' % prevMar], cwd=path.join(nativeDistDir, 'previous'), env=env) run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl', current_mar], cwd=path.join(nativeDistDir, 'current'), env=env) run_cmd( ['bash', '../../tools/update-packaging/make_incremental_update.sh', partial_mar, previous, current], cwd=nativeDistDir, env=env) if os.environ.get('MOZ_SIGN_CMD'): run_cmd(['bash', '-c', '%s -f gpg -f mar "%s"' % (os.environ['MOZ_SIGN_CMD'], partial_mar)], env=env) UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name)) retry(run_cmd, args=(["make", "upload", "AB_CD=%s" % locale, 'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],), kwargs={'cwd': localeSrcDir, 'env': env})
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts, defaultBranch='default'): remote = make_hg_url(HG, repo) retry(mercurial, args=(remote, reponame)) def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): # set relbranchChangesets=1 because tag() generates exactly 1 commit relbranchChangesets = 1 defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # We always want our tags pointing at the tip of the relbranch # so we need to grab the current revision after we've switched # branches and bumped versions. revision = get_revision(reponame) # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # This is the bump of the version on the default branch # We do it after the other one in order to get the tip of the # repository back on default, thus avoiding confusion. if len(bumpFiles) > 0: update(reponame, revision=defaultBranch) bump(reponame, bumpFiles, 'nextVersion') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) defaultBranchChangesets += 1 except subprocess.CalledProcessError, e: if e.returncode != 1 or "nothing changed" not in e.output: raise
def maybe_delete_repo(server, username, sshKey, repo, repoPath): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) try: log.info("Trying to open %s" % repo_url) urlopen(repo_url) except URLError: log.info("%s doesn't exist, not deleting" % reponame) else: log.info('Deleting %s' % reponame) retry(run_remote_cmd, args=('edit %s delete YES' % reponame, server, username, sshKey))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir, makeDirs, locales, ftpProduct, appName, version, appVersion, buildNumber, stageServer, stageUsername, stageSshKey, compareLocalesRepo, merge, platform, stage_platform, brand, mobileDirName): sourceRepoName = path.split(sourceRepo)[-1] nightlyDir = "candidates" localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales") # Even on Windows we need to use "/" as a separator for this because # compare-locales doesn"t work any other way l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"]) env = { "MOZ_OBJDIR": objdir, "MOZ_PKG_VERSION": version, "UPLOAD_HOST": stageServer, "UPLOAD_USER": stageUsername, "UPLOAD_SSH_KEY": stageSshKey, "UPLOAD_TO_TEMP": "1", # Android signing "JARSIGNER": os.path.join(os.getcwd(), "scripts", "release", "signing", "mozpass.py") } build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName) retry(mercurial, args=(sourceRepo, sourceRepoName)) update(sourceRepoName, revision=revision) l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, l10nRepoDir, makeDirs, localeSrcDir, env) fullCandidatesDir = makeCandidatesDir(appName, version, buildNumber, protocol='http', server=stageServer, nightlyDir=nightlyDir) input_env = retry(downloadReleaseBuilds, args=(stageServer, ftpProduct, brand, version, buildNumber, stage_platform, fullCandidatesDir)) env.update(input_env) print "env pre-locale: %s" % str(env) failed = [] for l in locales: try: # adding locale into builddir env["POST_UPLOAD_CMD"] = postUploadCmdPrefix( to_mobile_candidates=True, product=appName, version=version, builddir='%s/%s' % (stage_platform, l), buildNumber=buildNumber, nightly_dir=nightlyDir,) print "env post-locale: %s" % str(env) repackLocale(str(l), l10nRepoDir, l10nBaseRepo, revision, localeSrcDir, l10nIni, compareLocalesRepo, env, merge) except Exception, e: failed.append((l, format_exc()))
def update_bouncer_alias(tuxedoServerUrl, auth, version, related_product_template, alias): url = "%s/create_update_alias" % tuxedoServerUrl related_product = related_product_template % {"version": version} data = {"alias": alias, "related_product": related_product} log.info("Updating %s to point to %s using %s", alias, related_product, url) # Wrap the real call to hide credentials from retry's logging def do_update_bouncer_alias(): requests.post(url, data=data, auth=auth, config={"danger_mode": True}, verify=False) retry(do_update_bouncer_alias)
def submit_locale(task, config, balrog_auth): """Submit a release blob to balrog.""" from util.retry import retry # noqa: E402 upstream_artifacts = get_upstream_artifacts(task) # Read the manifest from disk manifest = get_manifest(config, upstream_artifacts) for e in manifest: # Get release metadata from manifest submitter, release = create_locale_submitter(e, balrog_auth, config) # Connect to balrog and submit the metadata retry(lambda: submitter.run(**release))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch, pushAttempts, defaultBranch='default'): remote = make_hg_url(HG, repo) retry(mercurial, args=(remote, reponame)) def bump_and_tag(repo, attempt, config, relbranch, revision, tags, defaultBranch): relbranchChangesets = len(tags) defaultBranchChangesets = 0 if relbranch in get_branches(reponame): update(reponame, revision=relbranch) else: update(reponame, revision=revision) run_cmd(['hg', 'branch', relbranch], cwd=reponame) if len(bumpFiles) > 0: # Bump files on the relbranch, if necessary bump(reponame, bumpFiles, 'version') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) relbranchChangesets += 1 revision = get_revision(reponame) except subprocess.CalledProcessError, e: # We only want to ignore exceptions caused by having nothing to # commit, which are OK. We still want to raise exceptions caused # by any other thing. if e.returncode != 1 or "nothing changed" not in e.output: raise # Create the desired tags on the relbranch tag(repo, revision, tags, config['hgUsername']) # This is the bump of the version on the default branch # We do it after the other one in order to get the tip of the # repository back on default, thus avoiding confusion. if len(bumpFiles) > 0: update(reponame, revision=defaultBranch) bump(reponame, bumpFiles, 'nextVersion') run_cmd(['hg', 'diff'], cwd=repo) try: get_output(['hg', 'commit', '-u', config['hgUsername'], '-m', getBumpCommitMessage(config['productName'], config['version'])], cwd=reponame) defaultBranchChangesets += 1 except subprocess.CalledProcessError, e: if e.returncode != 1 or "nothing changed" not in e.output: raise
def process_slave(slaveapi, slave, dryrun=False): try: info = get_slave(slaveapi, slave) # Ignore slaves without recent job information if not info["recent_jobs"]: log.info("%s - Skipping reboot because no recent jobs found", slave) return last_job_time = datetime.fromtimestamp( info["recent_jobs"][0]["endtime"]) # And also slaves that haven't been idle for more than the threshold if not (now - last_job_time).total_seconds() > IDLE_THRESHOLD: log.info( "%s - Skipping reboot because last job ended recently at %s", slave, get_formatted_time(last_job_time)) return if dryrun: log.info("%s - Last job ended at %s, would've rebooted", slave, get_formatted_time(last_job_time)) return else: log.info("%s - Last job ended at %s, rebooting", slave, get_formatted_time(last_job_time)) # We need to set a graceful shutdown for the slave on the off chance that # it picks up a job before us making the decision to reboot it, and the # reboot actually happening. In most cases this will happen nearly # instantly. log.debug("%s - Setting graceful shutdown", slave) url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add( "shutdown_buildslave") url.args["waittime"] = 30 r = retry(requests.post, args=(str(url), )).json() url.args["requestid"] = r["requestid"] while r["state"] in (PENDING, RUNNING): time.sleep(30) r = retry(requests.get, args=(str(url), )).json() if r["state"] == FAILURE: log.info("%s - Graceful shutdown failed, aborting reboot", slave) return log.info("%s - Graceful shutdown finished, rebooting", slave) url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add("reboot") retry(requests.post, args=(str(url), )) # Because SlaveAPI fully escalates reboots (all the way to IT bug filing), # there's no reason for us to watch for it to complete. log.info("%s - Reboot queued", slave) except: log.exception("%s - Caught exception while processing", slave)
def commitSVN(targetSVNDirectory, product, version, fakeCommit=False): """ Commit the change in the svn """ retval = os.getcwd() os.chdir(targetSVNDirectory) try: if not os.path.isdir(".svn"): raise Exception("Could not find the svn directory") commitMSG = "'" + product + " version " + version + "'" if not fakeCommit: retry(doCommitSVN, args=(commitMSG), attempts=3, sleeptime=0) finally: os.chdir(retval)
def process(self): """ Process this patchset, doing the following: 1. Check permissions on patchset 2. Clone the repository 3. Apply patches, with 3 attempts """ # 1. Check permissions on each patch outgoing = self.branch if not self.try_run else 'try' if not has_sufficient_permissions(self.user, outgoing): log.error('Insufficient permissions to push to %s.' % (outgoing)) self.add_comment('Insufficient permissions to push to %s.' % (outgoing)) return (False, '\n'.join(self.comments)) # 2. Clone the repository cloned_rev = None try: cloned_rev = clone_branch(self.branch, self.branch_url) except RetryException: log.error('[Branch %s] Could not clone from %s.' % (self.branch, self.branch_url)) self.add_comment('An error occurred while cloning %s.' % (self.branch_url)) return (False, '\n'.join(self.comments)) # 3. Apply patches, with 3 attempts try: # make 3 attempts so that # 1st is on current clone, # 2nd attempt is after an update -C, # 3rd attempt is a fresh clone retry(apply_and_push, attempts=3, retry_exceptions=(RetryException,), cleanup=RepoCleanup(self.branch, self.branch_url), args=(self.active_repo, self.push_url, self.apply_patches, 1), kwargs=dict(ssh_username=config['hg_username'], ssh_key=config['hg_ssh_key'], force=self.try_run)) # force only on try revision = get_revision(self.active_repo) shutil.rmtree(self.active_repo) for patch in self.patches: patch.delete() except (HgUtilError, RetryException, FailException), err: # Failed log.error('[PatchSet] Could not be applied and pushed.\n%s' % (err)) self.add_comment('Patchset could not be applied and pushed.' '\n%s' % (err)) return (False, '\n'.join(self.comments))
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision="default", merge=True): retry(mercurial, args=(repo, "compare-locales")) update("compare-locales", revision=revision) mergeDir = path.join(localeSrcDir, "merged") if path.exists(mergeDir): log.info("Deleting %s" % mergeDir) shutil.rmtree(mergeDir) run_cmd(["python", path.join("compare-locales", "scripts", "compare-locales"), "-m", mergeDir, l10nIni, l10nRepoDir, locale], env={"PYTHONPATH": path.join("compare-locales", "lib")})
def process(self): """ Process this patchset, doing the following: 1. Check permissions on patchset 2. Clone the repository 3. Apply patches, with 3 attempts """ # 1. Check permissions on each patch outgoing = self.branch if not self.try_run else 'try' if not has_sufficient_permissions(self.user, outgoing): log.error('Insufficient permissions to push to %s.' % (outgoing)) self.add_comment('Insufficient permissions to push to %s.' % (outgoing)) return (False, '\n'.join(self.comments)) # 2. Clone the repository cloned_rev = None try: cloned_rev = clone_branch(self.branch, self.branch_url) except RetryException: log.error('[Branch %s] Could not clone from %s.' % (self.branch, self.branch_url)) self.add_comment('An error occurred while cloning %s.' % (self.branch_url)) return (False, '\n'.join(self.comments)) # 3. Apply patches, with 3 attempts try: # make 3 attempts so that # 1st is on current clone, # 2nd attempt is after an update -C, # 3rd attempt is a fresh clone retry(apply_and_push, attempts=3, retry_exceptions=(RetryException, ), cleanup=RepoCleanup(self.branch, self.branch_url), args=(self.active_repo, self.push_url, self.apply_patches, 1), kwargs=dict(ssh_username=config['hg_username'], ssh_key=config['hg_ssh_key'], force=self.try_run)) # force only on try revision = get_revision(self.active_repo) shutil.rmtree(self.active_repo) for patch in self.patches: patch.delete() except (HgUtilError, RetryException, FailException), err: # Failed log.error('[PatchSet] Could not be applied and pushed.\n%s' % (err)) self.add_comment('Patchset could not be applied and pushed.' '\n%s' % (err)) return (False, '\n'.join(self.comments))
def update_bouncer_alias(tuxedoServerUrl, auth, version, related_product_template, alias): url = "%s/create_update_alias" % tuxedoServerUrl related_product = related_product_template % {"version": version} data = {"alias": alias, "related_product": related_product} log.info("Updating %s to point to %s using %s", alias, related_product, url) # Wrap the real call to hide credentials from retry's logging def do_update_bouncer_alias(): r = requests.post(url, data=data, auth=auth, verify=False) r.raise_for_status() retry(do_update_bouncer_alias)
def submit_toplevel(task, config, balrog_auth): """Push a top-level release blob to balrog.""" from util.retry import retry # noqa: E402 auth = balrog_auth partials = {} if task['payload'].get('partial_versions'): for v in task['payload']['partial_versions'].split(','): v = v.strip() # we have whitespace after the comma version, build_number = v.split("build") partials[version] = {"buildNumber": build_number} # XXX WNP - support someday? # currently we create and set these manually. open_url = None creator = create_creator( api_root=config['api_root'], auth=auth, dummy=config['dummy'], # these are set for bz2, which we don't support. complete_mar_filename_pattern=None, complete_mar_bouncer_product_pattern=None, ) pusher = create_pusher( api_root=config['api_root'], auth=auth, dummy=config['dummy'], ) retry(lambda: creator.run( appVersion=task['payload']['app_version'], productName=task['payload']['product'].capitalize(), version=task['payload']['version'], buildNumber=task['payload']['build_number'], updateChannels=task['payload']['channel_names'], ftpServer=task['payload']['archive_domain'], bouncerServer=task['payload']['download_domain'], enUSPlatforms=task['payload']['platforms'], hashFunction='sha512', openURL=open_url, partialUpdates=partials, requiresMirrors=task['payload']['require_mirrors'], )) retry(lambda: pusher.run( productName=task['payload']['product'].capitalize(), version=task['payload']['version'], build_number=task['payload']['build_number'], rule_ids=task['payload']['rules_to_update'], ))
def request(self, params=None, data=None, method='GET', url_template_vars={}): url = self.api_root + self.url_template % url_template_vars if method != 'GET' and method != 'HEAD': if not self.csrf_token or is_csrf_token_expired(self.csrf_token): res = self.session.request( method='HEAD', url=self.api_root + '/csrf_token', config=self.config, timeout=self.timeout, auth=self.auth) self.csrf_token = res.headers['X-CSRF-Token'] data['csrf_token'] = self.csrf_token log.debug('Request to %s' % url) log.debug('Data sent: %s' % data) try: return retry(self.session.request, sleeptime=5, max_sleeptime=15, retry_exceptions=(requests.HTTPError, requests.ConnectionError), attempts=self.retries, kwargs=dict(method=method, url=url, data=data, config=self.config, timeout=self.timeout, auth=self.auth, params=params) ) except requests.HTTPError, e: log.error('Caught HTTPError: %d %s' % (e.response.status_code, e.response.content), exc_info=True) raise
def get_production_slaves(slaveapi): url = furl(slaveapi) url.path.add("slaves") url.args["environment"] = "prod" url.args["enabled"] = 1 r = retry(requests.get, args=(str(url), )) return r.json()["slaves"]
def cleanOutgoingRevs(reponame, remote, username, sshKey): outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=username, ssh_key=sshKey)) for r in reversed(outgoingRevs): run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n', r[REVISION]], cwd=reponame)
def ssh(user, identity, host, remote_cmd, port=22): cmd = ['ssh', '-l', user] if identity: cmd.extend(['-i', identity]) cmd.extend(['-p', str(port), host, remote_cmd]) return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd, ))
def ssh(user, identity, host, remote_cmd, port=22): cmd = ['ssh', '-l', user] if identity: cmd.extend(['-i', identity]) cmd.extend(['-p', str(port), host, remote_cmd]) return retry(do_cmd, attempts=retries + 1, sleeptime=retry_sleep, args=(cmd,))
def get_production_slaves(slaveapi): url = furl(slaveapi) url.path.add("slaves") url.args["environment"] = "prod" url.args["enabled"] = 1 r = retry(requests.get, args=(str(url),)) return r.json()["slaves"]
def make_generic_get_request(page_url): """Make generic GET request to retrieve some page content""" def _get(): req = requests.get(page_url, timeout=60) req.raise_for_status() return req.content return retry(_get, attempts=5, sleeptime=1)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump, configsToOverride): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame), protocol='ssh') retry(mercurial, args=(repo_url, reponame)) def bump(repo, configsToBump, configsToOverride): """Process dynamic (version, buildNumber, etc.) variables in configsToBump, then append overrides files to both configsToBump and configsToOverride.""" # First pass. Bump variables in configsToBump. configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()] cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip'] cmd.extend(configs) run_cmd(cmd) # Second pass. Append override files to configsToBump and # configsToOverride. for config, overrides in \ configsToBump.items() + configsToOverride.items(): newContent = cat([path.join(repo, config)] + [path.join(repo, x) for x in overrides]) fh = open(path.join(repo, config), 'wb') fh.write(newContent) fh.close() run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo) def bump_wrapper(r, n): bump(r, configsToBump, configsToOverride) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, username, sshKey) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, bump_wrapper), kwargs=dict(ssh_username=username, ssh_key=sshKey)) tags = [] for configfile in configsToBump.keys(): config = readReleaseConfig(path.join(reponame, configfile)) tags.extend( getTags(config['baseTag'], config['buildNumber'], buildTag=True)) return tags
def tagRepo(repo, attempt, config, revision, tags): totalChangesets = len(tags) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions")
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump, configsToOverride): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame), protocol='ssh') retry(mercurial, args=(repo_url, reponame)) def bump(repo, configsToBump, configsToOverride): """Process dynamic (version, buildNumber, etc.) variables in configsToBump, then append overrides files to both configsToBump and configsToOverride.""" # First pass. Bump variables in configsToBump. configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()] cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip'] cmd.extend(configs) run_cmd(cmd) # Second pass. Append override files to configsToBump and # configsToOverride. for config, overrides in \ configsToBump.items() + configsToOverride.items(): newContent = cat([path.join(repo, config)] + [path.join(repo, x) for x in overrides]) fh = open(path.join(repo, config), 'wb') fh.write(newContent) fh.close() run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo) def bump_wrapper(r, n): bump(r, configsToBump, configsToOverride) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, username, sshKey) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, bump_wrapper), kwargs=dict(ssh_username=username, ssh_key=sshKey)) tags = [] for configfile in configsToBump.keys(): config = readReleaseConfig(path.join(reponame, configfile)) tags.extend(getTags(config['baseTag'], config['buildNumber'], buildTag=True)) return tags
def main(): parser = argparse.ArgumentParser() parser.add_argument("--manifest", required=True) parser.add_argument("-a", "--api-root", required=True, help="Balrog API root") parser.add_argument("-v", "--verbose", action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.INFO) parser.add_argument("--product", help="Override product name from application.ini") args = parser.parse_args() logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=args.loglevel) logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("boto").setLevel(logging.WARNING) balrog_username = os.environ.get("BALROG_USERNAME") balrog_password = os.environ.get("BALROG_PASSWORD") suffix = os.environ.get("BALROG_BLOB_SUFFIX") if not balrog_username and not balrog_password: raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment " "variables should be set") if not suffix: raise RuntimeError("BALROG_BLOB_SUFFIX environment variable should be set") manifest = json.load(open(args.manifest)) auth = (balrog_username, balrog_password) for e in manifest: complete_info = [{ "hash": e["hash"], "size": e["size"], }] submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth, suffix=suffix) productName = args.product or e["appName"] retry(lambda: submitter.run( platform=e["platform"], productName=productName, version=e["toVersion"], build_number=e["toBuildNumber"], appVersion=e["version"], extVersion=e["version"], buildID=e["to_buildid"], locale=e["locale"], hashFunction='sha512', completeInfo=complete_info), attempts=30, sleeptime=10, max_sleeptime=60, jitter=3, )
def scp(user, identity, host, files, remote_dir, port=22): cmd = ['scp'] if identity: cmd.extend(['-i', identity]) cmd.extend(['-P', str(port)]) cmd.extend(files) cmd.append("%s@%s:%s" % (user, host, remote_dir)) return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd, ))
def schedule(task, config, balrog_auth): """Schedule a release to ship on balrog channel(s)""" from util.retry import retry # noqa: E402 auth = balrog_auth scheduler = create_scheduler(api_root=config['api_root'], auth=auth, dummy=config['dummy']) args = [ task['payload']['product'].capitalize(), task['payload']['version'], task['payload']['build_number'], task['payload']['publish_rules'], task['payload']['release_eta'] or None, # Send None if release_eta is '' ] # XXX optionally append background_rate if/when we want to support it # XXX should we catch requests.HTTPError and raise a scriptworker # error? maybe not since balrogscript isn't py3 retry(lambda: scheduler.run(*args))
def scp(user, identity, host, files, remote_dir, port=22): cmd = ['scp'] if identity: cmd.extend(['-i', identity]) cmd.extend(['-P', str(port)]) cmd.extend(files) cmd.append("%s@%s:%s" % (user, host, remote_dir)) return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd,))
def process_slave(slaveapi, slave, dryrun=False): try: info = get_slave(slaveapi, slave) # Ignore slaves without recent job information if not info["recent_jobs"]: log.info("%s - Skipping reboot because no recent jobs found", slave) return last_job_time = datetime.fromtimestamp(info["recent_jobs"][0]["endtime"]) # And also slaves that haven't been idle for more than the threshold if not (now - last_job_time).total_seconds() > IDLE_THRESHOLD: log.info("%s - Skipping reboot because last job ended recently at %s", slave, get_formatted_time(last_job_time)) return if dryrun: log.info("%s - Last job ended at %s, would've rebooted", slave, get_formatted_time(last_job_time)) return else: log.info("%s - Last job ended at %s, rebooting", slave, get_formatted_time(last_job_time)) # We need to set a graceful shutdown for the slave on the off chance that # it picks up a job before us making the decision to reboot it, and the # reboot actually happening. In most cases this will happen nearly # instantly. log.debug("%s - Setting graceful shutdown", slave) url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add("shutdown_buildslave") url.args["waittime"] = 30 r = retry(requests.post, args=(str(url),)).json() url.args["requestid"] = r["requestid"] while r["state"] in (PENDING, RUNNING): time.sleep(30) r = retry(requests.get, args=(str(url),)).json() if r["state"] == FAILURE: log.info("%s - Graceful shutdown failed, aborting reboot", slave) return log.info("%s - Graceful shutdown finished, rebooting", slave) url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add("reboot") retry(requests.post, args=(str(url),)) # Because SlaveAPI fully escalates reboots (all the way to IT bug filing), # there's no reason for us to watch for it to complete. log.info("%s - Reboot queued", slave) except: log.exception("%s - Caught exception while processing", slave)
def retrieveFile(url, file_path): success = True url = urllib.quote(url, safe=':/') log.info('Downloading from %s' % url) log.info('To: %s', file_path) log.info('CWD: %s' % os.getcwd()) try: # use URLopener, which handles errors properly retry(StrictFancyURLopener().retrieve, kwargs=dict(url=url, filename=file_path)) except IOError: log.error("Error downloading %s" % url, exc_info=True) success = False try: os.remove(file_path) except OSError: log.info("Cannot remove %s" % file_path, exc_info=True) return success
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni, revision="default", merge=True): retry(mercurial, args=(repo, "compare-locales")) update("compare-locales", revision=revision) mergeDir = path.join(localeSrcDir, "merged") if path.exists(mergeDir): log.info("Deleting %s" % mergeDir) shutil.rmtree(mergeDir) run_cmd([ "python", path.join("compare-locales", "scripts", "compare-locales"), "-m", mergeDir, l10nIni, l10nRepoDir, locale ], env={"PYTHONPATH": path.join("compare-locales", "lib")})
def testRetryArgsPassed(self): args = (1, 'two', 3) kwargs = dict(foo='a', bar=7) ret = retry(_mirrorArgs, args=args, kwargs=kwargs.copy(), sleeptime=0, jitter=0) self.assertEqual(ret[0], args) self.assertEqual(ret[1], kwargs)
def main(): config = load_config(sys.argv[1:]) setup_logging(config['verbose']) balrog_auth, config = update_config(config) # hacking the tools repo dependency by first reading its location from # the config file and only then loading the module from subdfolder sys.path.insert(0, os.path.join(config['tools_location'], 'lib/python')) # Until we get rid of our tools dep, this import(s) will break flake8 E402 from util.retry import retry # noqa: E402 # Read the manifest from disk manifest = get_manifest(config) for e in manifest: # Get release metadata from manifest submitter, release = create_submitter(e, balrog_auth, config) # Connect to balrog and submit the metadata retry(lambda: submitter.run(**release))
def fetch_actions_json(task_id): queue = taskcluster.Queue() actions_url = queue.buildUrl("getLatestArtifact", task_id, 'public/actions.json') def _get(): q = requests.get(actions_url) q.raise_for_status() return q.json() return retry(_get)
def get_recent_action(slaveapi, slave, action): url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add(action) history = retry(requests.get, args=(str(url),)).json() results = [] for key in history.keys(): if not key == action: continue for item in history[action]: results.append(history[action][item]) return get_latest_result(results)
def get_recent_action(slaveapi, slave, action): url = furl(slaveapi) url.path.add("slaves").add(slave).add("actions").add(action) history = retry(requests.get, args=(str(url), )).json() results = [] for key in history.keys(): if not key == action: continue for item in history[action]: results.append(history[action][item]) return get_latest_result(results)
def tag_repo(server, username, sshKey, repo, repoPath, tags): reponame = get_repo_name(repo) repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame)) pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame), protocol='ssh') mercurial(repo_url, reponame) def do_tag(repo, tags): cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags run_cmd(cmd, cwd=repo) def do_tag_wrapper(r, n): do_tag(r, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, username, sshKey) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, do_tag_wrapper), kwargs=dict(ssh_username=username, ssh_key=sshKey))
def updateProductDetails(productName, version, productDetailsRepo, mozillaComRepo, svnSshKey, dryRun=False): """ Add a new version to the product details """ os.environ["SVN_SSH"] = "ssh -i %s" % svnSshKey pdDir = "product-details.svn" mcDir = "mozilla.com.svn" retry(checkoutSVN, args=(pdDir, productDetailsRepo), attempts=3) retry(checkoutSVN, args=(mcDir, mozillaComRepo), attempts=3) # Update the PHP files updateProductDetailFiles(pdDir, productName, version) # Export to json exportJSON(pdDir) # Commit to svn commitSVN(pdDir, productName, version, dryRun) # Get the svn revision of the p-d repository svnRev = getSVNrev(pdDir) # Update Mozilla.com updateRev(mcDir, svnRev) # commit Mozilla.com commitSVN(pdDir, productName, version, dryRun)
def find_decision_task_id(trust_domain, project, revision): decision_task_route = "{trust_domain}.v2.{project}.revision.{revision}.taskgraph.decision".format( trust_domain=trust_domain, project=project, revision=revision, ) index = taskcluster.Index() def _get(): return index.findTask(decision_task_route)["taskId"] return retry(_get)
def tagOtherRepo(config, repo, reponame, revision, pushAttempts): remote = make_hg_url(HG, repo) retry(mercurial, args=(remote, reponame)) def tagRepo(repo, attempt, config, revision, tags): totalChangesets = len(tags) tag(repo, revision, tags, config['hgUsername']) outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote, ssh_username=config['hgUsername'], ssh_key=config['hgSshKey'])) if len(outgoingRevs) != totalChangesets: raise Exception("Wrong number of outgoing revisions") pushRepo = make_hg_url(HG, repo, protocol='ssh') def tag_wrapper(r, n): tagRepo(r, n, config, revision, tags) def cleanup_wrapper(): cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'], config['hgSshKey']) retry(apply_and_push, cleanup=cleanup_wrapper, args=(reponame, pushRepo, tag_wrapper, pushAttempts), kwargs=dict(ssh_username=config['hgUsername'], ssh_key=config['hgSshKey']))