def updateProductDetails(productName,
version,
productDetailsRepo,
mozillaComRepo,
svnSshKey,
dryRun=False):
"""
Add a new version to the product details
"""
os.environ["SVN_SSH"] = "ssh -i %s" % svnSshKey
cwd = os.getcwd()
pdDir = path.join(cwd, "product-details.svn")
mcDir = path.join(cwd, "mozilla.com.svn")
retry(checkoutSVN, args=(pdDir, productDetailsRepo), attempts=3)
retry(checkoutSVN, args=(mcDir, mozillaComRepo), attempts=3)
# Update the PHP files
updateProductDetailFiles(pdDir, productName, version)
# Export to json
exportJSON(pdDir)
# Commit to svn
commitSVN(pdDir, productName, version, dryRun)
# Get the svn revision of the p-d repository
svnRev = getSVNrev(pdDir)
# Update Mozilla.com
updateRev(mcDir, svnRev)
# commit Mozilla.com
commitSVN(pdDir, productName, version, dryRun)
def make_generic_head_request(page_url):
"""Make generic HEAD request to check page existence"""
def _get():
req = requests.head(page_url, timeout=60)
req.raise_for_status()
retry(_get, attempts=5, sleeptime=1)
def main(name=None, config_path=None):
if name not in (None, '__main__'):
return
config = setup_config(config_path)
setup_logging(config['verbose'])
task = get_task(config)
validate_task_schema(config, task)
server = get_task_server(task, config)
balrog_auth, config = update_config(config, server)
config['upstream_artifacts'] = get_upstream_artifacts(task)
# hacking the tools repo dependency by first reading its location from
# the config file and only then loading the module from subdfolder
sys.path.insert(0, os.path.join(config['tools_location'], 'lib/python'))
# Until we get rid of our tools dep, this import(s) will break flake8 E402
from util.retry import retry # noqa: E402
# Read the manifest from disk
manifest = get_manifest(config)
for e in manifest:
# Get release metadata from manifest
submitter, release = create_submitter(e, balrog_auth, config)
# Connect to balrog and submit the metadata
retry(lambda: submitter.run(**release))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out,
kwargs=dict(src=reponame,
remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config[
'hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--manifest", required=True)
parser.add_argument("-a",
"--api-root",
required=True,
help="Balrog API root")
parser.add_argument("-v",
"--verbose",
action="store_const",
dest="loglevel",
const=logging.DEBUG,
default=logging.INFO)
parser.add_argument("--product",
help="Override product name from application.ini")
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.loglevel)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("boto").setLevel(logging.WARNING)
balrog_username = os.environ.get("BALROG_USERNAME")
balrog_password = os.environ.get("BALROG_PASSWORD")
suffix = os.environ.get("BALROG_BLOB_SUFFIX")
if not balrog_username and not balrog_password:
raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
"variables should be set")
if not suffix:
raise RuntimeError(
"BALROG_BLOB_SUFFIX environment variable should be set")
manifest = json.load(open(args.manifest))
auth = (balrog_username, balrog_password)
for e in manifest:
complete_info = [{
"hash": e["hash"],
"size": e["size"],
}]
submitter = ReleaseSubmitterV4(api_root=args.api_root,
auth=auth,
suffix=suffix)
productName = args.product or e["appName"]
retry(
lambda: submitter.run(platform=e["platform"],
productName=productName,
version=e["toVersion"],
build_number=e["toBuildNumber"],
appVersion=e["version"],
extVersion=e["version"],
buildID=e["to_buildid"],
locale=e["locale"],
hashFunction='sha512',
completeInfo=complete_info),
attempts=30,
sleeptime=10,
max_sleeptime=60,
jitter=3,
)
def make_generic_head_request(page_url):
"""Make generic HEAD request to check page existence"""
def _get():
req = requests.head(page_url, timeout=60)
req.raise_for_status()
retry(_get, attempts=5, sleeptime=1)
def do_reboot(slaveapi, slave):
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add("reboot")
retry(requests.post, args=(str(url),))
# Because SlaveAPI fully escalates reboots (all the way to IT bug filing),
# there's no reason for us to watch for it to complete.
log.info("%s - Reboot queued", slave)
return
def do_reboot(slaveapi, slave):
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add("reboot")
retry(requests.post, args=(str(url), ))
# Because SlaveAPI fully escalates reboots (all the way to IT bug filing),
# there's no reason for us to watch for it to complete.
log.info("%s - Reboot queued", slave)
return
def action_update(master):
print "sleeping 30 seconds to make sure that hg.m.o syncs NFS... ",
time.sleep(30)
print OK
with show('running'):
retry(run, args=('source bin/activate && make update',),
kwargs={'workdir': master['basedir']}, sleeptime=10,
retry_exceptions=(SystemExit,))
print OK, "updated %(hostname)s:%(basedir)s" % master
def action_update(master):
print "sleeping 30 seconds to make sure that hg.m.o syncs NFS... ",
time.sleep(30)
print OK
with show('running'):
retry(run, args=('source bin/activate && make update',),
kwargs={'workdir': master['basedir']}, sleeptime=10,
retry_exceptions=(SystemExit,))
print OK, "updated %(hostname)s:%(basedir)s" % master
def repackLocale(locale, l10nRepoDir, l10nBaseRepo, revision, localeSrcDir,
l10nIni, compareLocalesRepo, env, merge=True,
prevMar=None, productName=None, platform=None,
version=None, oldVersion=None):
repo = "/".join([l10nBaseRepo, locale])
localeDir = path.join(l10nRepoDir, locale)
retry(mercurial, args=(repo, localeDir))
update(localeDir, revision=revision)
compareLocales(compareLocalesRepo, locale, l10nRepoDir, localeSrcDir,
l10nIni, revision=revision, merge=merge)
env["AB_CD"] = locale
env["LOCALE_MERGEDIR"] = path.abspath(path.join(localeSrcDir, "merged"))
if sys.platform.startswith('win'):
env["LOCALE_MERGEDIR"] = windows2msys(env["LOCALE_MERGEDIR"])
if sys.platform.startswith('darwin'):
env["MOZ_PKG_PLATFORM"] = "mac"
run_cmd(["make", "installers-%s" % locale], cwd=localeSrcDir, env=env)
UPLOAD_EXTRA_FILES = []
if prevMar:
nativeDistDir = path.normpath(path.abspath(path.join(localeSrcDir,
'../../dist')))
posixDistDir = windows2msys(nativeDistDir)
mar = '%s/host/bin/mar' % posixDistDir
mbsdiff = '%s/host/bin/mbsdiff' % posixDistDir
current = '%s/current' % posixDistDir
previous = '%s/previous' % posixDistDir
updateDir = 'update/%s/%s' % (buildbot2ftp(platform), locale)
updateAbsDir = '%s/%s' % (posixDistDir, updateDir)
current_mar = '%s/%s-%s.complete.mar' % (updateAbsDir, productName, version)
partial_mar_name = '%s-%s-%s.partial.mar' % (productName, oldVersion,
version)
partial_mar = '%s/%s' % (updateAbsDir, partial_mar_name)
UPLOAD_EXTRA_FILES.append('%s/%s' % (updateDir, partial_mar_name))
env['MAR'] = mar
env['MBSDIFF'] = mbsdiff
run_cmd(['rm', '-rf', previous, current])
run_cmd(['mkdir', previous, current])
run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
'../../../../%s' % prevMar],
cwd=path.join(nativeDistDir, 'previous'), env=env)
run_cmd(['perl', '../../../tools/update-packaging/unwrap_full_update.pl',
current_mar], cwd=path.join(nativeDistDir, 'current'), env=env)
run_cmd(
['bash', '../../tools/update-packaging/make_incremental_update.sh',
partial_mar, previous, current], cwd=nativeDistDir, env=env)
if os.environ.get('MOZ_SIGN_CMD'):
run_cmd(['bash', '-c',
'%s -f gpg -f mar "%s"' %
(os.environ['MOZ_SIGN_CMD'], partial_mar)],
env=env)
UPLOAD_EXTRA_FILES.append('%s/%s.asc' % (updateDir, partial_mar_name))
retry(run_cmd,
args=(["make", "upload", "AB_CD=%s" % locale,
'UPLOAD_EXTRA_FILES=%s' % ' '.join(UPLOAD_EXTRA_FILES)],),
kwargs={'cwd': localeSrcDir, 'env': env})
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts, defaultBranch='default'):
remote = make_hg_url(HG, repo)
retry(mercurial, args=(remote, reponame))
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def maybe_delete_repo(server, username, sshKey, repo, repoPath):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
try:
log.info("Trying to open %s" % repo_url)
urlopen(repo_url)
except URLError:
log.info("%s doesn't exist, not deleting" % reponame)
else:
log.info('Deleting %s' % reponame)
retry(run_remote_cmd, args=('edit %s delete YES' % reponame, server,
username, sshKey))
def maybe_delete_repo(server, username, sshKey, repo, repoPath):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
try:
log.info("Trying to open %s" % repo_url)
urlopen(repo_url)
except URLError:
log.info("%s doesn't exist, not deleting" % reponame)
else:
log.info('Deleting %s' % reponame)
retry(run_remote_cmd,
args=('edit %s delete YES' % reponame, server, username, sshKey))
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, locales, ftpProduct,
appName, version, appVersion, buildNumber, stageServer,
stageUsername, stageSshKey, compareLocalesRepo, merge,
platform, stage_platform, brand, mobileDirName):
sourceRepoName = path.split(sourceRepo)[-1]
nightlyDir = "candidates"
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_PKG_VERSION": version,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
# Android signing
"JARSIGNER": os.path.join(os.getcwd(), "scripts", "release",
"signing", "mozpass.py")
}
build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName)
retry(mercurial, args=(sourceRepo, sourceRepoName))
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath,
l10nRepoDir, makeDirs, localeSrcDir, env)
fullCandidatesDir = makeCandidatesDir(appName, version, buildNumber,
protocol='http', server=stageServer,
nightlyDir=nightlyDir)
input_env = retry(downloadReleaseBuilds,
args=(stageServer, ftpProduct, brand, version,
buildNumber, stage_platform, fullCandidatesDir))
env.update(input_env)
print "env pre-locale: %s" % str(env)
failed = []
for l in locales:
try:
# adding locale into builddir
env["POST_UPLOAD_CMD"] = postUploadCmdPrefix(
to_mobile_candidates=True,
product=appName,
version=version,
builddir='%s/%s' % (stage_platform, l),
buildNumber=buildNumber,
nightly_dir=nightlyDir,)
print "env post-locale: %s" % str(env)
repackLocale(str(l), l10nRepoDir, l10nBaseRepo, revision,
localeSrcDir, l10nIni, compareLocalesRepo, env, merge)
except Exception, e:
failed.append((l, format_exc()))
def update_bouncer_alias(tuxedoServerUrl, auth, version, related_product_template, alias):
url = "%s/create_update_alias" % tuxedoServerUrl
related_product = related_product_template % {"version": version}
data = {"alias": alias, "related_product": related_product}
log.info("Updating %s to point to %s using %s", alias, related_product, url)
# Wrap the real call to hide credentials from retry's logging
def do_update_bouncer_alias():
requests.post(url, data=data, auth=auth, config={"danger_mode": True}, verify=False)
retry(do_update_bouncer_alias)
def submit_locale(task, config, balrog_auth):
"""Submit a release blob to balrog."""
from util.retry import retry # noqa: E402
upstream_artifacts = get_upstream_artifacts(task)
# Read the manifest from disk
manifest = get_manifest(config, upstream_artifacts)
for e in manifest:
# Get release metadata from manifest
submitter, release = create_locale_submitter(e, balrog_auth, config)
# Connect to balrog and submit the metadata
retry(lambda: submitter.run(**release))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts, defaultBranch='default'):
remote = make_hg_url(HG, repo)
retry(mercurial, args=(remote, reponame))
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
relbranchChangesets = len(tags)
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
revision = get_revision(reponame)
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def process_slave(slaveapi, slave, dryrun=False):
try:
info = get_slave(slaveapi, slave)
# Ignore slaves without recent job information
if not info["recent_jobs"]:
log.info("%s - Skipping reboot because no recent jobs found",
slave)
return
last_job_time = datetime.fromtimestamp(
info["recent_jobs"][0]["endtime"])
# And also slaves that haven't been idle for more than the threshold
if not (now - last_job_time).total_seconds() > IDLE_THRESHOLD:
log.info(
"%s - Skipping reboot because last job ended recently at %s",
slave, get_formatted_time(last_job_time))
return
if dryrun:
log.info("%s - Last job ended at %s, would've rebooted", slave,
get_formatted_time(last_job_time))
return
else:
log.info("%s - Last job ended at %s, rebooting", slave,
get_formatted_time(last_job_time))
# We need to set a graceful shutdown for the slave on the off chance that
# it picks up a job before us making the decision to reboot it, and the
# reboot actually happening. In most cases this will happen nearly
# instantly.
log.debug("%s - Setting graceful shutdown", slave)
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add(
"shutdown_buildslave")
url.args["waittime"] = 30
r = retry(requests.post, args=(str(url), )).json()
url.args["requestid"] = r["requestid"]
while r["state"] in (PENDING, RUNNING):
time.sleep(30)
r = retry(requests.get, args=(str(url), )).json()
if r["state"] == FAILURE:
log.info("%s - Graceful shutdown failed, aborting reboot", slave)
return
log.info("%s - Graceful shutdown finished, rebooting", slave)
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add("reboot")
retry(requests.post, args=(str(url), ))
# Because SlaveAPI fully escalates reboots (all the way to IT bug filing),
# there's no reason for us to watch for it to complete.
log.info("%s - Reboot queued", slave)
except:
log.exception("%s - Caught exception while processing", slave)
def commitSVN(targetSVNDirectory, product, version, fakeCommit=False):
"""
Commit the change in the svn
"""
retval = os.getcwd()
os.chdir(targetSVNDirectory)
try:
if not os.path.isdir(".svn"):
raise Exception("Could not find the svn directory")
commitMSG = "'" + product + " version " + version + "'"
if not fakeCommit:
retry(doCommitSVN, args=(commitMSG), attempts=3, sleeptime=0)
finally:
os.chdir(retval)
def process(self):
"""
Process this patchset, doing the following:
1. Check permissions on patchset
2. Clone the repository
3. Apply patches, with 3 attempts
"""
# 1. Check permissions on each patch
outgoing = self.branch if not self.try_run else 'try'
if not has_sufficient_permissions(self.user, outgoing):
log.error('Insufficient permissions to push to %s.'
% (outgoing))
self.add_comment('Insufficient permissions to push to %s.'
% (outgoing))
return (False, '\n'.join(self.comments))
# 2. Clone the repository
cloned_rev = None
try:
cloned_rev = clone_branch(self.branch, self.branch_url)
except RetryException:
log.error('[Branch %s] Could not clone from %s.'
% (self.branch, self.branch_url))
self.add_comment('An error occurred while cloning %s.'
% (self.branch_url))
return (False, '\n'.join(self.comments))
# 3. Apply patches, with 3 attempts
try:
# make 3 attempts so that
# 1st is on current clone,
# 2nd attempt is after an update -C,
# 3rd attempt is a fresh clone
retry(apply_and_push, attempts=3,
retry_exceptions=(RetryException,),
cleanup=RepoCleanup(self.branch, self.branch_url),
args=(self.active_repo, self.push_url,
self.apply_patches, 1),
kwargs=dict(ssh_username=config['hg_username'],
ssh_key=config['hg_ssh_key'],
force=self.try_run)) # force only on try
revision = get_revision(self.active_repo)
shutil.rmtree(self.active_repo)
for patch in self.patches:
patch.delete()
except (HgUtilError, RetryException, FailException), err:
# Failed
log.error('[PatchSet] Could not be applied and pushed.\n%s'
% (err))
self.add_comment('Patchset could not be applied and pushed.'
'\n%s' % (err))
return (False, '\n'.join(self.comments))
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni,
revision="default", merge=True):
retry(mercurial, args=(repo, "compare-locales"))
update("compare-locales", revision=revision)
mergeDir = path.join(localeSrcDir, "merged")
if path.exists(mergeDir):
log.info("Deleting %s" % mergeDir)
shutil.rmtree(mergeDir)
run_cmd(["python", path.join("compare-locales", "scripts",
"compare-locales"),
"-m", mergeDir,
l10nIni,
l10nRepoDir, locale],
env={"PYTHONPATH": path.join("compare-locales", "lib")})
def process(self):
"""
Process this patchset, doing the following:
1. Check permissions on patchset
2. Clone the repository
3. Apply patches, with 3 attempts
"""
# 1. Check permissions on each patch
outgoing = self.branch if not self.try_run else 'try'
if not has_sufficient_permissions(self.user, outgoing):
log.error('Insufficient permissions to push to %s.' % (outgoing))
self.add_comment('Insufficient permissions to push to %s.' %
(outgoing))
return (False, '\n'.join(self.comments))
# 2. Clone the repository
cloned_rev = None
try:
cloned_rev = clone_branch(self.branch, self.branch_url)
except RetryException:
log.error('[Branch %s] Could not clone from %s.' %
(self.branch, self.branch_url))
self.add_comment('An error occurred while cloning %s.' %
(self.branch_url))
return (False, '\n'.join(self.comments))
# 3. Apply patches, with 3 attempts
try:
# make 3 attempts so that
# 1st is on current clone,
# 2nd attempt is after an update -C,
# 3rd attempt is a fresh clone
retry(apply_and_push,
attempts=3,
retry_exceptions=(RetryException, ),
cleanup=RepoCleanup(self.branch, self.branch_url),
args=(self.active_repo, self.push_url, self.apply_patches,
1),
kwargs=dict(ssh_username=config['hg_username'],
ssh_key=config['hg_ssh_key'],
force=self.try_run)) # force only on try
revision = get_revision(self.active_repo)
shutil.rmtree(self.active_repo)
for patch in self.patches:
patch.delete()
except (HgUtilError, RetryException, FailException), err:
# Failed
log.error('[PatchSet] Could not be applied and pushed.\n%s' %
(err))
self.add_comment('Patchset could not be applied and pushed.'
'\n%s' % (err))
return (False, '\n'.join(self.comments))
def commitSVN(targetSVNDirectory, product, version, fakeCommit=False):
"""
Commit the change in the svn
"""
retval = os.getcwd()
os.chdir(targetSVNDirectory)
try:
if not os.path.isdir(".svn"):
raise Exception("Could not find the svn directory")
commitMSG = "'" + product + " version " + version + "'"
if not fakeCommit:
retry(doCommitSVN, args=(commitMSG), attempts=3, sleeptime=0)
finally:
os.chdir(retval)
def update_bouncer_alias(tuxedoServerUrl, auth, version,
related_product_template, alias):
url = "%s/create_update_alias" % tuxedoServerUrl
related_product = related_product_template % {"version": version}
data = {"alias": alias, "related_product": related_product}
log.info("Updating %s to point to %s using %s", alias, related_product,
url)
# Wrap the real call to hide credentials from retry's logging
def do_update_bouncer_alias():
r = requests.post(url, data=data, auth=auth, verify=False)
r.raise_for_status()
retry(do_update_bouncer_alias)
def submit_toplevel(task, config, balrog_auth):
"""Push a top-level release blob to balrog."""
from util.retry import retry # noqa: E402
auth = balrog_auth
partials = {}
if task['payload'].get('partial_versions'):
for v in task['payload']['partial_versions'].split(','):
v = v.strip() # we have whitespace after the comma
version, build_number = v.split("build")
partials[version] = {"buildNumber": build_number}
# XXX WNP - support someday?
# currently we create and set these manually.
open_url = None
creator = create_creator(
api_root=config['api_root'], auth=auth,
dummy=config['dummy'],
# these are set for bz2, which we don't support.
complete_mar_filename_pattern=None,
complete_mar_bouncer_product_pattern=None,
)
pusher = create_pusher(
api_root=config['api_root'], auth=auth,
dummy=config['dummy'],
)
retry(lambda: creator.run(
appVersion=task['payload']['app_version'],
productName=task['payload']['product'].capitalize(),
version=task['payload']['version'],
buildNumber=task['payload']['build_number'],
updateChannels=task['payload']['channel_names'],
ftpServer=task['payload']['archive_domain'],
bouncerServer=task['payload']['download_domain'],
enUSPlatforms=task['payload']['platforms'],
hashFunction='sha512',
openURL=open_url,
partialUpdates=partials,
requiresMirrors=task['payload']['require_mirrors'],
))
retry(lambda: pusher.run(
productName=task['payload']['product'].capitalize(),
version=task['payload']['version'],
build_number=task['payload']['build_number'],
rule_ids=task['payload']['rules_to_update'],
))
def request(self, params=None, data=None, method='GET', url_template_vars={}):
url = self.api_root + self.url_template % url_template_vars
if method != 'GET' and method != 'HEAD':
if not self.csrf_token or is_csrf_token_expired(self.csrf_token):
res = self.session.request(
method='HEAD', url=self.api_root + '/csrf_token',
config=self.config, timeout=self.timeout,
auth=self.auth)
self.csrf_token = res.headers['X-CSRF-Token']
data['csrf_token'] = self.csrf_token
log.debug('Request to %s' % url)
log.debug('Data sent: %s' % data)
try:
return retry(self.session.request, sleeptime=5, max_sleeptime=15,
retry_exceptions=(requests.HTTPError,
requests.ConnectionError),
attempts=self.retries,
kwargs=dict(method=method, url=url, data=data,
config=self.config, timeout=self.timeout,
auth=self.auth, params=params)
)
except requests.HTTPError, e:
log.error('Caught HTTPError: %d %s' %
(e.response.status_code, e.response.content),
exc_info=True)
raise
def get_production_slaves(slaveapi):
url = furl(slaveapi)
url.path.add("slaves")
url.args["environment"] = "prod"
url.args["enabled"] = 1
r = retry(requests.get, args=(str(url), ))
return r.json()["slaves"]
def cleanOutgoingRevs(reponame, remote, username, sshKey):
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=username,
ssh_key=sshKey))
for r in reversed(outgoingRevs):
run_cmd(['hg', '--config', 'extensions.mq=', 'strip', '-n',
r[REVISION]], cwd=reponame)
def ssh(user, identity, host, remote_cmd, port=22):
cmd = ['ssh', '-l', user]
if identity:
cmd.extend(['-i', identity])
cmd.extend(['-p', str(port), host, remote_cmd])
return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd, ))
def ssh(user, identity, host, remote_cmd, port=22):
cmd = ['ssh', '-l', user]
if identity:
cmd.extend(['-i', identity])
cmd.extend(['-p', str(port), host, remote_cmd])
return retry(do_cmd, attempts=retries + 1, sleeptime=retry_sleep, args=(cmd,))
def get_production_slaves(slaveapi):
url = furl(slaveapi)
url.path.add("slaves")
url.args["environment"] = "prod"
url.args["enabled"] = 1
r = retry(requests.get, args=(str(url),))
return r.json()["slaves"]
def make_generic_get_request(page_url):
"""Make generic GET request to retrieve some page content"""
def _get():
req = requests.get(page_url, timeout=60)
req.raise_for_status()
return req.content
return retry(_get, attempts=5, sleeptime=1)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
configsToOverride):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
retry(mercurial, args=(repo_url, reponame))
def bump(repo, configsToBump, configsToOverride):
"""Process dynamic (version, buildNumber, etc.) variables in
configsToBump, then append overrides files to both configsToBump and
configsToOverride."""
# First pass. Bump variables in configsToBump.
configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
cmd.extend(configs)
run_cmd(cmd)
# Second pass. Append override files to configsToBump and
# configsToOverride.
for config, overrides in \
configsToBump.items() + configsToOverride.items():
newContent = cat([path.join(repo, config)] +
[path.join(repo, x) for x in overrides])
fh = open(path.join(repo, config), 'wb')
fh.write(newContent)
fh.close()
run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo)
def bump_wrapper(r, n):
bump(r, configsToBump, configsToOverride)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, bump_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
tags = []
for configfile in configsToBump.keys():
config = readReleaseConfig(path.join(reponame, configfile))
tags.extend(
getTags(config['baseTag'], config['buildNumber'], buildTag=True))
return tags
def tagRepo(repo, attempt, config, revision, tags):
totalChangesets = len(tags)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
def make_generic_get_request(page_url):
"""Make generic GET request to retrieve some page content"""
def _get():
req = requests.get(page_url, timeout=60)
req.raise_for_status()
return req.content
return retry(_get, attempts=5, sleeptime=1)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
configsToOverride):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
protocol='ssh')
retry(mercurial, args=(repo_url, reponame))
def bump(repo, configsToBump, configsToOverride):
"""Process dynamic (version, buildNumber, etc.) variables in
configsToBump, then append overrides files to both configsToBump and
configsToOverride."""
# First pass. Bump variables in configsToBump.
configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
cmd.extend(configs)
run_cmd(cmd)
# Second pass. Append override files to configsToBump and
# configsToOverride.
for config, overrides in \
configsToBump.items() + configsToOverride.items():
newContent = cat([path.join(repo, config)] +
[path.join(repo, x) for x in overrides])
fh = open(path.join(repo, config), 'wb')
fh.write(newContent)
fh.close()
run_cmd(['hg', 'commit', '-m', 'Automatic config bump'],
cwd=repo)
def bump_wrapper(r, n):
bump(r, configsToBump, configsToOverride)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, bump_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
tags = []
for configfile in configsToBump.keys():
config = readReleaseConfig(path.join(reponame, configfile))
tags.extend(getTags(config['baseTag'], config['buildNumber'],
buildTag=True))
return tags
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--manifest", required=True)
parser.add_argument("-a", "--api-root", required=True,
help="Balrog API root")
parser.add_argument("-v", "--verbose", action="store_const",
dest="loglevel", const=logging.DEBUG,
default=logging.INFO)
parser.add_argument("--product", help="Override product name from application.ini")
args = parser.parse_args()
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=args.loglevel)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("boto").setLevel(logging.WARNING)
balrog_username = os.environ.get("BALROG_USERNAME")
balrog_password = os.environ.get("BALROG_PASSWORD")
suffix = os.environ.get("BALROG_BLOB_SUFFIX")
if not balrog_username and not balrog_password:
raise RuntimeError("BALROG_USERNAME and BALROG_PASSWORD environment "
"variables should be set")
if not suffix:
raise RuntimeError("BALROG_BLOB_SUFFIX environment variable should be set")
manifest = json.load(open(args.manifest))
auth = (balrog_username, balrog_password)
for e in manifest:
complete_info = [{
"hash": e["hash"],
"size": e["size"],
}]
submitter = ReleaseSubmitterV4(api_root=args.api_root, auth=auth,
suffix=suffix)
productName = args.product or e["appName"]
retry(lambda: submitter.run(
platform=e["platform"], productName=productName,
version=e["toVersion"],
build_number=e["toBuildNumber"],
appVersion=e["version"], extVersion=e["version"],
buildID=e["to_buildid"], locale=e["locale"],
hashFunction='sha512', completeInfo=complete_info),
attempts=30, sleeptime=10, max_sleeptime=60, jitter=3,
)
def scp(user, identity, host, files, remote_dir, port=22):
cmd = ['scp']
if identity:
cmd.extend(['-i', identity])
cmd.extend(['-P', str(port)])
cmd.extend(files)
cmd.append("%s@%s:%s" % (user, host, remote_dir))
return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd, ))
def schedule(task, config, balrog_auth):
"""Schedule a release to ship on balrog channel(s)"""
from util.retry import retry # noqa: E402
auth = balrog_auth
scheduler = create_scheduler(api_root=config['api_root'], auth=auth,
dummy=config['dummy'])
args = [
task['payload']['product'].capitalize(),
task['payload']['version'],
task['payload']['build_number'],
task['payload']['publish_rules'],
task['payload']['release_eta'] or None, # Send None if release_eta is ''
]
# XXX optionally append background_rate if/when we want to support it
# XXX should we catch requests.HTTPError and raise a scriptworker
# error? maybe not since balrogscript isn't py3
retry(lambda: scheduler.run(*args))
def scp(user, identity, host, files, remote_dir, port=22):
cmd = ['scp']
if identity:
cmd.extend(['-i', identity])
cmd.extend(['-P', str(port)])
cmd.extend(files)
cmd.append("%s@%s:%s" % (user, host, remote_dir))
return retry(do_cmd, attempts=retries, sleeptime=retry_sleep, args=(cmd,))
def process_slave(slaveapi, slave, dryrun=False):
try:
info = get_slave(slaveapi, slave)
# Ignore slaves without recent job information
if not info["recent_jobs"]:
log.info("%s - Skipping reboot because no recent jobs found", slave)
return
last_job_time = datetime.fromtimestamp(info["recent_jobs"][0]["endtime"])
# And also slaves that haven't been idle for more than the threshold
if not (now - last_job_time).total_seconds() > IDLE_THRESHOLD:
log.info("%s - Skipping reboot because last job ended recently at %s", slave, get_formatted_time(last_job_time))
return
if dryrun:
log.info("%s - Last job ended at %s, would've rebooted", slave, get_formatted_time(last_job_time))
return
else:
log.info("%s - Last job ended at %s, rebooting", slave, get_formatted_time(last_job_time))
# We need to set a graceful shutdown for the slave on the off chance that
# it picks up a job before us making the decision to reboot it, and the
# reboot actually happening. In most cases this will happen nearly
# instantly.
log.debug("%s - Setting graceful shutdown", slave)
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add("shutdown_buildslave")
url.args["waittime"] = 30
r = retry(requests.post, args=(str(url),)).json()
url.args["requestid"] = r["requestid"]
while r["state"] in (PENDING, RUNNING):
time.sleep(30)
r = retry(requests.get, args=(str(url),)).json()
if r["state"] == FAILURE:
log.info("%s - Graceful shutdown failed, aborting reboot", slave)
return
log.info("%s - Graceful shutdown finished, rebooting", slave)
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add("reboot")
retry(requests.post, args=(str(url),))
# Because SlaveAPI fully escalates reboots (all the way to IT bug filing),
# there's no reason for us to watch for it to complete.
log.info("%s - Reboot queued", slave)
except:
log.exception("%s - Caught exception while processing", slave)
def retrieveFile(url, file_path):
success = True
url = urllib.quote(url, safe=':/')
log.info('Downloading from %s' % url)
log.info('To: %s', file_path)
log.info('CWD: %s' % os.getcwd())
try:
# use URLopener, which handles errors properly
retry(StrictFancyURLopener().retrieve,
kwargs=dict(url=url, filename=file_path))
except IOError:
log.error("Error downloading %s" % url, exc_info=True)
success = False
try:
os.remove(file_path)
except OSError:
log.info("Cannot remove %s" % file_path, exc_info=True)
return success
def compareLocales(repo,
locale,
l10nRepoDir,
localeSrcDir,
l10nIni,
revision="default",
merge=True):
retry(mercurial, args=(repo, "compare-locales"))
update("compare-locales", revision=revision)
mergeDir = path.join(localeSrcDir, "merged")
if path.exists(mergeDir):
log.info("Deleting %s" % mergeDir)
shutil.rmtree(mergeDir)
run_cmd([
"python",
path.join("compare-locales", "scripts", "compare-locales"), "-m",
mergeDir, l10nIni, l10nRepoDir, locale
],
env={"PYTHONPATH": path.join("compare-locales", "lib")})
def testRetryArgsPassed(self):
args = (1, 'two', 3)
kwargs = dict(foo='a', bar=7)
ret = retry(_mirrorArgs,
args=args,
kwargs=kwargs.copy(),
sleeptime=0,
jitter=0)
self.assertEqual(ret[0], args)
self.assertEqual(ret[1], kwargs)
def main():
config = load_config(sys.argv[1:])
setup_logging(config['verbose'])
balrog_auth, config = update_config(config)
# hacking the tools repo dependency by first reading its location from
# the config file and only then loading the module from subdfolder
sys.path.insert(0, os.path.join(config['tools_location'], 'lib/python'))
# Until we get rid of our tools dep, this import(s) will break flake8 E402
from util.retry import retry # noqa: E402
# Read the manifest from disk
manifest = get_manifest(config)
for e in manifest:
# Get release metadata from manifest
submitter, release = create_submitter(e, balrog_auth, config)
# Connect to balrog and submit the metadata
retry(lambda: submitter.run(**release))
def fetch_actions_json(task_id):
queue = taskcluster.Queue()
actions_url = queue.buildUrl("getLatestArtifact", task_id, 'public/actions.json')
def _get():
q = requests.get(actions_url)
q.raise_for_status()
return q.json()
return retry(_get)
def retrieveFile(url, file_path):
success = True
url = urllib.quote(url, safe=':/')
log.info('Downloading from %s' % url)
log.info('To: %s', file_path)
log.info('CWD: %s' % os.getcwd())
try:
# use URLopener, which handles errors properly
retry(StrictFancyURLopener().retrieve,
kwargs=dict(url=url, filename=file_path))
except IOError:
log.error("Error downloading %s" % url, exc_info=True)
success = False
try:
os.remove(file_path)
except OSError:
log.info("Cannot remove %s" % file_path, exc_info=True)
return success
def get_recent_action(slaveapi, slave, action):
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add(action)
history = retry(requests.get, args=(str(url),)).json()
results = []
for key in history.keys():
if not key == action:
continue
for item in history[action]:
results.append(history[action][item])
return get_latest_result(results)
def fetch_actions_json(task_id):
queue = taskcluster.Queue()
actions_url = queue.buildUrl("getLatestArtifact", task_id,
'public/actions.json')
def _get():
q = requests.get(actions_url)
q.raise_for_status()
return q.json()
return retry(_get)
def get_recent_action(slaveapi, slave, action):
url = furl(slaveapi)
url.path.add("slaves").add(slave).add("actions").add(action)
history = retry(requests.get, args=(str(url), )).json()
results = []
for key in history.keys():
if not key == action:
continue
for item in history[action]:
results.append(history[action][item])
return get_latest_result(results)
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def updateProductDetails(productName, version, productDetailsRepo, mozillaComRepo, svnSshKey, dryRun=False):
"""
Add a new version to the product details
"""
os.environ["SVN_SSH"] = "ssh -i %s" % svnSshKey
pdDir = "product-details.svn"
mcDir = "mozilla.com.svn"
retry(checkoutSVN, args=(pdDir, productDetailsRepo), attempts=3)
retry(checkoutSVN, args=(mcDir, mozillaComRepo), attempts=3)
# Update the PHP files
updateProductDetailFiles(pdDir, productName, version)
# Export to json
exportJSON(pdDir)
# Commit to svn
commitSVN(pdDir, productName, version, dryRun)
# Get the svn revision of the p-d repository
svnRev = getSVNrev(pdDir)
# Update Mozilla.com
updateRev(mcDir, svnRev)
# commit Mozilla.com
commitSVN(pdDir, productName, version, dryRun)
def find_decision_task_id(trust_domain, project, revision):
decision_task_route = "{trust_domain}.v2.{project}.revision.{revision}.taskgraph.decision".format(
trust_domain=trust_domain,
project=project,
revision=revision,
)
index = taskcluster.Index()
def _get():
return index.findTask(decision_task_route)["taskId"]
return retry(_get)
def find_decision_task_id(trust_domain, project, revision):
decision_task_route = "{trust_domain}.v2.{project}.revision.{revision}.taskgraph.decision".format(
trust_domain=trust_domain,
project=project,
revision=revision,
)
index = taskcluster.Index()
def _get():
return index.findTask(decision_task_route)["taskId"]
return retry(_get)
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
retry(mercurial, args=(remote, reponame))
def tagRepo(repo, attempt, config, revision, tags):
totalChangesets = len(tags)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
