def verify_configs(configs_dir, revision, hghost, configs_repo, changesets,
filename):
"""Check the release_configs and l10n-changesets against tagged
revisions"""
release_config_file = path.join(configs_dir, 'mozilla', filename)
l10n_changesets_file = path.join(configs_dir, 'mozilla', changesets)
configs_url = make_hg_url(hghost, configs_repo, revision=revision,
filename=path.join('mozilla', filename))
l10n_url = make_hg_url(hghost, configs_repo, revision=revision,
filename=path.join('mozilla', changesets))
success = True
try:
official_configs = urllib2.urlopen(configs_url)
log.info("Comparing tagged revision %s to on-disk %s ..." % (
configs_url, filename))
if not compare(official_configs, release_config_file):
log.error("local configs do not match tagged revisions in repo")
success = False
error_tally.add('verify_configs')
l10n_changesets = urllib2.urlopen(l10n_url)
log.info("Comparing tagged revision %s to on-disk %s ..." % (
l10n_url, changesets))
if not compare(l10n_changesets, l10n_changesets_file):
log.error("local l10n-changesets do not match tagged revisions"
" in repo")
success = False
error_tally.add('verify_configs')
except urllib2.HTTPError:
log.error("cannot find configs in repo %s" % configs_url)
log.error("cannot find configs in repo %s" % l10n_url)
success = False
error_tally.add('verify_configs')
return success
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config[
'hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out,
kwargs=dict(src=reponame,
remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def verify_configs(revision, hghost, configs_repo, changesets, filename):
"""Check the release_configs and l10n-changesets against tagged revisions"""
configs_url = make_hg_url(hghost,
configs_repo,
revision=revision,
filename="mozilla/%s" % filename)
l10n_url = make_hg_url(hghost,
configs_repo,
revision=revision,
filename="mozilla/%s" % changesets)
success = True
try:
official_configs = urllib2.urlopen(configs_url)
log.info("Comparing tagged revision %s to on-disk %s ..." %
(configs_url, filename))
if not compare(official_configs, filename):
log.error("local configs do not match tagged revisions in repo")
success = False
error_tally.add('verify_configs')
l10n_changesets = urllib2.urlopen(l10n_url)
log.info("Comparing tagged revision %s to on-disk %s ..." %
(l10n_url, changesets))
if not compare(l10n_changesets, changesets):
log.error(
"local l10n-changesets do not match tagged revisions in repo")
success = False
error_tally.add('verify_configs')
except urllib2.HTTPError:
log.error("cannot find configs in repo %s" % configs_url)
log.error("cannot find configs in repo %s" % l10n_url)
success = False
error_tally.add('verify_configs')
return success
def testMakeHGUrlNoRevisionNoFilename(self):
repo_url = make_hg_url(
"hg.mozilla.org",
"/build/tools"
)
expected_url = "https://hg.mozilla.org/build/tools"
self.assertEquals(repo_url, expected_url)
def make_hg_get_request(repo_path,
revision,
filename=None,
hg_url='hg.mozilla.org'):
"""Wrapper to make a GET request for a specific URI under hg repo"""
url = make_hg_url(hg_url, repo_path, revision=revision, filename=filename)
return make_generic_get_request(url)
def testMakeHGUrl(self):
# construct an hg url specific to revision, branch and filename and try
# to pull it down
file_url = make_hg_url(
"hg.mozilla.org", "//build/tools/", revision="FIREFOX_3_6_12_RELEASE", filename="/lib/python/util/hg.py"
)
expected_url = "https://hg.mozilla.org/build/tools/raw-file/FIREFOX_3_6_12_RELEASE/lib/python/util/hg.py"
self.assertEquals(file_url, expected_url)
def testMakeHGUrlDifferentProtocol(self):
repo_url = make_hg_url(
"hg.mozilla.org",
"/build/tools",
protocol='ssh'
)
expected_url = "ssh://hg.mozilla.org/build/tools"
self.assertEquals(repo_url, expected_url)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
configsToOverride):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
retry(mercurial, args=(repo_url, reponame))
def bump(repo, configsToBump, configsToOverride):
"""Process dynamic (version, buildNumber, etc.) variables in
configsToBump, then append overrides files to both configsToBump and
configsToOverride."""
# First pass. Bump variables in configsToBump.
configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
cmd.extend(configs)
run_cmd(cmd)
# Second pass. Append override files to configsToBump and
# configsToOverride.
for config, overrides in \
configsToBump.items() + configsToOverride.items():
newContent = cat([path.join(repo, config)] +
[path.join(repo, x) for x in overrides])
fh = open(path.join(repo, config), 'wb')
fh.write(newContent)
fh.close()
run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo)
def bump_wrapper(r, n):
bump(r, configsToBump, configsToOverride)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, bump_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
tags = []
for configfile in configsToBump.keys():
config = readReleaseConfig(path.join(reponame, configfile))
tags.extend(
getTags(config['baseTag'], config['buildNumber'], buildTag=True))
return tags
def testMakeHGUrlNoFilename(self):
file_url = make_hg_url(
"hg.mozilla.org",
"/build/tools",
revision="default"
)
expected_url = "https://hg.mozilla.org/build/tools/rev/default"
self.assertEquals(file_url, expected_url)
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
configsToOverride):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
protocol='ssh')
retry(mercurial, args=(repo_url, reponame))
def bump(repo, configsToBump, configsToOverride):
"""Process dynamic (version, buildNumber, etc.) variables in
configsToBump, then append overrides files to both configsToBump and
configsToOverride."""
# First pass. Bump variables in configsToBump.
configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
cmd.extend(configs)
run_cmd(cmd)
# Second pass. Append override files to configsToBump and
# configsToOverride.
for config, overrides in \
configsToBump.items() + configsToOverride.items():
newContent = cat([path.join(repo, config)] +
[path.join(repo, x) for x in overrides])
fh = open(path.join(repo, config), 'wb')
fh.write(newContent)
fh.close()
run_cmd(['hg', 'commit', '-m', 'Automatic config bump'],
cwd=repo)
def bump_wrapper(r, n):
bump(r, configsToBump, configsToOverride)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, bump_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
tags = []
for configfile in configsToBump.keys():
config = readReleaseConfig(path.join(reponame, configfile))
tags.extend(getTags(config['baseTag'], config['buildNumber'],
buildTag=True))
return tags
def testMakeHGUrl(self):
# construct an hg url specific to revision, branch and filename and try
# to pull it down
file_url = make_hg_url("hg.mozilla.org",
'//build/tools/',
revision='FIREFOX_3_6_12_RELEASE',
filename="/lib/python/util/hg.py")
expected_url = "https://hg.mozilla.org/build/tools/raw-file/FIREFOX_3_6_12_RELEASE/lib/python/util/hg.py"
self.assertEquals(file_url, expected_url)
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts, defaultBranch='default'):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def maybe_delete_repo(server, username, sshKey, repo, repoPath):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
try:
log.info("Trying to open %s" % repo_url)
urlopen(repo_url)
except URLError:
log.info("%s doesn't exist, not deleting" % reponame)
else:
log.info('Deleting %s' % reponame)
retry(run_remote_cmd, args=('edit %s delete YES' % reponame, server,
username, sshKey))
def maybe_delete_repo(server, username, sshKey, repo, repoPath):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
try:
log.info("Trying to open %s" % repo_url)
urlopen(repo_url)
except URLError:
log.info("%s doesn't exist, not deleting" % reponame)
else:
log.info('Deleting %s' % reponame)
retry(run_remote_cmd,
args=('edit %s delete YES' % reponame, server, username, sshKey))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output([
'hg', 'commit', '-u', config['hgUsername'], '-m',
getBumpCommitMessage(config['productName'],
config['version'])
],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# Validate that the repository is only different from the remote in
# ways we expect.
outgoingRevs = out(src=reponame,
remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey'])
if len([r for r in outgoingRevs if r[BRANCH] == relbranch
]) != relbranchChangesets:
raise Exception("Incorrect number of revisions on %s" % relbranch)
if len(outgoingRevs) != relbranchChangesets:
raise Exception("Wrong number of outgoing revisions")
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts, defaultBranch='default'):
remote = make_hg_url(HG, repo)
retry(mercurial, args=(remote, reponame))
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
relbranchChangesets = len(tags)
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
revision = get_revision(reponame)
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
retry(mercurial, args=(remote, reponame))
def tagRepo(repo, attempt, config, revision, tags):
totalChangesets = len(tags)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def verify_repo(branch, revision, hghost):
"""Poll the hgweb interface for a given branch and revision to
make sure it exists"""
repo_url = make_hg_url(hghost, branch, revision=revision)
log.info("Checking for existence of %s..." % repo_url)
success = True
try:
repo_page = urllib2.urlopen(repo_url)
log.info("Got: %s !" % repo_page.geturl())
except urllib2.HTTPError:
log.error("Repo does not exist with required revision. Check again, or use -b to bypass")
success = False
error_tally.add('verify_repo')
return success
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs, appName, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences"""
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
types = {'+': 'release', '-': 'nightly'}
for platform,mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
# Create links to the two mozconfigs.
releaseConfig = make_hg_url(hghost, branch, 'http', revision, mozconfig)
urls.append(releaseConfig)
# The nightly one is the exact same URL, with the file part changed.
urls.append(releaseConfig.rstrip('release') + 'nightly')
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \
and clean_line in mozconfigWhitelist[branch][platform]:
continue
if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \
and clean_line in mozconfigWhitelist['nightly'][platform]:
continue
if line[0] == '-':
opposite = 'release'
else:
opposite = 'nightly'
log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line))
success = False
error_tally.add('verify_mozconfig')
else:
log.info("Missing mozconfigs to compare for %s" % platform)
error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform)
return success
def verify_mozconfigs(branch, version, hghost, product, platforms, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences"""
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
types = {'+': 'release', '-': 'nightly'}
tag = ''.join([product.upper(), "_", version.replace('.','_'), "_RELEASE"])
for platform in platforms:
urls = []
mozconfigs = []
for type in types.values():
urls.append(make_hg_url(hghost, 'build/buildbot-configs', 'http',
tag, os.path.join('mozilla2', platform,
branch, type,'mozconfig')))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0],mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \
and clean_line in mozconfigWhitelist[branch][platform]:
continue
if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \
and clean_line in mozconfigWhitelist['nightly'][platform]:
continue
if line[0] == '-':
opposite = 'release'
else:
opposite = 'nightly'
log.error("not in %s mozconfig's whitelist (%s/%s/%s) : %s" % (opposite, branch, platform, types[line[0]], clean_line))
success = False
error_tally.add('verify_mozconfig')
else:
log.info("Missing mozconfigs to compare for %s" % platform)
error_tally.add("verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare" % platform)
return success
def verify_repo(branch, revision, hghost):
"""Poll the hgweb interface for a given branch and revision to
make sure it exists"""
repo_url = make_hg_url(hghost, branch, revision=revision)
log.info("Checking for existence of %s..." % repo_url)
success = True
try:
repo_page = urllib2.urlopen(repo_url)
log.info("Got: %s !" % repo_page.geturl())
except urllib2.HTTPError:
log.error("Repo does not exist with required revision."
" Check again, or use -b to bypass")
success = False
error_tally.add('verify_repo')
return success
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# Validate that the repository is only different from the remote in
# ways we expect.
outgoingRevs = out(src=reponame, remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey'])
if len([r for r in outgoingRevs if r[BRANCH] == relbranch]) != relbranchChangesets:
raise Exception("Incorrect number of revisions on %s" % relbranch)
if len(outgoingRevs) != relbranchChangesets:
raise Exception("Wrong number of outgoing revisions")
def verify_build(sourceRepo, hghost):
""" Ensure that the bumpFiles match what the release config wants them to be"""
success = True
for filename, versions in sourceRepo['bumpFiles'].iteritems():
try:
url = make_hg_url(hghost, sourceRepo['path'],
revision=sourceRepo['revision'],
filename=filename)
found_version = urllib2.urlopen(url).read()
if not findVersion(found_version, versions['version']):
log.error("%s has incorrect version '%s' (expected '%s')" % \
(filename, found_version, versions['version']))
success = False
error_tally.add('verify_build')
except urllib2.HTTPError, inst:
log.error("cannot find %s. Check again, or -b to bypass" % inst.geturl())
success = False
error_tally.add('verify_build')
def verify_build(sourceRepo, hghost):
""" Ensure that the bumpFiles match what the release config wants them to be"""
success = True
for filename, versions in sourceRepo['bumpFiles'].iteritems():
try:
url = make_hg_url(hghost,
sourceRepo['path'],
revision=sourceRepo['revision'],
filename=filename)
found_version = urllib2.urlopen(url).read()
if not findVersion(found_version, versions['version']):
log.error("%s has incorrect version '%s' (expected '%s')" % \
(filename, found_version, versions['version']))
success = False
error_tally.add('verify_build')
except urllib2.HTTPError, inst:
log.error("cannot find %s. Check again, or -b to bypass" %
inst.geturl())
success = False
error_tally.add('verify_build')
def verify_l10n_changesets(hgHost, l10n_changesets):
"""Checks for the existance of all l10n changesets"""
success = True
locales = query_locale_revisions(l10n_changesets)
for locale in sorted(locales.keys()):
revision = locales[locale]
localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % {
'repoPath': releaseConfig['l10nRepoPath'].strip('/'),
'locale': locale,
'revision': revision,
}
locale_url = make_hg_url(hgHost, localePath, protocol='https')
log.info("Checking for existence l10n changeset %s %s in repo %s ..."
% (locale, revision, locale_url))
try:
urllib2.urlopen(locale_url)
except urllib2.HTTPError, e:
log.error("error checking l10n changeset %s: %d %s" % (locale_url, e.code, e.reason))
success = False
error_tally.add('verify_l10n')
def verify_l10n_changesets(hgHost, l10n_changesets):
"""Checks for the existance of all l10n changesets"""
success = True
locales = query_locale_revisions(l10n_changesets)
for locale in sorted(locales.keys()):
revision = locales[locale]
localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % {
'repoPath': releaseConfig['l10nRepoPath'].strip('/'),
'locale': locale,
'revision': revision,
}
locale_url = make_hg_url(hgHost, localePath, protocol='https')
log.info("Checking for existence l10n changeset %s %s in repo %s ..." %
(locale, revision, locale_url))
try:
urllib2.urlopen(locale_url)
except urllib2.HTTPError, e:
log.error("error checking l10n changeset %s: %d %s" %
(locale_url, e.code, e.reason))
success = False
error_tally.add('verify_l10n')
def verify_l10n_changesets(hgHost, l10n_changesets):
"""Checks for the existance of all l10n changesets"""
success = True
locales = query_locale_revisions(l10n_changesets)
for locale in sorted(locales.keys()):
revision = locales[locale]
localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % {
'repoPath': releaseConfig['l10nRepoPath'].strip('/'),
'locale': locale,
'revision': revision,
}
locale_url = make_hg_url(hgHost, localePath, protocol='https')
log.info("Checking for existence l10n changeset %s %s in repo %s ..."
% (locale, revision, locale_url))
success = retry(get_l10n_changesets,
kwargs=dict(locale_url=locale_url), attempts=3,
sleeptime=1, retry_exceptions=(urllib2.HTTPError,))
if not success:
error_tally.add('verify_l10n')
return success
def verify_l10n_changesets(hgHost, l10n_changesets):
"""Checks for the existance of all l10n changesets"""
success = True
locales = query_locale_revisions(l10n_changesets)
for locale in sorted(locales.keys()):
revision = locales[locale]
localePath = '%(repoPath)s/%(locale)s/file/%(revision)s' % {
'repoPath': releaseConfig['l10nRepoPath'].strip('/'),
'locale': locale,
'revision': revision,
}
locale_url = make_hg_url(hgHost, localePath, protocol='https')
log.info("Checking for existence l10n changeset %s %s in repo %s ..."
% (locale, revision, locale_url))
success = retry(get_l10n_changesets,
kwargs=dict(locale_url=locale_url), attempts=3,
sleeptime=1, retry_exceptions=(urllib2.HTTPError, urllib2.URLError))
if not success:
error_tally.add('verify_l10n')
return success
from build.download import downloadNightlyBuild
from build.l10n import repackLocale, l10nRepackPrep, getNightlyLocalesForChunk
import build.misc
from build.paths import getLatestDir
from build.upload import postUploadCmdPrefix
from release.info import readBranchConfig
from util.commands import run_cmd
from util.hg import mercurial, update, make_hg_url
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format="%(message)s")
log = logging.getLogger(__name__)
HG = "hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, "build/buildbot-configs")
class RepackError(Exception):
pass
def createRepacks(sourceRepo, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir,
makeDirs, locales, ftpProduct, stageServer, stageUsername,
stageSshKey, compareLocalesRepo, merge, platform,
stage_platform, mobileDirName, en_us_binary_url):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
from build.upload import postUploadCmdPrefix
from release.download import downloadReleaseBuilds, downloadUpdateIgnore404
from release.info import readReleaseConfig, readConfig, fileInfo
from release.l10n import getReleaseLocalesForChunk
from util.hg import mercurial, update, make_hg_url
from util.retry import retry
from util.commands import run_cmd
from release.info import getBuildID
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format="%(message)s")
log = logging.getLogger(__name__)
HG = "hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, "build/buildbot-configs")
class RepackError(Exception):
pass
def createRepacks(sourceRepo,
revision,
l10nRepoDir,
l10nBaseRepo,
mozconfigPath,
srcMozconfigPath,
objdir,
makeDirs,
appName,
sys.path.append(path.join(path.dirname(__file__), "../../lib/python"))
from build.download import downloadNightlyBuild
from build.l10n import repackLocale, l10nRepackPrep, getNightlyLocalesForChunk
import build.misc
from build.paths import getLatestDir
from build.upload import postUploadCmdPrefix
from release.info import readBranchConfig
from util.commands import run_cmd
from util.hg import mercurial, update, make_hg_url
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
log = logging.getLogger(__name__)
HG="hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO=make_hg_url(HG, "build/buildbot-configs")
class RepackError(Exception):
pass
def createRepacks(sourceRepo,l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, locales, ftpProduct,
stageServer, stageUsername, stageSshKey, compareLocalesRepo,
merge, platform, stage_platform, mobileDirName,
en_us_binary_url):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
log = logging.getLogger(__name__)
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python/vendor"))
from release.info import readReleaseConfig, readConfig
from release.paths import makeCandidatesDir, makeReleasesDir
from util.hg import update, make_hg_url, mercurial
from util.commands import run_remote_cmd
from util.transfer import scp
from util.retry import retry
import requests
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url("hg.mozilla.org", "build/buildbot-configs")
REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber", "stage_product")
DEFAULT_RSYNC_EXCLUDES = [
"--exclude=*tests*",
"--exclude=*crashreporter*",
"--exclude=*.log",
"--exclude=*.txt",
"--exclude=*unsigned*",
"--exclude=*update-backup*",
"--exclude=*partner-repacks*",
"--exclude=*.checksums",
"--exclude=*.checksums.asc",
"--exclude=logs",
"--exclude=jsshell*",
try:
import json
except ImportError:
import simplejson as json
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
log = logging.getLogger(__name__)
sys.path.append(path.join(path.dirname(__file__), "../../lib/python"))
from release.info import readReleaseConfig, readBranchConfig
from release.paths import makeCandidatesDir, makeReleasesDir
from util.hg import update, make_hg_url, mercurial
from util.commands import run_remote_cmd
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url('hg.mozilla.org',
'build/buildbot-configs')
REQUIRED_BRANCH_CONFIG = ("stage_server", "stage_username", "stage_ssh_key")
REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber")
DEFAULT_RSYNC_EXCLUDES = ['--exclude=*tests*',
'--exclude=*crashreporter*',
'--exclude=*.log',
'--exclude=*.txt',
'--exclude=*unsigned*',
'--exclude=*update-backup*',
'--exclude=*partner-repacks*',
'--exclude=*.checksums',
'--exclude=logs',
'--exclude=jsshell*',
]
sys.path.append(path.join(path.dirname(__file__), "../../lib/python"))
from build.l10n import repackLocale, l10nRepackPrep
import build.misc
from build.upload import postUploadCmdPrefix
from release.download import downloadReleaseBuilds, downloadUpdateIgnore404
from release.info import readReleaseConfig, readBranchConfig
from release.l10n import getReleaseLocalesForChunk
from util.hg import mercurial, update, make_hg_url
from util.retry import retry
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
log = logging.getLogger(__name__)
HG="hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO=make_hg_url(HG, "build/buildbot-configs")
class RepackError(Exception):
pass
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, appName, locales, product,
version, buildNumber, stageServer, stageUsername, stageSshKey,
compareLocalesRepo, merge, platform, brand,
generatePartials=False, oldVersion=None,
oldBuildNumber=None, appVersion=None):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
import sys
sys.path.append(path.join(path.dirname(__file__), "../../lib/python"))
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(message)s")
log = logging.getLogger(__name__)
from util.commands import run_cmd, get_output
from util.hg import mercurial, apply_and_push, update, get_revision, \
make_hg_url, out, BRANCH, REVISION, get_branches, cleanOutgoingRevs
from util.retry import retry
from build.versions import bumpFile
from release.info import readReleaseConfig, getTags, generateRelbranchName
from release.l10n import getL10nRepositories
HG="hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO=make_hg_url(HG, 'build/buildbot-configs')
DEFAULT_MAX_PUSH_ATTEMPTS=10
REQUIRED_CONFIG = ('version', 'appVersion', 'appName', 'productName',
'milestone', 'buildNumber', 'hgUsername', 'hgSshKey',
'baseTag', 'l10nRepoPath', 'sourceRepositories',
'l10nRevisionFile')
REQUIRED_SOURCE_REPO_KEYS = ('path', 'revision')
def getBumpCommitMessage(productName, version):
return 'Automated checkin: version bump for ' + productName + ' ' + \
version + ' release. CLOSED TREE a=release'
def getTagCommitMessage(revision, tag):
return "Added tag " + tag + " for changeset " + revision + \
". CLOSED TREE a=release"
def make_hg_get_request(repo_path, revision,
filename=None, hg_url='hg.mozilla.org'):
"""Wrapper to make a GET request for a specific URI under hg repo"""
url = make_hg_url(hg_url, repo_path, revision=revision, filename=filename)
return make_generic_get_request(url)
test_success = True
buildNumber = options.buildNumber
products = []
check_buildbot()
if not options.dryrun and not options.skip_reconfig:
from util.fabric.common import check_fabric, FabricHelper
check_fabric()
if options.configs_dir:
configs_dir = options.configs_dir
cleanup_configs = False
else:
cleanup_configs = True
configs_dir = mkdtemp()
remote = make_hg_url(HG, options.configs_repo_url)
retry(mercurial,
args=(remote, configs_dir),
kwargs={'branch': options.configs_branch})
update(configs_dir, options.configs_branch)
# https://bugzilla.mozilla.org/show_bug.cgi?id=678103#c5
# This goes through the list of config files in reverse order, which is a
# hacky way of making sure that the config file that's listed first is the
# one that's loaded in releaseConfig for the sendchange.
for releaseConfigFile in list(reversed(options.releaseConfigFiles)):
abs_release_config_file = path.join(configs_dir, 'mozilla',
releaseConfigFile)
releaseConfig = readReleaseConfig(abs_release_config_file)
products.append(releaseConfig['productName'])
def __init__(self, rmConfig, **kwargs):
BuildFactory.__init__(self, **kwargs)
self.addStep(
SetProperty(
name='set_topdir',
command=['pwd'],
property='topdir',
workdir='.',
))
self.addStep(
ShellCommand(
command=['rm', '-rvf', 'tools', 'buildbot-configs'],
workdir='.',
))
self.addStep(
ShellCommand(
command=[
'hg', 'clone',
make_hg_url(rmConfig['HG_HOST'], 'build/tools'), 'tools'
],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=[
rmConfig['PYTHON'], 'scripts/preproduction/repo_setup.py',
'-c', 'scripts/preproduction/repo_setup_config.py'
],
workdir='tools',
haltOnFailure=True,
))
self.addStep(
SetProperty(
property='previousSetupMakefile',
command='ls %s/Makefile 2>/dev/null || exit 0' %
rmConfig['releaseMasterDir'],
flunkOnFailure=False,
haltOnFailure=False,
warnOnFailure=True,
))
def previousSetupExists(step):
return \
step.build.getProperties().has_key('previousSetupMakefile') \
and len(step.build.getProperty('previousSetupMakefile')) > 0
self.addStep(
ShellCommand(
command=[
rmConfig['PYTHON'],
'tools/buildfarm/maintenance/buildbot-wrangler.py', 'stop',
'%s/master' % rmConfig['releaseMasterDir']
],
workdir=rmConfig['releaseMasterDir'],
flunkOnFailure=False,
doStepIf=previousSetupExists,
))
self.addStep(
ShellCommand(
command=['rm', '-rvf', rmConfig['releaseMasterDir']],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=['rm', '-rvf', 'buildbot-configs'],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=[
'hg', 'clone',
make_hg_url(rmConfig['HG_HOST'],
'%s/buildbot-configs' % rmConfig['HG_DIR']),
'buildbot-configs'
],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=[
'make', '-f', 'Makefile.setup',
'PYTHON=%s' % rmConfig['PYTHON'],
'VIRTUALENV=%s' % rmConfig['VIRTUALENV'],
'HG_DIR=%s' % rmConfig['HG_DIR'], 'MASTER_NAME=pp-release',
'MASTERS_JSON=/home/cltbld/conf/master_config.json',
'BASEDIR=%s' % rmConfig['releaseMasterDir'],
'BUILDBOTCUSTOM_BRANCH=%s' %
rmConfig['BUILDBOTCUSTOM_BRANCH'],
'BUILDBOTCONFIGS_BRANCH=%s' %
rmConfig['BUILDBOTCONFIGS_BRANCH'], 'virtualenv', 'deps',
'install-buildbot', 'master', 'master-makefile'
],
workdir='buildbot-configs',
env={
'PIP_DOWNLOAD_CACHE': WithProperties('%(topdir)s/cache'),
'CC': None,
'CXX': None
},
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=['make', 'checkconfig'],
workdir=rmConfig['releaseMasterDir'],
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=['touch', 'twistd.log'],
workdir='%s/master' % rmConfig['releaseMasterDir'],
))
self.addStep(
ShellCommand(
command=[
'bash', '-c',
'if [ -e ~/conf/passwords.py ]; then cp -fv ~/conf/passwords.py ./; fi'
],
workdir='%s/master' % rmConfig['releaseMasterDir'],
))
self.addStep(
ShellCommand(
command=[
rmConfig['PYTHON'],
'tools/buildfarm/maintenance/buildbot-wrangler.py',
'start',
'%s/master' % rmConfig['releaseMasterDir']
],
workdir=rmConfig['releaseMasterDir'],
))
for release_config in rmConfig['releaseConfigs']:
self.addStep(
SetProperty(
property='release_tag',
command=[
rmConfig['PYTHON'], '-c',
'execfile("%s"); \
print releaseConfig["baseTag"] + "_RELEASE"' %
release_config
],
workdir='%s/buildbot-configs' %
rmConfig['releaseMasterDir'],
))
self.addStep(
SetProperty(
property='release_branch',
command=[
rmConfig['PYTHON'], '-c',
'execfile("%s"); \
print releaseConfig["sourceRepositories"]["mozilla"]["path"]'
% release_config
],
workdir='%s/buildbot-configs' %
rmConfig['releaseMasterDir'],
))
self.addStep(
ShellCommand(
command=[
'buildbot', 'sendchange', '--username',
'preproduction', '--master',
rmConfig['releaseMasterHostPort'], '--branch',
WithProperties('%(release_branch)s'), '-p',
'products:firefox', '-p',
WithProperties('script_repo_revision:%(release_tag)s'),
'release_build'
],
workdir='.',
))
test_success = True
buildNumber = options.buildNumber
products = []
check_buildbot()
if not options.dryrun and not options.skip_reconfig:
from util.fabric.common import check_fabric, FabricHelper
check_fabric()
if options.configs_dir:
configs_dir = options.configs_dir
cleanup_configs = False
else:
cleanup_configs = True
configs_dir = mkdtemp()
remote = make_hg_url(HG, options.configs_repo_url)
retry(mercurial, args=(remote, configs_dir),
kwargs={'branch': options.configs_branch})
update(configs_dir, options.configs_branch)
# https://bugzilla.mozilla.org/show_bug.cgi?id=678103#c5
# This goes through the list of config files in reverse order, which is a
# hacky way of making sure that the config file that's listed first is the
# one that's loaded in releaseConfig for the sendchange.
for releaseConfigFile in list(reversed(options.releaseConfigFiles)):
abs_release_config_file = path.join(configs_dir, 'mozilla',
releaseConfigFile)
releaseConfig = readReleaseConfig(abs_release_config_file)
products.append(releaseConfig['productName'])
if not options.buildNumber:
from build.upload import postUploadCmdPrefix
from release.download import downloadReleaseBuilds
from release.info import readReleaseConfig, readBranchConfig
from release.l10n import getReleaseLocalesFromJsonForChunk
from release.paths import makeCandidatesDir
from util.commands import run_cmd
from util.hg import mercurial, update, make_hg_url
from util.retry import retry
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format="%(message)s")
log = logging.getLogger(__name__)
HG = "hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, "build/buildbot-configs")
class RepackError(Exception):
pass
def createRepacks(sourceRepo, revision, l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, locales, ftpProduct,
appName, version, appVersion, buildNumber, stageServer,
stageUsername, stageSshKey, compareLocalesRepo, merge,
platform, stage_platform, brand, mobileDirName):
sourceRepoName = path.split(sourceRepo)[-1]
nightlyDir = "candidates"
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
def __init__(self, rmConfig, **kwargs):
BuildFactory.__init__(self, **kwargs)
self.addStep(SetProperty(
name='set_topdir',
command=['pwd'],
property='topdir',
workdir='.',
))
self.addStep(ShellCommand(
command=['rm', '-rvf', 'tools', 'buildbot-configs'],
workdir='.',
))
self.addStep(ShellCommand(
command=['hg', 'clone',
make_hg_url(rmConfig['HG_HOST'], 'build/tools'),
'tools'],
workdir='.',
haltOnFailure=True,
))
self.addStep(ShellCommand(
command=[rmConfig['PYTHON'], 'scripts/preproduction/repo_setup.py', '-c',
'scripts/preproduction/repo_setup_config.py'],
workdir='tools',
haltOnFailure=True,
))
self.addStep(SetProperty(
property='previousSetupMakefile',
command='ls %s/Makefile 2>/dev/null || exit 0' %
rmConfig['releaseMasterDir'],
flunkOnFailure=False,
haltOnFailure=False,
warnOnFailure=True,
))
def previousSetupExists(step):
return \
step.build.getProperties().has_key('previousSetupMakefile') \
and len(step.build.getProperty('previousSetupMakefile')) > 0
self.addStep(ShellCommand(
command=[rmConfig['PYTHON'],
'tools/buildfarm/maintenance/buildbot-wrangler.py',
'stop', '%s/master' % rmConfig['releaseMasterDir']],
workdir=rmConfig['releaseMasterDir'],
flunkOnFailure=False,
doStepIf=previousSetupExists,
))
self.addStep(ShellCommand(
command=['rm', '-rvf', rmConfig['releaseMasterDir']],
workdir='.',
haltOnFailure=True,
))
self.addStep(ShellCommand(
command=['rm', '-rvf', 'buildbot-configs'],
workdir='.',
haltOnFailure=True,
))
self.addStep(ShellCommand(
command=['hg', 'clone',
make_hg_url(rmConfig['HG_HOST'],
'%s/buildbot-configs' % rmConfig['HG_DIR']),
'buildbot-configs'],
workdir='.',
haltOnFailure=True,
))
self.addStep(ShellCommand(
command=['make', '-f', 'Makefile.setup',
'PYTHON=%s' % rmConfig['PYTHON'],
'VIRTUALENV=%s' % rmConfig['VIRTUALENV'],
'HG_DIR=%s' % rmConfig['HG_DIR'],
'MASTER_NAME=pp-release',
'MASTERS_JSON=/home/cltbld/conf/master_config.json',
'BASEDIR=%s' % rmConfig['releaseMasterDir'],
'BUILDBOTCUSTOM_BRANCH=%s' %
rmConfig['BUILDBOTCUSTOM_BRANCH'],
'BUILDBOTCONFIGS_BRANCH=%s' %
rmConfig['BUILDBOTCONFIGS_BRANCH'],
'UNIVERSAL=1',
'virtualenv', 'deps', 'install-buildbot', 'master',
'master-makefile'],
workdir='buildbot-configs',
env={'PIP_DOWNLOAD_CACHE': WithProperties('%(topdir)s/cache'),
'CC': None,
'CXX': None
},
haltOnFailure=True,
))
self.addStep(ShellCommand(
command=['make', 'checkconfig'],
workdir=rmConfig['releaseMasterDir'],
haltOnFailure=True,
))
self.addStep(ShellCommand(
command=['touch', 'twistd.log'],
workdir='%s/master' % rmConfig['releaseMasterDir'],
))
self.addStep(ShellCommand(
command=[
'bash', '-c',
'if [ -e ~/conf/passwords.py ]; then cp -fv ~/conf/passwords.py ./; fi'
],
workdir='%s/master' % rmConfig['releaseMasterDir'],
))
self.addStep(ShellCommand(
command=[rmConfig['PYTHON'],
'tools/buildfarm/maintenance/buildbot-wrangler.py',
'start', '%s/master' % rmConfig['releaseMasterDir']],
workdir=rmConfig['releaseMasterDir'],
))
for release_config in rmConfig['releaseConfigs']:
self.addStep(SetProperty(
property='release_tag',
command=[rmConfig['PYTHON'], '-c',
'execfile("%s"); \
print releaseConfig["baseTag"] + "_RELEASE"' %
release_config],
workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'],
))
self.addStep(SetProperty(
property='release_branch',
command=[rmConfig['PYTHON'], '-c',
'execfile("%s"); \
print releaseConfig["sourceRepositories"]["mozilla"]["path"]' %
release_config],
workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'],
))
self.addStep(ShellCommand(
command=['buildbot', 'sendchange',
'--username', 'preproduction',
'--master', rmConfig['releaseMasterHostPort'],
'--branch', WithProperties('%(release_branch)s'),
'-p', 'products:firefox',
'-p', WithProperties(
'script_repo_revision:%(release_tag)s'),
'release_build'],
workdir='.',
))
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python/vendor"))
from distutils.version import LooseVersion
from release.updates.patcher import PatcherConfig
from release.l10n import makeReleaseRepackUrls
from release.platforms import buildbot2updatePlatforms, buildbot2ftp
from release.paths import makeReleasesDir, makeCandidatesDir
from release.info import readReleaseConfig
from util.retry import retry
from util.hg import mercurial, make_hg_url, update
from release.updates.verify import UpdateVerifyConfig
HG = "hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, 'build/buildbot-configs')
DEFAULT_MAX_PUSH_ATTEMPTS = 10
REQUIRED_CONFIG = ('productName', 'buildNumber', 'ausServerUrl',
'stagingServer')
FTP_SERVER_TEMPLATE = 'http://%s/pub/mozilla.org'
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
log = logging.getLogger()
def validate(options):
err = False
config = {}
if not path.exists(path.join('buildbot-configs', options.release_config)):
log.error("%s does not exist!" % options.release_config)
sys.path.append(path.join(path.dirname(__file__), "../../lib/python"))
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format="%(message)s")
log = logging.getLogger(__name__)
from release.info import readReleaseConfig, readBranchConfig
from util.hg import update, mercurial, make_hg_url
from util.commands import run_remote_cmd
from release.download import rsyncFilesByPattern, rsyncFiles
from release.signing import generateChecksums, signFiles
from release.paths import makeCandidatesDir
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url('hg.mozilla.org',
'build/buildbot-configs')
REQUIRED_BRANCH_CONFIG = ("stage_server", "stage_username", "stage_ssh_key")
REQUIRED_RELEASE_CONFIG = ("productName", "version", "buildNumber")
def validate(options, args):
if not options.configfile:
log.info("Must pass --configfile")
sys.exit(1)
releaseConfigFile = path.join("buildbot-configs", options.releaseConfig)
branchConfigFile = path.join("buildbot-configs", options.configfile)
branchConfigDir = path.dirname(branchConfigFile)
if not path.exists(branchConfigFile):
log.info("%s does not exist!" % branchConfigFile)
def main(options):
log.info('Loading config from %s' % options.config)
config = load_config(options.config)
if config.getboolean('release-runner', 'verbose'):
log_level=logging.DEBUG
else:
log_level=logging.INFO
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=log_level)
check_buildbot()
check_fabric()
# Shorthand
sendchange_master = config.get('release-runner', 'sendchange_master')
api_root = config.get('api', 'api_root')
username = config.get('api', 'username')
password = config.get('api', 'password')
hg_username = config.get('release-runner', 'hg_username')
hg_ssh_key = config.get('release-runner', 'hg_ssh_key')
buildbot_configs = config.get('release-runner', 'buildbot_configs')
buildbot_configs_branch = config.get('release-runner',
'buildbot_configs_branch')
buildbotcustom = config.get('release-runner', 'buildbotcustom')
buildbotcustom_branch = config.get('release-runner',
'buildbotcustom_branch')
tools = config.get('release-runner', 'tools')
tools_branch = config.get('release-runner', 'tools_branch')
masters_json = config.get('release-runner', 'masters_json')
staging = config.getboolean('release-runner', 'staging')
sleeptime = config.getint('release-runner', 'sleeptime')
notify_from = get_config(config, 'release-runner', 'notify_from', None)
notify_to = get_config(config, 'release-runner', 'notify_to', None)
ssh_username = get_config(config, 'release-runner', 'ssh_username', None)
ssh_key = get_config(config, 'release-runner', 'ssh_key', None)
if isinstance(notify_to, basestring):
notify_to = [x.strip() for x in notify_to.split(',')]
smtp_server = get_config(config, 'release-runner', 'smtp_server',
'localhost')
configs_workdir = 'buildbot-configs'
custom_workdir = 'buildbotcustom'
tools_workdir = 'tools'
configs_pushRepo = make_hg_url(HG, get_repo_path(buildbot_configs),
protocol='ssh')
custom_pushRepo = make_hg_url(HG, get_repo_path(buildbotcustom),
protocol='ssh')
tools_pushRepo = make_hg_url(HG, get_repo_path(tools), protocol='ssh')
rr = ReleaseRunner(api_root=api_root, username=username, password=password)
# Main loop waits for new releases, processes them and exits.
while True:
try:
log.debug('Fetching release requests')
rr.get_release_requests()
if rr.new_releases:
for release in rr.new_releases:
log.info('Got a new release request: %s' % release)
break
else:
log.debug('Sleeping for %d seconds before polling again' %
sleeptime)
time.sleep(sleeptime)
except Exception, e:
log.error("Caught exception when polling:", exc_info=True)
sys.exit(5)
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs,
nightly_mozconfigs, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and
compare to whitelist of known differences"""
branch_name = get_repo_name(branch)
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
for platform, mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
nightly_mozconfig = nightly_mozconfigs[platform]
mozconfig_paths = [mozconfig, nightly_mozconfig]
# Create links to the two mozconfigs.
for c in mozconfig, nightly_mozconfig:
urls.append(make_hg_url(hghost, branch, 'http', revision, c))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
# Nothing to compare against
return False
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
message = ""
if line[0] == '-':
# handle lines that move around in diff
if '+' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if '-' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning("%s not in %s %s!" % (
clean_line, platform,
mozconfigWhitelist['nightly'][platform]))
else:
log.error("Skipping line %s!" % line)
continue
message = "found in %s but not in %s: %s"
if line[0] == '-':
log.error(message % (mozconfig_paths[0],
mozconfig_paths[1], clean_line))
else:
log.error(message % (mozconfig_paths[1],
mozconfig_paths[0], clean_line))
success = False
else:
log.info("Missing mozconfigs to compare for %s" % platform)
return False
return success
def verify_mozconfigs(branch,
revision,
hghost,
product,
mozconfigs,
nightly_mozconfigs,
whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and
compare to whitelist of known differences"""
branch_name = get_repo_name(branch)
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
for platform, mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
nightly_mozconfig = nightly_mozconfigs[platform]
mozconfig_paths = [mozconfig, nightly_mozconfig]
# Create links to the two mozconfigs.
for c in mozconfig, nightly_mozconfig:
urls.append(make_hg_url(hghost, branch, 'http', revision, c))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
# Nothing to compare against
return False
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
message = ""
if line[0] == '-':
# handle lines that move around in diff
if '+' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if '-' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning(
"%s not in %s %s!" %
(clean_line, platform,
mozconfigWhitelist['nightly'][platform]))
else:
log.error("Skipping line %s!" % line)
continue
message = "found in %s but not in %s: %s"
if line[0] == '-':
log.error(message % (mozconfig_paths[0],
mozconfig_paths[1], clean_line))
else:
log.error(message % (mozconfig_paths[1],
mozconfig_paths[0], clean_line))
success = False
else:
log.info("Missing mozconfigs to compare for %s" % platform)
return False
return success
sys.path.append(path.join(path.dirname(__file__), "../../lib/python"))
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format="%(message)s")
log = logging.getLogger(__name__)
from util.commands import run_cmd, get_output
from util.hg import mercurial, apply_and_push, update, get_revision, \
make_hg_url, out, BRANCH, get_branches, cleanOutgoingRevs
from util.retry import retry
from build.versions import bumpFile
from release.info import readReleaseConfig, getTags, generateRelbranchName
from release.l10n import getL10nRepositories
HG = "hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, 'build/buildbot-configs')
DEFAULT_MAX_PUSH_ATTEMPTS = 10
REQUIRED_CONFIG = ('version', 'appVersion', 'appName', 'productName',
'buildNumber', 'hgUsername', 'hgSshKey', 'baseTag',
'l10nRepoPath', 'sourceRepositories', 'l10nRevisionFile')
REQUIRED_SOURCE_REPO_KEYS = ('path', 'revision')
def getBumpCommitMessage(productName, version):
return 'Automated checkin: version bump for ' + productName + ' ' + \
version + ' release. DONTBUILD CLOSED TREE a=release'
def getTagCommitMessage(revision, tags):
return "Added " + " ".join(tags) + " tag(s) for changeset " + revision + \
". DONTBUILD CLOSED TREE a=release"
import os
from os import path
import site
import logging
import sys
site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
from balrog.submitter.cli import ReleaseCreator, ReleasePusher
from release.info import readReleaseConfig
from util.retry import retry
from util.hg import mercurial, make_hg_url
HG = "hg.mozilla.org"
DEFAULT_BUILDBOT_CONFIGS_REPO = make_hg_url(HG, 'build/buildbot-configs')
REQUIRED_CONFIG = ('appVersion', 'productName', 'version', 'enUSPlatforms',
'testChannels', 'releaseChannel', 'baseTag', 'buildNumber',
'partialUpdates', 'stagingServer', 'bouncerServer', 'testChannelRuleIds')
def validate(options):
err = False
config = {}
if not path.exists(path.join('buildbot-configs', options.release_config)):
print "%s does not exist!" % options.release_config
sys.exit(1)
config = readReleaseConfig(path.join('buildbot-configs',
options.release_config))
for key in REQUIRED_CONFIG:
def testMakeHGUrlDifferentProtocol(self):
repo_url = make_hg_url("hg.mozilla.org",
"/build/tools",
protocol='ssh')
expected_url = "ssh://hg.mozilla.org/build/tools"
self.assertEquals(repo_url, expected_url)
def testMakeHGUrlNoFilename(self):
file_url = make_hg_url("hg.mozilla.org",
"/build/tools",
revision="default")
expected_url = "https://hg.mozilla.org/build/tools/rev/default"
self.assertEquals(file_url, expected_url)
def testMakeHGUrlNoRevisionNoFilename(self):
repo_url = make_hg_url("hg.mozilla.org", "/build/tools")
expected_url = "https://hg.mozilla.org/build/tools"
self.assertEquals(repo_url, expected_url)
def main(options):
log.info('Loading config from %s' % options.config)
config = load_config(options.config)
if config.getboolean('release-runner', 'verbose'):
log_level = logging.DEBUG
else:
log_level = logging.INFO
logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s",
level=log_level)
# Suppress logging of retry(), see bug 925321 for the details
logging.getLogger("util.retry").setLevel(logging.WARN)
check_buildbot()
check_fabric()
# Shorthand
sendchange_master = config.get('release-runner', 'sendchange_master')
api_root = config.get('api', 'api_root')
username = config.get('api', 'username')
password = config.get('api', 'password')
hg_host = config.get('release-runner', 'hg_host')
hg_username = config.get('release-runner', 'hg_username')
hg_ssh_key = config.get('release-runner', 'hg_ssh_key')
buildbot_configs = config.get('release-runner', 'buildbot_configs')
buildbot_configs_branch = config.get('release-runner',
'buildbot_configs_branch')
buildbotcustom = config.get('release-runner', 'buildbotcustom')
buildbotcustom_branch = config.get('release-runner',
'buildbotcustom_branch')
tools = config.get('release-runner', 'tools')
tools_branch = config.get('release-runner', 'tools_branch')
masters_json = config.get('release-runner', 'masters_json')
staging = config.getboolean('release-runner', 'staging')
sleeptime = config.getint('release-runner', 'sleeptime')
notify_from = get_config(config, 'release-runner', 'notify_from', None)
notify_to = get_config(config, 'release-runner', 'notify_to', None)
ssh_username = get_config(config, 'release-runner', 'ssh_username', None)
ssh_key = get_config(config, 'release-runner', 'ssh_key', None)
if isinstance(notify_to, basestring):
notify_to = [x.strip() for x in notify_to.split(',')]
smtp_server = get_config(config, 'release-runner', 'smtp_server',
'localhost')
configs_workdir = 'buildbot-configs'
custom_workdir = 'buildbotcustom'
tools_workdir = 'tools'
if "://" in buildbot_configs and not buildbot_configs.startswith("file"):
configs_pushRepo = make_hg_url(hg_host,
get_repo_path(buildbot_configs),
protocol='ssh')
else:
configs_pushRepo = buildbot_configs
if "://" in buildbotcustom and not buildbotcustom.startswith("file"):
custom_pushRepo = make_hg_url(hg_host,
get_repo_path(buildbotcustom),
protocol='ssh')
else:
custom_pushRepo = buildbotcustom
if "://" in tools and not tools.startswith("file"):
tools_pushRepo = make_hg_url(hg_host,
get_repo_path(tools),
protocol='ssh')
else:
tools_pushRepo = tools
rr = ReleaseRunner(api_root=api_root, username=username, password=password)
# Main loop waits for new releases, processes them and exits.
while True:
try:
log.debug('Fetching release requests')
rr.get_release_requests()
if rr.new_releases:
for release in rr.new_releases:
log.info('Got a new release request: %s' % release)
break
else:
log.debug('Sleeping for %d seconds before polling again' %
sleeptime)
time.sleep(sleeptime)
except Exception, e:
log.error("Caught exception when polling:", exc_info=True)
sys.exit(5)
def verify_mozconfigs(branch,
version,
hghost,
product,
platforms,
whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and compare to whitelist of known differences"""
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
types = {'+': 'release', '-': 'nightly'}
tag = ''.join(
[product.upper(), "_",
version.replace('.', '_'), "_RELEASE"])
for platform in platforms:
urls = []
mozconfigs = []
for type in types.values():
urls.append(
make_hg_url(
hghost, 'build/buildbot-configs', 'http', tag,
os.path.join('mozilla2', platform, branch, type,
'mozconfig')))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
if line[0] == '-' and mozconfigWhitelist.get(branch, {}).has_key(platform) \
and clean_line in mozconfigWhitelist[branch][platform]:
continue
if line[0] == '+' and mozconfigWhitelist.get('nightly', {}).has_key(platform) \
and clean_line in mozconfigWhitelist['nightly'][platform]:
continue
if line[0] == '-':
opposite = 'release'
else:
opposite = 'nightly'
log.error(
"not in %s mozconfig's whitelist (%s/%s/%s) : %s" %
(opposite, branch, platform, types[line[0]],
clean_line))
success = False
error_tally.add('verify_mozconfig')
else:
log.info("Missing mozconfigs to compare for %s" % platform)
error_tally.add(
"verify_mozconfigs: Confirm that %s does not have release/nightly mozconfigs to compare"
% platform)
return success
