def maybe_delete_repo(server, username, sshKey, repo, repoPath):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
try:
log.info("Trying to open %s" % repo_url)
urlopen(repo_url)
except URLError:
log.info("%s doesn't exist, not deleting" % reponame)
else:
log.info('Deleting %s' % reponame)
retry(run_remote_cmd,
args=('edit %s delete YES' % reponame, server, username, sshKey))
def maybe_delete_repo(server, username, sshKey, repo, repoPath):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
try:
log.info("Trying to open %s" % repo_url)
urlopen(repo_url)
except URLError:
log.info("%s doesn't exist, not deleting" % reponame)
else:
log.info('Deleting %s' % reponame)
retry(run_remote_cmd, args=('edit %s delete YES' % reponame, server,
username, sshKey))
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
configsToOverride):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
retry(mercurial, args=(repo_url, reponame))
def bump(repo, configsToBump, configsToOverride):
"""Process dynamic (version, buildNumber, etc.) variables in
configsToBump, then append overrides files to both configsToBump and
configsToOverride."""
# First pass. Bump variables in configsToBump.
configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
cmd.extend(configs)
run_cmd(cmd)
# Second pass. Append override files to configsToBump and
# configsToOverride.
for config, overrides in \
configsToBump.items() + configsToOverride.items():
newContent = cat([path.join(repo, config)] +
[path.join(repo, x) for x in overrides])
fh = open(path.join(repo, config), 'wb')
fh.write(newContent)
fh.close()
run_cmd(['hg', 'commit', '-m', 'Automatic config bump'], cwd=repo)
def bump_wrapper(r, n):
bump(r, configsToBump, configsToOverride)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, bump_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
tags = []
for configfile in configsToBump.keys():
config = readReleaseConfig(path.join(reponame, configfile))
tags.extend(
getTags(config['baseTag'], config['buildNumber'], buildTag=True))
return tags
def bump_configs(server, username, sshKey, repo, repoPath, configsToBump,
configsToOverride):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
protocol='ssh')
retry(mercurial, args=(repo_url, reponame))
def bump(repo, configsToBump, configsToOverride):
"""Process dynamic (version, buildNumber, etc.) variables in
configsToBump, then append overrides files to both configsToBump and
configsToOverride."""
# First pass. Bump variables in configsToBump.
configs = ['%s/%s' % (repo, x) for x in configsToBump.keys()]
cmd = ['python', BUMP_SCRIPT, '--bump-version', '--revision=tip']
cmd.extend(configs)
run_cmd(cmd)
# Second pass. Append override files to configsToBump and
# configsToOverride.
for config, overrides in \
configsToBump.items() + configsToOverride.items():
newContent = cat([path.join(repo, config)] +
[path.join(repo, x) for x in overrides])
fh = open(path.join(repo, config), 'wb')
fh.write(newContent)
fh.close()
run_cmd(['hg', 'commit', '-m', 'Automatic config bump'],
cwd=repo)
def bump_wrapper(r, n):
bump(r, configsToBump, configsToOverride)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, bump_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
tags = []
for configfile in configsToBump.keys():
config = readReleaseConfig(path.join(reponame, configfile))
tags.extend(getTags(config['baseTag'], config['buildNumber'],
buildTag=True))
return tags
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server, '%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def verify_mozconfigs(branch,
revision,
hghost,
product,
mozconfigs,
nightly_mozconfigs,
whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and
compare to whitelist of known differences"""
branch_name = get_repo_name(branch)
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
for platform, mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
nightly_mozconfig = nightly_mozconfigs[platform]
mozconfig_paths = [mozconfig, nightly_mozconfig]
# Create links to the two mozconfigs.
for c in mozconfig, nightly_mozconfig:
urls.append(make_hg_url(hghost, branch, 'http', revision, c))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
# Nothing to compare against
return False
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
message = ""
if line[0] == '-':
# handle lines that move around in diff
if '+' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if '-' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning(
"%s not in %s %s!" %
(clean_line, platform,
mozconfigWhitelist['nightly'][platform]))
else:
log.error("Skipping line %s!" % line)
continue
message = "found in %s but not in %s: %s"
if line[0] == '-':
log.error(message % (mozconfig_paths[0],
mozconfig_paths[1], clean_line))
else:
log.error(message % (mozconfig_paths[1],
mozconfig_paths[0], clean_line))
success = False
else:
log.info("Missing mozconfigs to compare for %s" % platform)
return False
return success
def clone_repo(server, username, sshKey, repo):
reponame = get_repo_name(repo)
log.info('Cloning %s to %s' % (repo, reponame))
retry(run_remote_cmd, args=('clone %s %s' % (reponame, repo),
server, username, sshKey))
def verify_mozconfigs(branch, revision, hghost, product, mozconfigs,
nightly_mozconfigs, whitelist=None):
"""Compare nightly mozconfigs for branch to release mozconfigs and
compare to whitelist of known differences"""
branch_name = get_repo_name(branch)
if whitelist:
mozconfigWhitelist = readConfig(whitelist, ['whitelist'])
else:
mozconfigWhitelist = {}
log.info("Comparing %s mozconfigs to nightly mozconfigs..." % product)
success = True
for platform, mozconfig in mozconfigs.items():
urls = []
mozconfigs = []
nightly_mozconfig = nightly_mozconfigs[platform]
mozconfig_paths = [mozconfig, nightly_mozconfig]
# Create links to the two mozconfigs.
for c in mozconfig, nightly_mozconfig:
urls.append(make_hg_url(hghost, branch, 'http', revision, c))
for url in urls:
try:
mozconfigs.append(urllib2.urlopen(url).readlines())
except urllib2.HTTPError as e:
log.error("MISSING: %s - ERROR: %s" % (url, e.msg))
# Nothing to compare against
return False
diffInstance = difflib.Differ()
if len(mozconfigs) == 2:
diffList = list(diffInstance.compare(mozconfigs[0], mozconfigs[1]))
for line in diffList:
clean_line = line[1:].strip()
if (line[0] == '-' or line[0] == '+') and len(clean_line) > 1:
# skip comment lines
if clean_line.startswith('#'):
continue
# compare to whitelist
message = ""
if line[0] == '-':
# handle lines that move around in diff
if '+' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get(branch_name, {}):
if clean_line in \
mozconfigWhitelist[branch_name][platform]:
continue
elif line[0] == '+':
if '-' + line[1:] in diffList:
continue
if platform in mozconfigWhitelist.get('nightly', {}):
if clean_line in \
mozconfigWhitelist['nightly'][platform]:
continue
else:
log.warning("%s not in %s %s!" % (
clean_line, platform,
mozconfigWhitelist['nightly'][platform]))
else:
log.error("Skipping line %s!" % line)
continue
message = "found in %s but not in %s: %s"
if line[0] == '-':
log.error(message % (mozconfig_paths[0],
mozconfig_paths[1], clean_line))
else:
log.error(message % (mozconfig_paths[1],
mozconfig_paths[0], clean_line))
success = False
else:
log.info("Missing mozconfigs to compare for %s" % platform)
return False
return success
def clone_repo(server, username, sshKey, repo):
reponame = get_repo_name(repo)
log.info('Cloning %s to %s' % (repo, reponame))
retry(run_remote_cmd,
args=('clone %s %s' % (reponame, repo), server, username, sshKey))