def clone_branch(branch, branch_url):
"""
Clone tip of the specified branch.
"""
remote = branch_url
# Set up the clean repository if it doesn't exist,
# otherwise, it will be updated.
clean = os.path.join('clean')
clean_repo = os.path.join(clean, branch)
if not os.access(clean, os.F_OK):
log_msg(os.getcwd())
os.mkdir(clean)
try:
mercurial(remote, clean_repo)
except subprocess.CalledProcessError as error:
log_msg('[Clone] error cloning \'%s\' into clean repository:\n%s'
% (remote, error))
return None
# Clone that clean repository to active and return that revision
active = os.path.join('active')
active_repo = os.path.join(active, branch)
if not os.access(active, os.F_OK):
os.mkdir(active)
elif os.access(active_repo, os.F_OK):
shutil.rmtree(active_repo)
try:
print 'Cloning from %s -----> %s' % (clean_repo, active_repo)
revision = mercurial(clean_repo, active_repo)
log_msg('[Clone] Cloned revision %s' %(revision), log.info)
except subprocess.CalledProcessError as error:
log_msg('[Clone] error cloning \'%s\' into active repository:\n%s'
% (remote, error))
return None
return revision
def testMercurialWithShareAndBundle(self):
# First create the bundle
bundle = os.path.join(self.tmpdir, 'bundle')
run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
# Create a commit
open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
# Wrap unbundle so we can tell if it got called
orig_unbundle = unbundle
try:
called = []
def new_unbundle(*args, **kwargs):
called.append(True)
return orig_unbundle(*args, **kwargs)
hg.unbundle = new_unbundle
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(
self.repodir, self.wc, shareBase=shareBase, bundles=[bundle])
self.assertEquals(called, [True])
self.assertEquals(
getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(
getRevisions(self.repodir), getRevisions(sharerepo))
finally:
hg.unbundle = orig_unbundle
def testMercurialChangeRepo(self):
# Create a new repo
old_env = os.environ.copy()
if 'HG_SHARE_BASE_DIR' in os.environ:
del os.environ['HG_SHARE_BASE_DIR']
try:
repo2 = os.path.join(self.tmpdir, 'repo2')
run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2],
env={'extra': 'change_repo'})
self.assertNotEqual(self.revisions, getRevisions(repo2))
# Clone the original repo
mercurial(self.repodir, self.wc)
self.assertEquals(getRevisions(self.wc), self.revisions)
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
# Clone the new one
mercurial(repo2, self.wc)
self.assertEquals(getRevisions(self.wc), getRevisions(repo2))
# Make sure our local file went away
self.failUnless(
not os.path.exists(os.path.join(self.wc, 'test.txt')))
finally:
os.environ.clear()
os.environ.update(old_env)
def testOutofSyncMirrorFailingMaster(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, "repo2")
clone(self.repodir, mirror)
shareBase = os.path.join(self.tmpdir, "share")
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror])
# Create a bundle
bundle = os.path.join(self.tmpdir, "bundle")
run_cmd(["hg", "bundle", "-a", bundle], cwd=self.repodir)
# Move our repodir out of the way so that pulling/cloning from it fails
os.rename(self.repodir, self.repodir + "-bad")
# Try and update to a non-existent revision using our mirror and
# bundle, with the master failing. We should fail
self.assertRaises(
subprocess.CalledProcessError,
mercurial,
self.repodir,
self.wc,
shareBase=shareBase,
mirrors=[mirror],
bundles=[bundle],
revision="1234567890",
)
def testMercurialChangeRepo(self):
# Create a new repo
old_env = os.environ.copy()
if 'HG_SHARE_BASE_DIR' in os.environ:
del os.environ['HG_SHARE_BASE_DIR']
try:
repo2 = os.path.join(self.tmpdir, 'repo2')
run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2])
self.assertNotEqual(self.revisions, getRevisions(repo2))
# Clone the original repo
mercurial(self.repodir, self.wc)
self.assertEquals(getRevisions(self.wc), self.revisions)
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
# Clone the new one
mercurial(repo2, self.wc)
self.assertEquals(getRevisions(self.wc), getRevisions(repo2))
# Make sure our local file went away
self.failUnless(
not os.path.exists(os.path.join(self.wc, 'test.txt')))
finally:
os.environ.clear()
os.environ.update(old_env)
def testMercurialWithShareAndBundle(self):
# First create the bundle
bundle = os.path.join(self.tmpdir, "bundle")
run_cmd(["hg", "bundle", "-a", bundle], cwd=self.repodir)
# Create a commit
open(os.path.join(self.repodir, "test.txt"), "w").write("hello!")
run_cmd(["hg", "add", "test.txt"], cwd=self.repodir)
run_cmd(["hg", "commit", "-m", "adding changeset"], cwd=self.repodir)
# Wrap unbundle so we can tell if it got called
orig_unbundle = unbundle
try:
called = []
def new_unbundle(*args, **kwargs):
called.append(True)
return orig_unbundle(*args, **kwargs)
hg.unbundle = new_unbundle
shareBase = os.path.join(self.tmpdir, "share")
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, bundles=[bundle])
self.assertEquals(called, [True])
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
finally:
hg.unbundle = orig_unbundle
def testOutofSyncMirrorFailingMaster(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, 'repo2')
clone(self.repodir, mirror)
shareBase = os.path.join(self.tmpdir, 'share')
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror])
# Create a bundle
bundle = os.path.join(self.tmpdir, 'bundle')
run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
# Move our repodir out of the way so that pulling/cloning from it fails
os.rename(self.repodir, self.repodir + "-bad")
# Try and update to a non-existent revision using our mirror and
# bundle, with the master failing. We should fail
self.assertRaises(subprocess.CalledProcessError,
mercurial,
self.repodir,
self.wc,
shareBase=shareBase,
mirrors=[mirror],
bundles=[bundle],
revision="1234567890")
def clone_branch(branch):
"""
Clone the tip of the specified branch.
"""
remote = '%s%s' % (config['hg_base_url'], branch)
# Set up the local/clean repository if it doesn't exist,
# otherwise, it will update.
clean = 'clean/%s' % (branch)
if not os.access('clean', os.F_OK):
os.mkdir('clean')
try:
mercurial(remote, clean)
except subprocess.CalledProcessError as error:
log_msg('[Clone] error cloning \'%s\' into local repository :\n%s'
%(remote,error))
return None
# Clone that local repository and return that revision
active = 'active/%s' % (branch)
if not os.access('active', os.F_OK):
os.mkdir('active')
elif os.access(active, os.F_OK):
shutil.rmtree(active)
try:
revision = mercurial(clean, active)
log_msg('[Clone] Cloned revision %s' % (revision))
except subprocess.CalledProcessError as error:
log_msg('[Clone] error cloning \'%s\' into active repository :\n%s'
%(remote,error))
return None
return revision
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out, kwargs=dict(src=reponame, remote=remote,
ssh_username=config[
'hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push, cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def testMercurialWithNewShare(self):
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
def tagOtherRepo(config, repo, reponame, revision, pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def tagRepo(repo, attempt, config, revision, tags):
# set totalChangesets=1 because tag() generates exactly 1 commit
totalChangesets = 1
# update to the desired revision first, then to the tip of revision's
# branch to avoid new head creation
update(repo, revision=revision)
update(repo)
tag(repo, revision, tags, config['hgUsername'])
outgoingRevs = retry(out,
kwargs=dict(src=reponame,
remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
if len(outgoingRevs) != totalChangesets:
raise Exception("Wrong number of outgoing revisions")
pushRepo = make_hg_url(HG, repo, protocol='ssh')
def tag_wrapper(r, n):
tagRepo(r, n, config, revision, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, config['hgUsername'],
config['hgSshKey'])
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, tag_wrapper, pushAttempts),
kwargs=dict(ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey']))
def testMercurialWithNewShare(self):
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
def testMercurialSkipPull(self):
# Clone once into our working copy
mercurial(self.repodir, self.wc)
# The second clone should avoid calling pull()
with patch('util.hg.pull') as patched_pull:
mercurial(self.repodir, self.wc, revision=self.revisions[-1])
self.assertEquals(patched_pull.call_count, 0)
def testMercurialSkipPull(self):
# Clone once into our working copy
mercurial(self.repodir, self.wc)
# The second clone should avoid calling pull()
with patch('util.hg.pull') as patched_pull:
mercurial(self.repodir, self.wc, revision=self.revisions[-1])
self.assertEquals(patched_pull.call_count, 0)
def testMercurialUpdateRev(self):
rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1])
self.assertEquals(rev, self.revisions[-1])
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
rev = mercurial(self.repodir, self.wc, revision=self.revisions[0])
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
def testMercurialUpdateRev(self):
rev = mercurial(self.repodir, self.wc, revision=self.revisions[-1])
self.assertEquals(rev, self.revisions[-1])
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
rev = mercurial(self.repodir, self.wc, revision=self.revisions[0])
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts, defaultBranch='default'):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags,
defaultBranch):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
defaultBranchChangesets = 0
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# This is the bump of the version on the default branch
# We do it after the other one in order to get the tip of the
# repository back on default, thus avoiding confusion.
if len(bumpFiles) > 0:
update(reponame, revision=defaultBranch)
bump(reponame, bumpFiles, 'nextVersion')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
defaultBranchChangesets += 1
except subprocess.CalledProcessError, e:
if e.returncode != 1 or "nothing changed" not in e.output:
raise
def testMercurialWithExistingShare(self):
shareBase = os.path.join(self.tmpdir, "share")
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, sharerepo)
open(os.path.join(self.repodir, "test.txt"), "w").write("hello!")
run_cmd(["hg", "add", "test.txt"], cwd=self.repodir)
run_cmd(["hg", "commit", "-m", "adding changeset"], cwd=self.repodir)
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
def testMercurialWithShareBaseInEnv(self):
shareBase = os.path.join(self.tmpdir, "share")
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
try:
os.environ["HG_SHARE_BASE_DIR"] = shareBase
mercurial(self.repodir, self.wc)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
finally:
del os.environ["HG_SHARE_BASE_DIR"]
def testMercurialWithExistingShare(self):
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, sharerepo)
open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
def testMercurialWithExistingShare(self):
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, sharerepo)
open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir), getRevisions(sharerepo))
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output([
'hg', 'commit', '-u', config['hgUsername'], '-m',
getBumpCommitMessage(config['productName'],
config['version'])
],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# Validate that the repository is only different from the remote in
# ways we expect.
outgoingRevs = out(src=reponame,
remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey'])
if len([r for r in outgoingRevs if r[BRANCH] == relbranch
]) != relbranchChangesets:
raise Exception("Incorrect number of revisions on %s" % relbranch)
if len(outgoingRevs) != relbranchChangesets:
raise Exception("Wrong number of outgoing revisions")
def testMercurialByRevWithShareAndMirror(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, 'repo2')
clone(self.repodir, mirror)
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror], clone_by_rev=True, revision=self.revisions[-1])
# We should only have the one revision
self.assertEquals(getRevisions(sharerepo), self.revisions[-1:])
self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
def testMercurialWithShareBaseInEnv(self):
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
try:
os.environ['HG_SHARE_BASE_DIR'] = shareBase
mercurial(self.repodir, self.wc)
self.assertEquals(getRevisions(self.repodir),
getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir),
getRevisions(sharerepo))
finally:
del os.environ['HG_SHARE_BASE_DIR']
def testMercurialShareOutgoing(self):
# ensure that outgoing changesets in a shared clone affect the shared history
repo5 = os.path.join(self.tmpdir, 'repo5')
repo6 = os.path.join(self.tmpdir, 'repo6')
mercurial(self.repodir, repo5)
share(repo5, repo6)
open(os.path.join(repo6, 'test.txt'), 'w').write("hello!")
# modify the history of the new clone
run_cmd(['hg', 'add', 'test.txt'], cwd=repo6)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=repo6)
self.assertNotEquals(self.revisions, getRevisions(repo6))
self.assertNotEquals(self.revisions, getRevisions(repo5))
self.assertEquals(getRevisions(repo5), getRevisions(repo6))
def testMercurialRelativeDir(self):
os.chdir(os.path.dirname(self.repodir))
repo = os.path.basename(self.repodir)
wc = os.path.basename(self.wc)
rev = mercurial(repo, wc, revision=self.revisions[-1])
self.assertEquals(rev, self.revisions[-1])
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
rev = mercurial(repo, wc)
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
def testMercurialShareOutgoing(self):
# ensure that outgoing changesets in a shared clone affect the shared
# history
repo5 = os.path.join(self.tmpdir, "repo5")
repo6 = os.path.join(self.tmpdir, "repo6")
mercurial(self.repodir, repo5)
share(repo5, repo6)
open(os.path.join(repo6, "test.txt"), "w").write("hello!")
# modify the history of the new clone
run_cmd(["hg", "add", "test.txt"], cwd=repo6)
run_cmd(["hg", "commit", "-m", "adding changeset"], cwd=repo6)
self.assertNotEquals(self.revisions, getRevisions(repo6))
self.assertNotEquals(self.revisions, getRevisions(repo5))
self.assertEquals(getRevisions(repo5), getRevisions(repo6))
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni,
revision="default", merge=True):
mercurial(repo, "compare-locales")
update("compare-locales", revision=revision)
mergeDir = path.join(localeSrcDir, "merged")
if path.exists(mergeDir):
log.info("Deleting %s" % mergeDir)
shutil.rmtree(mergeDir)
run_cmd(["python", path.join("compare-locales", "scripts",
"compare-locales"),
"-m", mergeDir,
l10nIni,
l10nRepoDir, locale],
env={"PYTHONPATH": path.join("compare-locales", "lib")})
def testMercurialShareOutgoing(self):
# ensure that outgoing changesets in a shared clone affect the shared
# history
repo5 = os.path.join(self.tmpdir, 'repo5')
repo6 = os.path.join(self.tmpdir, 'repo6')
mercurial(self.repodir, repo5)
share(repo5, repo6)
open(os.path.join(repo6, 'test.txt'), 'w').write("hello!")
# modify the history of the new clone
run_cmd(['hg', 'add', 'test.txt'], cwd=repo6)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=repo6)
self.assertNotEquals(self.revisions, getRevisions(repo6))
self.assertNotEquals(self.revisions, getRevisions(repo5))
self.assertEquals(getRevisions(repo5), getRevisions(repo6))
def testMercurialRelativeDir(self):
os.chdir(os.path.dirname(self.repodir))
repo = os.path.basename(self.repodir)
wc = os.path.basename(self.wc)
rev = mercurial(repo, wc, revision=self.revisions[-1])
self.assertEquals(rev, self.revisions[-1])
open(os.path.join(self.wc, 'test.txt'), 'w').write("hello!")
rev = mercurial(repo, wc)
self.assertEquals(rev, self.revisions[0])
# Make sure our local file didn't go away
self.failUnless(os.path.exists(os.path.join(self.wc, 'test.txt')))
def compareLocales(repo, locale, l10nRepoDir, localeSrcDir, l10nIni,
revision="default", merge=True):
mercurial(repo, "compare-locales")
update("compare-locales", revision=revision)
mergeDir = path.join(localeSrcDir, "merged")
if path.exists(mergeDir):
log.info("Deleting %s" % mergeDir)
shutil.rmtree(mergeDir)
run_cmd(["python", path.join("compare-locales", "scripts",
"compare-locales"),
"-m", mergeDir,
l10nIni,
l10nRepoDir, locale],
env={"PYTHONPATH": path.join("compare-locales", "lib")})
def createRepacks(sourceRepo, l10nRepoDir, l10nBaseRepo, mozconfigPath, objdir,
makeDirs, locales, ftpProduct, stageServer, stageUsername,
stageSshKey, compareLocalesRepo, merge, platform,
stage_platform, mobileDirName, en_us_binary_url):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR":
objdir,
"UPLOAD_HOST":
stageServer,
"UPLOAD_USER":
stageUsername,
"UPLOAD_SSH_KEY":
stageSshKey,
"UPLOAD_TO_TEMP":
"1",
"EN_US_BINARY_URL":
en_us_binary_url,
# Android signing
"JARSIGNER":
os.path.join(os.getcwd(), "scripts", "release", "signing",
"mozpass.py")
}
build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName)
mercurial(sourceRepo, sourceRepoName)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, l10nRepoDir,
makeDirs, localeSrcDir, env)
buildInfo = downloadNightlyBuild(localeSrcDir, env)
run_cmd(["hg", "update", "-r", buildInfo["gecko_revision"]],
cwd=sourceRepoName)
env["POST_UPLOAD_CMD"] = postUploadCmdPrefix(
to_latest=True,
branch="%s-%s-l10n" % (sourceRepoName, stage_platform),
product=ftpProduct)
err = False
for l in locales:
try:
repackLocale(l, l10nRepoDir, l10nBaseRepo, "default", localeSrcDir,
l10nIni, compareLocalesRepo, env, merge)
except Exception, e:
err = True
log.error("Error creating locale '%s': %s", l, e)
pass
def tagRepo(config, repo, reponame, revision, tags, bumpFiles, relbranch,
pushAttempts):
remote = make_hg_url(HG, repo)
mercurial(remote, reponame)
def bump_and_tag(repo, attempt, config, relbranch, revision, tags):
# set relbranchChangesets=1 because tag() generates exactly 1 commit
relbranchChangesets = 1
if relbranch in get_branches(reponame):
update(reponame, revision=relbranch)
else:
update(reponame, revision=revision)
run_cmd(['hg', 'branch', relbranch], cwd=reponame)
if len(bumpFiles) > 0:
# Bump files on the relbranch, if necessary
bump(reponame, bumpFiles, 'version')
run_cmd(['hg', 'diff'], cwd=repo)
try:
get_output(['hg', 'commit', '-u', config['hgUsername'],
'-m', getBumpCommitMessage(config['productName'], config['version'])],
cwd=reponame)
relbranchChangesets += 1
except subprocess.CalledProcessError, e:
# We only want to ignore exceptions caused by having nothing to
# commit, which are OK. We still want to raise exceptions caused
# by any other thing.
if e.returncode != 1 or "nothing changed" not in e.output:
raise
# We always want our tags pointing at the tip of the relbranch
# so we need to grab the current revision after we've switched
# branches and bumped versions.
revision = get_revision(reponame)
# Create the desired tags on the relbranch
tag(repo, revision, tags, config['hgUsername'])
# Validate that the repository is only different from the remote in
# ways we expect.
outgoingRevs = out(src=reponame, remote=remote,
ssh_username=config['hgUsername'],
ssh_key=config['hgSshKey'])
if len([r for r in outgoingRevs if r[BRANCH] == relbranch]) != relbranchChangesets:
raise Exception("Incorrect number of revisions on %s" % relbranch)
if len(outgoingRevs) != relbranchChangesets:
raise Exception("Wrong number of outgoing revisions")
def testBustedHgrc(self):
# Test that we can recover from hgrc being lost
mercurial(self.repodir, self.wc)
# Delete .hg/hgrc
os.unlink(os.path.join(self.wc, '.hg', 'hgrc'))
# path is busted now
p = path(self.wc)
self.assertEquals(p, None)
# cloning again should fix this up
mercurial(self.repodir, self.wc)
p = path(self.wc)
self.assertEquals(p, self.repodir)
def testShareUnrelated(self):
# Create a new repo
repo2 = os.path.join(self.tmpdir, 'repo2')
run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2])
self.assertNotEqual(self.revisions, getRevisions(repo2))
shareBase = os.path.join(self.tmpdir, 'share')
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
# Clone the new repo
mercurial(repo2, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.wc), getRevisions(repo2))
def testShareUnrelated(self):
# Create a new repo
repo2 = os.path.join(self.tmpdir, 'repo2')
run_cmd(['%s/init_hgrepo.sh' % os.path.dirname(__file__), repo2])
self.assertNotEqual(self.revisions, getRevisions(repo2))
shareBase = os.path.join(self.tmpdir, 'share')
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
# Clone the new repo
mercurial(repo2, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.wc), getRevisions(repo2))
def testBustedHgrc(self):
# Test that we can recover from hgrc being lost
mercurial(self.repodir, self.wc)
# Delete .hg/hgrc
os.unlink(os.path.join(self.wc, '.hg', 'hgrc'))
# path is busted now
p = path(self.wc)
self.assertEquals(p, None)
# cloning again should fix this up
mercurial(self.repodir, self.wc)
p = path(self.wc)
self.assertEquals(p, self.repodir)
def clone_branch(branch, branch_url):
"""
Clone tip of the specified branch.
"""
remote = branch_url
# Set up the clean repository if it doesn't exist,
# otherwise, it will be updated.
clean_path = os.path.join(config['work_dir'], 'clean')
clean_repo = os.path.join(clean_path, branch)
if not os.path.isdir(clean_path):
os.mkdir(clean_path)
try:
mercurial(remote, clean_repo, update_dest=False)
except subprocess.CalledProcessError, err:
log.error('[Clone] error cloning \'%s\' into clean repository:\n%s'
% (remote, err))
raise RetryException
return None
def testMercurialByRevWithShareAndMirror(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, 'repo2')
clone(self.repodir, mirror)
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir,
self.wc,
shareBase=shareBase,
mirrors=[mirror],
clone_by_rev=True,
revision=self.revisions[-1])
# We should only have the one revision
self.assertEquals(getRevisions(sharerepo), self.revisions[-1:])
self.assertEquals(getRevisions(self.wc), self.revisions[-1:])
def testShareReset(self):
shareBase = os.path.join(self.tmpdir, "share")
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
old_revs = self.revisions[:]
# Reset the repo
run_cmd(["%s/init_hgrepo.sh" % os.path.dirname(__file__), self.repodir])
self.assertNotEqual(old_revs, getRevisions(self.repodir))
# Try and update our working copy
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertNotEqual(old_revs, getRevisions(self.wc))
def clone_branch(branch, branch_url):
"""
Clone tip of the specified branch.
"""
remote = branch_url
# Set up the clean repository if it doesn't exist,
# otherwise, it will be updated.
clean_path = os.path.join(config['work_dir'], 'clean')
clean_repo = os.path.join(clean_path, branch)
if not os.path.isdir(clean_path):
os.mkdir(clean_path)
try:
mercurial(remote, clean_repo, update_dest=False)
except subprocess.CalledProcessError, err:
log.error('[Clone] error cloning \'%s\' into clean repository:\n%s' %
(remote, err))
raise RetryException
return None
def createRepacks(sourceRepo,l10nRepoDir, l10nBaseRepo,
mozconfigPath, objdir, makeDirs, locales, ftpProduct,
stageServer, stageUsername, stageSshKey, compareLocalesRepo,
merge, platform, stage_platform, mobileDirName,
en_us_binary_url):
sourceRepoName = path.split(sourceRepo)[-1]
localeSrcDir = path.join(sourceRepoName, objdir, mobileDirName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, mobileDirName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"EN_US_BINARY_URL": en_us_binary_url,
# Android signing
"JARSIGNER": os.path.join(os.getcwd(), "scripts", "release",
"signing", "mozpass.py")
}
build.misc.cleanupObjdir(sourceRepoName, objdir, mobileDirName)
mercurial(sourceRepo, sourceRepoName)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath,
l10nRepoDir, makeDirs, localeSrcDir, env)
buildInfo = downloadNightlyBuild(localeSrcDir, env)
run_cmd(["hg", "update", "-r", buildInfo["gecko_revision"]],
cwd=sourceRepoName)
env["POST_UPLOAD_CMD"] = postUploadCmdPrefix(
to_latest=True,
branch="%s-%s-l10n" % (sourceRepoName, stage_platform),
product=ftpProduct
)
err = False
for l in locales:
try:
repackLocale(l, l10nRepoDir, l10nBaseRepo, "default",
localeSrcDir, l10nIni, compareLocalesRepo, env, merge)
except Exception, e:
err = True
log.error("Error creating locale '%s': %s", l, e)
pass
def testMercurialWithShareAndMirror(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, 'repo2')
clone(self.repodir, mirror)
# Create a commit
open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror])
# Since we used the mirror, we should be missing a commit
self.assertNotEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertNotEquals(getRevisions(self.repodir), getRevisions(sharerepo))
self.assertEquals(getRevisions(mirror), getRevisions(self.wc))
def testMercurialWithShareAndMirror(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, "repo2")
clone(self.repodir, mirror)
# Create a commit
open(os.path.join(self.repodir, "test.txt"), "w").write("hello!")
run_cmd(["hg", "add", "test.txt"], cwd=self.repodir)
run_cmd(["hg", "commit", "-m", "adding changeset"], cwd=self.repodir)
shareBase = os.path.join(self.tmpdir, "share")
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror])
# Since we used the mirror, we should be missing a commit
self.assertNotEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertNotEquals(getRevisions(self.repodir), getRevisions(sharerepo))
self.assertEquals(getRevisions(mirror), getRevisions(self.wc))
def testShareReset(self):
shareBase = os.path.join(self.tmpdir, 'share')
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
old_revs = self.revisions[:]
# Reset the repo
run_cmd(
['%s/init_hgrepo.sh' % os.path.dirname(__file__), self.repodir])
self.assertNotEqual(old_revs, getRevisions(self.repodir))
# Try and update our working copy
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertNotEqual(old_revs, getRevisions(self.wc))
def testShareExtraFilesReset(self):
shareBase = os.path.join(self.tmpdir, "share")
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
# Reset the repo
run_cmd(["%s/init_hgrepo.sh" % os.path.dirname(__file__), self.repodir])
# Make the working repo have a new file. We need it to have an earlier
# timestamp to trigger the odd behavior in hg, so use '-d yesterday'
run_cmd(["touch", "-d", "yesterday", "newfile"], cwd=self.wc)
run_cmd(["hg", "add", "newfile"], cwd=self.wc)
run_cmd(["hg", "commit", "-m", '"add newfile"'], cwd=self.wc)
# Try and update our working copy
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertFalse(os.path.exists(os.path.join(self.wc, "newfile")))
def testAdjustPaths(self):
mercurial(self.repodir, self.wc)
# Make sure our default path is correct
self.assertEquals(path(self.wc), self.repodir)
# Add a comment, make sure it's still there if we don't change
# anything
hgrc = os.path.join(self.wc, '.hg', 'hgrc')
open(hgrc, 'a').write("# Hello world")
adjust_paths(self.wc, default=self.repodir)
self.assert_("Hello world" in open(hgrc).read())
# Add a path, and the comment goes away
adjust_paths(self.wc, test=self.repodir)
self.assert_("Hello world" not in open(hgrc).read())
# Make sure our paths are correct
self.assertEquals(path(self.wc), self.repodir)
self.assertEquals(path(self.wc, 'test'), self.repodir)
def testShareExtraFilesReset(self):
shareBase = os.path.join(self.tmpdir, 'share')
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
# Reset the repo
run_cmd(
['%s/init_hgrepo.sh' % os.path.dirname(__file__), self.repodir])
# Make the working repo have a new file. We need it to have an earlier
# timestamp to trigger the odd behavior in hg, so use '-d yesterday'
run_cmd(['touch', '-d', 'yesterday', 'newfile'], cwd=self.wc)
run_cmd(['hg', 'add', 'newfile'], cwd=self.wc)
run_cmd(['hg', 'commit', '-m', '"add newfile"'], cwd=self.wc)
# Try and update our working copy
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertFalse(os.path.exists(os.path.join(self.wc, 'newfile')))
def testAdjustPaths(self):
mercurial(self.repodir, self.wc)
# Make sure our default path is correct
self.assertEquals(path(self.wc), self.repodir)
# Add a comment, make sure it's still there if we don't change
# anything
hgrc = os.path.join(self.wc, '.hg', 'hgrc')
open(hgrc, 'a').write("# Hello world")
adjust_paths(self.wc, default=self.repodir)
self.assert_("Hello world" in open(hgrc).read())
# Add a path, and the comment goes away
adjust_paths(self.wc, test=self.repodir)
self.assert_("Hello world" not in open(hgrc).read())
# Make sure our paths are correct
self.assertEquals(path(self.wc), self.repodir)
self.assertEquals(path(self.wc, 'test'), self.repodir)
def testMercurialWithShareAndMirror(self):
# First create the mirror
mirror = os.path.join(self.tmpdir, 'repo2')
clone(self.repodir, mirror)
# Create a commit
open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase, mirrors=[mirror])
# Since we used the mirror, we should be missing a commit
self.assertNotEquals(getRevisions(self.repodir), getRevisions(self.wc))
self.assertNotEquals(getRevisions(self.repodir),
getRevisions(sharerepo))
self.assertEquals(getRevisions(mirror), getRevisions(self.wc))
def testShareExtraFilesReset(self):
shareBase = os.path.join(self.tmpdir, 'share')
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
# Reset the repo
run_cmd(
['%s/init_hgrepo.sh' % os.path.dirname(__file__), self.repodir])
# Make the working repo have a new file. We need it to have an earlier
# timestamp to trigger the odd behavior in hg, so use '-d yesterday'
run_cmd(['touch', '-d', 'yesterday', 'newfile'], cwd=self.wc)
run_cmd(['hg', 'add', 'newfile'], cwd=self.wc)
run_cmd(['hg', 'commit', '-m', '"add newfile"'], cwd=self.wc)
# Try and update our working copy
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertFalse(os.path.exists(os.path.join(self.wc, 'newfile')))
def testBustedHgrcWithShare(self):
# Test that we can recover from hgrc being lost
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir, self.wc, shareBase=shareBase)
# Delete .hg/hgrc
for d in sharerepo, self.wc:
f = os.path.join(d, '.hg', 'hgrc')
os.unlink(f)
# path is busted now
p = path(self.wc)
self.assertEquals(p, None)
# cloning again should fix this up
mercurial(self.repodir, self.wc, shareBase=shareBase)
p = path(self.wc)
self.assertEquals(p, self.repodir)
def testShareExtraFiles(self):
shareBase = os.path.join(self.tmpdir, 'share')
backup = os.path.join(self.tmpdir, 'backup')
# Clone the original repo
mercurial(self.repodir, self.wc, shareBase=shareBase)
clone(self.repodir, backup)
# Make the working repo have a new file. We need it to have an earlier
# timestamp (yesterday) to trigger the odd behavior in hg
newfile = os.path.join(self.wc, 'newfile')
touch(newfile, timestamp=yesterday_timestamp())
run_cmd(['hg', 'add', 'newfile'], cwd=self.wc)
run_cmd(['hg', 'commit', '-m', '"add newfile"'], cwd=self.wc)
# Reset the share base to remove the 'add newfile' commit. We
# overwrite repodir with the backup that doesn't have the commit,
# then clone the repodir to a throwaway dir to create the new
# shareBase. Now self.wc still points to shareBase, but the
# changeset that self.wc was on is lost.
shutil.rmtree(self.repodir)
shutil.rmtree(shareBase)
clone(backup, self.repodir)
throwaway = os.path.join(self.tmpdir, 'throwaway')
mercurial(self.repodir, throwaway, shareBase=shareBase)
# Try and update our working copy
mercurial(self.repodir, self.wc, shareBase=shareBase)
self.assertFalse(os.path.exists(os.path.join(self.wc, 'newfile')))
def tag_repo(server, username, sshKey, repo, repoPath, tags):
reponame = get_repo_name(repo)
repo_url = make_hg_url(server, '%s/%s' % (repoPath, reponame))
pushRepo = make_hg_url(server,
'%s/%s' % (repoPath, reponame),
protocol='ssh')
mercurial(repo_url, reponame)
def do_tag(repo, tags):
cmd = ['hg', 'tag', '-f', '-m', 'Automatic preproduction tag'] + tags
run_cmd(cmd, cwd=repo)
def do_tag_wrapper(r, n):
do_tag(r, tags)
def cleanup_wrapper():
cleanOutgoingRevs(reponame, pushRepo, username, sshKey)
retry(apply_and_push,
cleanup=cleanup_wrapper,
args=(reponame, pushRepo, do_tag_wrapper),
kwargs=dict(ssh_username=username, ssh_key=sshKey))
def testMercurialWithShareAndBundle(self):
# First create the bundle
bundle = os.path.join(self.tmpdir, 'bundle')
run_cmd(['hg', 'bundle', '-a', bundle], cwd=self.repodir)
# Create a commit
open(os.path.join(self.repodir, 'test.txt'), 'w').write('hello!')
run_cmd(['hg', 'add', 'test.txt'], cwd=self.repodir)
run_cmd(['hg', 'commit', '-m', 'adding changeset'], cwd=self.repodir)
# Wrap unbundle so we can tell if it got called
orig_unbundle = unbundle
try:
called = []
def new_unbundle(*args, **kwargs):
called.append(True)
return orig_unbundle(*args, **kwargs)
hg.unbundle = new_unbundle
shareBase = os.path.join(self.tmpdir, 'share')
sharerepo = os.path.join(shareBase, self.repodir.lstrip("/"))
os.mkdir(shareBase)
mercurial(self.repodir,
self.wc,
shareBase=shareBase,
bundles=[bundle])
self.assertEquals(called, [True])
self.assertEquals(getRevisions(self.repodir),
getRevisions(self.wc))
self.assertEquals(getRevisions(self.repodir),
getRevisions(sharerepo))
finally:
hg.unbundle = orig_unbundle
mercurial(remote, clean_repo, update_dest=False)
except subprocess.CalledProcessError, err:
log.error('[Clone] error cloning \'%s\' into clean repository:\n%s' %
(remote, err))
raise RetryException
return None
# Clone that clean repository to active and return that revision
active = os.path.join('active')
active_repo = os.path.join(active, branch)
if not os.path.isdir(active):
os.mkdir(active)
elif os.path.isdir(active_repo):
shutil.rmtree(active_repo)
try:
log.info('Cloning from %s -----> %s' % (clean_repo, active_repo))
revision = mercurial(clean_repo, active_repo)
log.info('[Clone] Cloned revision %s' % (revision))
except subprocess.CalledProcessError, err:
log.error('[Clone] error cloning \'%s\' into active repository:\n%s' %
(remote, err))
raise RetryException
return None
return revision
def valid_dictionary_structure(dict_, elements):
"""
Check that the given dictionary contains all elements.
"""
for element in elements:
action="store_true",
default=False,
help="Tag the source repo(s).")
parser.add_option("--tag-l10n",
dest="tag_l10n",
action="store_true",
default=False,
help="Tag the L10n repo(s).")
parser.add_option("--tag-other",
dest="tag_other",
action="store_true",
default=False,
help="Tag the other repo(s).")
options, args = parser.parse_args()
mercurial(options.buildbot_configs, 'buildbot-configs')
update('buildbot-configs', revision=options.release_tag)
config = validate(options, args)
configDir = path.dirname(options.configfile)
# We generate this upfront to ensure that it's consistent throughout all
# repositories that use it. However, in cases where a relbranch is provided
# for all repositories, it will not be used
generatedRelbranch = generateRelbranchName(config['version'])
if config.get('relbranchPrefix'):
generatedRelbranch = generateRelbranchName(
config['version'], prefix=config['relbranchPrefix'])
tags = getTags(config['baseTag'], config['buildNumber'])
l10nRevisionFile = path.join('buildbot-configs', configDir,
config['l10nRevisionFile'])
l10nRepos = getL10nRepositories(
dest="tooltool_script",
default=[],
action="append")
parser.add_option("--tooltool-url", dest="tooltool_urls", action="append")
parser.add_option("--use-pymake",
dest="use_pymake",
action="store_true",
default=False)
# todo: maybe read these from branch/release config? is that even possible for credentials file?
parser.add_option("--balrog-api-root", dest="balrog_api_root")
parser.add_option("--credentials-file", dest="credentials_file")
parser.add_option("--balrog-username", dest="balrog_username")
parser.add_option("--bucket-prefix", dest="bucket_prefix")
options, args = parser.parse_args()
mercurial(options.buildbotConfigs, "buildbot-configs")
update("buildbot-configs", revision=options.releaseTag)
sys.path.append(os.getcwd())
branchConfig, releaseConfig = validate(options, args)
sourceRepoInfo = releaseConfig["sourceRepositories"][
options.source_repo_key]
try:
brandName = releaseConfig["brandName"]
except KeyError:
brandName = releaseConfig["productName"].title()
platform = options.platform
if platform == "linux":
platform = "linux32"
mozconfig = path.join(get_repo_dirname(sourceRepoInfo["path"]),
def createRepacks(sourceRepo,
revision,
l10nRepoDir,
l10nBaseRepo,
mozconfigPath,
srcMozconfigPath,
objdir,
makeDirs,
appName,
locales,
product,
version,
buildNumber,
stageServer,
stageUsername,
stageSshKey,
ftpServer,
compareLocalesRepo,
merge,
platform,
brand,
appVersion,
generatePartials=False,
partialUpdates=None,
usePymake=False,
tooltoolManifest=None,
tooltool_script=None,
tooltool_urls=None,
balrog_submitter=None,
balrog_hash="sha512",
mozillaDir=None,
mozillaSrcDir=None,
bucket_prefix=None):
buildid = retry(getBuildID,
args=(platform, product, version, buildNumber,
'candidates', ftpServer))
log.info('Got buildid: %s' % buildid)
sourceRepoName = path.split(sourceRepo)[-1]
absObjdir = path.abspath(path.join(sourceRepoName, objdir))
localeSrcDir = path.join(absObjdir, appName, "locales")
# Even on Windows we need to use "/" as a separator for this because
# compare-locales doesn"t work any other way
l10nIni = "/".join([sourceRepoName, appName, "locales", "l10n.ini"])
env = {
"MOZ_OBJDIR": objdir,
"MOZ_MAKE_COMPLETE_MAR": "1",
"DOWNLOAD_HOST": ftpServer,
"UPLOAD_HOST": stageServer,
"UPLOAD_USER": stageUsername,
"UPLOAD_SSH_KEY": stageSshKey,
"UPLOAD_TO_TEMP": "1",
"MOZ_PKG_PRETTYNAMES": "1",
"MOZILLA_REV": os.getenv('MOZILLA_REV', ''),
"COMM_REV": os.getenv('COMM_REV', ''),
"LD_LIBRARY_PATH": os.getenv("LD_LIBRARY_PATH", ""),
"MBSDIFF_HOOK": os.getenv("MBSDIFF_HOOK", ""),
}
if appVersion is None or version != appVersion:
env["MOZ_PKG_VERSION"] = version
signed = False
if os.environ.get('MOZ_SIGN_CMD'):
env['MOZ_SIGN_CMD'] = os.environ['MOZ_SIGN_CMD']
signed = True
env['POST_UPLOAD_CMD'] = postUploadCmdPrefix(
to_candidates=True,
product=product,
version=version,
buildNumber=buildNumber,
signed=signed,
bucket_prefix=bucket_prefix,
)
if usePymake:
env['USE_PYMAKE'] = "1"
env['MOZILLA_OFFICIAL'] = "1"
env["MOZ_SIGN_CMD"] = "python " + \
path.join(os.getcwd(), "scripts", "release", "signing", "signtool.py").replace('\\', '\\\\\\\\') + \
" --cachedir " + \
path.join(os.getcwd(), "signing_cache").replace('\\', '\\\\\\\\') + \
" -t " + \
path.join(os.getcwd(), "token").replace('\\', '\\\\\\\\') + \
" -n " + \
path.join(os.getcwd(), "nonce").replace('\\', '\\\\\\\\') + \
" -c " + \
path.join(os.getcwd(), "scripts", "release", "signing", "host.cert").replace('\\', '\\\\\\\\')
signingServers = os.environ["MOZ_SIGN_CMD"].split("-H",
1)[1].split("-H")
for s in signingServers:
env["MOZ_SIGN_CMD"] += " -H %s" % s.strip()
build.misc.cleanupObjdir(sourceRepoName, objdir, appName)
mercurial(sourceRepo, sourceRepoName)
update(sourceRepoName, revision=revision)
l10nRepackPrep(sourceRepoName, objdir, mozconfigPath, srcMozconfigPath,
l10nRepoDir, makeDirs, env, tooltoolManifest,
tooltool_script, tooltool_urls)
input_env = retry(downloadReleaseBuilds,
args=(ftpServer, product, brand, version, buildNumber,
platform),
kwargs={
'signed': signed,
'usePymake': usePymake
})
env.update(input_env)
if product == "thunderbird" and platform == "macosx64":
# TODO: FIXME: HACK: KILLME:
# Terrible, terrible, terrible hack to work around bug 1234935 and make
# the build system happier
absMozillaSrcDir = path.abspath(
path.join(sourceRepoName, mozillaSrcDir))
run_cmd(['ln', '-sf', '../obj-l10n', absMozillaSrcDir])
failed = []
for l in locales:
try:
if generatePartials:
for oldVersion in partialUpdates:
oldBuildNumber = partialUpdates[oldVersion]['buildNumber']
partialUpdates[oldVersion]['mar'] = retry(
downloadUpdateIgnore404,
args=(ftpServer, product, oldVersion, oldBuildNumber,
platform, l))
checksums_file = repackLocale(
locale=l,
l10nRepoDir=l10nRepoDir,
l10nBaseRepo=l10nBaseRepo,
revision=revision,
localeSrcDir=localeSrcDir,
l10nIni=l10nIni,
compareLocalesRepo=compareLocalesRepo,
env=env,
absObjdir=absObjdir,
merge=merge,
productName=product,
platform=platform,
version=version,
partialUpdates=partialUpdates,
buildNumber=buildNumber,
stageServer=ftpServer,
mozillaDir=mozillaDir,
mozillaSrcDir=mozillaSrcDir)
if balrog_submitter:
# TODO: partials, after bug 797033 is fixed
checksums = parseChecksumsFile(open(checksums_file).read())
completeInfo = []
partialInfo = []
for f, info in checksums.iteritems():
if f.endswith('.complete.mar'):
completeInfo.append({
"size":
info["size"],
"hash":
info["hashes"][balrog_hash],
})
if f.endswith('.partial.mar'):
pathInfo = fileInfo(f, product.lower())
previousVersion = pathInfo["previousVersion"]
partialInfo.append({
"previousVersion":
previousVersion,
"previousBuildNumber":
partialUpdates[previousVersion]['buildNumber'],
"size":
info["size"],
"hash":
info["hashes"][balrog_hash],
})
if not completeInfo:
raise Exception("Couldn't find complete mar info")
retry(balrog_submitter.run,
kwargs={
'platform': platform,
'productName': product.capitalize(),
'appVersion': appVersion,
'version': version,
'build_number': buildNumber,
'locale': l,
'hashFunction': balrog_hash,
'extVersion': appVersion,
'buildID': buildid,
'completeInfo': completeInfo,
'partialInfo': partialInfo,
})
except Exception, e:
print_exc()
failed.append((l, format_exc()))
# Parse propsfile
if options.propsfile:
try:
import json
except ImportError:
import simplejson as json
js = json.load(open(options.propsfile))
if options.revision is None:
options.revision = js['sourcestamp']['revision']
if options.branch is None:
options.branch = js['sourcestamp']['branch']
# look for and clobber outgoing changesets
if options.outgoing:
if out(dest, repo):
remove_path(dest)
if options.shared_dir and out(options.shared_dir, repo):
remove_path(options.shared_dir)
got_revision = mercurial(repo,
dest,
options.branch,
options.revision,
shareBase=options.shared_dir,
clone_by_rev=options.clone_by_rev,
mirrors=options.mirrors,
bundles=options.bundles,
autoPurge=options.auto_purge)
print "Got revision %s" % got_revision
