def init(dest, bare=False):
"""Initializes an empty repository at dest. If dest exists and isn't empty, it will be removed.
If `bare` is True, then a bare repo will be created."""
if not os.path.isdir(dest):
log.info("removing %s", dest)
safe_unlink(dest)
else:
for f in os.listdir(dest):
f = os.path.join(dest, f)
log.info("removing %s", f)
if os.path.isdir(f):
remove_path(f)
else:
safe_unlink(f)
# Git will freak out if it tries to create intermediate directories for
# dest, and then they exist. We can hit this when pulling in multiple repos
# in parallel to shared repo paths that contain common parent directories
# Let's create all the directories first
try:
os.makedirs(dest)
except OSError, e:
if e.errno == 20:
# Not a directory error...one of the parents of dest isn't a
# directory
raise
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, timeout=1800):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
If this function runs for more than `timeout` seconds, the hg clone
subprocess will be terminated. This features allows to terminate hung clones
before buildbot kills the full jobs. When a timeout terminates the process,
the exception is caught by `retrier`.
Default timeout is 1800 seconds
"""
if os.path.exists(dest):
remove_path(dest)
cmd = ['clone', '--traceback']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def init(dest, bare=False):
"""Initializes an empty repository at dest. If dest exists and isn't empty, it will be removed.
If `bare` is True, then a bare repo will be created."""
if not os.path.isdir(dest):
log.info("removing %s", dest)
safe_unlink(dest)
else:
for f in os.listdir(dest):
f = os.path.join(dest, f)
log.info("removing %s", f)
if os.path.isdir(f):
remove_path(f)
else:
safe_unlink(f)
# Git will freak out if it tries to create intermediate directories for
# dest, and then they exist. We can hit this when pulling in multiple repos
# in parallel to shared repo paths that contain common parent directories
# Let's create all the directories first
try:
os.makedirs(dest)
except OSError, e:
if e.errno == 20:
# Not a directory error...one of the parents of dest isn't a
# directory
raise
def clone(repo,
dest,
refname=None,
mirrors=None,
shared=False,
update_dest=True):
"""Clones git repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `shared` is True, then git shared repos will be used
If `update_dest` is False, then no working copy will be created
"""
if os.path.exists(dest):
remove_path(dest)
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, refname, update_dest=update_dest)
return retval
except KeyboardInterrupt:
raise
except Exception:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
cmd = ['git', 'clone', '-q']
if not update_dest:
# TODO: Use --bare/--mirror here?
cmd.append('--no-checkout')
if refname:
cmd.extend(['--branch', refname])
if shared:
cmd.append('--shared')
cmd.extend([repo, dest])
run_cmd(cmd)
if update_dest:
return get_revision(dest)
def clone(repo, dest, refname=None, mirrors=None, shared=False, update_dest=True):
"""Clones git repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `shared` is True, then git shared repos will be used
If `update_dest` is False, then no working copy will be created
"""
if os.path.exists(dest):
remove_path(dest)
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, refname, update_dest=update_dest)
return retval
except KeyboardInterrupt:
raise
except Exception:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
cmd = ['git', 'clone', '-q']
if not update_dest:
# TODO: Use --bare/--mirror here?
cmd.append('--no-checkout')
if refname:
cmd.extend(['--branch', refname])
if shared:
cmd.append('--shared')
cmd.extend([repo, dest])
run_cmd(cmd)
if update_dest:
return get_revision(dest)
def git(repo, dest, refname=None, revision=None, update_dest=True,
shareBase=DefaultShareBase, mirrors=None, clean_dest=False):
"""Makes sure that `dest` is has `revision` or `refname` checked out from
`repo`.
Do what it takes to make that happen, including possibly clobbering
dest.
If `mirrors` is set, will try and use the mirrors before `repo`.
"""
if shareBase is DefaultShareBase:
shareBase = os.environ.get("GIT_SHARE_BASE_DIR", None)
if shareBase is not None:
repo_name = get_repo_name(repo)
share_dir = os.path.join(shareBase, repo_name)
else:
share_dir = None
if share_dir is not None and not is_git_repo(share_dir):
log.info("creating bare repo %s", share_dir)
try:
init(share_dir, bare=True)
os.utime(share_dir, None)
except Exception:
log.warning("couldn't create shared repo %s; disabling sharing", share_dir)
shareBase = None
share_dir = None
dest = os.path.abspath(dest)
if not is_git_repo(dest):
if os.path.exists(dest):
log.warning("%s doesn't appear to be a valid git directory; clobbering", dest)
remove_path(dest)
if share_dir is not None:
# Initialize the repo and set up the share
init(dest)
set_share(dest, share_dir)
else:
# Otherwise clone into dest
clone(repo, dest, refname=refname, mirrors=mirrors, update_dest=False)
# Make sure our share is pointing to the right place
if share_dir is not None:
lock_file = os.path.join(get_git_dir(share_dir), "index.lock")
if os.path.exists(lock_file):
log.info("removing %s", lock_file)
safe_unlink(lock_file)
set_share(dest, share_dir)
# If we're supposed to be updating to a revision, check if we
# have that revision already. If so, then there's no need to
# fetch anything.
do_fetch = False
if revision is None:
# we don't have a revision specified, so pull in everything
do_fetch = True
elif has_ref(dest, revision):
# revision is actually a ref name, so we need to run fetch
# to make sure we update the ref
do_fetch = True
elif not has_revision(dest, revision):
# we don't have this revision, so need to fetch it
do_fetch = True
if do_fetch:
if share_dir:
# Fetch our refs into our share
try:
# TODO: Handle fetching refnames like refs/tags/XXXX
if refname is None:
fetch(repo, share_dir, mirrors=mirrors)
else:
fetch(repo, share_dir, mirrors=mirrors, refname=refname)
except subprocess.CalledProcessError:
# Something went wrong!
# Clobber share_dir and re-raise
log.info("error fetching into %s - clobbering", share_dir)
remove_path(share_dir)
raise
try:
if refname is None:
fetch(share_dir, dest, fetch_remote="origin")
else:
fetch(share_dir, dest, fetch_remote="origin", refname=refname)
except subprocess.CalledProcessError:
log.info("clobbering %s", share_dir)
remove_path(share_dir)
log.info("error fetching into %s - clobbering", dest)
remove_path(dest)
raise
else:
try:
fetch(repo, dest, mirrors=mirrors, refname=refname)
except Exception:
log.info("error fetching into %s - clobbering", dest)
remove_path(dest)
raise
if update_dest:
log.info("Updating local copy refname: %s; revision: %s", refname, revision)
# Sometimes refname is passed in as a revision
if revision:
if not has_revision(dest, revision) and has_ref(dest, 'origin/%s' % revision):
log.info("Using %s as ref name instead of revision", revision)
refname = revision
revision = None
rev = update(dest, refname=refname, revision=revision)
if clean_dest:
clean(dest)
return rev
if clean_dest:
clean(dest)
def mercurial(repo, dest, branch=None, revision=None, update_dest=True,
shareBase=DefaultShareBase, allowUnsharedLocalClones=False,
clone_by_rev=False, mirrors=None, bundles=None, autoPurge=False):
"""Makes sure that `dest` is has `revision` or `branch` checked out from
`repo`.
Do what it takes to make that happen, including possibly clobbering
dest.
If allowUnsharedLocalClones is True and we're trying to use the share
extension but fail, then we will be able to clone from the shared repo to
our destination. If this is False, the default, then if we don't have the
share extension we will just clone from the remote repository.
If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.
This is slower, but useful when cloning repos with lots of heads.
If `mirrors` is set, will try and use the mirrors before `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it instead of doing a full clone. If successful, will pull in
new revisions from mirrors or the master repo. If unbundling fails, will
fall back to doing a regular clone from mirrors or the master repo.
"""
dest = os.path.abspath(dest)
if shareBase is DefaultShareBase:
shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
log.info("Reporting hg version in use")
cmd = ['hg', '-q', 'version']
run_cmd(cmd, cwd='.')
if shareBase:
# Check that 'hg share' works
try:
log.info("Checking if share extension works")
output = get_output(['hg', 'help', 'share'], dont_log=True)
if 'no commands defined' in output:
# Share extension is enabled, but not functional
log.info("Disabling sharing since share extension doesn't seem to work (1)")
shareBase = None
elif 'unknown command' in output:
# Share extension is disabled
log.info("Disabling sharing since share extension doesn't seem to work (2)")
shareBase = None
except subprocess.CalledProcessError:
# The command failed, so disable sharing
log.info("Disabling sharing since share extension doesn't seem to work (3)")
shareBase = None
# Check that our default path is correct
if os.path.exists(os.path.join(dest, '.hg')):
hgpath = path(dest, "default")
# Make sure that our default path is correct
if hgpath != _make_absolute(repo):
log.info("hg path isn't correct (%s should be %s); clobbering",
hgpath, _make_absolute(repo))
remove_path(dest)
# If the working directory already exists and isn't using share we update
# the working directory directly from the repo, ignoring the sharing
# settings
if os.path.exists(dest):
if not os.path.exists(os.path.join(dest, ".hg")):
log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)
remove_path(dest)
elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
try:
if autoPurge:
purge(dest)
return pull(repo, dest, update_dest=update_dest, branch=branch,
revision=revision,
mirrors=mirrors)
except subprocess.CalledProcessError:
log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
log.debug("Exception:", exc_info=True)
remove_path(dest)
# If that fails for any reason, and sharing is requested, we'll try to
# update the shared repository, and then update the working directory from
# that.
if shareBase:
sharedRepo = os.path.join(shareBase, get_repo_path(repo))
dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath')
if os.path.exists(sharedRepo):
hgpath = path(sharedRepo, "default")
# Make sure that our default path is correct
if hgpath != _make_absolute(repo):
log.info("hg path isn't correct (%s should be %s); clobbering",
hgpath, _make_absolute(repo))
# we need to clobber both the shared checkout and the dest,
# since hgrc needs to be in both places
remove_path(sharedRepo)
remove_path(dest)
if os.path.exists(dest_sharedPath):
# Make sure that the sharedpath points to sharedRepo
dest_sharedPath_data = os.path.normpath(
open(dest_sharedPath).read())
norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg'))
if dest_sharedPath_data != norm_sharedRepo:
# Clobber!
log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering",
dest_sharedPath_data, repo, norm_sharedRepo)
remove_path(dest)
try:
log.info("Updating shared repo")
mercurial(repo, sharedRepo, branch=branch, revision=revision,
update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,
mirrors=mirrors, bundles=bundles, autoPurge=False)
if os.path.exists(dest):
if autoPurge:
purge(dest)
return update(dest, branch=branch, revision=revision)
try:
log.info("Trying to share %s to %s", sharedRepo, dest)
return share(sharedRepo, dest, branch=branch, revision=revision)
except subprocess.CalledProcessError:
if not allowUnsharedLocalClones:
# Re-raise the exception so it gets caught below.
# We'll then clobber dest, and clone from original repo
raise
log.warning("Error calling hg share from %s to %s;"
"falling back to normal clone from shared repo",
sharedRepo, dest)
# Do a full local clone first, and then update to the
# revision we want
# This lets us use hardlinks for the local clone if the OS
# supports it
clone(sharedRepo, dest, update_dest=False,
mirrors=mirrors, bundles=bundles)
return update(dest, branch=branch, revision=revision)
except subprocess.CalledProcessError:
log.warning(
"Error updating %s from sharedRepo (%s): ", dest, sharedRepo)
log.debug("Exception:", exc_info=True)
remove_path(dest)
# end if shareBase
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
# Share isn't available or has failed, clone directly from the source
return clone(repo, dest, branch, revision,
update_dest=update_dest, mirrors=mirrors,
bundles=bundles, clone_by_rev=clone_by_rev)
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, mirrors=None, bundles=None):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it. If successful, will pull in new revisions from mirrors or
the master repo. If unbundling fails, will fall back to doing a regular
clone from mirrors or the master repo.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
"""
if os.path.exists(dest):
remove_path(dest)
if bundles:
log.info("Attempting to initialize clone with bundles")
for bundle in bundles:
if os.path.exists(dest):
remove_path(dest)
init(dest)
log.info("Trying to use bundle %s", bundle)
try:
if not unbundle(bundle, dest):
remove_path(dest)
continue
adjust_paths(dest, default=repo)
# Now pull / update
return pull(repo, dest, update_dest=update_dest,
mirrors=mirrors, revision=revision, branch=branch)
except Exception:
remove_path(dest)
log.exception("Problem unbundling/pulling from %s", bundle)
continue
else:
log.info("Using bundles failed; falling back to clone")
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, branch, revision,
update_dest=update_dest, clone_by_rev=clone_by_rev)
adjust_paths(dest, default=repo)
return retval
except:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# We may have a partial repo here; mercurial() copes with that
# We need to make sure our paths are correct though
if os.path.exists(os.path.join(dest, '.hg')):
adjust_paths(dest, default=repo)
return mercurial(repo, dest, branch, revision, autoPurge=True,
update_dest=update_dest, clone_by_rev=clone_by_rev)
cmd = ['hg', 'clone']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
run_cmd(cmd)
if update_dest:
return update(dest, branch, revision)
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, mirrors=None, bundles=None, timeout=1800):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it. If successful, will pull in new revisions from mirrors or
the master repo. If unbundling fails, will fall back to doing a regular
clone from mirrors or the master repo.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
If this function runs for more than `timeout` seconds, the hg clone
subprocess will be terminated. This features allows to terminate hung clones
before buildbot kills the full jobs. When a timeout terminates the process,
the exception is caught by `retrier`.
Default timeout is 1800 seconds
"""
if os.path.exists(dest):
remove_path(dest)
if bundles:
log.info("Attempting to initialize clone with bundles")
for bundle in bundles:
if os.path.exists(dest):
remove_path(dest)
init(dest)
log.info("Trying to use bundle %s", bundle)
try:
if not unbundle(bundle, dest):
remove_path(dest)
continue
adjust_paths(dest, default=repo)
# Now pull / update
return pull(repo, dest, update_dest=update_dest,
mirrors=mirrors, revision=revision, branch=branch)
except Exception:
remove_path(dest)
log.exception("Problem unbundling/pulling from %s", bundle)
continue
else:
log.info("Using bundles failed; falling back to clone")
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, branch, revision,
update_dest=update_dest, clone_by_rev=clone_by_rev)
adjust_paths(dest, default=repo)
return retval
except:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# We may have a partial repo here; mercurial() copes with that
# We need to make sure our paths are correct though
if os.path.exists(os.path.join(dest, '.hg')):
adjust_paths(dest, default=repo)
return mercurial(repo, dest, branch, revision, autoPurge=True,
update_dest=update_dest, clone_by_rev=clone_by_rev)
cmd = ['clone', '--traceback']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def clone(repo,
dest,
branch=None,
revision=None,
update_dest=True,
clone_by_rev=False,
mirrors=None,
bundles=None,
timeout=1800):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it. If successful, will pull in new revisions from mirrors or
the master repo. If unbundling fails, will fall back to doing a regular
clone from mirrors or the master repo.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
If this function runs for more than `timeout` seconds, the hg clone
subprocess will be terminated. This features allows to terminate hung clones
before buildbot kills the full jobs. When a timeout terminates the process,
the exception is caught by `retrier`.
Default timeout is 1800 seconds
"""
if os.path.exists(dest):
remove_path(dest)
if bundles:
log.info("Attempting to initialize clone with bundles")
for bundle in bundles:
if os.path.exists(dest):
remove_path(dest)
init(dest)
log.info("Trying to use bundle %s", bundle)
try:
if not unbundle(bundle, dest):
remove_path(dest)
continue
adjust_paths(dest, default=repo)
# Now pull / update
return pull(repo,
dest,
update_dest=update_dest,
mirrors=mirrors,
revision=revision,
branch=branch)
except Exception:
remove_path(dest)
log.exception("Problem unbundling/pulling from %s", bundle)
continue
else:
log.info("Using bundles failed; falling back to clone")
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror,
dest,
branch,
revision,
update_dest=update_dest,
clone_by_rev=clone_by_rev)
adjust_paths(dest, default=repo)
return retval
except:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# We may have a partial repo here; mercurial() copes with that
# We need to make sure our paths are correct though
if os.path.exists(os.path.join(dest, '.hg')):
adjust_paths(dest, default=repo)
return mercurial(repo,
dest,
branch,
revision,
autoPurge=True,
update_dest=update_dest,
clone_by_rev=clone_by_rev)
cmd = ['clone', '--traceback']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS,
sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE,
jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug(
"Encountered an HG error which requires extra sleep, sleeping for %.2fs",
sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def git(repo,
dest,
refname=None,
revision=None,
update_dest=True,
shareBase=DefaultShareBase,
mirrors=None,
clean_dest=False):
"""Makes sure that `dest` is has `revision` or `refname` checked out from
`repo`.
Do what it takes to make that happen, including possibly clobbering
dest.
If `mirrors` is set, will try and use the mirrors before `repo`.
"""
if shareBase is DefaultShareBase:
shareBase = os.environ.get("GIT_SHARE_BASE_DIR", None)
if shareBase is not None:
repo_name = get_repo_name(repo)
share_dir = os.path.join(shareBase, repo_name)
else:
share_dir = None
if share_dir is not None and not is_git_repo(share_dir):
log.info("creating bare repo %s", share_dir)
try:
init(share_dir, bare=True)
os.utime(share_dir, None)
except Exception:
log.warning("couldn't create shared repo %s; disabling sharing",
share_dir,
exc_info=True)
shareBase = None
share_dir = None
dest = os.path.abspath(dest)
log.info("Checking dest %s", dest)
if not is_git_repo(dest):
if os.path.exists(dest):
log.warning(
"%s doesn't appear to be a valid git directory; clobbering",
dest)
remove_path(dest)
if share_dir is not None:
# Initialize the repo and set up the share
init(dest)
set_share(dest, share_dir)
else:
# Otherwise clone into dest
clone(repo,
dest,
refname=refname,
mirrors=mirrors,
update_dest=False)
# Make sure our share is pointing to the right place
if share_dir is not None:
lock_file = os.path.join(get_git_dir(share_dir), "index.lock")
if os.path.exists(lock_file):
log.info("removing %s", lock_file)
safe_unlink(lock_file)
set_share(dest, share_dir)
# If we're supposed to be updating to a revision, check if we
# have that revision already. If so, then there's no need to
# fetch anything.
do_fetch = False
if revision is None:
# we don't have a revision specified, so pull in everything
do_fetch = True
elif has_ref(dest, revision):
# revision is actually a ref name, so we need to run fetch
# to make sure we update the ref
do_fetch = True
elif not has_revision(dest, revision):
# we don't have this revision, so need to fetch it
do_fetch = True
if do_fetch:
if share_dir:
# Fetch our refs into our share
try:
# TODO: Handle fetching refnames like refs/tags/XXXX
if refname is None:
fetch(repo, share_dir, mirrors=mirrors)
else:
fetch(repo, share_dir, mirrors=mirrors, refname=refname)
except subprocess.CalledProcessError:
# Something went wrong!
# Clobber share_dir and re-raise
log.info("error fetching into %s - clobbering", share_dir)
remove_path(share_dir)
raise
try:
if refname is None:
fetch(share_dir, dest, fetch_remote="origin")
else:
fetch(share_dir,
dest,
fetch_remote="origin",
refname=refname)
except subprocess.CalledProcessError:
log.info("clobbering %s", share_dir)
remove_path(share_dir)
log.info("error fetching into %s - clobbering", dest)
remove_path(dest)
raise
else:
try:
fetch(repo, dest, mirrors=mirrors, refname=refname)
except Exception:
log.info("error fetching into %s - clobbering", dest)
remove_path(dest)
raise
# Set our remote
set_remote(dest, 'origin', repo)
if update_dest:
log.info("Updating local copy refname: %s; revision: %s", refname,
revision)
# Sometimes refname is passed in as a revision
if revision:
if not has_revision(dest, revision) and has_ref(
dest, 'origin/%s' % revision):
log.info("Using %s as ref name instead of revision", revision)
refname = revision
revision = None
rev = update(dest, refname=refname, revision=revision)
if clean_dest:
clean(dest)
return rev
if clean_dest:
clean(dest)
def cleanupObjdir(sourceRepo, objdir, appName):
remove_path(path.join(sourceRepo, objdir, "dist", "upload"))
remove_path(path.join(sourceRepo, objdir, "dist", "update"))
remove_path(path.join(sourceRepo, objdir, appName, "locales", "merged"))
def mercurial(repo, dest, branch=None, revision=None, update_dest=True,
shareBase=DefaultShareBase, clone_by_rev=False,
autoPurge=False):
"""Makes sure that `dest` is has `revision` or `branch` checked out from
`repo`.
Do what it takes to make that happen, including possibly clobbering
dest.
If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.
This is slower, but useful when cloning repos with lots of heads.
"""
dest = os.path.abspath(dest)
if shareBase is DefaultShareBase:
shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
log.info("Reporting hg version in use")
cmd = ['hg', '-q', 'version']
run_cmd(cmd, cwd='.')
if shareBase:
# Check that 'hg share' works
try:
log.info("Checking if share extension works")
output = get_hg_output(['help', 'share'], dont_log=True)
if 'no commands defined' in output:
# Share extension is enabled, but not functional
log.info("Disabling sharing since share extension doesn't seem to work (1)")
shareBase = None
elif 'unknown command' in output:
# Share extension is disabled
log.info("Disabling sharing since share extension doesn't seem to work (2)")
shareBase = None
except subprocess.CalledProcessError:
# The command failed, so disable sharing
log.info("Disabling sharing since share extension doesn't seem to work (3)")
shareBase = None
# Check that our default path is correct
if os.path.exists(os.path.join(dest, '.hg')):
hgpath = path(dest, "default")
# Make sure that our default path is correct
if not hgpath or _make_absolute(hgpath) != _make_absolute(repo):
log.info("hg path isn't correct (%s should be %s); clobbering",
hgpath, _make_absolute(repo))
remove_path(dest)
# If the working directory already exists and isn't using share we update
# the working directory directly from the repo, ignoring the sharing
# settings
if os.path.exists(dest):
if not os.path.exists(os.path.join(dest, ".hg")):
log.warning("%s doesn't appear to be a valid hg directory; clobbering", dest)
remove_path(dest)
elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
try:
if autoPurge:
purge(dest)
if revision is not None and is_hg_cset(revision) and has_rev(dest, revision):
log.info("skipping pull since we already have %s", revision)
if update_dest:
return update(dest, branch=branch, revision=revision)
return revision
return pull(repo, dest, update_dest=update_dest, branch=branch,
revision=revision)
except subprocess.CalledProcessError:
log.warning("Error pulling changes into %s from %s; clobbering", dest, repo)
log.debug("Exception:", exc_info=True)
remove_path(dest)
# If that fails for any reason, and sharing is requested, we'll try to
# update the shared repository, and then update the working directory from
# that.
if shareBase:
sharedRepo = os.path.join(shareBase, get_repo_path(repo))
dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath')
if os.path.exists(sharedRepo):
hgpath = path(sharedRepo, "default")
# Make sure that our default path is correct
if hgpath != _make_absolute(repo):
log.info("hg path isn't correct (%s should be %s); clobbering",
hgpath, _make_absolute(repo))
# we need to clobber both the shared checkout and the dest,
# since hgrc needs to be in both places
remove_path(sharedRepo)
remove_path(dest)
if os.path.exists(dest_sharedPath):
# Make sure that the sharedpath points to sharedRepo
dest_sharedPath_data = os.path.normpath(
open(dest_sharedPath).read())
norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg'))
if dest_sharedPath_data != norm_sharedRepo:
# Clobber!
log.info("We're currently shared from %s, but are being requested to pull from %s (%s); clobbering",
dest_sharedPath_data, repo, norm_sharedRepo)
remove_path(dest)
try:
log.info("Updating shared repo")
mercurial(repo, sharedRepo, branch=branch, revision=revision,
update_dest=False, shareBase=None, clone_by_rev=clone_by_rev,
autoPurge=False)
if os.path.exists(dest):
# Bug 969689: Check to see if the dest repo is still on a valid
# commit. It is possible that the shared repo was cloberred out
# from under us, effectively stripping our active commit. This
# can cause 'hg status', 'hg purge', and the like to do
# incorrect things. If we detect this situation, then it's best
# to clobber and re-create dest.
parent = get_revision(dest)
if not parent:
log.info("Shared repo %s no longer has our parent cset; clobbering",
sharedRepo)
remove_path(dest)
else:
if autoPurge:
purge(dest)
return update(dest, branch=branch, revision=revision)
log.info("Trying to share %s to %s", sharedRepo, dest)
return share(sharedRepo, dest, branch=branch, revision=revision)
except subprocess.CalledProcessError:
log.warning(
"Error updating %s from sharedRepo (%s): ", dest, sharedRepo)
log.debug("Exception:", exc_info=True)
remove_path(dest)
# end if shareBase
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
# Share isn't available or has failed, clone directly from the source
return clone(repo, dest, branch, revision,
update_dest=update_dest, clone_by_rev=clone_by_rev)
def mercurial(repo,
dest,
branch=None,
revision=None,
update_dest=True,
shareBase=DefaultShareBase,
clone_by_rev=False,
autoPurge=False):
"""Makes sure that `dest` is has `revision` or `branch` checked out from
`repo`.
Do what it takes to make that happen, including possibly clobbering
dest.
If `clone_by_rev` is True, use 'hg clone -r <rev>' instead of 'hg clone'.
This is slower, but useful when cloning repos with lots of heads.
"""
dest = os.path.abspath(dest)
if shareBase is DefaultShareBase:
shareBase = os.environ.get("HG_SHARE_BASE_DIR", None)
log.info("Reporting hg version in use")
cmd = ['hg', '-q', 'version']
run_cmd(cmd, cwd='.')
if shareBase:
# Check that 'hg share' works
try:
log.info("Checking if share extension works")
output = get_hg_output(['help', 'share'], dont_log=True)
if 'no commands defined' in output:
# Share extension is enabled, but not functional
log.info(
"Disabling sharing since share extension doesn't seem to work (1)"
)
shareBase = None
elif 'unknown command' in output:
# Share extension is disabled
log.info(
"Disabling sharing since share extension doesn't seem to work (2)"
)
shareBase = None
except subprocess.CalledProcessError:
# The command failed, so disable sharing
log.info(
"Disabling sharing since share extension doesn't seem to work (3)"
)
shareBase = None
# Check that our default path is correct
if os.path.exists(os.path.join(dest, '.hg')):
hgpath = path(dest, "default")
# Make sure that our default path is correct
if not hgpath or _make_absolute(hgpath) != _make_absolute(repo):
log.info("hg path isn't correct (%s should be %s); clobbering",
hgpath, _make_absolute(repo))
remove_path(dest)
# If the working directory already exists and isn't using share we update
# the working directory directly from the repo, ignoring the sharing
# settings
if os.path.exists(dest):
if not os.path.exists(os.path.join(dest, ".hg")):
log.warning(
"%s doesn't appear to be a valid hg directory; clobbering",
dest)
remove_path(dest)
elif not os.path.exists(os.path.join(dest, ".hg", "sharedpath")):
try:
if autoPurge:
purge(dest)
if revision is not None and is_hg_cset(revision) and has_rev(
dest, revision):
log.info("skipping pull since we already have %s",
revision)
if update_dest:
return update(dest, branch=branch, revision=revision)
return revision
return pull(repo,
dest,
update_dest=update_dest,
branch=branch,
revision=revision)
except subprocess.CalledProcessError:
log.warning(
"Error pulling changes into %s from %s; clobbering", dest,
repo)
log.debug("Exception:", exc_info=True)
remove_path(dest)
# If that fails for any reason, and sharing is requested, we'll try to
# update the shared repository, and then update the working directory from
# that.
if shareBase:
sharedRepo = os.path.join(shareBase, get_repo_path(repo))
dest_sharedPath = os.path.join(dest, '.hg', 'sharedpath')
if os.path.exists(sharedRepo):
hgpath = path(sharedRepo, "default")
# Make sure that our default path is correct
if hgpath != _make_absolute(repo):
log.info("hg path isn't correct (%s should be %s); clobbering",
hgpath, _make_absolute(repo))
# we need to clobber both the shared checkout and the dest,
# since hgrc needs to be in both places
remove_path(sharedRepo)
remove_path(dest)
if os.path.exists(dest_sharedPath):
# Make sure that the sharedpath points to sharedRepo
dest_sharedPath_data = os.path.normpath(
open(dest_sharedPath).read())
norm_sharedRepo = os.path.normpath(os.path.join(sharedRepo, '.hg'))
if dest_sharedPath_data != norm_sharedRepo:
# Clobber!
log.info(
"We're currently shared from %s, but are being requested to pull from %s (%s); clobbering",
dest_sharedPath_data, repo, norm_sharedRepo)
remove_path(dest)
try:
log.info("Updating shared repo")
mercurial(repo,
sharedRepo,
branch=branch,
revision=revision,
update_dest=False,
shareBase=None,
clone_by_rev=clone_by_rev,
autoPurge=False)
if os.path.exists(dest):
# Bug 969689: Check to see if the dest repo is still on a valid
# commit. It is possible that the shared repo was cloberred out
# from under us, effectively stripping our active commit. This
# can cause 'hg status', 'hg purge', and the like to do
# incorrect things. If we detect this situation, then it's best
# to clobber and re-create dest.
parent = get_revision(dest)
if not parent:
log.info(
"Shared repo %s no longer has our parent cset; clobbering",
sharedRepo)
remove_path(dest)
else:
if autoPurge:
purge(dest)
return update(dest, branch=branch, revision=revision)
log.info("Trying to share %s to %s", sharedRepo, dest)
return share(sharedRepo, dest, branch=branch, revision=revision)
except subprocess.CalledProcessError:
log.warning("Error updating %s from sharedRepo (%s): ", dest,
sharedRepo)
log.debug("Exception:", exc_info=True)
remove_path(dest)
# end if shareBase
if not os.path.exists(os.path.dirname(dest)):
os.makedirs(os.path.dirname(dest))
# Share isn't available or has failed, clone directly from the source
return clone(repo,
dest,
branch,
revision,
update_dest=update_dest,
clone_by_rev=clone_by_rev)
def clone(repo,
dest,
branch=None,
revision=None,
update_dest=True,
clone_by_rev=False,
timeout=1800):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
If this function runs for more than `timeout` seconds, the hg clone
subprocess will be terminated. This features allows to terminate hung clones
before buildbot kills the full jobs. When a timeout terminates the process,
the exception is caught by `retrier`.
Default timeout is 1800 seconds
"""
if os.path.exists(dest):
remove_path(dest)
cmd = ['clone', '--traceback']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS,
sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE,
jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug(
"Encountered an HG error which requires extra sleep, sleeping for %.2fs",
sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def cleanupObjdir(sourceRepo, objdir, appName):
remove_path(path.join(sourceRepo, objdir, "dist", "upload"))
remove_path(path.join(sourceRepo, objdir, "dist", "update"))
remove_path(path.join(sourceRepo, objdir, appName, "locales", "merged"))
