def test_retrier_maxsleep(self):
with mock.patch("time.sleep") as sleep:
# Test that max sleep time works
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=30, sleepscale=2, jitter=0):
pass
expected = [mock.call(x) for x in (10, 20, 30, 30)]
self.assertEquals(sleep.call_args_list, expected)
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, timeout=1800):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
If this function runs for more than `timeout` seconds, the hg clone
subprocess will be terminated. This features allows to terminate hung clones
before buildbot kills the full jobs. When a timeout terminates the process,
the exception is caught by `retrier`.
Default timeout is 1800 seconds
"""
if os.path.exists(dest):
remove_path(dest)
cmd = ['clone', '--traceback']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def test_retrier_sleep(self):
"""Make sure retrier sleep is behaving"""
with mock.patch("time.sleep") as sleep:
# Test that normal sleep scaling works
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=0):
pass
expected = [mock.call(x) for x in (10, 20, 40, 80)]
self.assertEquals(sleep.call_args_list, expected)
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
"""Pulls changes from hg repo and places it in `dest`.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and pull from the mirrors first before
`repo`."""
if mirrors:
for mirror in mirrors:
try:
return pull(mirror, dest, update_dest=update_dest, **kwargs)
except:
log.exception("Problem pulling from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# Convert repo to an absolute path if it's a local repository
repo = _make_absolute(repo)
cmd = ['pull', '--traceback']
# Don't pass -r to "hg pull", except when it's a valid HG revision.
# Pulling using tag names is dangerous: it uses the local .hgtags, so if
# the tag has moved on the remote side you won't pull the new revision the
# remote tag refers to.
pull_kwargs = kwargs.copy()
if 'revision' in pull_kwargs and \
not is_hg_cset(pull_kwargs['revision']):
del pull_kwargs['revision']
cmd.extend(common_args(**pull_kwargs))
cmd.append(repo)
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS,
sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE,
jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, cwd=dest, include_stderr=True)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug(
"Encountered an HG error which requires extra sleep, sleeping for %.2fs",
sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
continue
raise
def test_retrier_jitter(self):
with mock.patch("time.sleep") as sleep:
# Test that jitter works
with mock.patch("random.randint") as randint:
randint.return_value = 3
for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=3):
randint.return_value *= -1
expected = [mock.call(x) for x in (7, 17, 31, 65)]
self.assertEquals(sleep.call_args_list, expected)
self.assertEquals(randint.call_args, mock.call(-3, 3))
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
"""Pulls changes from hg repo and places it in `dest`.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and pull from the mirrors first before
`repo`."""
if mirrors:
for mirror in mirrors:
try:
return pull(mirror, dest, update_dest=update_dest, **kwargs)
except:
log.exception("Problem pulling from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# Convert repo to an absolute path if it's a local repository
repo = _make_absolute(repo)
cmd = ['pull', '--traceback']
# Don't pass -r to "hg pull", except when it's a valid HG revision.
# Pulling using tag names is dangerous: it uses the local .hgtags, so if
# the tag has moved on the remote side you won't pull the new revision the
# remote tag refers to.
pull_kwargs = kwargs.copy()
if 'revision' in pull_kwargs and \
not is_hg_cset(pull_kwargs['revision']):
del pull_kwargs['revision']
cmd.extend(common_args(**pull_kwargs))
cmd.append(repo)
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, cwd=dest, include_stderr=True)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
continue
raise
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, mirrors=None, bundles=None):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it. If successful, will pull in new revisions from mirrors or
the master repo. If unbundling fails, will fall back to doing a regular
clone from mirrors or the master repo.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
"""
if os.path.exists(dest):
remove_path(dest)
if bundles:
log.info("Attempting to initialize clone with bundles")
for bundle in bundles:
if os.path.exists(dest):
remove_path(dest)
init(dest)
log.info("Trying to use bundle %s", bundle)
try:
if not unbundle(bundle, dest):
remove_path(dest)
continue
adjust_paths(dest, default=repo)
# Now pull / update
return pull(repo, dest, update_dest=update_dest,
mirrors=mirrors, revision=revision, branch=branch)
except Exception:
remove_path(dest)
log.exception("Problem unbundling/pulling from %s", bundle)
continue
else:
log.info("Using bundles failed; falling back to clone")
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, branch, revision,
update_dest=update_dest, clone_by_rev=clone_by_rev)
adjust_paths(dest, default=repo)
return retval
except:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# We may have a partial repo here; mercurial() copes with that
# We need to make sure our paths are correct though
if os.path.exists(os.path.join(dest, '.hg')):
adjust_paths(dest, default=repo)
return mercurial(repo, dest, branch, revision, autoPurge=True,
update_dest=update_dest, clone_by_rev=clone_by_rev)
cmd = ['clone']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS):
try:
get_hg_output(cmd=cmd, include_stderr=True)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def clone(repo, dest, branch=None, revision=None, update_dest=True,
clone_by_rev=False, mirrors=None, bundles=None):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
If `mirrors` is set, will try and clone from the mirrors before
cloning from `repo`.
If `bundles` is set, will try and download the bundle first and
unbundle it. If successful, will pull in new revisions from mirrors or
the master repo. If unbundling fails, will fall back to doing a regular
clone from mirrors or the master repo.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
"""
if os.path.exists(dest):
remove_path(dest)
if bundles:
log.info("Attempting to initialize clone with bundles")
for bundle in bundles:
if os.path.exists(dest):
remove_path(dest)
init(dest)
log.info("Trying to use bundle %s", bundle)
try:
if not unbundle(bundle, dest):
remove_path(dest)
continue
adjust_paths(dest, default=repo)
# Now pull / update
return pull(repo, dest, update_dest=update_dest,
mirrors=mirrors, revision=revision, branch=branch)
except Exception:
remove_path(dest)
log.exception("Problem unbundling/pulling from %s", bundle)
continue
else:
log.info("Using bundles failed; falling back to clone")
if mirrors:
log.info("Attempting to clone from mirrors")
for mirror in mirrors:
log.info("Cloning from %s", mirror)
try:
retval = clone(mirror, dest, branch, revision,
update_dest=update_dest, clone_by_rev=clone_by_rev)
adjust_paths(dest, default=repo)
return retval
except:
log.exception("Problem cloning from mirror %s", mirror)
continue
else:
log.info("Pulling from mirrors failed; falling back to %s", repo)
# We may have a partial repo here; mercurial() copes with that
# We need to make sure our paths are correct though
if os.path.exists(os.path.join(dest, '.hg')):
adjust_paths(dest, default=repo)
return mercurial(repo, dest, branch, revision, autoPurge=True,
update_dest=update_dest, clone_by_rev=clone_by_rev)
cmd = ['clone']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS):
try:
get_hg_output(cmd=cmd, include_stderr=True)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def test_jitter_bounds(self):
self.assertRaises(Exception, retrier(sleeptime=1, jitter=2))
def test_retrier(self):
"""Make sure retrier behaves properly"""
n = 0
for _ in retrier(attempts=5, sleeptime=0, jitter=0):
n += 1
self.assertEquals(n, 5)
def test_retrier(self):
"""Make sure retrier behaves properly"""
n = 0
for _ in retrier(attempts=5, sleeptime=0, jitter=0):
n += 1
self.assertEquals(n, 5)
def clone(repo,
dest,
branch=None,
revision=None,
update_dest=True,
clone_by_rev=False,
timeout=1800):
"""Clones hg repo and places it at `dest`, replacing whatever else is
there. The working copy will be empty.
If `revision` is set, only the specified revision and its ancestors will
be cloned.
If `update_dest` is set, then `dest` will be updated to `revision` if
set, otherwise to `branch`, otherwise to the head of default.
Regardless of how the repository ends up being cloned, the 'default' path
will point to `repo`.
If this function runs for more than `timeout` seconds, the hg clone
subprocess will be terminated. This features allows to terminate hung clones
before buildbot kills the full jobs. When a timeout terminates the process,
the exception is caught by `retrier`.
Default timeout is 1800 seconds
"""
if os.path.exists(dest):
remove_path(dest)
cmd = ['clone', '--traceback']
if not update_dest:
cmd.append('-U')
if clone_by_rev:
if revision:
cmd.extend(['-r', revision])
elif branch:
# hg >= 1.6 supports -b branch for cloning
ver = hg_ver()
if ver >= (1, 6, 0):
cmd.extend(['-b', branch])
cmd.extend([repo, dest])
exc = None
for _ in retrier(attempts=RETRY_ATTEMPTS,
sleeptime=RETRY_SLEEPTIME,
sleepscale=RETRY_SLEEPSCALE,
jitter=RETRY_JITTER):
try:
get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
break
except subprocess.CalledProcessError, e:
exc = sys.exc_info()
if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
log.debug(
"Encountered an HG error which requires extra sleep, sleeping for %.2fs",
sleeptime)
time.sleep(sleeptime)
if any(s in e.output for s in TRANSIENT_HG_ERRORS):
# This is ok, try again!
# Make sure the dest is clean
if os.path.exists(dest):
log.debug("deleting %s", dest)
remove_path(dest)
continue
raise
def test_jitter_bounds(self):
self.assertRaises(Exception, retrier(sleeptime=1, jitter=2))
