def test_retrier_maxsleep(self): with mock.patch("time.sleep") as sleep: # Test that max sleep time works for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=30, sleepscale=2, jitter=0): pass expected = [mock.call(x) for x in (10, 20, 30, 30)] self.assertEquals(sleep.call_args_list, expected)
def clone(repo, dest, branch=None, revision=None, update_dest=True, clone_by_rev=False, timeout=1800): """Clones hg repo and places it at `dest`, replacing whatever else is there. The working copy will be empty. If `revision` is set, only the specified revision and its ancestors will be cloned. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. Regardless of how the repository ends up being cloned, the 'default' path will point to `repo`. If this function runs for more than `timeout` seconds, the hg clone subprocess will be terminated. This features allows to terminate hung clones before buildbot kills the full jobs. When a timeout terminates the process, the exception is caught by `retrier`. Default timeout is 1800 seconds """ if os.path.exists(dest): remove_path(dest) cmd = ['clone', '--traceback'] if not update_dest: cmd.append('-U') if clone_by_rev: if revision: cmd.extend(['-r', revision]) elif branch: # hg >= 1.6 supports -b branch for cloning ver = hg_ver() if ver >= (1, 6, 0): cmd.extend(['-b', branch]) cmd.extend([repo, dest]) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME, sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER): try: get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT): sleeptime = _ * RETRY_EXTRA_WAIT_SCALE log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime) time.sleep(sleeptime) if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! # Make sure the dest is clean if os.path.exists(dest): log.debug("deleting %s", dest) remove_path(dest) continue raise
def test_retrier_sleep(self): """Make sure retrier sleep is behaving""" with mock.patch("time.sleep") as sleep: # Test that normal sleep scaling works for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=0): pass expected = [mock.call(x) for x in (10, 20, 40, 80)] self.assertEquals(sleep.call_args_list, expected)
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs): """Pulls changes from hg repo and places it in `dest`. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. If `mirrors` is set, will try and pull from the mirrors first before `repo`.""" if mirrors: for mirror in mirrors: try: return pull(mirror, dest, update_dest=update_dest, **kwargs) except: log.exception("Problem pulling from mirror %s", mirror) continue else: log.info("Pulling from mirrors failed; falling back to %s", repo) # Convert repo to an absolute path if it's a local repository repo = _make_absolute(repo) cmd = ['pull', '--traceback'] # Don't pass -r to "hg pull", except when it's a valid HG revision. # Pulling using tag names is dangerous: it uses the local .hgtags, so if # the tag has moved on the remote side you won't pull the new revision the # remote tag refers to. pull_kwargs = kwargs.copy() if 'revision' in pull_kwargs and \ not is_hg_cset(pull_kwargs['revision']): del pull_kwargs['revision'] cmd.extend(common_args(**pull_kwargs)) cmd.append(repo) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME, sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER): try: get_hg_output(cmd=cmd, cwd=dest, include_stderr=True) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT): sleeptime = _ * RETRY_EXTRA_WAIT_SCALE log.debug( "Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime) time.sleep(sleeptime) if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! continue raise
def test_retrier_jitter(self): with mock.patch("time.sleep") as sleep: # Test that jitter works with mock.patch("random.randint") as randint: randint.return_value = 3 for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=3): randint.return_value *= -1 expected = [mock.call(x) for x in (7, 17, 31, 65)] self.assertEquals(sleep.call_args_list, expected) self.assertEquals(randint.call_args, mock.call(-3, 3))
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs): """Pulls changes from hg repo and places it in `dest`. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. If `mirrors` is set, will try and pull from the mirrors first before `repo`.""" if mirrors: for mirror in mirrors: try: return pull(mirror, dest, update_dest=update_dest, **kwargs) except: log.exception("Problem pulling from mirror %s", mirror) continue else: log.info("Pulling from mirrors failed; falling back to %s", repo) # Convert repo to an absolute path if it's a local repository repo = _make_absolute(repo) cmd = ['pull', '--traceback'] # Don't pass -r to "hg pull", except when it's a valid HG revision. # Pulling using tag names is dangerous: it uses the local .hgtags, so if # the tag has moved on the remote side you won't pull the new revision the # remote tag refers to. pull_kwargs = kwargs.copy() if 'revision' in pull_kwargs and \ not is_hg_cset(pull_kwargs['revision']): del pull_kwargs['revision'] cmd.extend(common_args(**pull_kwargs)) cmd.append(repo) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME, sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER): try: get_hg_output(cmd=cmd, cwd=dest, include_stderr=True) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT): sleeptime = _ * RETRY_EXTRA_WAIT_SCALE log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime) time.sleep(sleeptime) if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! continue raise
def clone(repo, dest, branch=None, revision=None, update_dest=True, clone_by_rev=False, mirrors=None, bundles=None): """Clones hg repo and places it at `dest`, replacing whatever else is there. The working copy will be empty. If `revision` is set, only the specified revision and its ancestors will be cloned. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. If `mirrors` is set, will try and clone from the mirrors before cloning from `repo`. If `bundles` is set, will try and download the bundle first and unbundle it. If successful, will pull in new revisions from mirrors or the master repo. If unbundling fails, will fall back to doing a regular clone from mirrors or the master repo. Regardless of how the repository ends up being cloned, the 'default' path will point to `repo`. """ if os.path.exists(dest): remove_path(dest) if bundles: log.info("Attempting to initialize clone with bundles") for bundle in bundles: if os.path.exists(dest): remove_path(dest) init(dest) log.info("Trying to use bundle %s", bundle) try: if not unbundle(bundle, dest): remove_path(dest) continue adjust_paths(dest, default=repo) # Now pull / update return pull(repo, dest, update_dest=update_dest, mirrors=mirrors, revision=revision, branch=branch) except Exception: remove_path(dest) log.exception("Problem unbundling/pulling from %s", bundle) continue else: log.info("Using bundles failed; falling back to clone") if mirrors: log.info("Attempting to clone from mirrors") for mirror in mirrors: log.info("Cloning from %s", mirror) try: retval = clone(mirror, dest, branch, revision, update_dest=update_dest, clone_by_rev=clone_by_rev) adjust_paths(dest, default=repo) return retval except: log.exception("Problem cloning from mirror %s", mirror) continue else: log.info("Pulling from mirrors failed; falling back to %s", repo) # We may have a partial repo here; mercurial() copes with that # We need to make sure our paths are correct though if os.path.exists(os.path.join(dest, '.hg')): adjust_paths(dest, default=repo) return mercurial(repo, dest, branch, revision, autoPurge=True, update_dest=update_dest, clone_by_rev=clone_by_rev) cmd = ['clone'] if not update_dest: cmd.append('-U') if clone_by_rev: if revision: cmd.extend(['-r', revision]) elif branch: # hg >= 1.6 supports -b branch for cloning ver = hg_ver() if ver >= (1, 6, 0): cmd.extend(['-b', branch]) cmd.extend([repo, dest]) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS): try: get_hg_output(cmd=cmd, include_stderr=True) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! # Make sure the dest is clean if os.path.exists(dest): log.debug("deleting %s", dest) remove_path(dest) continue raise
def test_jitter_bounds(self): self.assertRaises(Exception, retrier(sleeptime=1, jitter=2))
def test_retrier(self): """Make sure retrier behaves properly""" n = 0 for _ in retrier(attempts=5, sleeptime=0, jitter=0): n += 1 self.assertEquals(n, 5)
def clone(repo, dest, branch=None, revision=None, update_dest=True, clone_by_rev=False, timeout=1800): """Clones hg repo and places it at `dest`, replacing whatever else is there. The working copy will be empty. If `revision` is set, only the specified revision and its ancestors will be cloned. If `update_dest` is set, then `dest` will be updated to `revision` if set, otherwise to `branch`, otherwise to the head of default. Regardless of how the repository ends up being cloned, the 'default' path will point to `repo`. If this function runs for more than `timeout` seconds, the hg clone subprocess will be terminated. This features allows to terminate hung clones before buildbot kills the full jobs. When a timeout terminates the process, the exception is caught by `retrier`. Default timeout is 1800 seconds """ if os.path.exists(dest): remove_path(dest) cmd = ['clone', '--traceback'] if not update_dest: cmd.append('-U') if clone_by_rev: if revision: cmd.extend(['-r', revision]) elif branch: # hg >= 1.6 supports -b branch for cloning ver = hg_ver() if ver >= (1, 6, 0): cmd.extend(['-b', branch]) cmd.extend([repo, dest]) exc = None for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME, sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER): try: get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout) break except subprocess.CalledProcessError, e: exc = sys.exc_info() if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT): sleeptime = _ * RETRY_EXTRA_WAIT_SCALE log.debug( "Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime) time.sleep(sleeptime) if any(s in e.output for s in TRANSIENT_HG_ERRORS): # This is ok, try again! # Make sure the dest is clean if os.path.exists(dest): log.debug("deleting %s", dest) remove_path(dest) continue raise