Ejemplo n.º 1
0
 def test_retrier_maxsleep(self):
     with mock.patch("time.sleep") as sleep:
         # Test that max sleep time works
         for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=30, sleepscale=2, jitter=0):
             pass
         expected = [mock.call(x) for x in (10, 20, 30, 30)]
         self.assertEquals(sleep.call_args_list, expected)
Ejemplo n.º 2
0
def clone(repo, dest, branch=None, revision=None, update_dest=True,
          clone_by_rev=False, timeout=1800):
    """Clones hg repo and places it at `dest`, replacing whatever else is
    there.  The working copy will be empty.

    If `revision` is set, only the specified revision and its ancestors will
    be cloned.

    If `update_dest` is set, then `dest` will be updated to `revision` if
    set, otherwise to `branch`, otherwise to the head of default.

    Regardless of how the repository ends up being cloned, the 'default' path
    will point to `repo`.

    If this function runs for more than `timeout` seconds, the hg clone
    subprocess will be terminated. This features allows to terminate hung clones
    before buildbot kills the full jobs. When a timeout terminates the process,
    the exception is caught by `retrier`.

    Default timeout is 1800 seconds
    """
    if os.path.exists(dest):
        remove_path(dest)

    cmd = ['clone', '--traceback']
    if not update_dest:
        cmd.append('-U')

    if clone_by_rev:
        if revision:
            cmd.extend(['-r', revision])
        elif branch:
            # hg >= 1.6 supports -b branch for cloning
            ver = hg_ver()
            if ver >= (1, 6, 0):
                cmd.extend(['-b', branch])

    cmd.extend([repo, dest])
    exc = None
    for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME,
                     sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER):
        try:
            get_hg_output(cmd=cmd, include_stderr=True, timeout=timeout)
            break
        except subprocess.CalledProcessError, e:
            exc = sys.exc_info()

            if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
                sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
                log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime)
                time.sleep(sleeptime)

            if any(s in e.output for s in TRANSIENT_HG_ERRORS):
                # This is ok, try again!
                # Make sure the dest is clean
                if os.path.exists(dest):
                    log.debug("deleting %s", dest)
                    remove_path(dest)
                continue
            raise
Ejemplo n.º 3
0
 def test_retrier_sleep(self):
     """Make sure retrier sleep is behaving"""
     with mock.patch("time.sleep") as sleep:
         # Test that normal sleep scaling works
         for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=0):
             pass
         expected = [mock.call(x) for x in (10, 20, 40, 80)]
         self.assertEquals(sleep.call_args_list, expected)
Ejemplo n.º 4
0
 def test_retrier_jitter(self):
     with mock.patch("time.sleep") as sleep:
         # Test that jitter works
         with mock.patch("random.randint") as randint:
             randint.return_value = 3
             for _ in retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=2, jitter=3):
                 randint.return_value *= -1
             expected = [mock.call(x) for x in (7, 17, 31, 65)]
             self.assertEquals(sleep.call_args_list, expected)
             self.assertEquals(randint.call_args, mock.call(-3, 3))
Ejemplo n.º 5
0
def pull(repo, dest, update_dest=True, mirrors=None, **kwargs):
    """Pulls changes from hg repo and places it in `dest`.

    If `update_dest` is set, then `dest` will be updated to `revision` if
    set, otherwise to `branch`, otherwise to the head of default.

    If `mirrors` is set, will try and pull from the mirrors first before
    `repo`."""

    if mirrors:
        for mirror in mirrors:
            try:
                return pull(mirror, dest, update_dest=update_dest, **kwargs)
            except:
                log.exception("Problem pulling from mirror %s", mirror)
                continue
        else:
            log.info("Pulling from mirrors failed; falling back to %s", repo)

    # Convert repo to an absolute path if it's a local repository
    repo = _make_absolute(repo)
    cmd = ['pull', '--traceback']
    # Don't pass -r to "hg pull", except when it's a valid HG revision.
    # Pulling using tag names is dangerous: it uses the local .hgtags, so if
    # the tag has moved on the remote side you won't pull the new revision the
    # remote tag refers to.
    pull_kwargs = kwargs.copy()
    if 'revision' in pull_kwargs and \
       not is_hg_cset(pull_kwargs['revision']):
        del pull_kwargs['revision']

    cmd.extend(common_args(**pull_kwargs))

    cmd.append(repo)
    exc = None
    for _ in retrier(attempts=RETRY_ATTEMPTS, sleeptime=RETRY_SLEEPTIME,
                     sleepscale=RETRY_SLEEPSCALE, jitter=RETRY_JITTER):
        try:
            get_hg_output(cmd=cmd, cwd=dest, include_stderr=True)
            break
        except subprocess.CalledProcessError, e:
            exc = sys.exc_info()

            if any(s in e.output for s in TRANSIENT_HG_ERRORS_EXTRA_WAIT):
                sleeptime = _ * RETRY_EXTRA_WAIT_SCALE
                log.debug("Encountered an HG error which requires extra sleep, sleeping for %.2fs", sleeptime)
                time.sleep(sleeptime)

            if any(s in e.output for s in TRANSIENT_HG_ERRORS):
                # This is ok, try again!
                continue
            raise
Ejemplo n.º 6
0
def clone(repo, dest, branch=None, revision=None, update_dest=True,
          clone_by_rev=False, mirrors=None, bundles=None):
    """Clones hg repo and places it at `dest`, replacing whatever else is
    there.  The working copy will be empty.

    If `revision` is set, only the specified revision and its ancestors will
    be cloned.

    If `update_dest` is set, then `dest` will be updated to `revision` if
    set, otherwise to `branch`, otherwise to the head of default.

    If `mirrors` is set, will try and clone from the mirrors before
    cloning from `repo`.

    If `bundles` is set, will try and download the bundle first and
    unbundle it. If successful, will pull in new revisions from mirrors or
    the master repo. If unbundling fails, will fall back to doing a regular
    clone from mirrors or the master repo.

    Regardless of how the repository ends up being cloned, the 'default' path
    will point to `repo`.
    """
    if os.path.exists(dest):
        remove_path(dest)

    if bundles:
        log.info("Attempting to initialize clone with bundles")
        for bundle in bundles:
            if os.path.exists(dest):
                remove_path(dest)
            init(dest)
            log.info("Trying to use bundle %s", bundle)
            try:
                if not unbundle(bundle, dest):
                    remove_path(dest)
                    continue
                adjust_paths(dest, default=repo)
                # Now pull / update
                return pull(repo, dest, update_dest=update_dest,
                            mirrors=mirrors, revision=revision, branch=branch)
            except Exception:
                remove_path(dest)
                log.exception("Problem unbundling/pulling from %s", bundle)
                continue
        else:
            log.info("Using bundles failed; falling back to clone")

    if mirrors:
        log.info("Attempting to clone from mirrors")
        for mirror in mirrors:
            log.info("Cloning from %s", mirror)
            try:
                retval = clone(mirror, dest, branch, revision,
                               update_dest=update_dest, clone_by_rev=clone_by_rev)
                adjust_paths(dest, default=repo)
                return retval
            except:
                log.exception("Problem cloning from mirror %s", mirror)
                continue
        else:
            log.info("Pulling from mirrors failed; falling back to %s", repo)
            # We may have a partial repo here; mercurial() copes with that
            # We need to make sure our paths are correct though
            if os.path.exists(os.path.join(dest, '.hg')):
                adjust_paths(dest, default=repo)
            return mercurial(repo, dest, branch, revision, autoPurge=True,
                             update_dest=update_dest, clone_by_rev=clone_by_rev)

    cmd = ['clone']
    if not update_dest:
        cmd.append('-U')

    if clone_by_rev:
        if revision:
            cmd.extend(['-r', revision])
        elif branch:
            # hg >= 1.6 supports -b branch for cloning
            ver = hg_ver()
            if ver >= (1, 6, 0):
                cmd.extend(['-b', branch])

    cmd.extend([repo, dest])
    exc = None
    for _ in retrier(attempts=RETRY_ATTEMPTS):
        try:
            get_hg_output(cmd=cmd, include_stderr=True)
            break
        except subprocess.CalledProcessError, e:
            exc = sys.exc_info()
            if any(s in e.output for s in TRANSIENT_HG_ERRORS):
                # This is ok, try again!
                # Make sure the dest is clean
                if os.path.exists(dest):
                    log.debug("deleting %s", dest)
                    remove_path(dest)
                continue
            raise
Ejemplo n.º 7
0
 def test_retrier(self):
     """Make sure retrier behaves properly"""
     n = 0
     for _ in retrier(attempts=5, sleeptime=0, jitter=0):
         n += 1
     self.assertEquals(n, 5)
Ejemplo n.º 8
0
 def test_jitter_bounds(self):
     self.assertRaises(Exception, retrier(sleeptime=1, jitter=2))