def clone_git_repo(self):
        tenacity.retry(
            lambda: subprocess.run(
                ["git", "clone", "--quiet", self.repo_url, self.git_repo_path],
                check=True,
            ),
            wait=tenacity.wait_fixed(30),
            stop=tenacity.stop_after_attempt(5),
        )()

        try:
            tenacity.retry(
                lambda: subprocess.run(
                    ["git", "pull", "--quiet", self.repo_url, "master"],
                    cwd=self.git_repo_path,
                    capture_output=True,
                    check=True,
                ),
                wait=tenacity.wait_fixed(30),
                stop=tenacity.stop_after_attempt(5),
            )()
        except subprocess.CalledProcessError as e:
            # When the repo is empty.
            if b"Couldn't find remote ref master" in e.stdout:
                pass
    def _blocking_poll(self, timeout=None):
        """Poll and wait for the Future to be resolved.

        Args:
            timeout (int): How long to wait for the operation to complete.
                If None, wait indefinitely.
        """
        if self._result_set:
            return

        retry_on = tenacity.retry_if_result(
            functools.partial(operator.is_not, True))
        # Use exponential backoff with jitter.
        wait_on = (tenacity.wait_exponential(multiplier=1, max=10) +
                   tenacity.wait_random(0, 1))

        if timeout is None:
            retry = tenacity.retry(retry=retry_on, wait=wait_on)
        else:
            retry = tenacity.retry(retry=retry_on,
                                   wait=wait_on,
                                   stop=tenacity.stop_after_delay(timeout))

        try:
            retry(self.done)()
        except tenacity.RetryError as exc:
            six.raise_from(
                concurrent.futures.TimeoutError(
                    'Operation did not complete within the designated '
                    'timeout.'), exc)
Beispiel #3
0
    def clone_git_repo(self, repo_url, repo_dir):
        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                )
            )()

            logger.info(f"{repo_dir} cloned")

        logger.info(f"Fetching {repo_dir}")

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "fetch", "--quiet"],
                cwd=repo_dir,
                capture_output=True,
                check=True,
            )
        )()

        logger.info(f"{repo_dir} fetched")
Beispiel #4
0
    def test_retry_type_annotations(self):
        """The decorator should maintain types of decorated functions."""
        # Just in case this is run with unit-test, return early for py2
        if sys.version_info < (3, 0):
            return

        # Function-level import because we can't install this for python 2.
        from typeguard import check_type

        def num_to_str(number):
            # type: (int) -> str
            return str(number)

        # equivalent to a raw @retry decoration
        with_raw = retry(num_to_str)
        with_raw_result = with_raw(1)

        # equivalent to a @retry(...) decoration
        with_constructor = retry()(num_to_str)
        with_constructor_result = with_raw(1)

        # These raise TypeError exceptions if they fail
        check_type("with_raw", with_raw, typing.Callable[[int], str])
        check_type("with_raw_result", with_raw_result, str)
        check_type("with_constructor", with_constructor, typing.Callable[[int],
                                                                         str])
        check_type("with_constructor_result", with_constructor_result, str)
Beispiel #5
0
    def clone_git_repo(self,
                       repo_url,
                       repo_dir,
                       rev="origin/branches/default/tip"):
        logger.info(f"Cloning {repo_url}...")

        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(lambda: subprocess.run(
                ["git", "clone", "--quiet", repo_url, repo_dir], check=True))()

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(lambda: subprocess.run(
            ["git", "fetch"],
            cwd=repo_dir,
            capture_output=True,
            check=True,
        ))()

        subprocess.run(["git", "checkout", rev],
                       cwd=repo_dir,
                       capture_output=True,
                       check=True)
Beispiel #6
0
    def wait(self):
        """Blocks until mountebank is accessible (or waiting times out).
        """
        def wait():
            requests.get(self.mountebank_url).raise_for_status()

        retry(wait=wait_fixed(1), stop=stop_after_attempt(10))(wait)()
Beispiel #7
0
    def clone_git_repo(self, repo_url, repo_dir):
        if not os.path.exists(repo_dir):
            tenacity.retry(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                ),
                wait=tenacity.wait_fixed(30),
                stop=tenacity.stop_after_attempt(5),
            )()

            logger.info(f"{repo_dir} cloned")

        logger.info(f"Pulling and updating {repo_dir}")

        tenacity.retry(
            lambda: subprocess.run(
                ["git", "pull", "--quiet", repo_url, "master"],
                cwd=repo_dir,
                capture_output=True,
                check=True,
            ),
            wait=tenacity.wait_fixed(30),
            stop=tenacity.stop_after_attempt(5),
        )()

        logger.info(f"{repo_dir} pulled and updated")
Beispiel #8
0
    async def multi_tasks(cls,
                          tasks: Union[Iterable, Iterator],
                          to_do_func: Optional[Callable] = None,
                          concur_req: int = 4,
                          rate: float = 1.5,
                          logger: Optional[logging.Logger] = None):
        '''
        Template for multiTasking

        TODO
            1. asyncio.Semaphore
            2. unit func
        '''
        cls.init_logger('UnsyncFetch', logger)
        cls.retry_kwargs['after'] = after_log(cls.logger, logging.WARNING)
        cls.http_download = retry(cls.http_download, **cls.retry_kwargs)
        cls.ftp_download = retry(cls.ftp_download, **cls.retry_kwargs)
        semaphore = asyncio.Semaphore(concur_req)
        if to_do_func is None:
            tasks = [
                cls.fetch_file(semaphore, method, info, path, rate)
                for method, info, path in tasks
            ]
        else:
            tasks = [
                cls.fetch_file(semaphore, method, info, path,
                               rate).then(to_do_func)
                for method, info, path in tasks
            ]
        # return await asyncio.gather(*tasks)
        return [
            await fob
            for fob in tqdm(asyncio.as_completed(tasks), total=len(tasks))
        ]
    def generate(self):
        db_path = os.path.join("data", self.git_repo_path)
        db.register(
            db_path,
            "https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/",
            VERSION,
        )

        is_old_version = db.is_old_schema(db_path)

        with ThreadPoolExecutorResult(max_workers=2) as executor:
            cloner = executor.submit(repository.clone, self.repo_dir)
            cloner.add_done_callback(
                lambda future: logger.info("mozilla-central cloned")
            )

            git_user = get_secret("GIT_USER")
            git_password = get_secret("GIT_PASSWORD")

            repo_push_url = self.repo_url.replace(
                "https://", f"https://{git_user}:{git_password}@"
            )

            if not is_old_version:
                executor.submit(self.clone_git_repo)
            else:
                executor.submit(self.init_git_repo)

        tenacity.retry(
            lambda: subprocess.run(
                ["git", "config", "--global", "http.postBuffer", "12M"], check=True
            ),
            wait=tenacity.wait_fixed(30),
            stop=tenacity.stop_after_attempt(5),
        )()

        push_args = ["git", "push", repo_push_url, "master"]
        if is_old_version:
            push_args.append("--force")

        done = False
        while not done:
            done = generator.generate(
                self.repo_dir,
                self.git_repo_path,
                limit=COMMITS_STEP,
                tokenize=self.tokenize,
                remove_comments=self.remove_comments,
            )

            tenacity.retry(
                lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True),
                wait=tenacity.wait_fixed(30),
                stop=tenacity.stop_after_attempt(5),
            )()

            # We are not using db.upload as we don't need to upload the git repo.
            upload_s3([f"{db_path}.version"])
Beispiel #10
0
    def generate(self):
        db_path = os.path.join("data", self.git_repo_path)
        db.register(
            db_path,
            f"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.microannotate_{self.git_repo_path}.latest/artifacts/public/",
            VERSION,
        )

        # TODO: Check the version again once we can run tasks for longer (https://bugzilla.mozilla.org/show_bug.cgi?id=1604175).
        is_old_version = False  # db.is_old_schema(db_path)

        with ThreadPoolExecutorResult(max_workers=2) as executor:
            cloner = executor.submit(repository.clone, self.repo_dir)
            cloner.add_done_callback(
                lambda future: logger.info("mozilla-central cloned"))

            git_user = get_secret("GIT_USER")
            git_password = get_secret("GIT_PASSWORD")

            repo_push_url = self.repo_url.replace(
                "https://", f"https://{git_user}:{git_password}@")

            if not is_old_version:
                executor.submit(self.clone_git_repo)
            else:
                executor.submit(self.init_git_repo)

        tenacity.retry(
            lambda: subprocess.run(
                ["git", "config", "--global", "http.postBuffer", "12M"],
                check=True),
            wait=tenacity.wait_fixed(30),
            stop=tenacity.stop_after_attempt(5),
        )()

        push_args = ["git", "push", repo_push_url, "master"]
        if is_old_version:
            push_args.append("--force")

        done = False
        while not done:
            done = generator.generate(
                self.repo_dir,
                self.git_repo_path,
                limit=COMMITS_STEP,
                tokenize=self.tokenize,
                remove_comments=self.remove_comments,
            )

            tenacity.retry(
                lambda: subprocess.run(
                    push_args, cwd=self.git_repo_path, check=True),
                wait=tenacity.wait_fixed(30),
                stop=tenacity.stop_after_attempt(5),
            )()
Beispiel #11
0
    def clone_git_repo(self, repo_url, repo_dir):
        if not os.path.exists(repo_dir):
            retry(lambda: subprocess.run(
                ["git", "clone", "--quiet", repo_url, repo_dir], check=True))

        retry(lambda: subprocess.run(
            ["git", "pull", "--quiet", repo_url, "master"],
            cwd=repo_dir,
            capture_output=True,
            check=True,
        ))
Beispiel #12
0
    def __init__(
        self,
        base_url: str,
        default_params: Optional[dict] = None,
        default_headers: Optional[dict] = None,
        timeout: Optional[float] = None,
        ttl_dns_cache: Optional[float] = None,
        max_retries: Optional[int] = 2,
        retry_delay: Optional[float] = 0.5,
        proxy_url: Optional[str] = None,
    ):
        super().__init__()
        if base_url is None:
            raise RuntimeError(
                f'`base_url` must be passed for {self.__class__.__name__} constructor'
            )
        self.base_url = base_url.rstrip('/')
        self.ttl_dns_cache = ttl_dns_cache
        self.proxy_url = proxy_url
        self.session = None
        self.default_params = CIMultiDict(filter_none(default_params or {}))
        self.default_headers = default_headers or {}
        self.timeout = timeout
        self.max_retries = max_retries
        self.retry_delay = retry_delay

        self.request = retry(
            retry=retry_if_exception_type(self.temporary_errors),
            stop=stop_after_attempt(max_retries)
            if max_retries is not None else None,
            wait=wait_fixed(retry_delay),
            before_sleep=before_sleep_log(logging.getLogger('aiobaseclient'),
                                          logging.WARNING),
            reraise=True,
        )(self.request)
Beispiel #13
0
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2):
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_result(lambda x: x is None),
                          wait=tenacity.wait_exponential(
                              multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Beispiel #14
0
def retry_upon_exception(exc, delay, max_delay, max_attempts):
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_exception_type(exc),
                          wait=tenacity.wait_exponential(
                                multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Beispiel #15
0
 def decorated_function(*args, **kwargs) -> _RT:
     return tenacity.retry(
         retry=tenacity.retry_if_exception(is_retriable),
         stop=tenacity.stop.stop_after_attempt(_RETRY_UTILS_MAX_RETRIES),
         wait=tenacity.wait.wait_exponential(max=_RETRY_UTILS_MAX_RETRIES),
         after=tenacity.after.after_log(_LOGGER, logging.DEBUG),
         reraise=True)(function)(*args, **kwargs)
Beispiel #16
0
    def __init__(self,
                 queue,
                 destination,
                 namegen,
                 add_metadata=False,
                 dump_json=False,
                 dump_only=False,
                 pbar=None,
                 session=None):

        super(InstaDownloader, self).__init__()

        self.queue = queue
        self.destination = destination
        self.namegen = namegen
        self.session = session or requests.Session()
        self.pbar = pbar

        self.dump_only = dump_only
        self.dump_json = dump_json or dump_only
        self.add_metadata = add_metadata

        self._killed = False
        self._downloading = None

        retry = tenacity.retry(**self._tenacity_options)
        self._DOWNLOAD_METHODS = {
            "GraphImage": retry(self._download_image),
            "GraphVideo": retry(self._download_video),
            "GraphSidecar": self._download_sidecar,
        }
Beispiel #17
0
def retry_on_conflict(func):
    wrapper = tenacity.retry(stop=tenacity.stop_after_attempt(11),
                             wait=tenacity.wait_random(max=0.002),
                             retry=tenacity.retry_if_exception_type(
                                 exception.ConcurrentTransaction),
                             reraise=True)
    return wrapper(func)
Beispiel #18
0
 def _retry(func, retry_if_result=is_false, **kwargs):
     kwargs.setdefault('retry', tenacity.retry_if_result(retry_if_result))
     kwargs.setdefault('wait', tenacity.wait_fixed(1))
     kwargs.setdefault('stop', tenacity.stop_after_delay(10))
     decorator = tenacity.retry(**kwargs)
     decorated_func = decorator(func)
     return decorated_func()
Beispiel #19
0
 def decorator_f(self, *args, **kwargs):
     retry_args = getattr(self, 'retry_args', None)
     if retry_args is None:
         return fun(self, *args, **kwargs)
     multiplier = retry_args.get('multiplier', 1)
     min_limit = retry_args.get('min', 1)
     max_limit = retry_args.get('max', 1)
     stop_after_delay = retry_args.get('stop_after_delay', 10)
     tenacity_logger = tenacity.before_log(
         self.log, logging.DEBUG) if self.log else None
     default_kwargs = {
         'wait':
         tenacity.wait_exponential(multiplier=multiplier,
                                   max=max_limit,
                                   min=min_limit),
         'retry':
         tenacity.retry_if_exception(should_retry),
         'stop':
         tenacity.stop_after_delay(stop_after_delay),
         'before':
         tenacity_logger,
         'after':
         tenacity_logger,
     }
     return tenacity.retry(**default_kwargs)(fun)(self, *args,
                                                  **kwargs)
Beispiel #20
0
def retry_on_conflict(func):
    wrapper = tenacity.retry(stop=tenacity.stop_after_attempt(11),
                             wait=tenacity.wait_random_exponential(
                                 multiplier=0.5, max=60),
                             retry=tenacity.retry_if_exception_type(
                                 exception.ConcurrentTransaction),
                             reraise=True)
    return wrapper(func)
Beispiel #21
0
    def channel(self):
        if not self._connection or not self._channel.is_open:
            stop = stop_after_delay(self.config.market_expiration_time)
            wait = wait_exponential(max=5)
            retry = tenacity.retry(stop=stop, wait=wait)
            retry(self.ensure_connected)()

        return self._channel
Beispiel #22
0
def test_retry(stop_max_delay=None, **kwargs):
    k = {"wait": _default_wait, "retry": lambda x: False}
    if kwargs:
        for key in kwargs:
            k[key] = kwargs[key]
    if stop_max_delay not in (True, False, None):
        k['stop'] = stop.stop_after_delay(stop_max_delay)
    return tenacity.retry(**k)
def svn_retry():
    return retry(
        retry=retry_if_exception(is_retryable_svn_exception),
        wait=wait_exponential(exp_base=SVN_RETRY_WAIT_EXP_BASE),
        stop=stop_after_attempt(max_attempt_number=SVN_RETRY_MAX_ATTEMPTS),
        before_sleep=before_sleep_log(logger, logging.DEBUG),
        reraise=True,
    )
Beispiel #24
0
 def make_retry_decorator(
         retries: int, delay: float
 ) -> typing.Callable[[typing.Callable], typing.Callable]:
     return retry(wait=wait_fixed(delay),
                  retry=(retry_if_result(lambda res: res.status >= 500)
                         | retry_if_exception_type(
                             exception_types=aiohttp.ClientError)),
                  stop=stop_after_attempt(retries + 1))
Beispiel #25
0
 def wrapper(*args, **kwargs):
     retried = retry(*retry_args, **retry_kwargs)(f)
     # invoke the function using retries
     result = retried(*args, **kwargs)
     # if the result is a Deferred, wrap the inner
     # callback(s) in retries
     if isinstance(result, Deferred):
         result.callbacks[:] = [(
             (
                 retry(*retry_args, **retry_kwargs)(callback),
                 callbackArgs,
                 callbackKeywords,
             ),
             errback_spec,
         ) for (callback, callbackArgs,
                callbackKeywords), errback_spec in result.callbacks]
     return result
Beispiel #26
0
def retry_random_upon_exception(exc, delay=0.5, max_delay=5,
                                max_attempts=DEFAULT_MAX_ATTEMPTS):
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_exception_type(exc),
                          wait=tenacity.wait_random_exponential(
                              multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Beispiel #27
0
def retry_random_upon_exception(exc, delay=0.5, max_delay=5,
                                max_attempts=DEFAULT_MAX_ATTEMPTS):
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_exception_type(exc),
                          wait=tenacity.wait_random_exponential(
                              multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Beispiel #28
0
 async def fetch(self):
     """
     Fetch and return data.
     Calls self._fetch_ with a retry mechanism
     """
     attempter = retry(**self._retry_config)(self._fetch_)
     res = await attempter()
     return res
Beispiel #29
0
def retry_on_conflict(func):
    wrapper = tenacity.retry(
        stop=tenacity.stop_after_attempt(11),
        wait=tenacity.wait_random(max=0.002),
        retry=tenacity.retry_if_exception_type(exception.ConcurrentTransaction),
        reraise=True,
    )
    return wrapper(func)
Beispiel #30
0
def _safe_mongo_call(max_retries, retry_interval):
    return tenacity.retry(
        retry=tenacity.retry_if_exception_type(
            pymongo.errors.AutoReconnect),
        wait=tenacity.wait_fixed(retry_interval),
        stop=(tenacity.stop_after_attempt(max_retries) if max_retries >= 0
              else tenacity.stop_never)
    )
Beispiel #31
0
 def wrapped(*args, **kwargs):
     self = args[0]
     new_fn = tenacity.retry(
         reraise=True,
         retry=tenacity.retry_if_result(_ovsdb_result_pending),
         wait=tenacity.wait_exponential(multiplier=0.01, max=1),
         stop=tenacity.stop_after_delay(self.vsctl_timeout))(fn)
     return new_fn(*args, **kwargs)
Beispiel #32
0
 def decorator(fun: T):
     default_kwargs = {
         'wait': tenacity.wait_exponential(multiplier=1, max=300),
         'retry': retry_if_operation_in_progress(),
         'before': tenacity.before_log(log, logging.DEBUG),
         'after': tenacity.after_log(log, logging.DEBUG),
     }
     default_kwargs.update(**kwargs)
     return cast(T, tenacity.retry(*args, **default_kwargs)(fun))
Beispiel #33
0
 def decorator(fun: Callable):
     default_kwargs = {
         'wait': tenacity.wait_exponential(multiplier=1, max=100),
         'retry': retry_if_temporary_quota(),
         'before': tenacity.before_log(log, logging.DEBUG),
         'after': tenacity.after_log(log, logging.DEBUG),
     }
     default_kwargs.update(**kwargs)
     return tenacity.retry(*args, **default_kwargs)(fun)
Beispiel #34
0
 def wrapped(*args, **kwargs):
     self = args[0]
     new_fn = tenacity.retry(
         reraise=True,
         retry=tenacity.retry_if_result(_ofport_result_pending),
         wait=tenacity.wait_exponential(multiplier=0.01, max=1),
         stop=tenacity.stop_after_delay(
             self.vsctl_timeout))(fn)
     return new_fn(*args, **kwargs)
Beispiel #35
0
def retry_upon_none_result(max_attempts, delay=0.5, max_delay=2, random=False):
    if random:
        wait_func = tenacity.wait_exponential(
            multiplier=delay, max=max_delay)
    else:
        wait_func = tenacity.wait_random_exponential(
            multiplier=delay, max=max_delay)
    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_result(lambda x: x is None),
                          wait=wait_func,
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Beispiel #36
0
    def send_notification(self, target, ctxt, message, version, retry=None):
        if retry is None:
            retry = self._pika_engine.default_notification_retry_attempts

        def on_exception(ex):
            if isinstance(ex, (pika_drv_exc.ExchangeNotFoundException,
                               pika_drv_exc.RoutingException)):
                LOG.warning("Problem during sending notification. %s", ex)
                try:
                    self._declare_notification_queue_binding(target)
                except pika_drv_exc.ConnectionException as e:
                    LOG.warning("Problem during declaring notification queue "
                                "binding. %s", e)
                return True
            elif isinstance(ex, (pika_drv_exc.ConnectionException,
                                 pika_drv_exc.MessageRejectedException)):
                LOG.warning("Problem during sending notification. %s", ex)
                return True
            else:
                return False

        if retry:
            retrier = tenacity.retry(
                stop=(tenacity.stop_never if retry == -1 else
                      tenacity.stop_after_attempt(retry)),
                retry=tenacity.retry_if_exception(on_exception),
                wait=tenacity.wait_fixed(
                    self._pika_engine.notification_retry_delay
                )
            )
        else:
            retrier = None

        msg = pika_drv_msg.PikaOutgoingMessage(self._pika_engine, message,
                                               ctxt)
        return msg.send(
            exchange=(
                target.exchange or
                self._pika_engine.default_notification_exchange
            ),
            routing_key=target.topic,
            confirm=True,
            mandatory=True,
            persistent=self._pika_engine.notification_persistence,
            retrier=retrier
        )
Beispiel #37
0
def retry_upon_exception_exclude_error_codes(
    exc, excluded_errors, delay, max_delay, max_attempts):
    """Retry with the configured exponential delay, unless the exception error
    code is in the given list
    """
    def retry_if_not_error_codes(e):
        # return True only for BadRequests without error codes or with error
        # codes not in the exclude list
        if isinstance(e, exc):
            error_code = _get_bad_request_error_code(e)
            if error_code and error_code not in excluded_errors:
                return True
        return False

    return tenacity.retry(reraise=True,
                          retry=tenacity.retry_if_exception(
                                retry_if_not_error_codes),
                          wait=tenacity.wait_exponential(
                                multiplier=delay, max=max_delay),
                          stop=tenacity.stop_after_attempt(max_attempts),
                          before=_log_before_retry, after=_log_after_retry)
Beispiel #38
0
def retry_on_stale_data_error(func):
    wrapper = tenacity.retry(
        stop=tenacity.stop_after_attempt(3),
        retry=tenacity.retry_if_exception_type(exc.StaleDataError),
        reraise=True)
    return wrapper(func)
Beispiel #39
0
def retry(stop_max_delay=None, **kwargs):
    k = {"wait": _default_wait, "retry": lambda x: False}
    if stop_max_delay not in (True, False, None):
        k['stop'] = stop.stop_after_delay(stop_max_delay)
    return tenacity.retry(**k)
Beispiel #40
0
    def request(self,
                method,
                additional_headers=None,
                retry=True,
                timeout=None,
                auth=None,
                use_gzip_encoding=None,
                params=None,
                max_attempts=None,
                **kwargs):
        """
        Make an HTTP request by calling self._request with backoff retry.

        :param method: request method
        :type method: str
        :param additional_headers: additional headers to include in the request
        :type additional_headers: dict[str, str]
        :param retry: boolean indicating whether to retry if the request fails
        :type retry: boolean
        :param timeout: timeout in seconds, overrides default_timeout_secs
        :type timeout: float
        :param timeout: timeout in seconds
        :type timeout: float
        :param auth: auth scheme for the request
        :type auth: requests.auth.AuthBase
        :param use_gzip_encoding: boolean indicating whether to pass gzip
                                  encoding in the request headers or not
        :type use_gzip_encoding: boolean | None
        :param params: additional params to include in the request
        :type params: str | dict[str, T] | None
        :param max_attempts: maximum number of attempts to try for any request
        :type max_attempts: int
        :param kwargs: additional arguments to pass to requests.request
        :type kwargs: dict[str, T]
        :return: HTTP response
        :rtype: requests.Response
        """
        request = self._request

        if retry:
            if max_attempts is None:
                max_attempts = self.default_max_attempts

            # We retry only when it makes sense: either due to a network
            # partition (e.g. connection errors) or if the request failed
            # due to a server error such as 500s, timeouts, and so on.
            request = tenacity.retry(
                stop=tenacity.stop_after_attempt(max_attempts),
                wait=tenacity.wait_exponential(),
                retry=tenacity.retry_if_exception_type((
                    requests.exceptions.Timeout,
                    requests.exceptions.ConnectionError,
                    MesosServiceUnavailableException,
                    MesosInternalServerErrorException,
                )),
                reraise=True,
            )(request)

        try:
            return request(
                method=method,
                additional_headers=additional_headers,
                timeout=timeout,
                auth=auth,
                use_gzip_encoding=use_gzip_encoding,
                params=params,
                **kwargs
            )
        # If the request itself failed, an exception subclassed from
        # RequestException will be raised. Catch this and reraise as
        # MesosException since we want the caller to be able to catch
        # and handle this.
        except requests.exceptions.RequestException as err:
            raise MesosException('Request failed', err)
    def reply(self, reply=None, failure=None):
        """Send back reply to the RPC client
        :param reply: Dictionary, reply. In case of exception should be None
        :param failure: Tuple, should be a sys.exc_info() tuple.
            Should be None if RPC request was successfully processed.

        :return RpcReplyPikaIncomingMessage, message with reply
        """

        if self.reply_q is None:
            return

        reply_outgoing_message = RpcReplyPikaOutgoingMessage(
            self._pika_engine, self.msg_id, reply=reply, failure_info=failure,
            content_type=self._content_type,
        )

        def on_exception(ex):
            if isinstance(ex, pika_drv_exc.ConnectionException):
                LOG.warning(
                    "Connectivity related problem during reply sending. %s",
                    ex
                )
                return True
            else:
                return False

        if self._pika_engine.rpc_reply_retry_attempts:
            retrier = tenacity.retry(
                stop=(
                    tenacity.stop_never
                    if self._pika_engine.rpc_reply_retry_attempts == -1 else
                    tenacity.stop_after_attempt(
                        self._pika_engine.rpc_reply_retry_attempts
                    )
                ),
                retry=tenacity.retry_if_exception(on_exception),
                wait=tenacity.wait_fixed(
                    self._pika_engine.rpc_reply_retry_delay
                )
            )
        else:
            retrier = None

        try:
            timeout = (None if self.expiration_time is None else
                       max(self.expiration_time - time.time(), 0))
            with timeutils.StopWatch(duration=timeout) as stopwatch:
                reply_outgoing_message.send(
                    reply_q=self.reply_q,
                    stopwatch=stopwatch,
                    retrier=retrier
                )
            LOG.debug(
                "Message [id:'%s'] replied to '%s'.", self.msg_id, self.reply_q
            )
        except Exception:
            LOG.exception(
                "Message [id:'%s'] wasn't replied to : %s", self.msg_id,
                self.reply_q
            )
Beispiel #42
0
    def send(self, target, ctxt, message, wait_for_reply=None, timeout=None,
             retry=None):
        with timeutils.StopWatch(duration=timeout) as stopwatch:
            if retry is None:
                retry = self._pika_engine.default_rpc_retry_attempts

            exchange = self._pika_engine.get_rpc_exchange_name(
                target.exchange
            )

            def on_exception(ex):
                if isinstance(ex, pika_drv_exc.ExchangeNotFoundException):
                    # it is desired to create exchange because if we sent to
                    # exchange which is not exists, we get ChannelClosed
                    # exception and need to reconnect
                    try:
                        self._declare_rpc_exchange(exchange, stopwatch)
                    except pika_drv_exc.ConnectionException as e:
                        LOG.warning("Problem during declaring exchange. %s", e)
                    return True
                elif isinstance(ex, (pika_drv_exc.ConnectionException,
                                     exceptions.MessageDeliveryFailure)):
                    LOG.warning("Problem during message sending. %s", ex)
                    return True
                else:
                    return False

            if retry:
                retrier = tenacity.retry(
                    stop=(tenacity.stop_never if retry == -1 else
                          tenacity.stop_after_attempt(retry)),
                    retry=tenacity.retry_if_exception(on_exception),
                    wait=tenacity.wait_fixed(self._pika_engine.rpc_retry_delay)
                )
            else:
                retrier = None

            if target.fanout:
                return self.cast_all_workers(
                    exchange, target.topic, ctxt, message, stopwatch, retrier
                )

            routing_key = self._pika_engine.get_rpc_queue_name(
                target.topic, target.server, retrier is None
            )

            msg = pika_drv_msg.RpcPikaOutgoingMessage(self._pika_engine,
                                                      message, ctxt)
            try:
                reply = msg.send(
                    exchange=exchange,
                    routing_key=routing_key,
                    reply_listener=(
                        self._reply_listener if wait_for_reply else None
                    ),
                    stopwatch=stopwatch,
                    retrier=retrier
                )
            except pika_drv_exc.ExchangeNotFoundException as ex:
                try:
                    self._declare_rpc_exchange(exchange, stopwatch)
                except pika_drv_exc.ConnectionException as e:
                    LOG.warning("Problem during declaring exchange. %s", e)
                raise ex

            if reply is not None:
                if reply.failure is not None:
                    raise reply.failure

                return reply.result
Beispiel #43
0
        setattr(klass, '_libvirt_connection', connection)
    return connection


def is_disconnection_exception(e):
    if not libvirt:
        return False
    return (isinstance(e, libvirt.libvirtError)
            and e.get_error_code() in (libvirt.VIR_ERR_SYSTEM_ERROR,
                                       libvirt.VIR_ERR_INTERNAL_ERROR)
            and e.get_error_domain() in (libvirt.VIR_FROM_REMOTE,
                                         libvirt.VIR_FROM_RPC))


retry_on_disconnect = tenacity.retry(
    retry=tenacity.retry_if_exception(is_disconnection_exception),
    stop=tenacity.stop_after_attempt(2))


def raise_nodata_if_unsupported(method):
    def inner(in_self, instance, *args, **kwargs):
        try:
            return method(in_self, instance, *args, **kwargs)
        except libvirt.libvirtError as e:
            # NOTE(sileht): At this point libvirt connection error
            # have been reraise as tenacity.RetryError()
            msg = _('Failed to inspect instance %(instance_uuid)s stats, '
                    'can not get info from libvirt: %(error)s') % {
                        "instance_uuid": instance.id,
                        "error": e}
            raise virt_inspector.NoDataException(msg)