Beispiel #1
0
    def request(self,
                url,
                get_params=None,
                first_request_time=None,
                retry_counter=0,
                requests_kwargs=None,
                post_json=None,
                dry_run=None):
        """Performs HTTP GET/POST with credentials, returning the body as
        JSON.

        :param url: URL path for the request. Should begin with a slash.
        :type url: string

        :param get_params: HTTP GET parameters.
        :type get_params: dict or list of key/value tuples

        :param first_request_time: The time of the first request (None if no
            retries have occurred).
        :type first_request_time: datetime.datetime

        :param retry_counter: The number of this retry, or zero for first attempt.
        :type retry_counter: int

        :param requests_kwargs: Same extra keywords arg for requests as per
            __init__, but provided here to allow overriding internally on a
            per-request basis.
        :type requests_kwargs: dict

        :param post_json: HTTP POST parameters. Only specified by calling method.
        :type post_json: dict

        :param dry_run: If 'true', only prints URL and parameters. 'true' or 'false'.
        :type dry_run: string

        :raises ApiError: when the API returns an error.
        :raises Timeout: if the request timed out.
            
        :rtype: dict from JSON response.
        """

        if not first_request_time:
            first_request_time = datetime.now()

        elapsed = datetime.now() - first_request_time
        if elapsed > self._retry_timeout:
            raise exceptions.Timeout()

        if retry_counter > 0:
            # 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
            # starting at 0.5s when retry_counter=1. The first retry will occur
            # at 1, so subtract that first.
            delay_seconds = 1.5**(retry_counter - 1)

            # Jitter this value by 50% and pause.
            time.sleep(delay_seconds * (random.random() + 0.5))

        authed_url = self._generate_auth_url(
            url,
            get_params,
        )

        # Default to the client-level self.requests_kwargs, with method-level
        # requests_kwargs arg overriding.
        requests_kwargs = requests_kwargs or {}
        final_requests_kwargs = dict(self._requests_kwargs, **requests_kwargs)

        # Determine GET/POST.
        requests_method = self._session.get

        if post_json is not None:
            requests_method = self._session.post
            final_requests_kwargs["json"] = post_json

        # Only print URL and parameters for dry_run
        if dry_run:
            print("url:\n{}\nHeaders:\n{}".format(
                self._base_url + authed_url,
                json.dumps(final_requests_kwargs, indent=2)))
            return

        try:
            response = requests_method(self._base_url + authed_url,
                                       **final_requests_kwargs)
            self._req = response.request

        except requests.exceptions.Timeout:
            raise exceptions.Timeout()

        if response.status_code in _RETRIABLE_STATUSES:
            # Retry request.
            warnings.warn('Server down.\nRetrying for the {0}{1} time.'.format(
                retry_counter + 1, get_ordinal(retry_counter + 1)),
                          UserWarning,
                          stacklevel=1)

            return self.request(url, get_params, first_request_time,
                                retry_counter + 1, requests_kwargs, post_json)

        try:
            result = self._get_body(response)

            return result
        except exceptions._RetriableRequest as e:
            if isinstance(e, exceptions._OverQueryLimit
                          ) and not self._retry_over_query_limit:
                raise

            warnings.warn(
                'Rate limit exceeded. Retrying for the {0}{1} time.'.format(
                    retry_counter + 1, get_ordinal(retry_counter + 1)),
                UserWarning,
                stacklevel=1)
            # Retry request.
            return self.request(url, get_params, first_request_time,
                                retry_counter + 1, requests_kwargs, post_json)
    def request(self,
                url,
                params,
                first_request_time=None,
                retry_counter=0,
                requests_kwargs=None,
                post_json=None,
                dry_run=None):
        """Performs HTTP GET/POST with credentials, returning the body as
        JSON.

        :param url: URL path for the request. Should begin with a slash.
        :type url: string

        :param params: HTTP GET parameters.
        :type params: dict or list of key/value tuples

        :param first_request_time: The time of the first request (None if no
            retries have occurred).
        :type first_request_time: datetime.datetime

        :param retry_counter: The number of this retry, or zero for first attempt.
        :type retry_counter: int

        :param requests_kwargs: Same extra keywords arg for requests as per
            __init__, but provided here to allow overriding internally on a
            per-request basis.
        :type requests_kwargs: dict

        :param post_json: HTTP POST parameters. Only specified by calling method.
        :type post_json: dict

        :param dry_run: If 'true', only prints URL and parameters. 'true' or 'false'.
        :type dry_run: string

        :raises ApiError: when the API returns an error.
        :raises Timeout: if the request timed out.
        :raises TransportError: when something went wrong while trying to
            execute a request.
            
        :rtype: dict from JSON response.
        """

        if not first_request_time:
            first_request_time = datetime.now()

        elapsed = datetime.now() - first_request_time
        if elapsed > self.retry_timeout:
            raise exceptions.Timeout()

        if retry_counter > 0:
            # 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
            # starting at 0.5s when retry_counter=1. The first retry will occur
            # at 1, so subtract that first.
            delay_seconds = 1.5**(retry_counter - 1)

            # Jitter this value by 50% and pause.
            time.sleep(delay_seconds * (random.random() + 0.5))

        authed_url = self._generate_auth_url(
            url,
            params,
        )

        # Default to the client-level self.requests_kwargs, with method-level
        # requests_kwargs arg overriding.
        requests_kwargs = requests_kwargs or {}
        final_requests_kwargs = dict(self.requests_kwargs, **requests_kwargs)

        # Check if the time of the nth previous query (where n is
        # queries_per_second) is under a second ago - if so, sleep for
        # the difference.
        if self.sent_times and len(self.sent_times) == self.queries_per_minute:
            elapsed_since_earliest = time.time() - self.sent_times[0]
            if elapsed_since_earliest < 60:
                print(
                    "Request limit of {} per minute exceeded. Wait for {} seconds"
                    .format(self.queries_per_minute,
                            60 - elapsed_since_earliest))
                time.sleep(60 - elapsed_since_earliest)

        # Determine GET/POST.
        # post_json is so far only sent from matrix call
        requests_method = self.session.get

        if post_json is not None:
            requests_method = self.session.post
            final_requests_kwargs["json"] = post_json

        # Only print URL and parameters for dry_run
        if dry_run:
            print("url:\n{}\nParameters:\n{}".format(
                self.base_url + authed_url, final_requests_kwargs))
            return

        try:
            response = requests_method(self.base_url + authed_url,
                                       **final_requests_kwargs)
        except requests.exceptions.Timeout:
            raise exceptions.Timeout()
        except Exception as e:
            raise exceptions.TransportError(e)

        if response.status_code in _RETRIABLE_STATUSES:
            # Retry request.
            print('Server down.\nRetrying for the {}th time.'.format(
                retry_counter + 1))

            return self.request(url, params, first_request_time,
                                retry_counter + 1, requests_kwargs, post_json)

        try:
            result = self._get_body(response)
            self.sent_times.append(time.time())
            return result
        except exceptions._RetriableRequest as e:
            if isinstance(e, exceptions._OverQueryLimit
                          ) and not self.retry_over_query_limit:
                raise

            print('Rate limit exceeded.\nRetrying for the {}th time.'.format(
                retry_counter + 1))
            # Retry request.
            return self.request(url, params, first_request_time,
                                retry_counter + 1, requests_kwargs, post_json)
        except:
            raise