async def makeHttpRequest(self, method, route, payload=None, session=None, **kwargs):
        """ Make an HTTP Request for the API endpoint.  This method wraps
        the logic about doing failure retry and passes off the actual work
        of doing an HTTP request to another method."""

        url = self.makeFullUrl(route, **kwargs)
        log.debug('Full URL used is: %s', url)

        hawkExt = self.makeHawkExt()

        # Serialize payload if given
        if payload is not None and isinstance(payload, dict):
            payload = utils.dumpJson(payload)

        # Do a loop of retries
        retry = -1  # we plus first in the loop, and attempt 1 is retry 0
        retries = self.options['maxRetries']
        with self.contextSession(session=session) as session:
            while True:
                retry += 1
                await asyncio.sleep(utils.calculateSleepTime(retry))
                headers = self.makeHeaders(method, url, payload, hawkExt)
                log.debug('Making attempt %d', retry)
                try:
                    return await self._makeHttpRequest(method, url, payload,
                                                       headers, session=session)
                except (exceptions.TaskclusterConnectionError,
                        exceptions.TaskclusterRestFailure) as exc:
                    if retry < retries:
                        log.warn("Retrying because of %s" % str(exc))
                        continue
                    raise
async def makeHttpRequest(method, url, payload, headers, retries=utils.MAX_RETRIES,
                          session=None, args=(), kwargs=None):
    """ Make an HTTP request and retry it until success, return request """
    retry = -1
    response = None
    while True:
        retry += 1
        # if this isn't the first retry then we sleep
        asyncio.sleep(utils.calculateSleepTime(retry))

        # Seek payload to start, if it is a file
        if hasattr(payload, 'seek'):
            payload.seek(0)

        log.debug('Making attempt %d', retry)
        try:
            response = await makeSingleHttpRequest(method, url, payload, headers,
                                                   session, args=args, kwargs=kwargs)
        except aiohttp.ClientError as rerr:
            if retry < retries:
                log.warn('Retrying because of: %s' % rerr)
                continue
            # raise a connection exception
            raise rerr
        except ValueError as rerr:
            log.warn('ValueError from aiohttp: redirect to non-http or https')
            raise rerr
        except RuntimeError as rerr:
            log.warn('RuntimeError from aiohttp: session closed')
            raise rerr
        # Handle non 2xx status code and retry if possible
        status = response.status
        if 500 <= status and status < 600 and retry < retries:
            log.warn('Retrying because of: %d status' % status)
            continue
        if status >= 200 and status < 300:
            return response
        raise exceptions.TaskclusterRestFailure("Unknown Server Error", superExc=None)
Пример #3
0
    def _makeHttpRequest(self, method, route, payload):
        """ Make an HTTP Request for the API endpoint.  This method wraps
        the logic about doing failure retry and passes off the actual work
        of doing an HTTP request to another method."""

        url = self._constructUrl(route)
        log.debug('Full URL used is: %s', url)

        hawkExt = self.makeHawkExt()

        # Serialize payload if given
        if payload is not None:
            payload = utils.dumpJson(payload)

        # Do a loop of retries
        retry = -1  # we plus first in the loop, and attempt 1 is retry 0
        retries = self.options['maxRetries']
        while retry < retries:
            retry += 1
            # if this isn't the first retry then we sleep
            if retry > 0:
                time.sleep(utils.calculateSleepTime(retry))
            # Construct header
            if self._hasCredentials():
                sender = mohawk.Sender(
                    credentials={
                        'id': self.options['credentials']['clientId'],
                        'key': self.options['credentials']['accessToken'],
                        'algorithm': 'sha256',
                    },
                    ext=hawkExt if hawkExt else {},
                    url=url,
                    content=payload if payload else '',
                    content_type='application/json' if payload else '',
                    method=method,
                )

                headers = {'Authorization': sender.request_header}
            else:
                log.debug('Not using hawk!')
                headers = {}
            if payload:
                # Set header for JSON if payload is given, note that we serialize
                # outside this loop.
                headers['Content-Type'] = 'application/json'

            log.debug('Making attempt %d', retry)
            try:
                response = utils.makeSingleHttpRequest(method, url, payload,
                                                       headers)
            except requests.exceptions.RequestException as rerr:
                if retry < retries:
                    log.warn('Retrying because of: %s' % rerr)
                    continue
                # raise a connection exception
                raise exceptions.TaskclusterConnectionError(
                    "Failed to establish connection", superExc=rerr)

            # Handle non 2xx status code and retry if possible
            status = response.status_code
            if status == 204:
                return None

            # Catch retryable errors and go to the beginning of the loop
            # to do the retry
            if 500 <= status and status < 600 and retry < retries:
                log.warn('Retrying because of a %s status code' % status)
                continue

            # Throw errors for non-retryable errors
            if status < 200 or status >= 300:
                data = {}
                try:
                    data = response.json()
                except:
                    pass  # Ignore JSON errors in error messages
                # Find error message
                message = "Unknown Server Error"
                if isinstance(data, dict):
                    message = data.get('message')
                else:
                    if status == 401:
                        message = "Authentication Error"
                    elif status == 500:
                        message = "Internal Server Error"
                # Raise TaskclusterAuthFailure if this is an auth issue
                if status == 401:
                    raise exceptions.TaskclusterAuthFailure(message,
                                                            status_code=status,
                                                            body=data,
                                                            superExc=None)
                # Raise TaskclusterRestFailure for all other issues
                raise exceptions.TaskclusterRestFailure(message,
                                                        status_code=status,
                                                        body=data,
                                                        superExc=None)

            # Try to load JSON
            try:
                return response.json()
            except ValueError:
                return {"response": response}

        # This code-path should be unreachable
        assert False, "Error from last retry should have been raised!"
Пример #4
0
    def _makeHttpRequest(self, method, route, payload):
        """ Make an HTTP Request for the API endpoint.  This method wraps
        the logic about doing failure retry and passes off the actual work
        of doing an HTTP request to another method."""

        url = self._constructUrl(route)
        log.debug('Full URL used is: %s', url)

        hawkExt = self.makeHawkExt()

        # Serialize payload if given
        if payload is not None:
            payload = utils.dumpJson(payload)

        # Do a loop of retries
        retry = -1  # we plus first in the loop, and attempt 1 is retry 0
        retries = self.options['maxRetries']
        while retry < retries:
            retry += 1
            # if this isn't the first retry then we sleep
            if retry > 0:
                time.sleep(utils.calculateSleepTime(retry))
            # Construct header
            if self._hasCredentials():
                sender = mohawk.Sender(
                    credentials={
                        'id': self.options['credentials']['clientId'],
                        'key': self.options['credentials']['accessToken'],
                        'algorithm': 'sha256',
                    },
                    ext=hawkExt if hawkExt else {},
                    url=url,
                    content=payload if payload else '',
                    content_type='application/json' if payload else '',
                    method=method,
                )

                headers = {'Authorization': sender.request_header}
            else:
                log.debug('Not using hawk!')
                headers = {}
            if payload:
                # Set header for JSON if payload is given, note that we serialize
                # outside this loop.
                headers['Content-Type'] = 'application/json'

            log.debug('Making attempt %d', retry)
            try:
                response = utils.makeSingleHttpRequest(method, url, payload, headers)
            except requests.exceptions.RequestException as rerr:
                if retry < retries:
                    log.warn('Retrying because of: %s' % rerr)
                    continue
                # raise a connection exception
                raise exceptions.TaskclusterConnectionError(
                    "Failed to establish connection",
                    superExc=rerr
                )

            # Handle non 2xx status code and retry if possible
            status = response.status_code
            if status == 204:
                return None

            # Catch retryable errors and go to the beginning of the loop
            # to do the retry
            if 500 <= status and status < 600 and retry < retries:
                log.warn('Retrying because of a %s status code' % status)
                continue

            # Throw errors for non-retryable errors
            if status < 200 or status >= 300:
                data = {}
                try:
                    data = response.json()
                except Exception:
                    pass  # Ignore JSON errors in error messages
                # Find error message
                message = "Unknown Server Error"
                if isinstance(data, dict):
                    message = data.get('message')
                else:
                    if status == 401:
                        message = "Authentication Error"
                    elif status == 500:
                        message = "Internal Server Error"
                # Raise TaskclusterAuthFailure if this is an auth issue
                if status == 401:
                    raise exceptions.TaskclusterAuthFailure(
                        message,
                        status_code=status,
                        body=data,
                        superExc=None
                    )
                # Raise TaskclusterRestFailure for all other issues
                raise exceptions.TaskclusterRestFailure(
                    message,
                    status_code=status,
                    body=data,
                    superExc=None
                )

            # Try to load JSON
            try:
                return response.json()
            except ValueError:
                return {"response": response}

        # This code-path should be unreachable
        assert False, "Error from last retry should have been raised!"