def _makeHttpRequest(self, method, url, payload, headers): """Synchronous, requests-based """ try: response = utils.makeSingleHttpRequest(method, url, payload, headers) except requests.exceptions.RequestException as rerr: raise exceptions.TaskclusterConnectionError( "Failed to establish connection", superExc=rerr ) # Handle non 2xx status code and retry if possible try: response.raise_for_status() if response.status_code == 204: return None except requests.exceptions.RequestException as rerr: status = response.status_code # Parse messages from errors data = None try: data = response.json() except: pass # Ignore JSON errors in error messages self._raiseHttpError(status, data, rerr) # Try to load JSON try: return response.json() except ValueError: return {"response": response}
def test_failure(self): @httmock.all_requests def response_content(url, requet): return {'status_code': 404} with httmock.HTTMock(response_content): d = subject.makeSingleHttpRequest('GET', 'http://www.example.com', {}, {}) with self.assertRaises(requests.exceptions.RequestException): d.raise_for_status()
def test_success_no_payload(self): @httmock.all_requests def response_content(url, request): return {'status_code': 200, 'content': {}} with httmock.HTTMock(response_content): d = subject.makeSingleHttpRequest('GET', 'http://www.example.com', {}, {}) self.assertEqual(d.json(), {}) self.assertEqual(d.status_code, 200) d.raise_for_status()
def test_success_payload(): @httmock.all_requests def response_content(url, request): assert request.body == 'i=j' return {'status_code': 200, 'content': {'k': 'l'}} with httmock.HTTMock(response_content): d = subject.makeSingleHttpRequest('GET', 'http://www.example.com', {'i': 'j'}, {}) assert d.json() == {'k': 'l'} assert d.status_code == 200 d.raise_for_status()
def test_redirect_response(): @httmock.all_requests def response_content(url, request): return { 'status_code': 303, 'headers': { 'Location': 'https://nosuch.example.com' }, 'content': {} } with httmock.HTTMock(response_content): d = subject.makeSingleHttpRequest('GET', 'http://www.example.com', None, {}) assert d.json() == {} assert d.status_code == 303
def _makeHttpRequest(self, method, route, payload): """ Make an HTTP Request for the API endpoint. This method wraps the logic about doing failure retry and passes off the actual work of doing an HTTP request to another method.""" url = self._constructUrl(route) log.debug('Full URL used is: %s', url) hawkExt = self.makeHawkExt() # Serialize payload if given if payload is not None: payload = utils.dumpJson(payload) # Do a loop of retries retry = -1 # we plus first in the loop, and attempt 1 is retry 0 retries = self.options['maxRetries'] while retry < retries: retry += 1 # if this isn't the first retry then we sleep if retry > 0: time.sleep(utils.calculateSleepTime(retry)) # Construct header if self._hasCredentials(): sender = mohawk.Sender( credentials={ 'id': self.options['credentials']['clientId'], 'key': self.options['credentials']['accessToken'], 'algorithm': 'sha256', }, ext=hawkExt if hawkExt else {}, url=url, content=payload if payload else '', content_type='application/json' if payload else '', method=method, ) headers = {'Authorization': sender.request_header} else: log.debug('Not using hawk!') headers = {} if payload: # Set header for JSON if payload is given, note that we serialize # outside this loop. headers['Content-Type'] = 'application/json' log.debug('Making attempt %d', retry) try: response = utils.makeSingleHttpRequest(method, url, payload, headers) except requests.exceptions.RequestException as rerr: if retry < retries: log.warn('Retrying because of: %s' % rerr) continue # raise a connection exception raise exceptions.TaskclusterConnectionError( "Failed to establish connection", superExc=rerr) # Handle non 2xx status code and retry if possible status = response.status_code if status == 204: return None # Catch retryable errors and go to the beginning of the loop # to do the retry if 500 <= status and status < 600 and retry < retries: log.warn('Retrying because of a %s status code' % status) continue # Throw errors for non-retryable errors if status < 200 or status >= 300: data = {} try: data = response.json() except: pass # Ignore JSON errors in error messages # Find error message message = "Unknown Server Error" if isinstance(data, dict): message = data.get('message') else: if status == 401: message = "Authentication Error" elif status == 500: message = "Internal Server Error" # Raise TaskclusterAuthFailure if this is an auth issue if status == 401: raise exceptions.TaskclusterAuthFailure(message, status_code=status, body=data, superExc=None) # Raise TaskclusterRestFailure for all other issues raise exceptions.TaskclusterRestFailure(message, status_code=status, body=data, superExc=None) # Try to load JSON try: return response.json() except ValueError: return {"response": response} # This code-path should be unreachable assert False, "Error from last retry should have been raised!"
def _makeHttpRequest(self, method, route, payload): """ Make an HTTP Request for the API endpoint. This method wraps the logic about doing failure retry and passes off the actual work of doing an HTTP request to another method.""" url = self._constructUrl(route) log.debug('Full URL used is: %s', url) hawkExt = self.makeHawkExt() # Serialize payload if given if payload is not None: payload = utils.dumpJson(payload) # Do a loop of retries retry = -1 # we plus first in the loop, and attempt 1 is retry 0 retries = self.options['maxRetries'] while retry < retries: retry += 1 # if this isn't the first retry then we sleep if retry > 0: time.sleep(utils.calculateSleepTime(retry)) # Construct header if self._hasCredentials(): sender = mohawk.Sender( credentials={ 'id': self.options['credentials']['clientId'], 'key': self.options['credentials']['accessToken'], 'algorithm': 'sha256', }, ext=hawkExt if hawkExt else {}, url=url, content=payload if payload else '', content_type='application/json' if payload else '', method=method, ) headers = {'Authorization': sender.request_header} else: log.debug('Not using hawk!') headers = {} if payload: # Set header for JSON if payload is given, note that we serialize # outside this loop. headers['Content-Type'] = 'application/json' log.debug('Making attempt %d', retry) try: response = utils.makeSingleHttpRequest(method, url, payload, headers) except requests.exceptions.RequestException as rerr: if retry < retries: log.warn('Retrying because of: %s' % rerr) continue # raise a connection exception raise exceptions.TaskclusterConnectionError( "Failed to establish connection", superExc=rerr ) # Handle non 2xx status code and retry if possible status = response.status_code if status == 204: return None # Catch retryable errors and go to the beginning of the loop # to do the retry if 500 <= status and status < 600 and retry < retries: log.warn('Retrying because of a %s status code' % status) continue # Throw errors for non-retryable errors if status < 200 or status >= 300: data = {} try: data = response.json() except Exception: pass # Ignore JSON errors in error messages # Find error message message = "Unknown Server Error" if isinstance(data, dict): message = data.get('message') else: if status == 401: message = "Authentication Error" elif status == 500: message = "Internal Server Error" # Raise TaskclusterAuthFailure if this is an auth issue if status == 401: raise exceptions.TaskclusterAuthFailure( message, status_code=status, body=data, superExc=None ) # Raise TaskclusterRestFailure for all other issues raise exceptions.TaskclusterRestFailure( message, status_code=status, body=data, superExc=None ) # Try to load JSON try: return response.json() except ValueError: return {"response": response} # This code-path should be unreachable assert False, "Error from last retry should have been raised!"
def _makeHttpRequest(self, method, route, payload): """ Make an HTTP Request for the API endpoint. This method wraps the logic about doing failure retry and passes off the actual work of doing an HTTP request to another method.""" baseUrl = self.options['baseUrl'] # urljoin ignores the last param of the baseUrl if the base url doesn't end # in /. I wonder if it's better to just do something basic like baseUrl + # route instead if not baseUrl.endswith('/'): baseUrl += '/' url = urllib.parse.urljoin(baseUrl, route.lstrip('/')) log.debug('Full URL used is: %s', url) hawkExt = self.makeHawkExt() # Serialize payload if given if payload is not None: payload = utils.dumpJson(payload) # Do a loop of retries retry = -1 # we plus first in the loop, and attempt 1 is retry 0 retries = self.options['maxRetries'] while retry < retries: retry += 1 # if this isn't the first retry then we sleep if retry > 0: snooze = float(retry * retry) / 10.0 log.info('Sleeping %0.2f seconds for exponential backoff', snooze) time.sleep(snooze) # Construct header if self._hasCredentials(): sender = mohawk.Sender( credentials={ 'id': self.options['credentials']['clientId'], 'key': self.options['credentials']['accessToken'], 'algorithm': 'sha256', }, ext=hawkExt if hawkExt else {}, url=url, content=payload if payload else '', content_type='application/json' if payload else '', method=method, ) headers = {'Authorization': sender.request_header} else: log.debug('Not using hawk!') headers = {} if payload: # Set header for JSON if payload is given, note that we serialize # outside this loop. headers['Content-Type'] = 'application/json' log.debug('Making attempt %d', retry) try: response = utils.makeSingleHttpRequest(method, url, payload, headers) except requests.exceptions.RequestException as rerr: if retry < retries: log.warn('Retrying because of: %s' % rerr) continue # raise a connection exception raise exceptions.TaskclusterConnectionError( "Failed to establish connection", superExc=rerr ) # Handle non 2xx status code and retry if possible try: response.raise_for_status() if response.status_code == 204: return None except requests.exceptions.RequestException as rerr: status = response.status_code if 500 <= status and status < 600 and retry < retries: log.warn('Retrying because of: %s' % rerr) continue # Parse messages from errors data = {} try: data = response.json() except: pass # Ignore JSON errors in error messages # Find error message message = "Unknown Server Error" if isinstance(data, dict): message = data.get('message') else: if status == 401: message = "Authentication Error" elif status == 500: message = "Internal Server Error" # Raise TaskclusterAuthFailure if this is an auth issue if status == 401: raise exceptions.TaskclusterAuthFailure( message, status_code=status, body=data, superExc=rerr ) # Raise TaskclusterRestFailure for all other issues raise exceptions.TaskclusterRestFailure( message, status_code=status, body=data, superExc=rerr ) # Try to load JSON try: return response.json() except ValueError: return {"response": response} # This code-path should be unreachable assert False, "Error from last retry should have been raised!"