def test_no_retry_fails_fast(self): http = HttpMockSequence([({ 'status': '500' }, ''), ({ 'status': '200' }, '{}')]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = lambda _: self.fail( 'sleep should not have been called.') try: request.execute() self.fail('Should have raised an exception.') except HttpError: pass
def test_media_io_base_next_chunk_no_retry_403_not_configured(self): fd = BytesIO(b"i am png") upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) http = HttpMockSequence([ ({'status': '403'}, NOT_CONFIGURED_RESPONSE), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, headers={}, resumable=upload) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute(num_retries=3) request._sleep.assert_not_called()
def test_media_io_base_next_chunk_no_retry_403_not_configured(self): fd = BytesIO(b"i am png") upload = MediaIoBaseUpload(fd=fd, mimetype='image/png', chunksize=500, resumable=True) http = HttpMockSequence([({ 'status': '403' }, NOT_CONFIGURED_RESPONSE), ({ 'status': '200' }, '{}')]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, headers={}, resumable=upload) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute(num_retries=3) request._sleep.assert_not_called()
def test_retry(self): num_retries = 5 resp_seq = [({'status': '500'}, '')] * num_retries resp_seq.append(({'status': '200'}, '{}')) http = HttpMockSequence(resp_seq) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=num_retries) self.assertEqual(num_retries, len(sleeptimes)) for retry_num in xrange(num_retries): self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_turn_get_into_post(self): def _postproc(resp, content): return content http = HttpMockSequence([ ({ 'status': '200' }, 'echo_request_body'), ({ 'status': '200' }, 'echo_request_headers'), ]) # Send a long query parameter. query = {'q': 'a' * MAX_URI_LENGTH + '?&'} req = HttpRequest(http, _postproc, 'http://example.com?' + urlencode(query), method='GET', body=None, headers={}, methodId='foo', resumable=None) # Query parameters should be sent in the body. response = req.execute() self.assertEqual(b'q=' + b'a' * MAX_URI_LENGTH + b'%3F%26', response) # Extra headers should be set. response = req.execute() self.assertEqual('GET', response['x-http-method-override']) self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length']) self.assertEqual('application/x-www-form-urlencoded', response['content-type'])
def test_retry(self): num_retries = 5 resp_seq = [({'status': '500'}, '')] * (num_retries - 3) resp_seq.append(({'status': '403'}, RATE_LIMIT_EXCEEDED_RESPONSE)) resp_seq.append(({'status': '403'}, USER_RATE_LIMIT_EXCEEDED_RESPONSE)) resp_seq.append(({'status': '429'}, '')) resp_seq.append(({'status': '200'}, '{}')) http = HttpMockSequence(resp_seq) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=num_retries) self.assertEqual(num_retries, len(sleeptimes)) for retry_num in range(num_retries): self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
def test_media_io_base_next_chunk_retries(self): try: import io except ImportError: return f = open(datafile('small.png'), 'r') fd = io.BytesIO(f.read()) upload = MediaIoBaseUpload(fd=fd, mimetype='image/png', chunksize=500, resumable=True) # Simulate 5XXs for both the request that creates the resumable upload and # the upload itself. http = HttpMockSequence([ ({ 'status': '500' }, ''), ({ 'status': '500' }, ''), ({ 'status': '503' }, ''), ({ 'status': '200', 'location': 'location' }, ''), ({ 'status': '500' }, ''), ({ 'status': '500' }, ''), ({ 'status': '503' }, ''), ({ 'status': '200' }, '{}'), ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, headers={}, resumable=upload) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=3) self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_empty_content_type(self): """Test for #284""" http = HttpMock(None, headers={'status': 200}) uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, _postproc_none, uri, method=method, headers={'content-type': ''}) request.execute() self.assertEqual('', http.headers.get('content-type'))
def test_empty_content_type(self): """Test for #284""" http = HttpMock(None, headers={'status': 200}) uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, _postproc_none, uri, method=method, headers={'content-type': ''}) request.execute() self.assertEqual('', http.headers.get('content-type'))
def log_and_send(self, desc: str, request: http.HttpRequest) -> Response: log.info(desc) log.debug(pprint.pformat(request)) response = request.execute() log.debug(pprint.pformat(response)) self.save_credentials() # They may have been refreshed in the process. return response
def exponentialBackoff(request: HttpRequest, time: int = 0, tries: int = 0, max_tries: int = 8): """Periodically retry a failed request over an increasing amount of time to handle errors related to rate limits, network volume, or response time. https://developers.google.com/drive/api/v3/handle-errors """ try: response = None while response is None: status, response = request.next_chunk() if status: print( f'Uploading: {round(status.resumable_progress / status.total_size * 100, 2)}%' ) file = request.execute(num_retries=2) return file except HttpError as e: print(e) if tries < max_tries: print(f'Trying again in {time}') t = Timer(time, exponentialBackoff, args=(request, 2**(tries + 1) + random.random(), tries + 1, max_tries)) t.start() else: raise Exception('Max tries reached.')
def _execute_request( self, request: http.HttpRequest ) -> Mapping[Text, Sequence[Mapping[Text, Any]]]: result = request.execute() if 'error' in result: raise ValueError(result['error']) return result
def test_ensure_response_callback(self): m = JsonModel() request = HttpRequest( None, m.response, 'https://www.googleapis.com/someapi/v1/collection/?foo=bar', method='POST', body='{}', headers={'content-type': 'application/json'}) h = HttpMockSequence([ ({'status': 200}, '{}')]) responses = [] def _on_response(resp, responses=responses): responses.append(resp) request.add_response_callback(_on_response) request.execute(http=h) self.assertEqual(1, len(responses))
def test_unicode(self): http = HttpMock(datafile('zoo.json'), headers={'status': '200'}) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request.execute() self.assertEqual(uri, http.uri) self.assertEqual(str, type(http.uri)) self.assertEqual(method, http.method) self.assertEqual(str, type(http.method))
def create(cluster_api: discovery.Resource, creds: Credentials, request: HttpRequest, project_id: str) -> "Optional[Cluster]": '''create cluster Note that this is a blocking call. Args: cluster_api: cluster api client cred: credentials request: cluster creation http request project_id: project id zone: zone Returns: Cluster instance on success, None otherwise ''' daemonset_url = utils.nvidia_daemonset_url(NodeImage.COS) body = json.loads(request.body) zone = body['cluster']['zone'] cluster_name = body['cluster']['name'] # execute rsp = request.execute() if rsp is None: logging.error('error: could not create cluster') return # wait for creation operation to complete operation_name = rsp['name'] rsp = utils.wait_for_operation( cluster_api, 'projects/{}/locations/{}/operations/{}'.format( project_id, zone, operation_name)) if rsp['status'] != OpStatus.DONE.value: logging.error('error creating cluster {}!'.format(cluster_name)) return # get our newly-created cluster cluster = Cluster.get(name=cluster_name, project_id=project_id, zone=zone, creds=creds) if cluster is None: logging.error( 'error: unable to connect to cluster {}'.format(cluster_name)) logging.error( 'nvidia-driver daemonset not applied, to do this manually:') logging.error('kubectl apply -f {}'.format(daemonset_url)) return logging.info('created cluster {} successfully'.format(cluster_name)) logging.info('applying nvidia driver daemonset...') rsp = cluster.apply_daemonset_from_url( daemonset_url, lambda x: yaml.load(x, Loader=yaml.FullLoader)) return cluster
def test_unicode(self): http = HttpMock(datafile('zoo.json'), headers={'status': '200'}) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request.execute() self.assertEqual(uri, http.uri) self.assertEqual(str, type(http.uri)) self.assertEqual(method, http.method) self.assertEqual(str, type(http.method))
def test_retry_connection_errors_non_resumable(self): model = JsonModel() request = HttpRequest( HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/json_api_endpoint') request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 response = request.execute(num_retries=3) self.assertEqual({u'foo': u'bar'}, response)
def test_no_retry_connection_errors(self): model = JsonModel() request = HttpRequest( HttpMockWithNonRetriableErrors(1, {'status': '200'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/json_api_endpoint') request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 with self.assertRaises(socket.error): response = request.execute(num_retries=3)
def test_retry_connection_errors_non_resumable(self): model = JsonModel() request = HttpRequest( HttpMockWithErrors(3, {'status': '200'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/json_api_endpoint') request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 response = request.execute(num_retries=3) self.assertEqual({u'foo': u'bar'}, response)
def test_no_retry_connection_errors(self): model = JsonModel() request = HttpRequest( HttpMockWithNonRetriableErrors(1, {'status': '200'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/json_api_endpoint') request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 with self.assertRaises(socket.error): response = request.execute(num_retries=3)
def test_media_io_base_next_chunk_retries(self): try: import io except ImportError: return f = open(datafile('small.png'), 'r') fd = io.BytesIO(f.read()) upload = MediaIoBaseUpload( fd=fd, mimetype='image/png', chunksize=500, resumable=True) # Simulate 5XXs for both the request that creates the resumable upload and # the upload itself. http = HttpMockSequence([ ({'status': '500'}, ''), ({'status': '500'}, ''), ({'status': '503'}, ''), ({'status': '200', 'location': 'location'}, ''), ({'status': '500'}, ''), ({'status': '500'}, ''), ({'status': '503'}, ''), ({'status': '200'}, '{}'), ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, headers={}, resumable=upload) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=3) self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
def test_no_retry_401_fails_fast(self): http = HttpMockSequence([({ 'status': '401' }, ''), ({ 'status': '200' }, '{}')]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute() request._sleep.assert_not_called()
def test_no_retry_401_fails_fast(self): http = HttpMockSequence([ ({'status': '401'}, ''), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = mock.MagicMock() with self.assertRaises(HttpError): request.execute() request._sleep.assert_not_called()
def test_no_retry_succeeds(self): num_retries = 5 resp_seq = [({'status': '200'}, '{}')] * (num_retries) http = HttpMockSequence(resp_seq) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest(http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) sleeptimes = [] request._sleep = lambda x: sleeptimes.append(x) request._rand = lambda: 10 request.execute(num_retries=num_retries) self.assertEqual(0, len(sleeptimes))
def test_turn_get_into_post(self): def _postproc(resp, content): return content http = HttpMockSequence([ ({'status': '200'}, 'echo_request_body'), ({'status': '200'}, 'echo_request_headers'), ]) # Send a long query parameter. query = { 'q': 'a' * MAX_URI_LENGTH + '?&' } req = HttpRequest( http, _postproc, 'http://example.com?' + urllib.urlencode(query), method='GET', body=None, headers={}, methodId='foo', resumable=None) # Query parameters should be sent in the body. response = req.execute() self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response) # Extra headers should be set. response = req.execute() self.assertEqual('GET', response['x-http-method-override']) self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length']) self.assertEqual( 'application/x-www-form-urlencoded', response['content-type'])
def execute_request(request: http.HttpRequest) -> Any: """Executes an HTTP request and return its response. This method executes an HTTP request and retries if the response error code is capable of being retried. Refer to `retry_utils.py` module to see the list of error codes for which the retry will be attempted. Args: request: HTTP request to be executed. Returns: response: Response from the HTTP request. """ response = request.execute() return response
def test_no_retry_fails_fast(self): http = HttpMockSequence([ ({'status': '500'}, ''), ({'status': '200'}, '{}') ]) model = JsonModel() uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar' method = u'POST' request = HttpRequest( http, model.response, uri, method=method, body=u'{}', headers={'content-type': 'application/json'}) request._rand = lambda: 1.0 request._sleep = lambda _: self.fail('sleep should not have been called.') try: request.execute() self.fail('Should have raised an exception.') except HttpError: pass
def test_retry_connection_errors_resumable(self): with open(datafile('small.png'), 'rb') as small_png_file: small_png_fd = BytesIO(small_png_file.read()) upload = MediaIoBaseUpload(fd=small_png_fd, mimetype='image/png', chunksize=500, resumable=True) model = JsonModel() request = HttpRequest( HttpMockWithErrors( 3, {'status': '200', 'location': 'location'}, '{"foo": "bar"}'), model.response, u'https://www.example.com/file_upload', method='POST', resumable=upload) request._sleep = lambda _x: 0 # do nothing request._rand = lambda: 10 response = request.execute(num_retries=3) self.assertEqual({u'foo': u'bar'}, response)
def _send_whole_upload(request: HttpRequest): '''Upload a non-resumable media file. @params: request: HttpRequest, a MediaUpload object. @return: tuple(bool, whether or not the upload succeeded response, the result of the executed request (or None) ''' if not request or not isinstance(request, HttpRequest): return (False, None) try: resp = request.execute(num_retries=2) return (True, resp) except (HttpLib2Error | HttpError) as err: print('Upload failed:', err) return (False, None)
def _exponential_backoff(request: HttpRequest) -> Dict[Any, Any]: """ Экспоненциальная выдержка для 500 и 503 ошибки. :param request: Запрос к API Google Drive :return: Результат запроса """ for n in range(0, 5): try: return request.execute() except errors.HttpError as e: if not re.search(r"HttpError (500|503)", str(e)): log.error(f"Failed to complete request to Google Drive " f"API due to error: {str(e)}") raise e if n == 4: error_text = ("Failed to complete request to " "Google Drive API in 5 attempts due " "to 500 or 503 error") log.error(error_text) raise Exception(error_text) log.debug( f"HTTP error: {str(e)[11:14]}. Trying again. Attempt {n + 1}" ) time.sleep((2**n) + random.random())
def google_api_call(request: HttpRequest): try: return request.execute() except Exception: logger.error('Call to Google Cloud Resource Manager API Failed!', sys.exc_info()[0])
def logAndExecute(request: HttpRequest): """Executes the given request while adding debug logs for the request and response.""" logging.debug('Request: {}'.format(request.to_json())) response = request.execute() logging.debug('Response: {}'.format(response)) return response