Example #1
0
    def test_no_retry_fails_fast(self):
        http = HttpMockSequence([({
            'status': '500'
        }, ''), ({
            'status': '200'
        }, '{}')])
        model = JsonModel()
        uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
        method = u'POST'
        request = HttpRequest(http,
                              model.response,
                              uri,
                              method=method,
                              body=u'{}',
                              headers={'content-type': 'application/json'})

        request._rand = lambda: 1.0
        request._sleep = lambda _: self.fail(
            'sleep should not have been called.')

        try:
            request.execute()
            self.fail('Should have raised an exception.')
        except HttpError:
            pass
Example #2
0
    def test_turn_get_into_post(self):
        def _postproc(resp, content):
            return content

        http = HttpMockSequence([
            ({
                'status': '200'
            }, 'echo_request_body'),
            ({
                'status': '200'
            }, 'echo_request_headers'),
        ])

        # Send a long query parameter.
        query = {'q': 'a' * MAX_URI_LENGTH + '?&'}
        req = HttpRequest(http,
                          _postproc,
                          'http://example.com?' + urllib.urlencode(query),
                          method='GET',
                          body=None,
                          headers={},
                          methodId='foo',
                          resumable=None)

        # Query parameters should be sent in the body.
        response = req.execute()
        self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)

        # Extra headers should be set.
        response = req.execute()
        self.assertEqual('GET', response['x-http-method-override'])
        self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
        self.assertEqual('application/x-www-form-urlencoded',
                         response['content-type'])
  def test_retry(self):
    num_retries = 5
    resp_seq = [({'status': '500'}, '')] * num_retries
    resp_seq.append(({'status': '200'}, '{}'))

    http = HttpMockSequence(resp_seq)
    model = JsonModel()
    uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
    method = u'POST'
    request = HttpRequest(
        http,
        model.response,
        uri,
        method=method,
        body=u'{}',
        headers={'content-type': 'application/json'})

    sleeptimes = []
    request._sleep = lambda x: sleeptimes.append(x)
    request._rand = lambda: 10

    request.execute(num_retries=num_retries)

    self.assertEqual(num_retries, len(sleeptimes))
    for retry_num in xrange(num_retries):
      self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
Example #4
0
    def test_retry(self):
        num_retries = 5
        resp_seq = [({'status': '500'}, '')] * num_retries
        resp_seq.append(({'status': '200'}, '{}'))

        http = HttpMockSequence(resp_seq)
        model = JsonModel()
        uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
        method = u'POST'
        request = HttpRequest(http,
                              model.response,
                              uri,
                              method=method,
                              body=u'{}',
                              headers={'content-type': 'application/json'})

        sleeptimes = []
        request._sleep = lambda x: sleeptimes.append(x)
        request._rand = lambda: 10

        request.execute(num_retries=num_retries)

        self.assertEqual(num_retries, len(sleeptimes))
        for retry_num in xrange(num_retries):
            self.assertEqual(10 * 2**(retry_num + 1), sleeptimes[retry_num])
Example #5
0
    def test_media_io_base_next_chunk_retries(self):
        try:
            import io
        except ImportError:
            return

        f = open(datafile('small.png'), 'r')
        fd = io.BytesIO(f.read())
        upload = MediaIoBaseUpload(fd=fd,
                                   mimetype='image/png',
                                   chunksize=500,
                                   resumable=True)

        # Simulate 5XXs for both the request that creates the resumable upload and
        # the upload itself.
        http = HttpMockSequence([
            ({
                'status': '500'
            }, ''),
            ({
                'status': '500'
            }, ''),
            ({
                'status': '503'
            }, ''),
            ({
                'status': '200',
                'location': 'location'
            }, ''),
            ({
                'status': '500'
            }, ''),
            ({
                'status': '500'
            }, ''),
            ({
                'status': '503'
            }, ''),
            ({
                'status': '200'
            }, '{}'),
        ])

        model = JsonModel()
        uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
        method = u'POST'
        request = HttpRequest(http,
                              model.response,
                              uri,
                              method=method,
                              headers={},
                              resumable=upload)

        sleeptimes = []
        request._sleep = lambda x: sleeptimes.append(x)
        request._rand = lambda: 10

        request.execute(num_retries=3)
        self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
Example #6
0
    def _make_request(self, request: HttpRequest) -> Dict:
        retryable_error_codes = [500, 503]
        current_delay = 0.1  #100ms

        while True:
            try:
                return request.execute()
            except ConnectionResetError:
                message = "Lost connection to the engine."
            except HttpError as raw_err:
                err = json.loads(raw_err.content).get('error')
                message = err.get('message')
                # Raise RuntimeError for exceptions that are not retryable.
                # Otherwise, pass through to retry.
                if not err.get('code') in retryable_error_codes:
                    raise EngineException(message) from raw_err

            current_delay *= 2
            if current_delay > self.max_retry_delay:
                raise TimeoutError(
                    'Reached max retry attempts for error: {}'.format(message))
            if (self.verbose):
                print(message, file=sys.stderr)
                print('Waiting ',
                      current_delay,
                      'seconds before retrying.',
                      file=sys.stderr)
            time.sleep(current_delay)
Example #7
0
 def test_unicode(self):
     http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
     model = JsonModel()
     uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
     method = u'POST'
     request = HttpRequest(http,
                           model.response,
                           uri,
                           method=method,
                           body=u'{}',
                           headers={'content-type': 'application/json'})
     request.execute()
     self.assertEqual(uri, http.uri)
     self.assertEqual(str, type(http.uri))
     self.assertEqual(method, http.method)
     self.assertEqual(str, type(http.method))
 def test_ensure_response_callback(self):
   m = JsonModel()
   request = HttpRequest(
       None,
       m.response,
       'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
       method='POST',
       body='{}',
       headers={'content-type': 'application/json'})
   h = HttpMockSequence([ ({'status': 200}, '{}')])
   responses = []
   def _on_response(resp, responses=responses):
     responses.append(resp)
   request.add_response_callback(_on_response)
   request.execute(http=h)
   self.assertEqual(1, len(responses))
Example #9
0
 def test_ensure_response_callback(self):
   m = JsonModel()
   request = HttpRequest(
       None,
       m.response,
       'https://www.googleapis.com/someapi/v1/collection/?foo=bar',
       method='POST',
       body='{}',
       headers={'content-type': 'application/json'})
   h = HttpMockSequence([ ({'status': 200}, '{}')])
   responses = []
   def _on_response(resp, responses=responses):
     responses.append(resp)
   request.add_response_callback(_on_response)
   request.execute(http=h)
   self.assertEqual(1, len(responses))
 def test_unicode(self):
   http = HttpMock(datafile('zoo.json'), headers={'status': '200'})
   model = JsonModel()
   uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
   method = u'POST'
   request = HttpRequest(
       http,
       model.response,
       uri,
       method=method,
       body=u'{}',
       headers={'content-type': 'application/json'})
   request.execute()
   self.assertEqual(uri, http.uri)
   self.assertEqual(str, type(http.uri))
   self.assertEqual(method, http.method)
   self.assertEqual(str, type(http.method))
  def test_media_io_base_next_chunk_retries(self):
    try:
      import io
    except ImportError:
      return

    f = open(datafile('small.png'), 'r')
    fd = io.BytesIO(f.read())
    upload = MediaIoBaseUpload(
        fd=fd, mimetype='image/png', chunksize=500, resumable=True)

    # Simulate 5XXs for both the request that creates the resumable upload and
    # the upload itself.
    http = HttpMockSequence([
      ({'status': '500'}, ''),
      ({'status': '500'}, ''),
      ({'status': '503'}, ''),
      ({'status': '200', 'location': 'location'}, ''),
      ({'status': '500'}, ''),
      ({'status': '500'}, ''),
      ({'status': '503'}, ''),
      ({'status': '200'}, '{}'),
    ])

    model = JsonModel()
    uri = u'https://www.googleapis.com/someapi/v1/upload/?foo=bar'
    method = u'POST'
    request = HttpRequest(
        http,
        model.response,
        uri,
        method=method,
        headers={},
        resumable=upload)

    sleeptimes = []
    request._sleep = lambda x: sleeptimes.append(x)
    request._rand = lambda: 10

    request.execute(num_retries=3)
    self.assertEqual([20, 40, 80, 20, 40, 80], sleeptimes)
Example #12
0
  def test_turn_get_into_post(self):

    def _postproc(resp, content):
      return content

    http = HttpMockSequence([
      ({'status': '200'},
        'echo_request_body'),
      ({'status': '200'},
        'echo_request_headers'),
      ])

    # Send a long query parameter.
    query = {
        'q': 'a' * MAX_URI_LENGTH + '?&'
        }
    req = HttpRequest(
        http,
        _postproc,
        'http://example.com?' + urllib.parse.urlencode(query),
        method='GET',
        body=None,
        headers={},
        methodId='foo',
        resumable=None)

    # Query parameters should be sent in the body.
    response = req.execute()
    self.assertEqual('q=' + 'a' * MAX_URI_LENGTH + '%3F%26', response)

    # Extra headers should be set.
    response = req.execute()
    response = simplejson.loads(response.decode())
    self.assertEqual('GET', response['x-http-method-override'])
    self.assertEqual(str(MAX_URI_LENGTH + 8), response['content-length'])
    self.assertEqual(
        'application/x-www-form-urlencoded', response['content-type'])
  def test_no_retry_fails_fast(self):
    http = HttpMockSequence([
        ({'status': '500'}, ''),
        ({'status': '200'}, '{}')
        ])
    model = JsonModel()
    uri = u'https://www.googleapis.com/someapi/v1/collection/?foo=bar'
    method = u'POST'
    request = HttpRequest(
        http,
        model.response,
        uri,
        method=method,
        body=u'{}',
        headers={'content-type': 'application/json'})

    request._rand = lambda: 1.0
    request._sleep = lambda _: self.fail('sleep should not have been called.')

    try:
      request.execute()
      self.fail('Should have raised an exception.')
    except HttpError:
      pass
 def _execute_with_repeat(self,
                          request: HttpRequest,
                          retry_num: int = 10) -> Any:
     for i in range(retry_num):
         try:
             response = request.execute()
             break
         except HttpError:
             self._logger.warning(
                 'An http error occurs during execution, retrying...')
             self._logger.warning('Error information: {}'.format(
                 sys.exc_info()))
     else:
         raise HttpError(
             'http request failed after {} trying.'.format(retry_num))
     return response