def _to_url(cls, url_path):
try:
return u'{0}{1}'.format(cls.BASE_URL, url_path)
except Exception as e:
write_error_message(url_path)
write_exception(e)
raise e
def _to_url(cls, url_path):
try:
return u'{0}{1}'.format(cls.BASE_URL, url_path)
except Exception as e:
write_error_message(url_path)
write_exception(e)
raise e
def _wait_for_response(self, requests, to_json):
"""Issue a batch of requests and wait for the responses.
Args:
requests - A list of requests
to_json - A boolean, should the responses be returned as JSON blobs
Returns:
A list of dicts if to_json, a list of grequest.response otherwise
"""
all_responses = []
for request, response in zip(requests, grequests.map(requests)):
if not response:
response = MultiRequest._FakeResponse(request, '<UNKNOWN>')
if 200 != response.status_code:
write_error_message('url[{0}] status_code[{1}]'.format(
response.request.url, response.status_code))
if to_json:
all_responses.append(response.json())
else:
all_responses.append(response)
return all_responses
def test_write_error_message(self):
message = 'Look for me in validation'
expected = '[ERROR] Look for me in validation\n'
write_error_message(message)
output = self._stringio.getvalue()
T.assert_equal(output, expected)
def test_write_error_message(self):
message = 'Look for me in validation'
expected = '[ERROR] Look for me in validation\n'
write_error_message(message)
output = self._stringio.getvalue()
T.assert_equal(output, expected)
def _wait_for_response(self, requests, to_json):
"""Issue a batch of requests and wait for the responses.
Args:
requests - A list of requests
to_json - A boolean, should the responses be returned as JSON blobs
Returns:
A list of dicts if to_json, a list of grequest.response otherwise
"""
all_responses = []
for retry in range(self._max_retry):
try:
responses = grequests.map(requests)
valid_responses = [
response for response in responses if response
]
failed_auth_responses = [
response for response in responses
if response.status_code == 403
]
if failed_auth_responses:
raise ConnectionError(
'Credentials not authorized to access URL')
if len(valid_responses) != len(requests):
continue
else:
break
except ConnectionError:
raise
except:
pass
if retry == self._max_retry:
raise ConnectionError(
'Unable to complete batch of requests within max_retry retries'
)
for request, response in zip(requests, responses):
if not response:
# should have caught this earlier, but if not ...
raise ConnectionError(
'Request to {0} had an empty response'.format(request.url))
if 200 != response.status_code:
write_error_message('url[{0}] status_code[{1}]'.format(
response.request.url, response.status_code))
if to_json:
all_responses.append(response.json())
else:
all_responses.append(response)
return all_responses
def _to_urls(cls, fmt_url_path, url_path_args):
url_paths = []
for path_arg in url_path_args:
try:
url_paths.append(fmt_url_path.format(path_arg))
except Exception as e:
write_error_message(path_arg)
write_exception(e)
raise e
return [cls._to_url(url_path) for url_path in url_paths]
def _to_urls(cls, fmt_url_path, url_path_args):
url_paths = []
for path_arg in url_path_args:
try:
url_paths.append(fmt_url_path.format(path_arg))
except Exception as e:
write_error_message(path_arg)
write_exception(e)
raise e
return [cls._to_url(url_path) for url_path in url_paths]
def wrapper(*args, **kwargs):
try:
result = fn(*args, **kwargs)
return result
except InvalidRequestError as e:
write_exception(e)
if hasattr(e, 'request'):
write_error_message('request {0}'.format(repr(e.request)))
if hasattr(e, 'response'):
write_error_message('response {0}'.format(repr(e.response)))
raise e
def wrapper(*args, **kwargs):
try:
result = fn(*args, **kwargs)
return result
except InvalidRequestError as e:
write_exception(e)
if hasattr(e, 'request'):
write_error_message('request {0}'.format(repr(e.request)))
if hasattr(e, 'response'):
write_error_message('response {0}'.format(repr(e.response)))
raise e
def _wait_for_response(self, requests, to_json):
"""Issue a batch of requests and wait for the responses.
Args:
requests - A list of requests
to_json - A boolean, should the responses be returned as JSON blobs
Returns:
A list of dicts if to_json, a list of grequest.response otherwise
"""
all_responses = []
for retry in range(self._max_retry):
try:
logging.debug('Try #{0}'.format(retry + 1))
# TODO retry only for the responses that have not finished successfully yet
responses = grequests.map(requests, self._handle_exception)
valid_responses = [response for response in responses if response]
if any(response is not None and response.status_code == 403 for response in responses):
raise InvalidRequestError('Access forbidden')
if len(valid_responses) == len(requests):
break
logging.warning('Try #{0}. Expected {1} valid response(s) but only got {2}.'.format(
retry + 1, len(requests), len(valid_responses)))
if retry + 1 == self._max_retry:
raise InvalidRequestError('Unable to complete batch of requests within {0} retries'.format(self._max_retry))
except InvalidRequestError:
raise
except Exception as e:
# log the exception for the informative purposes and pass to the next iteration
logging.exception('Try #{0}. Exception occured: {1}. Retrying.'.format(retry + 1, e))
pass
for request, response in zip(requests, responses):
if 200 != response.status_code:
write_error_message('url[{0}] status_code[{1}]'.format(response.request.url, response.status_code))
if to_json:
try:
all_responses.append(response.json())
except ValueError:
raise InvalidRequestError('Expected JSON response from: {0}'.format(response.request.url))
else:
all_responses.append(response)
return all_responses
def _wait_for_response(self, requests, to_json):
"""Issue a batch of requests and wait for the responses.
Args:
requests - A list of requests
to_json - A boolean, should the responses be returned as JSON blobs
Returns:
A list of dicts if to_json, a list of grequest.response otherwise
"""
all_responses = []
for retry in range(self._max_retry):
try:
responses = grequests.map(requests)
valid_responses = [response for response in responses if response]
failed_auth_responses = [response for response in responses if response.status_code == 403]
if failed_auth_responses:
raise ConnectionError('Credentials not authorized to access URL')
if len(valid_responses) != len(requests):
continue
else:
break
except ConnectionError:
raise
except:
pass
if retry == self._max_retry:
raise ConnectionError('Unable to complete batch of requests within max_retry retries')
for request, response in zip(requests, responses):
if not response:
# should have caught this earlier, but if not ...
raise ConnectionError('Request to {0} had an empty response'.format(request.url))
if 200 != response.status_code:
write_error_message('url[{0}] status_code[{1}]'.format(response.request.url, response.status_code))
if to_json:
all_responses.append(response.json())
else:
all_responses.append(response)
return all_responses
def _wait_for_response(self, requests, to_json):
"""Issue a batch of requests and wait for the responses.
Args:
requests - A list of requests
to_json - A boolean, should the responses be returned as JSON blobs
Returns:
A list of dicts if to_json, a list of grequest.response otherwise
"""
all_responses = []
for request, response in zip(requests, grequests.map(requests)):
if not response:
response = MultiRequest._FakeResponse(request, '<UNKNOWN>')
if 200 != response.status_code:
write_error_message('url[{0}] status_code[{1}]'.format(response.request.url, response.status_code))
if to_json:
all_responses.append(response.json())
else:
all_responses.append(response)
return all_responses
