def test_retry_on_retry_eligible_failures_lowercase_verbs(
mock_session, m_sleep):
expected_call_count = 0
max_calls = 3
api_response = {'key': 'value'}
session_context = mock_session.return_value.__enter__.return_value
session_context.send.return_value.json.return_value = api_response
for verb in RETRY_VERBS:
for code in RETRY_CODES:
expected_call_count += max_calls
session_context.send.return_value.status_code = code
request_info = dict(
params={
'secondParameter': 'b',
'firstParameter': 'a'
},
json={},
url='https://api.civisanalytics.com/wobble/wubble',
method=verb.lower())
request = Request(**request_info)
pre_request = session_context.prepare_request(request)
retry_request(verb, pre_request, session_context, max_calls)
assert session_context.send.call_count == expected_call_count
def test_no_retry_on_connection_error(mock_session):
expected_call_count = 0
api_response = {'key': 'value'}
session_context = mock_session.return_value.__enter__.return_value
session_context.send.return_value.json.return_value = api_response
for verb in RETRY_VERBS:
expected_call_count += 1
request_info = dict(params={
'secondParameter': 'b',
'firstParameter': 'a'
},
json={},
url='https://api.civisanalytics.com/wobble/wubble',
method=verb)
request = Request(**request_info)
pre_request = session_context.prepare_request(request)
session_context.send.side_effect = ConnectionError()
try:
retry_request(verb, pre_request, session_context, 3)
except ConnectionError:
pass
assert session_context.send.call_count == expected_call_count
def _make_request(self,
method,
path=None,
params=None,
data=None,
**kwargs):
url = self._build_path(path)
with self._lock:
with open_session(**self._session_kwargs) as sess:
request = Request(method,
url,
json=data,
params=params,
**kwargs)
pre_request = sess.prepare_request(request)
response = retry_request(method, pre_request, sess,
MAX_RETRIES)
if response.status_code == 401:
auth_error = response.headers["www-authenticate"]
raise CivisAPIKeyError(auth_error) from CivisAPIError(response)
if not response.ok:
raise CivisAPIError(response)
return response
def get_api_spec(api_key, api_version="1.0", user_agent="civis-python"):
"""Download the Civis API specification.
Parameters
----------
api_key : str
Your API key obtained from the Civis Platform.
api_version : string, optional
The version of endpoints to call. May instantiate multiple client
objects with different versions. Currently only "1.0" is supported.
user_agent : string, optional
Provide this user agent to the the Civis API, along with an
API client version tag and ``requests`` version tag.
"""
if api_version == "1.0":
with open_session(api_key, user_agent=user_agent) as sess:
request = Request('GET', "{}endpoints".format(get_base_url()))
pre_request = sess.prepare_request(request)
response = retry_request('get', pre_request, sess, MAX_RETRIES)
else:
msg = "API specification for api version {} cannot be found"
raise ValueError(msg.format(api_version))
if response.status_code in (401, 403):
msg = "{} error downloading API specification. API key may be expired."
raise requests.exceptions.HTTPError(msg.format(response.status_code))
response.raise_for_status()
spec = response.json(object_pairs_hook=OrderedDict)
return spec
def test_no_retry_on_post_success(mock_session):
expected_call_count = 1
max_calls = 3
api_response = {'key': 'value'}
session_context = mock_session.return_value.__enter__.return_value
session_context.send.return_value.json.return_value = api_response
session_context.send.return_value.status_code = 200
request_info = dict(params={
'secondParameter': 'b',
'firstParameter': 'a'
},
json={},
url='https://api.civisanalytics.com/wobble/wubble',
method='POST')
request = Request(**request_info)
pre_request = session_context.prepare_request(request)
retry_request('post', pre_request, session_context, max_calls)
assert session_context.send.call_count == expected_call_count
def test_no_retry_on_success(mock_session):
expected_call_count = 0
api_response = {'key': 'value'}
session_context = mock_session.return_value.__enter__.return_value
session_context.send.return_value.json.return_value = api_response
for verb in RETRY_VERBS:
expected_call_count += 1
session_context.send.return_value.status_code = 200
request_info = dict(params={
'secondParameter': 'b',
'firstParameter': 'a'
},
json={},
url='https://api.civisanalytics.com/wobble/wubble',
method=verb)
request = Request(**request_info)
pre_request = session_context.prepare_request(request)
retry_request(verb, pre_request, session_context, 3)
assert session_context.send.call_count == expected_call_count
def test_retry_respect_retry_after_headers(mock_session):
expected_call_count = 0
max_calls = 2
retry_after = 1
api_response = {'key': 'value'}
session_context = mock_session.return_value.__enter__.return_value
session_context.send.return_value.json.return_value = api_response
session_context.send.return_value.status_code = 429
session_context.send.return_value.headers = {
'Retry-After': str(retry_after)
}
for verb in [
'HEAD', 'TRACE', 'GET', 'PUT', 'OPTIONS', 'DELETE', 'POST', 'head',
'trace', 'get', 'put', 'options', 'delete', 'post'
]:
expected_call_count += max_calls
request_info = dict(params={
'secondParameter': 'b',
'firstParameter': 'a'
},
json={},
url='https://api.civisanalytics.com/wobble/wubble',
method=verb)
request = Request(**request_info)
pre_request = session_context.prepare_request(request)
start_time = datetime.now().timestamp()
retry_request(verb, pre_request, session_context, max_calls)
end_time = datetime.now().timestamp()
duration = end_time - start_time
assert session_context.send.call_count == expected_call_count
assert floor(duration) == retry_after * (max_calls - 1)
def invoke(method, path, op, *args, **kwargs):
"""
If json_output is in `kwargs` then the output is json. Otherwise, it is
yaml.
"""
# Remove None b/c click passes everything in as None if it's not set.
kwargs = {k: v for k, v in kwargs.items() if v is not None}
json_output = kwargs.pop('json_output', False)
# Construct the body of the request.
body = {}
body_params = [p for p in op['parameters'] if p['in'] == 'body']
if body_params:
if len(body_params) != 1:
raise ValueError(
"There can be only one body parameter, "
f"but {len(body_params)} are found: {body_params}")
props = body_params[0]['schema']['properties']
param_map = param_case_map(props.keys())
body = {param_map[k]: v for k, v in kwargs.items() if k in param_map}
# Construct the query part of the request.
query_names = {p['name'] for p in op['parameters'] if p['in'] == 'query'}
param_map = param_case_map(query_names)
query = {param_map[k]: v for k, v in kwargs.items() if k in param_map}
# Make the request.
request_info = dict(params=query,
json=body,
url=get_base_api_url() + path.format(**kwargs),
method=method)
with open_session(get_api_key(), user_agent=CLI_USER_AGENT) as sess:
request = Request(**request_info)
pre_request = sess.prepare_request(request)
response = retry_request(method, pre_request, sess, MAX_RETRIES)
# Print the response to stderr and set exit code to 1 if there was an error
output_file = sys.stdout
exit_code = 0
if not (200 <= response.status_code < 300):
output_file = sys.stderr
exit_code = 1
# Print the output, if there is any.
# For commands such as DELETE /scripts/containers/{script_id}/runs/{id},
# response ends up being " " here.
try:
if json_output:
json.dump(response.json(), output_file)
else:
yaml.safe_dump(response.json(),
output_file,
default_flow_style=False)
output_file.flush()
# json throws a ValueError if it is passed a blank string to load.
except ValueError as e:
# If the message was not blank, print an error message.
# Otherwise, do nothing.
if response.text.strip():
print("Error parsing response: {}".format(e), file=sys.stderr)
sys.exit(exit_code)