def test_check_status_acceptable_codes(self):
try:
response = requests.Response()
response.status_code = 200
HttpRequests.check_status(response)
except requests.HTTPError as e:
pytest.fail(str(e))
try:
response = requests.Response()
response.status_code = 202
HttpRequests.check_status(response)
except requests.HTTPError as e:
pytest.fail(str(e))
def test_create_submission_envelope(self, requests_mock, test_data):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes'
)
def _request_callback(request, context):
context.status_code = 201
return {
'submissionDate': '2018-08-22T15:10:11.993Z',
'updateDate': '2018-08-22T15:10:11.993Z',
'user': '******',
'lastModifiedUser': '******',
'uuid': {
'uuid': 'fake-uuid'
},
'events': [],
'stagingDetails': None,
'submissionState': 'Pending',
'triggersAnalysis': True,
'submissionErrors': [],
'open': True,
'_links': test_data.links_json['_links'],
}
requests_mock.post(envelope_url, json=_request_callback)
with HttpRequestsManager():
envelope = submit.create_submission_envelope(
envelope_url, test_data.headers, HttpRequests())
assert envelope['user'] == 'fake1@clients'
assert requests_mock.call_count == 1
def test_get_analysis_process_finds_existing_analysis_process(
self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 200
analysis_process = test_data.analysis_process
return {
'_embedded': {
'processes': [{
'content': analysis_process
}]
}
}
requests_mock.get(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
analysis_process = submit.get_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process_id,
HttpRequests(),
)
assert (analysis_process['content']['process_core']['process_id'] ==
test_data.analysis_process_id)
def test_post_records_details(self, requests_mock):
def callback(request):
response = requests.Response()
response.status_code = 200
return {}
url = 'https://fake_url'
js = json.dumps({'foo': 'bar'})
adapter = req_mock.Adapter()
session = requests.Session()
session.mount('mock', adapter)
adapter.add_matcher(callback)
requests_mock.post(url, json=js)
with HttpRequestsManager() as temp_dir:
HttpRequests().post(url, json=js)
file_path = temp_dir + '/request_001.txt'
assert os.path.isfile(file_path) is True
assert os.stat(file_path).st_size > 0
with open(file_path) as f:
first_line = f.readline()
parts = first_line.strip().split(' ')
assert len(parts) == 2
assert parts[0] == 'POST'
assert parts[1] == 'https://fake_url'
rest_of_file = f.read()
response_js = json.loads(rest_of_file)
assert 'foo' in response_js
assert response_js['foo'] == 'bar'
def create_ss2_se_input_tsv(bundle_uuid,
bundle_version,
dss_url,
input_tsv_name='inputs.tsv'):
"""Create TSV of Smart-seq2-SingleEnd inputs.
Args:
bundle_uuid (str): The bundle uuid
bundle_version (str): The bundle version
dss_url (str): The url for the DCP Data Storage Service
input_tsv_name (str): The file name of the input TSV file. By default, it's set to 'inputs.tsv',
which will be consumed by the pipelines.
Returns:
None: this function will write the TSV file of cloud paths for the input files.
Raises:
requests.HTTPError: for 4xx errors or 5xx errors beyond the timeout
"""
fastq_url, sample_id = _get_content_for_ss2_se_input_tsv(
bundle_uuid, bundle_version, dss_url, HttpRequests())
print('Creating input map')
with open(input_tsv_name, 'w') as f:
f.write('fastq\tsample_id\n')
f.write('{0}\t{1}\n'.format(fastq_url, sample_id))
print('Wrote input map to disk.')
def test_run_retry_if_error(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 500
return {}
requests_mock.get(test_data.envelope_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 3
def test_run(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 200
return test_data.envelope_json
requests_mock.get(test_data.envelope_url, json=_request_callback)
with HttpRequestsManager():
response = getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 1
def test_get_retries_on_connection_error(self, requests_mock):
def callback(request, response):
raise requests.ConnectionError()
url = 'https://fake_url'
requests_mock.get(url, json=callback)
with pytest.raises(requests.ConnectionError), HttpRequestsManager():
HttpRequests().get(url)
assert requests_mock.call_count == 3
def test_run_retry_if_read_timeout_error_occurs(self, requests_mock,
test_data):
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(test_data.envelope_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 3
def test_get_immediately_raises_exception_on_404(self, requests_mock):
def callback(request, response):
response.status_code = 404
return {}
url = 'https://fake_url'
requests_mock.get(url, json=callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
HttpRequests().get(url)
assert requests_mock.call_count == 1
def test_get_envelope_url_retries_on_error(self, requests_mock, test_data):
submit_url = 'https://api.ingest.dev.data.humancellatlas.org/'
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(submit_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
submit.get_envelope_url(submit_url, test_data.headers,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_succeeds_on_200(self, requests_mock):
def callback(request, response):
response.status_code = 200
return {}
url = 'https://fake_url'
requests_mock.get(url, json=callback)
try:
with HttpRequestsManager():
HttpRequests().get(url)
except requests.HTTPError as e:
pytest.fail(str(e))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-envelope_url', required=True)
parser.add_argument('-output', default='upload_urn.txt')
args = parser.parse_args()
try:
urn = run(args.envelope_url, HttpRequests())
except RetryError:
message = 'Timed out while trying to get urn.'
raise ValueError(message)
with open('upload_urn.txt', 'w') as f:
f.write(urn)
def test_get_envelope_url_retries_on_read_timeout_error(
self, requests_mock, test_data):
submit_url = 'https://api.ingest.dev.data.humancellatlas.org/'
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(submit_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
submit.get_envelope_url(submit_url, test_data.headers,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_auth_token(self, requests_mock, test_data):
url = "https://test.auth0"
def _request_callback(request, context):
context.status_code = 200
return test_data.AUTH_TOKEN
requests_mock.post(url, json=_request_callback)
with HttpRequestsManager():
token = dcp_utils.get_auth_token(HttpRequests(), url=url)
assert token == test_data.AUTH_TOKEN
def test_wait_for_valid_status_retries_if_not_valid(self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 200
return {'submissionState': 'Validating'}
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(RetryError), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_wait_for_valid_status_retries_on_error(self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_metadata_retries_on_failure(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(test_data.cromwell_metadata_url, json=_request_callback)
with patch(
'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers',
side_effect=mocked_get_auth_headers,
), pytest.raises(requests.HTTPError), HttpRequestsManager():
get_analysis_workflow_metadata.get_metadata(
test_data.base_url, test_data.workflow_id, HttpRequests()
)
assert requests_mock.call_count == 3
def test_get_envelope_url(self, requests_mock, test_data):
submit_url = "https://api.ingest.dev.data.humancellatlas.org/"
def _request_callback(request, context):
context.status_code = 200
return test_data.links_json
requests_mock.get(submit_url, json=_request_callback)
with HttpRequestsManager():
envelope_url = submit.get_envelope_url(submit_url,
test_data.headers,
HttpRequests())
expected = "https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes"
assert envelope_url == expected
assert requests_mock.call_count == 1
def test_wait_for_valid_status_retries_on_read_timeout_error(
self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_check_status_bad_codes(self):
with pytest.raises(requests.HTTPError):
response = requests.Response()
response.status_code = 404
HttpRequests.check_status(response)
with pytest.raises(requests.HTTPError):
response = requests.Response()
response.status_code = 409
HttpRequests.check_status(response)
with pytest.raises(requests.HTTPError):
response = requests.Response()
response.status_code = 500
HttpRequests.check_status(response)
with pytest.raises(requests.HTTPError):
response = requests.Response()
response.status_code = 301
HttpRequests.check_status(response)
def test_get_metadata_using_caas(self, requests_mock, test_data, tmpdir):
current_file_path = tmpdir.join('metadata.json')
def _request_callback(request, context):
context.status_code = 200
return {'workflowName': 'TestWorkflow'}
requests_mock.get(test_data.caas_metadata_url, json=_request_callback)
with patch(
'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers',
side_effect=mocked_get_auth_headers,
), tmpdir.as_cwd(), HttpRequestsManager():
get_analysis_workflow_metadata.get_metadata(
test_data.caas_base_url, test_data.workflow_id, HttpRequests())
assert requests_mock.call_count == 1
assert current_file_path.read() is not None
def test_add_input_bundles(self, requests_mock, test_data):
input_bundles_url = 'https://api.ingest.dev.data.humancellatlas.org/processes/abcde/bundleReferences'
def _request_callback(request, context):
context.status_code = 201
return {}
requests_mock.put(input_bundles_url, json=_request_callback)
with HttpRequestsManager():
submit.add_input_bundles(
input_bundles_url,
test_data.headers,
test_data.analysis_process,
HttpRequests(),
)
assert requests_mock.call_count == 1
def test_creates_dummy_files_even_when_record_not_on(self, requests_mock):
def callback(request):
response = requests.Response()
response.status_code = 200
return {}
url = 'https://fake_url'
adapter = req_mock.Adapter()
session = requests.Session()
session.mount('mock', adapter)
adapter.add_matcher(callback)
requests_mock.get(url)
with HttpRequestsManager(False) as temp_dir:
HttpRequests().get(url)
assert os.path.isfile(temp_dir + '/request_000.txt') is True
assert os.path.isfile(temp_dir + '/response_000.txt') is True
def test_does_not_record_when_var_not_set(self, requests_mock):
def callback(request):
response = requests.Response()
response.status_code = 200
return {}
url = 'https://fake_url'
adapter = req_mock.Adapter()
session = requests.Session()
session.mount('mock', adapter)
adapter.add_matcher(callback)
requests_mock.get(url)
with HttpRequestsManager(False) as temp_dir:
HttpRequests().get(url)
file_path = temp_dir + '/request_001.txt'
assert os.path.isfile(file_path) is False
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--analysis_output_path', required=True)
parser.add_argument('--cromwell_url', required=True)
args = parser.parse_args()
print('Using analysis output path: {0}'.format(args.analysis_output_path))
# Get the workflow id and metadata, write them to files
workflow_id = get_analysis_workflow_id(
analysis_output_path=args.analysis_output_path)
get_metadata(
cromwell_url=args.cromwell_url,
workflow_id=workflow_id,
http_requests=HttpRequests(),
)
def test_add_input_bundles_retries_on_read_timeout_error(
self, requests_mock, test_data):
input_bundles_url = 'https://api.ingest.dev.data.humancellatlas.org/processes/abcde/bundleReferences'
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.put(input_bundles_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
submit.add_input_bundles(
input_bundles_url,
test_data.headers,
test_data.analysis_process,
HttpRequests(),
)
assert requests_mock.call_count == 3
def test_get_analysis_process_returns_None(self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 200
return {}
requests_mock.get(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
analysis_process = submit.get_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process_id,
HttpRequests(),
)
assert analysis_process is None
def test_wait_for_valid_status_with_invalid_envelope(self, requests_mock):
status = 'Invalid'
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 200
return {'submissionState': status}
requests_mock.get(envelope_url, json=_request_callback)
with HttpRequestsManager():
result = confirm_submission.wait_for_valid_status(
envelope_url, HttpRequests())
assert result.get('submissionState') == status
assert requests_mock.call_count == 1
def test_attributes_initialized_for_empty_strings(self):
with HttpRequestsManager():
os.environ[http_requests.RECORD_HTTP_REQUESTS] = ''
os.environ[http_requests.HTTP_RECORD_DIR] = ''
os.environ[http_requests.RETRY_MAX_TRIES] = ''
os.environ[http_requests.RETRY_MAX_INTERVAL] = ''
os.environ[http_requests.RETRY_MULTIPLIER] = ''
os.environ[http_requests.RETRY_TIMEOUT] = ''
os.environ[http_requests.INDIVIDUAL_REQUEST_TIMEOUT] = ''
hr = HttpRequests(write_dummy_files=False)
assert hr.should_record is False
assert hr.record_dir == '.'
assert hr.retry_timeout == 7200
assert hr.individual_request_timeout == 60
assert hr.retry_max_tries == 1e4
assert hr.retry_multiplier == 1
assert hr.retry_max_interval == 60
assert hr.individual_request_timeout == 60
