def test_get_analysis_process_finds_existing_analysis_process(
self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 200
analysis_process = test_data.analysis_process
return {
'_embedded': {
'processes': [{
'content': analysis_process
}]
}
}
requests_mock.get(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
analysis_process = submit.get_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process_id,
HttpRequests(),
)
assert (analysis_process['content']['process_core']['process_id'] ==
test_data.analysis_process_id)
def test_create_submission_envelope(self, requests_mock, test_data):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes'
)
def _request_callback(request, context):
context.status_code = 201
return {
'submissionDate': '2018-08-22T15:10:11.993Z',
'updateDate': '2018-08-22T15:10:11.993Z',
'user': '******',
'lastModifiedUser': '******',
'uuid': {
'uuid': 'fake-uuid'
},
'events': [],
'stagingDetails': None,
'submissionState': 'Pending',
'triggersAnalysis': True,
'submissionErrors': [],
'open': True,
'_links': test_data.links_json['_links'],
}
requests_mock.post(envelope_url, json=_request_callback)
with HttpRequestsManager():
envelope = submit.create_submission_envelope(
envelope_url, test_data.headers, HttpRequests())
assert envelope['user'] == 'fake1@clients'
assert requests_mock.call_count == 1
def test_run(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 200
return test_data.envelope_json
requests_mock.get(test_data.envelope_url, json=_request_callback)
with HttpRequestsManager():
response = getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 1
def test_run_retry_if_error(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 500
return {}
requests_mock.get(test_data.envelope_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 3
def test_run_retry_if_read_timeout_error_occurs(self, requests_mock,
test_data):
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(test_data.envelope_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 3
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-envelope_url', required=True)
parser.add_argument('-output', default='upload_urn.txt')
args = parser.parse_args()
try:
urn = run(args.envelope_url, HttpRequests())
except RetryError:
message = 'Timed out while trying to get urn.'
raise ValueError(message)
with open('upload_urn.txt', 'w') as f:
f.write(urn)
def test_get_auth_token(self, requests_mock, test_data):
url = "https://test.auth0"
def _request_callback(request, context):
context.status_code = 200
return test_data.AUTH_TOKEN
requests_mock.post(url, json=_request_callback)
with HttpRequestsManager():
token = dcp_utils.get_auth_token(HttpRequests(), url=url)
assert token == test_data.AUTH_TOKEN
def test_get_metadata_retries_on_failure(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(test_data.cromwell_metadata_url, json=_request_callback)
with patch(
'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers',
side_effect=mocked_get_auth_headers,
), pytest.raises(requests.HTTPError), HttpRequestsManager():
get_analysis_workflow_metadata.get_metadata(
test_data.base_url, test_data.workflow_id, HttpRequests()
)
assert requests_mock.call_count == 3
def test_wait_for_valid_status_retries_on_error(self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_wait_for_valid_status_retries_if_not_valid(self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 200
return {'submissionState': 'Validating'}
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(RetryError), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_wait_for_valid_status_retries_on_read_timeout_error(
self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_metadata_using_caas(self, requests_mock, test_data, tmpdir):
current_file_path = tmpdir.join('metadata.json')
def _request_callback(request, context):
context.status_code = 200
return {'workflowName': 'TestWorkflow'}
requests_mock.get(test_data.caas_metadata_url, json=_request_callback)
with patch(
'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers',
side_effect=mocked_get_auth_headers,
), tmpdir.as_cwd(), HttpRequestsManager():
get_analysis_workflow_metadata.get_metadata(
test_data.caas_base_url, test_data.workflow_id, HttpRequests())
assert requests_mock.call_count == 1
assert current_file_path.read() is not None
def test_get_analysis_process_returns_None(self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 200
return {}
requests_mock.get(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
analysis_process = submit.get_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process_id,
HttpRequests(),
)
assert analysis_process is None
def test_wait_for_valid_status_with_invalid_envelope(self, requests_mock):
status = 'Invalid'
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 200
return {'submissionState': status}
requests_mock.get(envelope_url, json=_request_callback)
with HttpRequestsManager():
result = confirm_submission.wait_for_valid_status(
envelope_url, HttpRequests())
assert result.get('submissionState') == status
assert requests_mock.call_count == 1
def test_add_analysis_process(self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 201
return test_data.analysis_process
requests_mock.post(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
res = submit.add_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process,
HttpRequests(),
)
assert requests_mock.call_count == 1
assert res == test_data.analysis_process