def test_post_records_details(self, requests_mock):
def callback(request):
response = requests.Response()
response.status_code = 200
return {}
url = 'https://fake_url'
js = json.dumps({'foo': 'bar'})
adapter = req_mock.Adapter()
session = requests.Session()
session.mount('mock', adapter)
adapter.add_matcher(callback)
requests_mock.post(url, json=js)
with HttpRequestsManager() as temp_dir:
HttpRequests().post(url, json=js)
file_path = temp_dir + '/request_001.txt'
assert os.path.isfile(file_path) is True
assert os.stat(file_path).st_size > 0
with open(file_path) as f:
first_line = f.readline()
parts = first_line.strip().split(' ')
assert len(parts) == 2
assert parts[0] == 'POST'
assert parts[1] == 'https://fake_url'
rest_of_file = f.read()
response_js = json.loads(rest_of_file)
assert 'foo' in response_js
assert response_js['foo'] == 'bar'
def test_create_submission_envelope(self, requests_mock, test_data):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes'
)
def _request_callback(request, context):
context.status_code = 201
return {
'submissionDate': '2018-08-22T15:10:11.993Z',
'updateDate': '2018-08-22T15:10:11.993Z',
'user': '******',
'lastModifiedUser': '******',
'uuid': {
'uuid': 'fake-uuid'
},
'events': [],
'stagingDetails': None,
'submissionState': 'Pending',
'triggersAnalysis': True,
'submissionErrors': [],
'open': True,
'_links': test_data.links_json['_links'],
}
requests_mock.post(envelope_url, json=_request_callback)
with HttpRequestsManager():
envelope = submit.create_submission_envelope(
envelope_url, test_data.headers, HttpRequests())
assert envelope['user'] == 'fake1@clients'
assert requests_mock.call_count == 1
def test_get_analysis_process_finds_existing_analysis_process(
self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 200
analysis_process = test_data.analysis_process
return {
'_embedded': {
'processes': [{
'content': analysis_process
}]
}
}
requests_mock.get(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
analysis_process = submit.get_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process_id,
HttpRequests(),
)
assert (analysis_process['content']['process_core']['process_id'] ==
test_data.analysis_process_id)
def test_get_ss2_paired_end_inputs_to_hash(
self,
mock_sample_id,
mock_bundle,
mock_ncbi_taxon_id,
test_ss2_bundle_vx,
test_fastq1_manifest_entry,
test_fastq2_manifest_entry,
):
mock_sample_id.return_value = 'fake_id'
mock_bundle.return_value = test_ss2_bundle_vx
mock_ncbi_taxon_id.return_value = ReferenceId.Human.value
with HttpRequestsManager():
inputs_to_hash = smartseq2.get_ss2_paired_end_inputs_to_hash(
bundle_uuid='bundle_id',
bundle_version='bundle_version',
dss_url='foo_url',
)
fastq1_hashes = f'{test_fastq1_manifest_entry.sha1}{test_fastq1_manifest_entry.sha256}{test_fastq1_manifest_entry.s3_etag}{test_fastq1_manifest_entry.crc32c}'
fastq2_hashes = f'{test_fastq2_manifest_entry.sha1}{test_fastq2_manifest_entry.sha256}{test_fastq2_manifest_entry.s3_etag}{test_fastq2_manifest_entry.crc32c}'
assert inputs_to_hash == (
'fake_id',
ReferenceId.Human.value,
fastq1_hashes,
fastq2_hashes,
)
def test_get_retries_on_connection_error(self, requests_mock):
def callback(request, response):
raise requests.ConnectionError()
url = 'https://fake_url'
requests_mock.get(url, json=callback)
with pytest.raises(requests.ConnectionError), HttpRequestsManager():
HttpRequests().get(url)
assert requests_mock.call_count == 3
def test_run_retry_if_error(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 500
return {}
requests_mock.get(test_data.envelope_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 3
def test_run(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 200
return test_data.envelope_json
requests_mock.get(test_data.envelope_url, json=_request_callback)
with HttpRequestsManager():
response = getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 1
def test_get_immediately_raises_exception_on_404(self, requests_mock):
def callback(request, response):
response.status_code = 404
return {}
url = 'https://fake_url'
requests_mock.get(url, json=callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
HttpRequests().get(url)
assert requests_mock.call_count == 1
def test_run_retry_if_read_timeout_error_occurs(self, requests_mock,
test_data):
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(test_data.envelope_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
getter.run(test_data.envelope_url, HttpRequests())
assert requests_mock.call_count == 3
def test_exit_deletes_environment_var(self):
with HttpRequestsManager() as temp_dir:
pass
assert http_requests.HTTP_RECORD_DIR not in os.environ
assert http_requests.RECORD_HTTP_REQUESTS not in os.environ
assert http_requests.RETRY_MAX_TRIES not in os.environ
assert http_requests.RETRY_MAX_INTERVAL not in os.environ
assert http_requests.RETRY_TIMEOUT not in os.environ
assert http_requests.RETRY_MULTIPLIER not in os.environ
assert http_requests.INDIVIDUAL_REQUEST_TIMEOUT not in os.environ
def test_get_cellranger_inputs(self, mock_sample_id, mock_bundle,
mock_ncbi_taxon_id, test_tenx_bundle_vx):
mock_sample_id.return_value = 'fake_id'
mock_bundle.return_value = test_tenx_bundle_vx
mock_ncbi_taxon_id.return_value = ReferenceId.Human.value
with HttpRequestsManager():
cellranger.create_cellranger_input_tsv(uuid='bundle_uuid',
version='bundle_version',
dss_url='foo_url')
expected_fastqs = [
'gs://org-hca-dss-checkout-integration/bundles/3eebea0c-8b80-4007-a860-6802a215276d.2018-10-05T145809.216048Z/R1.fastq.gz',
'gs://org-hca-dss-checkout-integration/bundles/3eebea0c-8b80-4007-a860-6802a215276d.2018-10-05T145809.216048Z/R2.fastq.gz',
'gs://org-hca-dss-checkout-integration/bundles/3eebea0c-8b80-4007-a860-6802a215276d.2018-10-05T145809.216048Z/I1.fastq.gz',
]
expected_fastq_names = [
'fake_id_S1_L001_R1_001.fastq.gz',
'fake_id_S1_L001_R2_001.fastq.gz',
'fake_id_S1_L001_I1_001.fastq.gz',
]
expected_total_estimated_cells = 10000
expected_reference_name = 'GRCh38'
expected_transcriptome_tar_gz = 'gs://hca-dcp-mint-test-data/reference/GRCh38_Gencode/GRCh38_GencodeV27_Primary_CellRanger.tar'
with open('sample_id.txt') as f:
sample_id = f.read().strip()
assert sample_id == 'fake_id'
with open('expect_cells.txt') as f:
total_estimated_cells = f.read().strip()
assert int(total_estimated_cells) == expected_total_estimated_cells
with open('fastqs.txt') as f:
actual_fastqs = f.readlines()
for idx, url in enumerate(actual_fastqs):
assert actual_fastqs[idx].strip() == expected_fastqs[idx]
with open('fastq_names.txt') as f:
actual_fastq_names = f.readlines()
for idx, url in enumerate(actual_fastq_names):
assert actual_fastq_names[idx].strip() == expected_fastq_names[idx]
with open('reference_name.txt') as f:
actual_ref_name = f.read().strip()
assert actual_ref_name == expected_reference_name
with open('transcriptome_tar_gz.txt') as f:
actual_transcriptome_tar_gz = f.read().strip()
assert actual_transcriptome_tar_gz == expected_transcriptome_tar_gz
os.remove('fastqs.txt')
os.remove('fastq_names.txt')
os.remove('sample_id.txt')
os.remove('expect_cells.txt')
os.remove('reference_name.txt')
os.remove('transcriptome_tar_gz.txt')
def test_get_envelope_url_retries_on_error(self, requests_mock, test_data):
submit_url = 'https://api.ingest.dev.data.humancellatlas.org/'
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(submit_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
submit.get_envelope_url(submit_url, test_data.headers,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_succeeds_on_200(self, requests_mock):
def callback(request, response):
response.status_code = 200
return {}
url = 'https://fake_url'
requests_mock.get(url, json=callback)
try:
with HttpRequestsManager():
HttpRequests().get(url)
except requests.HTTPError as e:
pytest.fail(str(e))
def test_get_envelope_url_retries_on_read_timeout_error(
self, requests_mock, test_data):
submit_url = 'https://api.ingest.dev.data.humancellatlas.org/'
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(submit_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
submit.get_envelope_url(submit_url, test_data.headers,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_auth_token(self, requests_mock, test_data):
url = "https://test.auth0"
def _request_callback(request, context):
context.status_code = 200
return test_data.AUTH_TOKEN
requests_mock.post(url, json=_request_callback)
with HttpRequestsManager():
token = dcp_utils.get_auth_token(HttpRequests(), url=url)
assert token == test_data.AUTH_TOKEN
def test_wait_for_valid_status_retries_if_not_valid(self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 200
return {'submissionState': 'Validating'}
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(RetryError), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_wait_for_valid_status_retries_on_error(self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(requests.HTTPError), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_get_metadata_retries_on_failure(self, requests_mock, test_data):
def _request_callback(request, context):
context.status_code = 500
return {'status': 'error', 'message': 'Internal Server Error'}
requests_mock.get(test_data.cromwell_metadata_url, json=_request_callback)
with patch(
'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers',
side_effect=mocked_get_auth_headers,
), pytest.raises(requests.HTTPError), HttpRequestsManager():
get_analysis_workflow_metadata.get_metadata(
test_data.base_url, test_data.workflow_id, HttpRequests()
)
assert requests_mock.call_count == 3
def test_get_envelope_url(self, requests_mock, test_data):
submit_url = "https://api.ingest.dev.data.humancellatlas.org/"
def _request_callback(request, context):
context.status_code = 200
return test_data.links_json
requests_mock.get(submit_url, json=_request_callback)
with HttpRequestsManager():
envelope_url = submit.get_envelope_url(submit_url,
test_data.headers,
HttpRequests())
expected = "https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes"
assert envelope_url == expected
assert requests_mock.call_count == 1
def test_wait_for_valid_status_retries_on_read_timeout_error(
self, requests_mock):
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.get(envelope_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
confirm_submission.wait_for_valid_status(envelope_url,
HttpRequests())
assert requests_mock.call_count == 3
def test_enter_sets_environment_vars(self):
with HttpRequestsManager() as temp_dir:
assert http_requests.HTTP_RECORD_DIR in os.environ
assert os.environ[http_requests.HTTP_RECORD_DIR] == temp_dir
assert http_requests.RECORD_HTTP_REQUESTS in os.environ
assert os.environ[http_requests.RECORD_HTTP_REQUESTS] == 'true'
assert http_requests.RETRY_MAX_TRIES in os.environ
assert os.environ[http_requests.RETRY_MAX_TRIES] == '3'
assert http_requests.RETRY_MAX_INTERVAL in os.environ
assert os.environ[http_requests.RETRY_MAX_INTERVAL] == '10'
assert http_requests.RETRY_TIMEOUT in os.environ
assert os.environ[http_requests.RETRY_TIMEOUT] == '1'
assert http_requests.RETRY_MULTIPLIER in os.environ
assert os.environ[http_requests.RETRY_MULTIPLIER] == '0.01'
assert http_requests.INDIVIDUAL_REQUEST_TIMEOUT in os.environ
assert os.environ[http_requests.INDIVIDUAL_REQUEST_TIMEOUT] == '1'
def test_creates_dummy_files_even_when_record_not_on(self, requests_mock):
def callback(request):
response = requests.Response()
response.status_code = 200
return {}
url = 'https://fake_url'
adapter = req_mock.Adapter()
session = requests.Session()
session.mount('mock', adapter)
adapter.add_matcher(callback)
requests_mock.get(url)
with HttpRequestsManager(False) as temp_dir:
HttpRequests().get(url)
assert os.path.isfile(temp_dir + '/request_000.txt') is True
assert os.path.isfile(temp_dir + '/response_000.txt') is True
def test_get_metadata_using_caas(self, requests_mock, test_data, tmpdir):
current_file_path = tmpdir.join('metadata.json')
def _request_callback(request, context):
context.status_code = 200
return {'workflowName': 'TestWorkflow'}
requests_mock.get(test_data.caas_metadata_url, json=_request_callback)
with patch(
'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers',
side_effect=mocked_get_auth_headers,
), tmpdir.as_cwd(), HttpRequestsManager():
get_analysis_workflow_metadata.get_metadata(
test_data.caas_base_url, test_data.workflow_id, HttpRequests())
assert requests_mock.call_count == 1
assert current_file_path.read() is not None
def test_does_not_record_when_var_not_set(self, requests_mock):
def callback(request):
response = requests.Response()
response.status_code = 200
return {}
url = 'https://fake_url'
adapter = req_mock.Adapter()
session = requests.Session()
session.mount('mock', adapter)
adapter.add_matcher(callback)
requests_mock.get(url)
with HttpRequestsManager(False) as temp_dir:
HttpRequests().get(url)
file_path = temp_dir + '/request_001.txt'
assert os.path.isfile(file_path) is False
def test_add_input_bundles(self, requests_mock, test_data):
input_bundles_url = 'https://api.ingest.dev.data.humancellatlas.org/processes/abcde/bundleReferences'
def _request_callback(request, context):
context.status_code = 201
return {}
requests_mock.put(input_bundles_url, json=_request_callback)
with HttpRequestsManager():
submit.add_input_bundles(
input_bundles_url,
test_data.headers,
test_data.analysis_process,
HttpRequests(),
)
assert requests_mock.call_count == 1
def test_get_analysis_process_returns_None(self, requests_mock, test_data):
analysis_process_url = (
'https://api.ingest.dev.data.humancellatlas.org/abcde/processes')
def _request_callback(request, context):
context.status_code = 200
return {}
requests_mock.get(analysis_process_url, json=_request_callback)
with HttpRequestsManager():
analysis_process = submit.get_analysis_process(
analysis_process_url,
test_data.headers,
test_data.analysis_process_id,
HttpRequests(),
)
assert analysis_process is None
def test_wait_for_valid_status_with_invalid_envelope(self, requests_mock):
status = 'Invalid'
envelope_url = (
'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde'
)
def _request_callback(request, context):
context.status_code = 200
return {'submissionState': status}
requests_mock.get(envelope_url, json=_request_callback)
with HttpRequestsManager():
result = confirm_submission.wait_for_valid_status(
envelope_url, HttpRequests())
assert result.get('submissionState') == status
assert requests_mock.call_count == 1
def test_add_input_bundles_retries_on_read_timeout_error(
self, requests_mock, test_data):
input_bundles_url = 'https://api.ingest.dev.data.humancellatlas.org/processes/abcde/bundleReferences'
def _request_callback(request, context):
context.status_code = 500
raise requests.ReadTimeout
requests_mock.put(input_bundles_url, json=_request_callback)
with pytest.raises(requests.ReadTimeout), HttpRequestsManager():
submit.add_input_bundles(
input_bundles_url,
test_data.headers,
test_data.analysis_process,
HttpRequests(),
)
assert requests_mock.call_count == 3
def test_optimus_inputs_to_hash(
self,
mock_sample_id,
mock_bundle,
mock_ncbi_taxon_id,
mock_lib_construction_method,
test_tenx_bundle_vx,
test_tenx_bundle_manifest_vx,
):
mock_sample_id.return_value = 'fake_id'
mock_bundle.return_value = test_tenx_bundle_vx
mock_ncbi_taxon_id.return_value = ReferenceId.Human.value
mock_lib_construction_method.return_value = (
optimus.LibraryConstructionMethod.tenX_v2.value)
with HttpRequestsManager():
inputs = optimus.get_optimus_inputs_to_hash(
uuid='bundle_uuid',
version='bundle_version',
dss_url='foo_url')
read1_manifest_json = [
f for f in test_tenx_bundle_manifest_vx
if f['name'] == 'R1.fastq.gz'
][0]
read2_manifest_json = [
f for f in test_tenx_bundle_manifest_vx
if f['name'] == 'R2.fastq.gz'
][0]
index1_manifest_json = [
f for f in test_tenx_bundle_manifest_vx
if f['name'] == 'I1.fastq.gz'
][0]
read1_manifest = ManifestEntry.from_json(read1_manifest_json)
read2_manifest = ManifestEntry.from_json(read2_manifest_json)
index1_manifest = ManifestEntry.from_json(index1_manifest_json)
r1_hashes = f'{read1_manifest.sha1}{read1_manifest.sha256}{read1_manifest.s3_etag}{read1_manifest.crc32c}'
r2_hashes = f'{read2_manifest.sha1}{read2_manifest.sha256}{read2_manifest.s3_etag}{read2_manifest.crc32c}'
i1_hashes = f'{index1_manifest.sha1}{index1_manifest.sha256}{index1_manifest.s3_etag}{index1_manifest.crc32c}'
expected_chemistry = optimus.Chemistry.tenX_v2.value
assert inputs == (
'fake_id',
ReferenceId.Human.value,
f'{r1_hashes}{r2_hashes}{i1_hashes}',
expected_chemistry,
)
def test_attributes_initialized_for_empty_strings(self):
with HttpRequestsManager():
os.environ[http_requests.RECORD_HTTP_REQUESTS] = ''
os.environ[http_requests.HTTP_RECORD_DIR] = ''
os.environ[http_requests.RETRY_MAX_TRIES] = ''
os.environ[http_requests.RETRY_MAX_INTERVAL] = ''
os.environ[http_requests.RETRY_MULTIPLIER] = ''
os.environ[http_requests.RETRY_TIMEOUT] = ''
os.environ[http_requests.INDIVIDUAL_REQUEST_TIMEOUT] = ''
hr = HttpRequests(write_dummy_files=False)
assert hr.should_record is False
assert hr.record_dir == '.'
assert hr.retry_timeout == 7200
assert hr.individual_request_timeout == 60
assert hr.retry_max_tries == 1e4
assert hr.retry_multiplier == 1
assert hr.retry_max_interval == 60
assert hr.individual_request_timeout == 60
