def test_check_status_acceptable_codes(self): try: response = requests.Response() response.status_code = 200 HttpRequests.check_status(response) except requests.HTTPError as e: pytest.fail(str(e)) try: response = requests.Response() response.status_code = 202 HttpRequests.check_status(response) except requests.HTTPError as e: pytest.fail(str(e))
def test_create_submission_envelope(self, requests_mock, test_data): envelope_url = ( 'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes' ) def _request_callback(request, context): context.status_code = 201 return { 'submissionDate': '2018-08-22T15:10:11.993Z', 'updateDate': '2018-08-22T15:10:11.993Z', 'user': '******', 'lastModifiedUser': '******', 'uuid': { 'uuid': 'fake-uuid' }, 'events': [], 'stagingDetails': None, 'submissionState': 'Pending', 'triggersAnalysis': True, 'submissionErrors': [], 'open': True, '_links': test_data.links_json['_links'], } requests_mock.post(envelope_url, json=_request_callback) with HttpRequestsManager(): envelope = submit.create_submission_envelope( envelope_url, test_data.headers, HttpRequests()) assert envelope['user'] == 'fake1@clients' assert requests_mock.call_count == 1
def test_get_analysis_process_finds_existing_analysis_process( self, requests_mock, test_data): analysis_process_url = ( 'https://api.ingest.dev.data.humancellatlas.org/abcde/processes') def _request_callback(request, context): context.status_code = 200 analysis_process = test_data.analysis_process return { '_embedded': { 'processes': [{ 'content': analysis_process }] } } requests_mock.get(analysis_process_url, json=_request_callback) with HttpRequestsManager(): analysis_process = submit.get_analysis_process( analysis_process_url, test_data.headers, test_data.analysis_process_id, HttpRequests(), ) assert (analysis_process['content']['process_core']['process_id'] == test_data.analysis_process_id)
def test_post_records_details(self, requests_mock): def callback(request): response = requests.Response() response.status_code = 200 return {} url = 'https://fake_url' js = json.dumps({'foo': 'bar'}) adapter = req_mock.Adapter() session = requests.Session() session.mount('mock', adapter) adapter.add_matcher(callback) requests_mock.post(url, json=js) with HttpRequestsManager() as temp_dir: HttpRequests().post(url, json=js) file_path = temp_dir + '/request_001.txt' assert os.path.isfile(file_path) is True assert os.stat(file_path).st_size > 0 with open(file_path) as f: first_line = f.readline() parts = first_line.strip().split(' ') assert len(parts) == 2 assert parts[0] == 'POST' assert parts[1] == 'https://fake_url' rest_of_file = f.read() response_js = json.loads(rest_of_file) assert 'foo' in response_js assert response_js['foo'] == 'bar'
def create_ss2_se_input_tsv(bundle_uuid, bundle_version, dss_url, input_tsv_name='inputs.tsv'): """Create TSV of Smart-seq2-SingleEnd inputs. Args: bundle_uuid (str): The bundle uuid bundle_version (str): The bundle version dss_url (str): The url for the DCP Data Storage Service input_tsv_name (str): The file name of the input TSV file. By default, it's set to 'inputs.tsv', which will be consumed by the pipelines. Returns: None: this function will write the TSV file of cloud paths for the input files. Raises: requests.HTTPError: for 4xx errors or 5xx errors beyond the timeout """ fastq_url, sample_id = _get_content_for_ss2_se_input_tsv( bundle_uuid, bundle_version, dss_url, HttpRequests()) print('Creating input map') with open(input_tsv_name, 'w') as f: f.write('fastq\tsample_id\n') f.write('{0}\t{1}\n'.format(fastq_url, sample_id)) print('Wrote input map to disk.')
def test_run_retry_if_error(self, requests_mock, test_data): def _request_callback(request, context): context.status_code = 500 return {} requests_mock.get(test_data.envelope_url, json=_request_callback) with pytest.raises(requests.HTTPError), HttpRequestsManager(): getter.run(test_data.envelope_url, HttpRequests()) assert requests_mock.call_count == 3
def test_run(self, requests_mock, test_data): def _request_callback(request, context): context.status_code = 200 return test_data.envelope_json requests_mock.get(test_data.envelope_url, json=_request_callback) with HttpRequestsManager(): response = getter.run(test_data.envelope_url, HttpRequests()) assert requests_mock.call_count == 1
def test_get_retries_on_connection_error(self, requests_mock): def callback(request, response): raise requests.ConnectionError() url = 'https://fake_url' requests_mock.get(url, json=callback) with pytest.raises(requests.ConnectionError), HttpRequestsManager(): HttpRequests().get(url) assert requests_mock.call_count == 3
def test_run_retry_if_read_timeout_error_occurs(self, requests_mock, test_data): def _request_callback(request, context): context.status_code = 500 raise requests.ReadTimeout requests_mock.get(test_data.envelope_url, json=_request_callback) with pytest.raises(requests.ReadTimeout), HttpRequestsManager(): getter.run(test_data.envelope_url, HttpRequests()) assert requests_mock.call_count == 3
def test_get_immediately_raises_exception_on_404(self, requests_mock): def callback(request, response): response.status_code = 404 return {} url = 'https://fake_url' requests_mock.get(url, json=callback) with pytest.raises(requests.HTTPError), HttpRequestsManager(): HttpRequests().get(url) assert requests_mock.call_count == 1
def test_get_envelope_url_retries_on_error(self, requests_mock, test_data): submit_url = 'https://api.ingest.dev.data.humancellatlas.org/' def _request_callback(request, context): context.status_code = 500 return {'status': 'error', 'message': 'Internal Server Error'} requests_mock.get(submit_url, json=_request_callback) with pytest.raises(requests.HTTPError), HttpRequestsManager(): submit.get_envelope_url(submit_url, test_data.headers, HttpRequests()) assert requests_mock.call_count == 3
def test_get_succeeds_on_200(self, requests_mock): def callback(request, response): response.status_code = 200 return {} url = 'https://fake_url' requests_mock.get(url, json=callback) try: with HttpRequestsManager(): HttpRequests().get(url) except requests.HTTPError as e: pytest.fail(str(e))
def main(): parser = argparse.ArgumentParser() parser.add_argument('-envelope_url', required=True) parser.add_argument('-output', default='upload_urn.txt') args = parser.parse_args() try: urn = run(args.envelope_url, HttpRequests()) except RetryError: message = 'Timed out while trying to get urn.' raise ValueError(message) with open('upload_urn.txt', 'w') as f: f.write(urn)
def test_get_envelope_url_retries_on_read_timeout_error( self, requests_mock, test_data): submit_url = 'https://api.ingest.dev.data.humancellatlas.org/' def _request_callback(request, context): context.status_code = 500 raise requests.ReadTimeout requests_mock.get(submit_url, json=_request_callback) with pytest.raises(requests.ReadTimeout), HttpRequestsManager(): submit.get_envelope_url(submit_url, test_data.headers, HttpRequests()) assert requests_mock.call_count == 3
def test_get_auth_token(self, requests_mock, test_data): url = "https://test.auth0" def _request_callback(request, context): context.status_code = 200 return test_data.AUTH_TOKEN requests_mock.post(url, json=_request_callback) with HttpRequestsManager(): token = dcp_utils.get_auth_token(HttpRequests(), url=url) assert token == test_data.AUTH_TOKEN
def test_wait_for_valid_status_retries_if_not_valid(self, requests_mock): envelope_url = ( 'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde' ) def _request_callback(request, context): context.status_code = 200 return {'submissionState': 'Validating'} requests_mock.get(envelope_url, json=_request_callback) with pytest.raises(RetryError), HttpRequestsManager(): confirm_submission.wait_for_valid_status(envelope_url, HttpRequests()) assert requests_mock.call_count == 3
def test_wait_for_valid_status_retries_on_error(self, requests_mock): envelope_url = ( 'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde' ) def _request_callback(request, context): context.status_code = 500 return {'status': 'error', 'message': 'Internal Server Error'} requests_mock.get(envelope_url, json=_request_callback) with pytest.raises(requests.HTTPError), HttpRequestsManager(): confirm_submission.wait_for_valid_status(envelope_url, HttpRequests()) assert requests_mock.call_count == 3
def test_get_metadata_retries_on_failure(self, requests_mock, test_data): def _request_callback(request, context): context.status_code = 500 return {'status': 'error', 'message': 'Internal Server Error'} requests_mock.get(test_data.cromwell_metadata_url, json=_request_callback) with patch( 'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers', side_effect=mocked_get_auth_headers, ), pytest.raises(requests.HTTPError), HttpRequestsManager(): get_analysis_workflow_metadata.get_metadata( test_data.base_url, test_data.workflow_id, HttpRequests() ) assert requests_mock.call_count == 3
def test_get_envelope_url(self, requests_mock, test_data): submit_url = "https://api.ingest.dev.data.humancellatlas.org/" def _request_callback(request, context): context.status_code = 200 return test_data.links_json requests_mock.get(submit_url, json=_request_callback) with HttpRequestsManager(): envelope_url = submit.get_envelope_url(submit_url, test_data.headers, HttpRequests()) expected = "https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes" assert envelope_url == expected assert requests_mock.call_count == 1
def test_wait_for_valid_status_retries_on_read_timeout_error( self, requests_mock): envelope_url = ( 'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde' ) def _request_callback(request, context): context.status_code = 500 raise requests.ReadTimeout requests_mock.get(envelope_url, json=_request_callback) with pytest.raises(requests.ReadTimeout), HttpRequestsManager(): confirm_submission.wait_for_valid_status(envelope_url, HttpRequests()) assert requests_mock.call_count == 3
def test_check_status_bad_codes(self): with pytest.raises(requests.HTTPError): response = requests.Response() response.status_code = 404 HttpRequests.check_status(response) with pytest.raises(requests.HTTPError): response = requests.Response() response.status_code = 409 HttpRequests.check_status(response) with pytest.raises(requests.HTTPError): response = requests.Response() response.status_code = 500 HttpRequests.check_status(response) with pytest.raises(requests.HTTPError): response = requests.Response() response.status_code = 301 HttpRequests.check_status(response)
def test_get_metadata_using_caas(self, requests_mock, test_data, tmpdir): current_file_path = tmpdir.join('metadata.json') def _request_callback(request, context): context.status_code = 200 return {'workflowName': 'TestWorkflow'} requests_mock.get(test_data.caas_metadata_url, json=_request_callback) with patch( 'pipeline_tools.shared.submission.get_analysis_workflow_metadata.get_auth_headers', side_effect=mocked_get_auth_headers, ), tmpdir.as_cwd(), HttpRequestsManager(): get_analysis_workflow_metadata.get_metadata( test_data.caas_base_url, test_data.workflow_id, HttpRequests()) assert requests_mock.call_count == 1 assert current_file_path.read() is not None
def test_add_input_bundles(self, requests_mock, test_data): input_bundles_url = 'https://api.ingest.dev.data.humancellatlas.org/processes/abcde/bundleReferences' def _request_callback(request, context): context.status_code = 201 return {} requests_mock.put(input_bundles_url, json=_request_callback) with HttpRequestsManager(): submit.add_input_bundles( input_bundles_url, test_data.headers, test_data.analysis_process, HttpRequests(), ) assert requests_mock.call_count == 1
def test_creates_dummy_files_even_when_record_not_on(self, requests_mock): def callback(request): response = requests.Response() response.status_code = 200 return {} url = 'https://fake_url' adapter = req_mock.Adapter() session = requests.Session() session.mount('mock', adapter) adapter.add_matcher(callback) requests_mock.get(url) with HttpRequestsManager(False) as temp_dir: HttpRequests().get(url) assert os.path.isfile(temp_dir + '/request_000.txt') is True assert os.path.isfile(temp_dir + '/response_000.txt') is True
def test_does_not_record_when_var_not_set(self, requests_mock): def callback(request): response = requests.Response() response.status_code = 200 return {} url = 'https://fake_url' adapter = req_mock.Adapter() session = requests.Session() session.mount('mock', adapter) adapter.add_matcher(callback) requests_mock.get(url) with HttpRequestsManager(False) as temp_dir: HttpRequests().get(url) file_path = temp_dir + '/request_001.txt' assert os.path.isfile(file_path) is False
def main(): parser = argparse.ArgumentParser() parser.add_argument('--analysis_output_path', required=True) parser.add_argument('--cromwell_url', required=True) args = parser.parse_args() print('Using analysis output path: {0}'.format(args.analysis_output_path)) # Get the workflow id and metadata, write them to files workflow_id = get_analysis_workflow_id( analysis_output_path=args.analysis_output_path) get_metadata( cromwell_url=args.cromwell_url, workflow_id=workflow_id, http_requests=HttpRequests(), )
def test_add_input_bundles_retries_on_read_timeout_error( self, requests_mock, test_data): input_bundles_url = 'https://api.ingest.dev.data.humancellatlas.org/processes/abcde/bundleReferences' def _request_callback(request, context): context.status_code = 500 raise requests.ReadTimeout requests_mock.put(input_bundles_url, json=_request_callback) with pytest.raises(requests.ReadTimeout), HttpRequestsManager(): submit.add_input_bundles( input_bundles_url, test_data.headers, test_data.analysis_process, HttpRequests(), ) assert requests_mock.call_count == 3
def test_get_analysis_process_returns_None(self, requests_mock, test_data): analysis_process_url = ( 'https://api.ingest.dev.data.humancellatlas.org/abcde/processes') def _request_callback(request, context): context.status_code = 200 return {} requests_mock.get(analysis_process_url, json=_request_callback) with HttpRequestsManager(): analysis_process = submit.get_analysis_process( analysis_process_url, test_data.headers, test_data.analysis_process_id, HttpRequests(), ) assert analysis_process is None
def test_wait_for_valid_status_with_invalid_envelope(self, requests_mock): status = 'Invalid' envelope_url = ( 'https://api.ingest.dev.data.humancellatlas.org/submissionEnvelopes/abcde' ) def _request_callback(request, context): context.status_code = 200 return {'submissionState': status} requests_mock.get(envelope_url, json=_request_callback) with HttpRequestsManager(): result = confirm_submission.wait_for_valid_status( envelope_url, HttpRequests()) assert result.get('submissionState') == status assert requests_mock.call_count == 1
def test_attributes_initialized_for_empty_strings(self): with HttpRequestsManager(): os.environ[http_requests.RECORD_HTTP_REQUESTS] = '' os.environ[http_requests.HTTP_RECORD_DIR] = '' os.environ[http_requests.RETRY_MAX_TRIES] = '' os.environ[http_requests.RETRY_MAX_INTERVAL] = '' os.environ[http_requests.RETRY_MULTIPLIER] = '' os.environ[http_requests.RETRY_TIMEOUT] = '' os.environ[http_requests.INDIVIDUAL_REQUEST_TIMEOUT] = '' hr = HttpRequests(write_dummy_files=False) assert hr.should_record is False assert hr.record_dir == '.' assert hr.retry_timeout == 7200 assert hr.individual_request_timeout == 60 assert hr.retry_max_tries == 1e4 assert hr.retry_multiplier == 1 assert hr.retry_max_interval == 60 assert hr.individual_request_timeout == 60