def test_create_file_conflict(self, mock_requests_post, mock_create_session, mock_get_link, mock_get_file): ingest_api = IngestApi(token_manager=self.token_manager) mock_get_file.return_value = { '_embedded': { 'files': [{ 'content': { 'attr': 'value' }, '_links': { 'self': { 'href': 'existing-file-url' } } }] } } mock_get_link.return_value = 'url/sub/id/files' mock_create_session.return_value.patch.return_value.json.return_value = 'response' mock_requests_post.return_value.status_code = requests.codes.conflict api_url = mock_ingest_api_url submission_id = mock_submission_envelope_id submission_url = api_url + "/" + submission_id filename = "mock-filename" file = ingest_api.create_file(submission_url, filename, {'attr2': 'value2'}) self.assertEqual(file, 'response') mock_requests_post.assert_called_once() mock_create_session.return_value.patch \ .assert_called_with('existing-file-url', headers={'Content-type': 'application/json', 'Authorization': 'Bearer token'}, json={'content': {'attr': 'value', 'attr2': 'value2'}})
def test_create_file(self): api_url = mock_ingest_api_url submission_id = mock_submission_envelope_id submission_url = api_url + "/" + submission_id filename = "mock-filename" # mock the load_root() with patch('ingest.api.ingestapi.IngestApi.get_root_url' ) as mock_load_root: root_links = dict() root_links["file"] = {"href": api_url + "/files"} root_links["submissionEnvelopes"] = { "href": api_url + "/submissionEnvelopes" } mock_load_root.return_value = root_links ingest_api = IngestApi(api_url) ingest_api.submission_links[submission_url] = { 'files': { 'href': submission_url + "/files" } } with patch( 'ingest.api.ingestapi.optimistic_session') as mock_session: mock_session.return_value = MagicMock() ingest_api.createFile(submission_url, filename, "{}") mock_session.assert_called_once_with( f'{submission_url}/files/{filename}')
def upload_spreadsheet(): try: logger.info("Uploading spreadsheet") token = _check_token() path = _save_spreadsheet() ingest_api = IngestApi() ingest_api.set_token(token) importer = XlsImporter(ingest_api) project = _check_for_project(ingest_api) project_uuid = None if project and project.get('uuid'): project_uuid = project.get('uuid').get('uuid') submission_url = ingest_api.createSubmission(token) _submit_spreadsheet_data(importer, path, submission_url, project_uuid) return create_upload_success_response(submission_url) except SpreadsheetUploadError as spreadsheetUploadError: return create_upload_failure_response(spreadsheetUploadError.http_code, spreadsheetUploadError.message, spreadsheetUploadError.details) except Exception as err: logger.error(traceback.format_exc()) return create_upload_failure_response( 500, "We experienced a problem while uploading your spreadsheet", str(err))
def test_create_file(self, mock_requests_post, mock_create_session, mock_get_link): ingest_api = IngestApi(token_manager=self.token_manager) mock_get_link.return_value = 'url/sub/id/files' mock_requests_post.return_value.json.return_value = { 'uuid': 'file-uuid' } mock_requests_post.return_value.status_code = requests.codes.ok api_url = mock_ingest_api_url submission_id = mock_submission_envelope_id submission_url = api_url + "/" + submission_id filename = "mock-filename" file = ingest_api.create_file(submission_url, filename, {}) self.assertEqual(file, {'uuid': 'file-uuid'}) mock_requests_post.assert_called_with('url/sub/id/files', headers={ 'Content-type': 'application/json', 'Authorization': 'Bearer token' }, json={ 'fileName': 'mock-filename', 'content': {} }, params={})
def __init__(self, graph, submission_uuid): super().__init__(graph) self._logger.info( f"Started ingest hydrator for for submission [{submission_uuid}]") self._ingest_api = IngestApi(Config['INGEST_API']) project_url = self._ingest_api.get_submission_by_uuid( submission_uuid)['_links']['relatedProjects']['href'] project = self._ingest_api.get( project_url).json()['_embedded']['projects'][0] self._logger.info( f"Found project for submission {project['uuid']['uuid']}") self._entities = {} for submission in self.fetch_submissions_in_project(project): self._logger.info( f"Found submission for project with uuid {submission['uuid']['uuid']}" ) for entity in self.build_entities_from_submission(submission): self._entities[entity['uuid']] = entity self._nodes = self.get_nodes() self._edges = self.get_edges()
def test_get_submission_by_uuid(self, mock_create_session, mock_get_link): mock_get_link.return_value = 'url/{?uuid}' ingest_api = IngestApi(token_manager=self.token_manager) mock_create_session.return_value.get.return_value.json.return_value = { 'uuid': 'submission-uuid' } submission = ingest_api.get_submission_by_uuid('uuid') self.assertEqual(submission, {'uuid': 'submission-uuid'})
def get_latest_submittable_schemas(self, ingest_api_url): ingest_api = IngestApi(url=ingest_api_url) urls = [] for schema in ingest_api.get_schemas(high_level_entity="type", latest_only=True): url = schema["_links"]["json-schema"]["href"] urls.append(url) return urls
def configure_ingest_client(self): gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') self.s2s_token_client = S2STokenClient( ServiceCredential.from_file(gcp_credentials_file), INGEST_API_JWT_AUDIENCE) self.token_manager = TokenManager(self.s2s_token_client) self.ingest_api = IngestApi(url=INGEST_API, token_manager=self.token_manager)
def __init__(self): log_format = ' %(asctime)s - %(name)s - %(levelname)s in %(filename)s:%(lineno)s %(funcName)s(): %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=log_format) self.logger = logging.getLogger(__name__) self.ingest_api = IngestApi() self.staging_api = StagingApi()
def test_get_latest_schema_url__empty_result(self, mock_session): # given ingest_api = IngestApi(token_manager=self.token_manager) ingest_api.get_schemas = MagicMock(return_value=[]) # when result = ingest_api.get_latest_schema_url('type', 'project', 'project') # then self.assertEqual(result, None)
def __init__(self, graph, subid): super().__init__(graph) self._subid = subid self._logger.info(f"started ingest hydrator for subid [{self._subid}]") self._ingest_api = IngestApi(Config['INGEST_API']) self._entities = self.fetch_submission(subid) self._nodes = self.get_nodes() self._edges = self.get_edges()
def test_get_submission_by_uuid(self): api_url = mock_ingest_api_url mock_submission_uuid = "mock-submission-uuid" submissions_url = api_url + "/submissionEnvelopes" submission_search_uri = submissions_url + "/search" findByUuidRel = "findByUuid" findByUuidHref = submission_search_uri + "/findByUuidHref{?uuid}" ingestapi = IngestApi(api_url, dict()) with patch('ingest.api.ingestapi.IngestApi.get_link_from_resource_url' ) as mock_get_url_for_link: def mock_get_url_for_link_patch(*args, **kwargs): if args[0] == submission_search_uri and args[ 1] == findByUuidRel: return findByUuidHref mock_get_url_for_link.side_effect = mock_get_url_for_link_patch with patch( 'ingest.api.ingestapi.requests.get') as mock_requests_get: def mock_get_side_effect(*args, **kwargs): mock_response = {} mock_response_payload = {} if args[0] == submission_search_uri + "/findByUuidHref" \ and 'params' in kwargs \ and 'uuid' in kwargs['params'] \ and kwargs['params']['uuid'] == mock_submission_uuid: mock_response['status_code'] = 200 mock_response_payload = { "uuid": { "uuid": mock_submission_uuid } } else: mock_response['status_code'] = 404 mock_response['json'] = lambda _self: mock_response_payload mock_response['text'] = json.dumps(mock_response_payload) def raise_for_status(): raise Exception("test failed") mock_response[ 'raise_for_status'] = lambda _self: raise_for_status() return type("MockResponse", (), mock_response)() mock_requests_get.side_effect = mock_get_side_effect assert 'uuid' in ingestapi.getSubmissionByUuid( mock_submission_uuid)
def setUp(self): self.deployment = os.environ.get('DEPLOYMENT_ENV', None) if self.deployment not in DEPLOYMENTS: raise RuntimeError(f'DEPLOYMENT_ENV environment variable must be one of {DEPLOYMENTS}') self.ingest_client_api = IngestApi(url=f"https://api.ingest.{self.deployment}.data.humancellatlas.org") self.s2s_token_client = S2STokenClient() gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') self.s2s_token_client.setup_from_file(gcp_credentials_file) self.token_manager = TokenManager(self.s2s_token_client) self.ingest_broker = IngestUIAgent(self.deployment) self.ingest_api = IngestApiAgent(deployment=self.deployment)
class SpreadsheetImport(TestCase): def setUp(self): self.test_data_path = os.path.dirname(os.path.realpath(__file__)) self.configure_ingest_client() def configure_ingest_client(self): gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') self.s2s_token_client = S2STokenClient( ServiceCredential.from_file(gcp_credentials_file), INGEST_API_JWT_AUDIENCE) self.token_manager = TokenManager(self.s2s_token_client) self.ingest_api = IngestApi(url=INGEST_API, token_manager=self.token_manager) def test_spreadsheet_import(self): self.metadata_spreadsheet_path = os.path.join(self.test_data_path, SPREADSHEET_FILE) download_file(SPREADSHEET_LOCATION, self.metadata_spreadsheet_path) importer = XlsImporter(self.ingest_api) submission_resource = self.ingest_api.create_submission() submission_url = submission_resource["_links"]["self"]["href"].rsplit( "{")[0] submission, _ = importer.import_file(self.metadata_spreadsheet_path, submission_url, False) entities_by_type = {} for entity in submission.get_entities(): entity_type = entity.type if not entities_by_type.get(entity_type): entities_by_type[entity_type] = [] entities_by_type[entity_type].append(entity) files = list(self.ingest_api.get_entities(submission_url, 'files')) biomaterials = list( self.ingest_api.get_entities(submission_url, 'biomaterials')) protocols = list( self.ingest_api.get_entities(submission_url, 'protocols')) processes = list( self.ingest_api.get_entities(submission_url, 'processes')) self.assertEquals(len(files), len(entities_by_type['file'])) self.assertEquals(len(biomaterials), len(entities_by_type['biomaterial'])) self.assertEquals(len(protocols), len(entities_by_type['protocol'])) self.assertEquals(len(processes), len(entities_by_type['process'])) def tearDown(self) -> None: if self.metadata_spreadsheet_path: delete_file(self.metadata_spreadsheet_path)
def test_get_all(self, mock_create_session): # given ingest_api = IngestApi(token_manager=self.token_manager) mocked_responses = { 'url?page=0&size=3': { "page": { "size": 3, "totalElements": 5, "totalPages": 2, "number": 0 }, "_embedded": { "bundleManifests": [{ "attr": "value" }, { "attr": "value" }, { "attr": "value" }] }, "_links": { "next": { 'href': 'url?page=1&size=3' } } }, 'url?page=1&size=3': { "page": { "size": 3, "totalElements": 5, "totalPages": 2, "number": 1 }, "_embedded": { "bundleManifests": [{ "attr": "value" }, { "attr": "value" }] }, "_links": {} } } mock_create_session.return_value.get = lambda url, headers: self._create_mock_response( url, mocked_responses) # when entities = ingest_api.get_all('url?page=0&size=3', "bundleManifests") self.assertEqual(len(list(entities)), 5)
def get_submission_files(submission_id): ingest_api = IngestApi() response = ingest_api.getFiles(submission_id) files = [] if '_embedded' in response and 'files' in response['_embedded']: files = response['_embedded']['files'] file_page = None if 'page' in response: file_page = response['page'] file_page['len'] = len(files) return render_template('submission-files-table.html', files=files, filePage=file_page, helper=HTML_HELPER)
def test_create_staging_job_success(self, mock_post, mock_session): # given ingest_api = IngestApi(token_manager=self.token_manager) ingest_api.get_staging_jobs_url = MagicMock(return_value='url') mock_post.return_value.json.return_value = { 'staging-area-uuid': 'uuid' } mock_post.return_value.status_code = requests.codes.ok # when staging_job = ingest_api.create_staging_job('uuid', 'filename', 'metadata-uuid') self.assertEqual(staging_job, {'staging-area-uuid': 'uuid'})
def test_create_staging_job_failure(self, mock_post, mock_session): # given ingest_api = IngestApi(token_manager=self.token_manager) ingest_api.get_staging_jobs_url = MagicMock(return_value='url') mock_post.return_value.json.return_value = { 'staging-area-uuid': 'uuid' } mock_post.return_value.status_code = requests.codes.ok mock_post.return_value.raise_for_status = MagicMock( side_effect=HTTPError()) # when with self.assertRaises(HTTPError): ingest_api.create_staging_job('uuid', 'filename', 'metadata_uuid')
def setup_manifest_receiver() -> Thread: ingest_client = IngestApi() with Connection(DEFAULT_RABBIT_URL) as conn: bundle_exchange = Exchange(EXCHANGE, type=EXCHANGE_TYPE) bundle_queues = [ Queue(ASSAY_QUEUE_MANIFEST, bundle_exchange, routing_key=ASSAY_ROUTING_KEY) ] conf = { 'exchange': EXCHANGE, 'routing_key': ASSAY_COMPLETED_ROUTING_KEY, 'retry': True, 'retry_policy': RETRY_POLICY } manifest_generator = ManifestGenerator( ingest_client, GraphCrawler(MetadataService(ingest_client))) exporter = ManifestExporter(ingest_api=ingest_client, manifest_generator=manifest_generator) manifest_receiver = ManifestReceiver(conn, bundle_queues, exporter=exporter, publish_config=conf) manifest_process = Thread(target=manifest_receiver.run) manifest_process.start() return manifest_process
def import_spreadsheet(self, xls_filename): self._logger.debug("importing spreadsheet") ingest_api = IngestApi(url=Config['INGEST_API']) importer = XlsImporter(ingest_api) return importer.dry_run_import_file(file_path=self._xls_filename)
def test_get_related_entities_count(self, mock_create_session): # given ingest_api = IngestApi(token_manager=self.token_manager) mocked_responses = { 'https://url/project/files': { "page": { "size": 3, "totalElements": 5, "totalPages": 2, "number": 1 }, "_embedded": { "files": [{ "attr": "value" }, { "attr": "value" }, { "attr": "value" }, { "attr": "value" }, { "attr": "value" }] }, "_links": {} } } mock_entity = { "_links": { "self": { "href": "https://url/project/1" }, "files": { "href": "https://url/project/files", } } } mock_create_session.return_value.get = lambda url, headers: self._create_mock_response( url, mocked_responses) # when count = ingest_api.get_related_entities_count('files', mock_entity, 'files') self.assertEqual(count, 5)
def project_summary(project_uuid): project = IngestApi().getProjectByUuid(project_uuid) summary = SummaryService().summary_for_project(project) return app.response_class(response=jsonpickle.encode(summary, unpicklable=False), status=200, mimetype='application/json')
def submission_summary(submission_uuid): submission = IngestApi().getSubmissionByUuid(submission_uuid) summary = SummaryService().summary_for_submission(submission) return app.response_class(response=jsonpickle.encode(summary, unpicklable=False), status=200, mimetype='application/json')
def test_get_latest_schema_url(self, mock_session): # given ingest_api = IngestApi(token_manager=self.token_manager) latest_schema_url = 'latest-project-schema-url' ingest_api.get_schemas = MagicMock(return_value=[{ '_links': { 'json-schema': { 'href': latest_schema_url } } }]) # when result = ingest_api.get_latest_schema_url('type', 'project', 'project') # then self.assertEqual(result, latest_schema_url)
class StagingManager: def __init__(self): log_format = ' %(asctime)s - %(name)s - %(levelname)s in %(filename)s:%(lineno)s %(funcName)s(): %(message)s' logging.basicConfig(stream=sys.stdout, level=logging.INFO, format=log_format) self.logger = logging.getLogger(__name__) self.ingest_api = IngestApi() self.staging_api = StagingApi() def create_upload_area(self, body): message = json.loads(body) if "documentId" in message: submission_id = message["documentId"] submission_url = self.ingest_api.get_submission_url(submission_id) uuid = self.ingest_api.get_object_uuid(submission_url) self.logger.info("Creating upload area for submission " + uuid) upload_area_credentials = self.staging_api.createStagingArea(uuid) self.logger.info( "Upload area created! patching creds to subs envelope " + json.dumps(upload_area_credentials)) self.ingest_api.update_staging_details( submission_url, uuid, upload_area_credentials["uri"]) def delete_upload_area(self, body): message = json.loads(body) if "documentId" in message: submission_id = message["documentId"] submission_url = self.ingest_api.get_submission_url(submission_id) submission_uuid = self.ingest_api.get_object_uuid(submission_url) self.logger.info( "Trying to delete the upload area for submission_uuid: " + submission_uuid) if self.staging_api.hasStagingArea(submission_uuid): self.staging_api.deleteStagingArea(submission_uuid) self.logger.info("Upload area deleted!") self.set_submission_to_complete(submission_id) else: self.logger.error("There is no upload area found.") def set_submission_to_complete(self, submission_id): for i in range(1, 5): try: self.ingest_api.update_submission_state( submission_id, 'complete') self.logger.info('Submission status is set to COMPLETE') except Exception: self.logger.info( "failed to set state of submission {0} to Complete, retrying..." .format(submission_id)) time.sleep(1)
def get_hca_entity_types( ingest_api_url="http://api.ingest.data.humancellatlas.org"): ''' takes a while to run so only run once per ingest. ''' ingest_api = IngestApi(url=ingest_api_url) res = ingest_api.get_schemas(high_level_entity="type", latest_only=True) hca_schemas = {} for schema in res: concreteEntity = schema.get('concreteEntity') domainEntity = schema.get('domainEntity') if domainEntity not in hca_schemas: hca_schemas[domainEntity] = [concreteEntity + '_json'] else: hca_schemas[domainEntity].append(concreteEntity + '_json') return hca_schemas
def index(): submissions = [] try: submissions = IngestApi().getSubmissions() except Exception as e: flash("Can't connect to Ingest API!!", "alert-danger") return render_template('index.html', submissions=submissions, helper=HTML_HELPER)
def __init__(self, connection, validation_queue, graph, test_path): self.connection = connection self.validation_queue = validation_queue self._graph = graph self._test_path = test_path if Config["INGEST_API"] == "http://localhost:8080" or not ( Config["GOOGLE_APPLICATION_CREDENTIALS"] and Config["INGEST_JWT_AUDIENCE"]): self._ingest_api = IngestApi(Config['INGEST_API']) else: s2s_token_client = S2STokenClient( credential=ServiceCredential.from_file( Config['GOOGLE_APPLICATION_CREDENTIALS']), audience=Config['INGEST_JWT_AUDIENCE']) token_manager = TokenManager(s2s_token_client) self._ingest_api = IngestApi(Config['INGEST_API'], token_manager=token_manager) self._logger = logging.getLogger(__name__)
def test_create_file(self): api_url = mock_ingest_api_url submission_id = mock_submission_envelope_id submission_url = api_url + "/" + submission_id filename = "mock-filename" file_create_url = submission_url + "/files/" + filename # mock the load_root() with patch('ingest.api.ingestapi.IngestApi.get_root_url' ) as mock_load_root: root_links = dict() root_links["file"] = {"href": api_url + "/files"} root_links["submissionEnvelopes"] = { "href": api_url + "/submissionEnvelopes" } mock_load_root.return_value = root_links ingestapi = IngestApi(api_url) ingestapi.submission_links[submission_url] = { 'files': { 'href': submission_url + "/files" } } with patch('ingest.api.ingestapi.requests.post') as mock_post: def mock_post_side_effect(*args, **kwargs): mock_response = {} mock_response['json'] = lambda _self: {} mock_response['text'] = "{}" mock_response['raise_for_status'] = MagicMock() if args[0] == file_create_url: mock_response['status_code'] = 201 else: mock_response['status_code'] = 404 return type("MockResponse", (), mock_response)() mock_post.side_effect = mock_post_side_effect ingestapi.createFile(submission_url, filename, "{}")
def create_upload_success_response(submission_url): ingest_api = IngestApi() submission_uuid = ingest_api.getObjectUuid(submission_url) display_id = submission_uuid or '<UUID not generated yet>' submission_id = submission_url.rsplit('/', 1)[-1] data = { "message": "We’ve got your spreadsheet, and we’re currently importing and validating the data. Nothing else for you to do - check back later.", "details": { "submission_url": submission_url, "submission_uuid": submission_uuid, "display_uuid": display_id, "submission_id": submission_id } } success_response = app.response_class(response=json.dumps(data), status=201, mimetype='application/json') return success_response