def test_create_file_conflict(self, mock_requests_post,
mock_create_session, mock_get_link,
mock_get_file):
ingest_api = IngestApi(token_manager=self.token_manager)
mock_get_file.return_value = {
'_embedded': {
'files': [{
'content': {
'attr': 'value'
},
'_links': {
'self': {
'href': 'existing-file-url'
}
}
}]
}
}
mock_get_link.return_value = 'url/sub/id/files'
mock_create_session.return_value.patch.return_value.json.return_value = 'response'
mock_requests_post.return_value.status_code = requests.codes.conflict
api_url = mock_ingest_api_url
submission_id = mock_submission_envelope_id
submission_url = api_url + "/" + submission_id
filename = "mock-filename"
file = ingest_api.create_file(submission_url, filename,
{'attr2': 'value2'})
self.assertEqual(file, 'response')
mock_requests_post.assert_called_once()
mock_create_session.return_value.patch \
.assert_called_with('existing-file-url',
headers={'Content-type': 'application/json', 'Authorization': 'Bearer token'},
json={'content': {'attr': 'value', 'attr2': 'value2'}})
def test_create_file(self):
api_url = mock_ingest_api_url
submission_id = mock_submission_envelope_id
submission_url = api_url + "/" + submission_id
filename = "mock-filename"
# mock the load_root()
with patch('ingest.api.ingestapi.IngestApi.get_root_url'
) as mock_load_root:
root_links = dict()
root_links["file"] = {"href": api_url + "/files"}
root_links["submissionEnvelopes"] = {
"href": api_url + "/submissionEnvelopes"
}
mock_load_root.return_value = root_links
ingest_api = IngestApi(api_url)
ingest_api.submission_links[submission_url] = {
'files': {
'href': submission_url + "/files"
}
}
with patch(
'ingest.api.ingestapi.optimistic_session') as mock_session:
mock_session.return_value = MagicMock()
ingest_api.createFile(submission_url, filename, "{}")
mock_session.assert_called_once_with(
f'{submission_url}/files/{filename}')
def upload_spreadsheet():
try:
logger.info("Uploading spreadsheet")
token = _check_token()
path = _save_spreadsheet()
ingest_api = IngestApi()
ingest_api.set_token(token)
importer = XlsImporter(ingest_api)
project = _check_for_project(ingest_api)
project_uuid = None
if project and project.get('uuid'):
project_uuid = project.get('uuid').get('uuid')
submission_url = ingest_api.createSubmission(token)
_submit_spreadsheet_data(importer, path, submission_url, project_uuid)
return create_upload_success_response(submission_url)
except SpreadsheetUploadError as spreadsheetUploadError:
return create_upload_failure_response(spreadsheetUploadError.http_code,
spreadsheetUploadError.message,
spreadsheetUploadError.details)
except Exception as err:
logger.error(traceback.format_exc())
return create_upload_failure_response(
500, "We experienced a problem while uploading your spreadsheet",
str(err))
def test_create_file(self, mock_requests_post, mock_create_session,
mock_get_link):
ingest_api = IngestApi(token_manager=self.token_manager)
mock_get_link.return_value = 'url/sub/id/files'
mock_requests_post.return_value.json.return_value = {
'uuid': 'file-uuid'
}
mock_requests_post.return_value.status_code = requests.codes.ok
api_url = mock_ingest_api_url
submission_id = mock_submission_envelope_id
submission_url = api_url + "/" + submission_id
filename = "mock-filename"
file = ingest_api.create_file(submission_url, filename, {})
self.assertEqual(file, {'uuid': 'file-uuid'})
mock_requests_post.assert_called_with('url/sub/id/files',
headers={
'Content-type':
'application/json',
'Authorization':
'Bearer token'
},
json={
'fileName': 'mock-filename',
'content': {}
},
params={})
def __init__(self, graph, submission_uuid):
super().__init__(graph)
self._logger.info(
f"Started ingest hydrator for for submission [{submission_uuid}]")
self._ingest_api = IngestApi(Config['INGEST_API'])
project_url = self._ingest_api.get_submission_by_uuid(
submission_uuid)['_links']['relatedProjects']['href']
project = self._ingest_api.get(
project_url).json()['_embedded']['projects'][0]
self._logger.info(
f"Found project for submission {project['uuid']['uuid']}")
self._entities = {}
for submission in self.fetch_submissions_in_project(project):
self._logger.info(
f"Found submission for project with uuid {submission['uuid']['uuid']}"
)
for entity in self.build_entities_from_submission(submission):
self._entities[entity['uuid']] = entity
self._nodes = self.get_nodes()
self._edges = self.get_edges()
def test_get_submission_by_uuid(self, mock_create_session, mock_get_link):
mock_get_link.return_value = 'url/{?uuid}'
ingest_api = IngestApi(token_manager=self.token_manager)
mock_create_session.return_value.get.return_value.json.return_value = {
'uuid': 'submission-uuid'
}
submission = ingest_api.get_submission_by_uuid('uuid')
self.assertEqual(submission, {'uuid': 'submission-uuid'})
def get_latest_submittable_schemas(self, ingest_api_url):
ingest_api = IngestApi(url=ingest_api_url)
urls = []
for schema in ingest_api.get_schemas(high_level_entity="type",
latest_only=True):
url = schema["_links"]["json-schema"]["href"]
urls.append(url)
return urls
def configure_ingest_client(self):
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client = S2STokenClient(
ServiceCredential.from_file(gcp_credentials_file),
INGEST_API_JWT_AUDIENCE)
self.token_manager = TokenManager(self.s2s_token_client)
self.ingest_api = IngestApi(url=INGEST_API,
token_manager=self.token_manager)
def __init__(self):
log_format = ' %(asctime)s - %(name)s - %(levelname)s in %(filename)s:%(lineno)s %(funcName)s(): %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=log_format)
self.logger = logging.getLogger(__name__)
self.ingest_api = IngestApi()
self.staging_api = StagingApi()
def test_get_latest_schema_url__empty_result(self, mock_session):
# given
ingest_api = IngestApi(token_manager=self.token_manager)
ingest_api.get_schemas = MagicMock(return_value=[])
# when
result = ingest_api.get_latest_schema_url('type', 'project', 'project')
# then
self.assertEqual(result, None)
def __init__(self, graph, subid):
super().__init__(graph)
self._subid = subid
self._logger.info(f"started ingest hydrator for subid [{self._subid}]")
self._ingest_api = IngestApi(Config['INGEST_API'])
self._entities = self.fetch_submission(subid)
self._nodes = self.get_nodes()
self._edges = self.get_edges()
def test_get_submission_by_uuid(self):
api_url = mock_ingest_api_url
mock_submission_uuid = "mock-submission-uuid"
submissions_url = api_url + "/submissionEnvelopes"
submission_search_uri = submissions_url + "/search"
findByUuidRel = "findByUuid"
findByUuidHref = submission_search_uri + "/findByUuidHref{?uuid}"
ingestapi = IngestApi(api_url, dict())
with patch('ingest.api.ingestapi.IngestApi.get_link_from_resource_url'
) as mock_get_url_for_link:
def mock_get_url_for_link_patch(*args, **kwargs):
if args[0] == submission_search_uri and args[
1] == findByUuidRel:
return findByUuidHref
mock_get_url_for_link.side_effect = mock_get_url_for_link_patch
with patch(
'ingest.api.ingestapi.requests.get') as mock_requests_get:
def mock_get_side_effect(*args, **kwargs):
mock_response = {}
mock_response_payload = {}
if args[0] == submission_search_uri + "/findByUuidHref" \
and 'params' in kwargs \
and 'uuid' in kwargs['params'] \
and kwargs['params']['uuid'] == mock_submission_uuid:
mock_response['status_code'] = 200
mock_response_payload = {
"uuid": {
"uuid": mock_submission_uuid
}
}
else:
mock_response['status_code'] = 404
mock_response['json'] = lambda _self: mock_response_payload
mock_response['text'] = json.dumps(mock_response_payload)
def raise_for_status():
raise Exception("test failed")
mock_response[
'raise_for_status'] = lambda _self: raise_for_status()
return type("MockResponse", (), mock_response)()
mock_requests_get.side_effect = mock_get_side_effect
assert 'uuid' in ingestapi.getSubmissionByUuid(
mock_submission_uuid)
def setUp(self):
self.deployment = os.environ.get('DEPLOYMENT_ENV', None)
if self.deployment not in DEPLOYMENTS:
raise RuntimeError(f'DEPLOYMENT_ENV environment variable must be one of {DEPLOYMENTS}')
self.ingest_client_api = IngestApi(url=f"https://api.ingest.{self.deployment}.data.humancellatlas.org")
self.s2s_token_client = S2STokenClient()
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client.setup_from_file(gcp_credentials_file)
self.token_manager = TokenManager(self.s2s_token_client)
self.ingest_broker = IngestUIAgent(self.deployment)
self.ingest_api = IngestApiAgent(deployment=self.deployment)
class SpreadsheetImport(TestCase):
def setUp(self):
self.test_data_path = os.path.dirname(os.path.realpath(__file__))
self.configure_ingest_client()
def configure_ingest_client(self):
gcp_credentials_file = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')
self.s2s_token_client = S2STokenClient(
ServiceCredential.from_file(gcp_credentials_file),
INGEST_API_JWT_AUDIENCE)
self.token_manager = TokenManager(self.s2s_token_client)
self.ingest_api = IngestApi(url=INGEST_API,
token_manager=self.token_manager)
def test_spreadsheet_import(self):
self.metadata_spreadsheet_path = os.path.join(self.test_data_path,
SPREADSHEET_FILE)
download_file(SPREADSHEET_LOCATION, self.metadata_spreadsheet_path)
importer = XlsImporter(self.ingest_api)
submission_resource = self.ingest_api.create_submission()
submission_url = submission_resource["_links"]["self"]["href"].rsplit(
"{")[0]
submission, _ = importer.import_file(self.metadata_spreadsheet_path,
submission_url, False)
entities_by_type = {}
for entity in submission.get_entities():
entity_type = entity.type
if not entities_by_type.get(entity_type):
entities_by_type[entity_type] = []
entities_by_type[entity_type].append(entity)
files = list(self.ingest_api.get_entities(submission_url, 'files'))
biomaterials = list(
self.ingest_api.get_entities(submission_url, 'biomaterials'))
protocols = list(
self.ingest_api.get_entities(submission_url, 'protocols'))
processes = list(
self.ingest_api.get_entities(submission_url, 'processes'))
self.assertEquals(len(files), len(entities_by_type['file']))
self.assertEquals(len(biomaterials),
len(entities_by_type['biomaterial']))
self.assertEquals(len(protocols), len(entities_by_type['protocol']))
self.assertEquals(len(processes), len(entities_by_type['process']))
def tearDown(self) -> None:
if self.metadata_spreadsheet_path:
delete_file(self.metadata_spreadsheet_path)
def test_get_all(self, mock_create_session):
# given
ingest_api = IngestApi(token_manager=self.token_manager)
mocked_responses = {
'url?page=0&size=3': {
"page": {
"size": 3,
"totalElements": 5,
"totalPages": 2,
"number": 0
},
"_embedded": {
"bundleManifests": [{
"attr": "value"
}, {
"attr": "value"
}, {
"attr": "value"
}]
},
"_links": {
"next": {
'href': 'url?page=1&size=3'
}
}
},
'url?page=1&size=3': {
"page": {
"size": 3,
"totalElements": 5,
"totalPages": 2,
"number": 1
},
"_embedded": {
"bundleManifests": [{
"attr": "value"
}, {
"attr": "value"
}]
},
"_links": {}
}
}
mock_create_session.return_value.get = lambda url, headers: self._create_mock_response(
url, mocked_responses)
# when
entities = ingest_api.get_all('url?page=0&size=3', "bundleManifests")
self.assertEqual(len(list(entities)), 5)
def get_submission_files(submission_id):
ingest_api = IngestApi()
response = ingest_api.getFiles(submission_id)
files = []
if '_embedded' in response and 'files' in response['_embedded']:
files = response['_embedded']['files']
file_page = None
if 'page' in response:
file_page = response['page']
file_page['len'] = len(files)
return render_template('submission-files-table.html',
files=files,
filePage=file_page,
helper=HTML_HELPER)
def test_create_staging_job_success(self, mock_post, mock_session):
# given
ingest_api = IngestApi(token_manager=self.token_manager)
ingest_api.get_staging_jobs_url = MagicMock(return_value='url')
mock_post.return_value.json.return_value = {
'staging-area-uuid': 'uuid'
}
mock_post.return_value.status_code = requests.codes.ok
# when
staging_job = ingest_api.create_staging_job('uuid', 'filename',
'metadata-uuid')
self.assertEqual(staging_job, {'staging-area-uuid': 'uuid'})
def test_create_staging_job_failure(self, mock_post, mock_session):
# given
ingest_api = IngestApi(token_manager=self.token_manager)
ingest_api.get_staging_jobs_url = MagicMock(return_value='url')
mock_post.return_value.json.return_value = {
'staging-area-uuid': 'uuid'
}
mock_post.return_value.status_code = requests.codes.ok
mock_post.return_value.raise_for_status = MagicMock(
side_effect=HTTPError())
# when
with self.assertRaises(HTTPError):
ingest_api.create_staging_job('uuid', 'filename', 'metadata_uuid')
def setup_manifest_receiver() -> Thread:
ingest_client = IngestApi()
with Connection(DEFAULT_RABBIT_URL) as conn:
bundle_exchange = Exchange(EXCHANGE, type=EXCHANGE_TYPE)
bundle_queues = [
Queue(ASSAY_QUEUE_MANIFEST,
bundle_exchange,
routing_key=ASSAY_ROUTING_KEY)
]
conf = {
'exchange': EXCHANGE,
'routing_key': ASSAY_COMPLETED_ROUTING_KEY,
'retry': True,
'retry_policy': RETRY_POLICY
}
manifest_generator = ManifestGenerator(
ingest_client, GraphCrawler(MetadataService(ingest_client)))
exporter = ManifestExporter(ingest_api=ingest_client,
manifest_generator=manifest_generator)
manifest_receiver = ManifestReceiver(conn,
bundle_queues,
exporter=exporter,
publish_config=conf)
manifest_process = Thread(target=manifest_receiver.run)
manifest_process.start()
return manifest_process
def import_spreadsheet(self, xls_filename):
self._logger.debug("importing spreadsheet")
ingest_api = IngestApi(url=Config['INGEST_API'])
importer = XlsImporter(ingest_api)
return importer.dry_run_import_file(file_path=self._xls_filename)
def test_get_related_entities_count(self, mock_create_session):
# given
ingest_api = IngestApi(token_manager=self.token_manager)
mocked_responses = {
'https://url/project/files': {
"page": {
"size": 3,
"totalElements": 5,
"totalPages": 2,
"number": 1
},
"_embedded": {
"files": [{
"attr": "value"
}, {
"attr": "value"
}, {
"attr": "value"
}, {
"attr": "value"
}, {
"attr": "value"
}]
},
"_links": {}
}
}
mock_entity = {
"_links": {
"self": {
"href": "https://url/project/1"
},
"files": {
"href": "https://url/project/files",
}
}
}
mock_create_session.return_value.get = lambda url, headers: self._create_mock_response(
url, mocked_responses)
# when
count = ingest_api.get_related_entities_count('files', mock_entity,
'files')
self.assertEqual(count, 5)
def project_summary(project_uuid):
project = IngestApi().getProjectByUuid(project_uuid)
summary = SummaryService().summary_for_project(project)
return app.response_class(response=jsonpickle.encode(summary,
unpicklable=False),
status=200,
mimetype='application/json')
def submission_summary(submission_uuid):
submission = IngestApi().getSubmissionByUuid(submission_uuid)
summary = SummaryService().summary_for_submission(submission)
return app.response_class(response=jsonpickle.encode(summary,
unpicklable=False),
status=200,
mimetype='application/json')
def test_get_latest_schema_url(self, mock_session):
# given
ingest_api = IngestApi(token_manager=self.token_manager)
latest_schema_url = 'latest-project-schema-url'
ingest_api.get_schemas = MagicMock(return_value=[{
'_links': {
'json-schema': {
'href': latest_schema_url
}
}
}])
# when
result = ingest_api.get_latest_schema_url('type', 'project', 'project')
# then
self.assertEqual(result, latest_schema_url)
class StagingManager:
def __init__(self):
log_format = ' %(asctime)s - %(name)s - %(levelname)s in %(filename)s:%(lineno)s %(funcName)s(): %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=log_format)
self.logger = logging.getLogger(__name__)
self.ingest_api = IngestApi()
self.staging_api = StagingApi()
def create_upload_area(self, body):
message = json.loads(body)
if "documentId" in message:
submission_id = message["documentId"]
submission_url = self.ingest_api.get_submission_url(submission_id)
uuid = self.ingest_api.get_object_uuid(submission_url)
self.logger.info("Creating upload area for submission " + uuid)
upload_area_credentials = self.staging_api.createStagingArea(uuid)
self.logger.info(
"Upload area created! patching creds to subs envelope " +
json.dumps(upload_area_credentials))
self.ingest_api.update_staging_details(
submission_url, uuid, upload_area_credentials["uri"])
def delete_upload_area(self, body):
message = json.loads(body)
if "documentId" in message:
submission_id = message["documentId"]
submission_url = self.ingest_api.get_submission_url(submission_id)
submission_uuid = self.ingest_api.get_object_uuid(submission_url)
self.logger.info(
"Trying to delete the upload area for submission_uuid: " +
submission_uuid)
if self.staging_api.hasStagingArea(submission_uuid):
self.staging_api.deleteStagingArea(submission_uuid)
self.logger.info("Upload area deleted!")
self.set_submission_to_complete(submission_id)
else:
self.logger.error("There is no upload area found.")
def set_submission_to_complete(self, submission_id):
for i in range(1, 5):
try:
self.ingest_api.update_submission_state(
submission_id, 'complete')
self.logger.info('Submission status is set to COMPLETE')
except Exception:
self.logger.info(
"failed to set state of submission {0} to Complete, retrying..."
.format(submission_id))
time.sleep(1)
def get_hca_entity_types(
ingest_api_url="http://api.ingest.data.humancellatlas.org"):
'''
takes a while to run so only run once per ingest.
'''
ingest_api = IngestApi(url=ingest_api_url)
res = ingest_api.get_schemas(high_level_entity="type", latest_only=True)
hca_schemas = {}
for schema in res:
concreteEntity = schema.get('concreteEntity')
domainEntity = schema.get('domainEntity')
if domainEntity not in hca_schemas:
hca_schemas[domainEntity] = [concreteEntity + '_json']
else:
hca_schemas[domainEntity].append(concreteEntity + '_json')
return hca_schemas
def index():
submissions = []
try:
submissions = IngestApi().getSubmissions()
except Exception as e:
flash("Can't connect to Ingest API!!", "alert-danger")
return render_template('index.html',
submissions=submissions,
helper=HTML_HELPER)
def __init__(self, connection, validation_queue, graph, test_path):
self.connection = connection
self.validation_queue = validation_queue
self._graph = graph
self._test_path = test_path
if Config["INGEST_API"] == "http://localhost:8080" or not (
Config["GOOGLE_APPLICATION_CREDENTIALS"]
and Config["INGEST_JWT_AUDIENCE"]):
self._ingest_api = IngestApi(Config['INGEST_API'])
else:
s2s_token_client = S2STokenClient(
credential=ServiceCredential.from_file(
Config['GOOGLE_APPLICATION_CREDENTIALS']),
audience=Config['INGEST_JWT_AUDIENCE'])
token_manager = TokenManager(s2s_token_client)
self._ingest_api = IngestApi(Config['INGEST_API'],
token_manager=token_manager)
self._logger = logging.getLogger(__name__)
def test_create_file(self):
api_url = mock_ingest_api_url
submission_id = mock_submission_envelope_id
submission_url = api_url + "/" + submission_id
filename = "mock-filename"
file_create_url = submission_url + "/files/" + filename
# mock the load_root()
with patch('ingest.api.ingestapi.IngestApi.get_root_url'
) as mock_load_root:
root_links = dict()
root_links["file"] = {"href": api_url + "/files"}
root_links["submissionEnvelopes"] = {
"href": api_url + "/submissionEnvelopes"
}
mock_load_root.return_value = root_links
ingestapi = IngestApi(api_url)
ingestapi.submission_links[submission_url] = {
'files': {
'href': submission_url + "/files"
}
}
with patch('ingest.api.ingestapi.requests.post') as mock_post:
def mock_post_side_effect(*args, **kwargs):
mock_response = {}
mock_response['json'] = lambda _self: {}
mock_response['text'] = "{}"
mock_response['raise_for_status'] = MagicMock()
if args[0] == file_create_url:
mock_response['status_code'] = 201
else:
mock_response['status_code'] = 404
return type("MockResponse", (), mock_response)()
mock_post.side_effect = mock_post_side_effect
ingestapi.createFile(submission_url, filename, "{}")
def create_upload_success_response(submission_url):
ingest_api = IngestApi()
submission_uuid = ingest_api.getObjectUuid(submission_url)
display_id = submission_uuid or '<UUID not generated yet>'
submission_id = submission_url.rsplit('/', 1)[-1]
data = {
"message":
"We’ve got your spreadsheet, and we’re currently importing and validating the data. Nothing else for you to do - check back later.",
"details": {
"submission_url": submission_url,
"submission_uuid": submission_uuid,
"display_uuid": display_id,
"submission_id": submission_id
}
}
success_response = app.response_class(response=json.dumps(data),
status=201,
mimetype='application/json')
return success_response
