def test_generate_direct_links_link_not_found_error(self):
# given
spreadsheet_json = {
'project': {
'dummy-project-id': {
'content': {
'key': 'project_1'
}
}
},
'file': {
'file_id_1': {
'content': {
'key': 'file_1'
},
'links_by_entity': {
'biomaterial': ['biomaterial_id_1'],
'protocol': ['protocol_id_1', 'protocol_id_2']
}
}
}
}
entity_map = EntityMap.load(spreadsheet_json)
entity_linker = EntityLinker(self.mocked_template_manager, entity_map)
with self.assertRaises(LinkedEntityNotFound) as context:
entity_linker.handle_links_from_spreadsheet()
self.assertEqual('biomaterial', context.exception.entity)
self.assertEqual('biomaterial_id_1', context.exception.id)
def test_load(self):
# given:
spreadsheet_json = _create_spreadsheet_json()
# when:
entity_map = EntityMap.load(spreadsheet_json)
# then:
self.assertEqual(['project', 'biomaterial', 'file', 'protocol'],
list(entity_map.get_entity_types()))
# and:
# TODO shouldn't entity id's be unique and that there's no need to specify entity type?
biomaterial1 = entity_map.get_entity('biomaterial', 'biomaterial_id_1')
self._assert_correct_entity(biomaterial1,
entity_id='biomaterial_id_1',
entity_type='biomaterial',
content={'key': 'biomaterial_1'})
# and:
biomaterial2 = entity_map.get_entity('biomaterial', 'biomaterial_id_2')
links = {
'biomaterial': ['biomaterial_id_1'],
'process': ['process_id_1']
}
self._assert_correct_entity(biomaterial2,
entity_id='biomaterial_id_2',
entity_type='biomaterial',
content={'key': 'biomaterial_2'},
links=links)
# and:
protocol1 = entity_map.get_entity('protocol', 'protocol_id_1')
self.assertEqual({'key': 'protocol_1'}, protocol1.content)
def dry_run_import_file(self, file_path, project_uuid=None):
spreadsheet_json, template_mgr, errors = self.generate_json(file_path, project_uuid)
if errors:
return None, errors
entity_map = EntityMap.load(spreadsheet_json)
entity_linker = EntityLinker(template_mgr, entity_map)
entity_linker.handle_links_from_spreadsheet()
return entity_map, []
def test_handle_links_from_spreadsheet__with_external_links(self):
# given
with open(
os.path.dirname(__file__) +
'/spreadsheet_with_external_links.json') as file:
spreadsheet_json = json.load(file)
mocked_template_manager = MagicMock(name='template_manager')
mocked_template_manager.get_schema_url = MagicMock(return_value='url')
self.mocked_template_manager = mocked_template_manager
entity_map = EntityMap.load(spreadsheet_json)
entity_linker = EntityLinker(self.mocked_template_manager, entity_map)
# when
output = entity_linker.handle_links_from_spreadsheet()
# then
lib_prep_protocol = output.get_entity('protocol',
'librep-protocol-uuid')
self.assertTrue(lib_prep_protocol.is_linking_reference)
self.assertTrue(lib_prep_protocol.is_reference)
seq_protocol = output.get_entity('protocol', 'seq-protocol-uuid')
self.assertTrue(seq_protocol.is_linking_reference)
self.assertTrue(seq_protocol.is_reference)
cell_suspension = output.get_entity('biomaterial',
'cell-suspension-uuid')
self.assertTrue(cell_suspension.is_linking_reference)
self.assertTrue(cell_suspension.is_reference)
file1 = output.get_entity('file', 'seq-file-uuid-1')
self.assertFalse(file1.is_linking_reference)
self.assertTrue(file1.is_reference)
file2 = output.get_entity('file', 'seq-file-uuid-2')
self.assertFalse(file2.is_linking_reference)
self.assertTrue(file2.is_reference)
assay_process = output.get_entity('process', 'assay_process-uuid')
self.assertTrue(assay_process.is_linking_reference)
self.assertTrue(assay_process.is_reference)
assay_process_content = {
'process_core': {
'process_description': 'desc',
'process_id': 'assay_process'
},
'schema_type': 'process',
'describedBy': 'url'
}
self.assertEqual(assay_process.content, assay_process_content)
def test_load__is_reference(self):
# given:
spreadsheet_json = {
'biomaterial': {
'biomaterial_uuid': {
'content': {
'key': 'value'
},
'is_reference': True
}
}
}
# when:
entity_map = EntityMap.load(spreadsheet_json)
# then:
self.assertEqual(['biomaterial'], list(entity_map.get_entity_types()))
def test_generate_direct_links_multiple_process_links(self):
# given
spreadsheet_json = {
'project': {
'dummy-project-id': {
'content': {
'key': 'project_1'
}
}
},
'biomaterial': {
'biomaterial_id_1': {
'content': {
'key': 'biomaterial_1'
},
'links_by_entity': {
'process': ['process_id_1', 'process_id_2']
}
}
},
'process': {
'process_id_1': {
'content': {
'key': 'process_1'
}
},
'process_id_2': {
'content': {
'key': 'process_2'
}
}
}
}
entity_map = EntityMap.load(spreadsheet_json)
entity_linker = EntityLinker(self.mocked_template_manager, entity_map)
with self.assertRaises(MultipleProcessesFound) as context:
entity_linker.handle_links_from_spreadsheet()
self.assertEqual('biomaterial', context.exception.from_entity.type)
self.assertEqual(['process_id_1', 'process_id_2'],
context.exception.process_ids)
def test_generate_direct_links_invalid_spreadsheet_link(self):
# given
spreadsheet_json = {
'project': {
'dummy-project-id': {
'content': {
'key': 'project_1'
}
}
},
'biomaterial': {
'biomaterial_id_1': {
'content': {
'key': 'biomaterial_1'
},
'links_by_entity': {
'file': ['file_id_1']
}
}
},
'file': {
'file_id_1': {
'content': {
'key': 'file_1'
}
}
}
}
entity_map = EntityMap.load(spreadsheet_json)
entity_linker = EntityLinker(self.mocked_template_manager, entity_map)
with self.assertRaises(InvalidLinkInSpreadsheet) as context:
entity_linker.handle_links_from_spreadsheet()
self.assertEqual('biomaterial', context.exception.from_entity.type)
self.assertEqual('file', context.exception.link_entity_type)
self.assertEqual('biomaterial_id_1', context.exception.from_entity.id)
self.assertEqual('file_id_1', context.exception.link_entity_id)
def import_file(self, file_path, submission_url, is_update=False, project_uuid=None, update_project=False) -> Tuple[
Submission, TemplateManager]:
try:
if project_uuid:
self.submitter.link_submission_to_project(project_uuid, submission_url)
submission = None
template_mgr = None
spreadsheet_json, template_mgr, errors = self.generate_json(file_path, is_update, project_uuid=project_uuid,
update_project=update_project)
entity_map = EntityMap.load(spreadsheet_json)
self.ingest_api.delete_submission_errors(submission_url)
if errors:
self.report_errors(submission_url, errors)
elif is_update:
self.submitter.update_entities(entity_map)
else:
entity_linker = EntityLinker(template_mgr, entity_map)
entity_linker.handle_links_from_spreadsheet()
submission = self._submit_new_entities(entity_map, submission_url)
project = entity_map.get_project()
if project and project_uuid and update_project:
self.submitter.update_entity(project)
except HTTPError as httpError:
self.logger.exception(httpError)
status = httpError.response.status_code
text = httpError.response.text
importer_error = ImporterError(f'Received an HTTP {status} from {httpError.request.url}: {text}')
self.ingest_api.create_submission_error(submission_url, importer_error.getJSON())
return None, template_mgr
except Exception as e:
self.ingest_api.create_submission_error(submission_url, ImporterError(str(e)).getJSON())
self.logger.error(str(e), exc_info=True)
return None, template_mgr
finally:
self.logger.info(f'Submission in {submission_url} is done!')
return submission, template_mgr
def test_load__is_linking_reference(self):
# given:
spreadsheet_json = {
'biomaterial': {
'biomaterial_id': {
'content': {
'key': 'biomaterial_3'
},
'links_by_entity': {
'biomaterial': ['biomaterial_id_2'],
'process': ['process_id_2']
},
'external_links_by_entity': {
'biomaterial': ['biomaterial_uuid']
},
},
}
}
# when:
entity_map = EntityMap.load(spreadsheet_json)
# then:
self.assertEqual(['biomaterial'], list(entity_map.get_entity_types()))
def test_generate_direct_links_file_to_file_no_process(self):
# given
spreadsheet_json = {
'project': {
'dummy-project-id': {
'content': {
'key': 'project_1'
}
}
},
'file': {
'file_id_1': {
'content': {
'key': 'file_1'
}
},
'file_id_2': {
'content': {
'key': 'file_2'
},
'links_by_entity': {
'file': ['file_id_1'],
'protocol': ['protocol_id_1', 'protocol_id_2']
}
}
},
'protocol': {
'protocol_id_1': {
'content': {
'key': 'protocol_1'
}
},
'protocol_id_2': {
'content': {
'key': 'protocol_2'
}
}
}
}
expected_json = {
'project': {
'dummy-project-id': {
'content': {
'key': 'project_1'
}
}
},
'file': {
'file_id_1': {
'content': {
'key': 'file_1'
},
'direct_links': [{
'entity': 'process',
'id': 'process_id_1',
'relationship': 'inputToProcesses'
}, {
'entity': 'project',
'id': 'dummy-project-id',
'relationship': 'project',
'is_collection': False
}]
},
'file_id_2': {
'content': {
'key': 'file_2'
},
'links_by_entity': {
'file': ['file_id_1'],
},
'direct_links': [{
'entity': 'process',
'id': 'process_id_1',
'relationship': 'derivedByProcesses'
}, {
'entity': 'project',
'id': 'dummy-project-id',
'relationship': 'project',
'is_collection': False
}]
}
},
'process': {
'process_id_1': {
'content': {
'key': 'process_1'
},
'direct_links': [{
'entity': 'project',
'id': 'dummy-project-id',
'relationship': 'projects'
}, {
'entity': 'project',
'id': 'dummy-project-id',
'relationship': 'project',
'is_collection': False
}, {
'entity': 'protocol',
'id': 'protocol_id_1',
'relationship': 'protocols'
}, {
'entity': 'protocol',
'id': 'protocol_id_2',
'relationship': 'protocols'
}]
}
},
'protocol': {
'protocol_id_1': {
'content': {
'key': 'protocol_1'
},
'direct_links': [{
'entity': 'project',
'id': 'dummy-project-id',
'relationship': 'project',
'is_collection': False
}]
},
'protocol_id_2': {
'content': {
'key': 'protocol_2'
},
'direct_links': [{
'entity': 'project',
'id': 'dummy-project-id',
'relationship': 'project',
'is_collection': False
}]
}
}
}
entity_map = EntityMap.load(spreadsheet_json)
entity_linker = EntityLinker(self.mocked_template_manager, entity_map)
output = entity_linker.handle_links_from_spreadsheet()
self._assert_equal_direct_links(expected_json, output)