def test_do_import_no_project(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
expected_error = {
'location': 'File',
'type': 'NoProjectFound',
'detail': 'The spreadsheet should be associated to a project.'
}
# and:
item = MetadataEntity(concrete_type='product',
domain_type='product',
object_id=910)
worksheet_importer.do_import = MagicMock(side_effect=[([item],
no_errors)])
# and:
workbook = create_test_workbook('Item')
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json, errors = workbook_importer.do_import(
IngestWorkbook(workbook), is_update=False)
# then:
self.assertIn(
expected_error, errors,
f'Errors expected to contain {NoProjectFound.__name__}.')
def setup_workbook_with_project_worksheets(self,
worksheet_importer_constructor):
self.template_mgr.get_concrete_type = MagicMock(return_value='project')
worksheet_importer = WorkbookImporter(self.template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
# and:
project = MetadataEntity(domain_type='project',
concrete_type='project',
content={'description': 'test project'})
jsmith = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'John',
'email': '*****@*****.**'
}
})
ppan = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'Peter',
'email': '*****@*****.**'
}
})
worksheet_importer.do_import = MagicMock(
side_effect=[([project], no_errors), ([jsmith, ppan], no_errors)])
# and:
workbook = create_test_workbook('Project', 'Project - Contributors')
ingest_workbook = IngestWorkbook(workbook)
return ingest_workbook
def test_get_module_field_name(self):
# given:
workbook = create_test_workbook('Product - Reviews',
'User - SN Profiles',
'Log - file-names', 'Account')
# and: simple
reviews_sheet = workbook['Product - Reviews']
reviews = IngestWorksheet(reviews_sheet)
# and: with space in between
sn_profiles_sheet = workbook['User - SN Profiles']
sn_profiles = IngestWorksheet(sn_profiles_sheet)
# and: with hyphen
file_names_sheet = workbook['Log - file-names']
file_names = IngestWorksheet(file_names_sheet)
# and: not module worksheet
account_sheet = workbook['Account']
account = IngestWorksheet(account_sheet)
# expect:
self.assertEqual('reviews', reviews.get_module_field_name())
self.assertEqual('sn_profiles', sn_profiles.get_module_field_name())
self.assertEqual('file_names', file_names.get_module_field_name())
self.assertIsNone(account.get_module_field_name())
def setUp(self, worksheet_importer_constructor) -> None:
self.template_mgr = MagicMock(name='template_manager')
self.template_mgr.template.json_schemas = self.mock_json_schemas
self.concrete_type_map = {'Project': 'project', 'Users': 'users'}
self.template_mgr.get_concrete_type = lambda key: self.concrete_type_map.get(
key)
self.worksheet_importer = WorksheetImporter(self.template_mgr)
worksheet_importer_constructor.return_value = self.worksheet_importer
self.workbook_importer = WorkbookImporter(self.template_mgr)
self.workbook = create_test_workbook('Project', 'Users')
self.ingest_workbook = IngestWorkbook(self.workbook)
def test_get_title(self):
# given:
workbook = create_test_workbook('User', 'User - SN Profiles')
user_sheet = workbook['User']
sn_profiles_sheet = workbook['User - SN Profiles']
# and:
user = IngestWorksheet(user_sheet)
sn_profiles = IngestWorksheet(sn_profiles_sheet)
# expect:
self.assertEqual('User', user.title)
self.assertEqual('User - SN Profiles', sn_profiles.title)
def test_is_module_tab(self):
# given:
workbook = create_test_workbook('Product', 'Product - History')
product_sheet = workbook['Product']
history_sheet = workbook['Product - History']
# and:
product = IngestWorksheet(product_sheet)
history = IngestWorksheet(history_sheet)
# expect:
self.assertFalse(product.is_module_tab())
self.assertTrue(history.is_module_tab())
def test_add_schemas_worksheet(self):
# given
sheets = ['A', 'B', 'C', 'X']
workbook = create_test_workbook(*sheets)
wb = IngestWorkbook(workbook)
# when
wb.add_schemas_worksheet(['schema1', 'schema2', 'schema3'])
# then
self.assertEqual(wb.workbook['Schemas'].cell(1, 1).value, 'Schemas')
self.assertEqual(wb.workbook['Schemas'].cell(2, 1).value, 'schema1')
self.assertEqual(wb.workbook['Schemas'].cell(3, 1).value, 'schema2')
self.assertEqual(wb.workbook['Schemas'].cell(4, 1).value, 'schema3')
def test_do_import_update_project_but_no_project_worksheet(self):
# given:
workbook = create_test_workbook('Users')
# when:
spreadsheet_json, errors = self.workbook_importer.do_import(
IngestWorkbook(workbook),
project_uuid='project-uuid',
update_project=True,
is_update=False)
# then:
self.assertEqual(len(errors), 1)
self.assertEqual('NoProjectWorksheet', errors[0].get('type'))
def test_add_entity_uuids(self):
# given:
entities = [Entity('type', 'AA', 'content', concrete_type='a', ingest_json={'uuid': {'uuid': 'A-1-uuid'}},
spreadsheet_location={'row_index': 5, 'worksheet_title': 'A'}),
Entity('type', 'AA', 'content', concrete_type='a', ingest_json={'uuid': {'uuid': 'A-2-uuid'}},
spreadsheet_location={'row_index': 6, 'worksheet_title': 'A'}),
Entity('type', 'AA', 'content', concrete_type='a', ingest_json={'uuid': {'uuid': 'A-3-uuid'}},
spreadsheet_location={'row_index': 7, 'worksheet_title': 'A'}),
Entity('type', 'BB', 'content', concrete_type='b', ingest_json={'uuid': {'uuid': 'B-1-uuid'}},
spreadsheet_location={'row_index': 5, 'worksheet_title': 'B'}),
Entity('type', 'BB', 'content', concrete_type='b', ingest_json={'uuid': {'uuid': 'B-2-uuid'}},
spreadsheet_location={'row_index': 6, 'worksheet_title': 'B'}),
Entity('type', 'BB', 'content', concrete_type='b', ingest_json={'uuid': {'uuid': 'B-3-uuid'}},
spreadsheet_location={'row_index': 7, 'worksheet_title': 'B'}),
Entity('type', 'CC', 'content', concrete_type='c', ingest_json={'uuid': {'uuid': 'C-1-uuid'}},
spreadsheet_location={'row_index': 5, 'worksheet_title': 'C'}),
Entity('type', 'CC', 'content', concrete_type='c', ingest_json={'uuid': {'uuid': 'C-2-uuid'}},
spreadsheet_location={'row_index': 6, 'worksheet_title': 'C'}),
Entity('type', 'CC', 'content', concrete_type='c', ingest_json={'uuid': {'uuid': 'C-3-uuid'}},
spreadsheet_location={'row_index': 7, 'worksheet_title': 'C'}),
Entity('type', 'XX', 'content', concrete_type='x', ingest_json={'uuid': {'uuid': 'X-3-uuid'}})]
mock_submission = Mock('submission')
mock_submission.get_entities = Mock(return_value=entities)
sheets = ['A', 'B', 'C', 'X']
workbook = create_test_workbook(*sheets)
wb = IngestWorkbook(workbook)
# when
wb.add_entity_uuids(mock_submission)
# then
self.assertEqual(wb.workbook['A'].cell(4, 1).value, 'a.uuid')
self.assertEqual(wb.workbook['A'].cell(5, 1).value, 'A-1-uuid')
self.assertEqual(wb.workbook['A'].cell(6, 1).value, 'A-2-uuid')
self.assertEqual(wb.workbook['A'].cell(7, 1).value, 'A-3-uuid')
self.assertEqual(wb.workbook['B'].cell(4, 1).value, 'b.uuid')
self.assertEqual(wb.workbook['B'].cell(5, 1).value, 'B-1-uuid')
self.assertEqual(wb.workbook['B'].cell(6, 1).value, 'B-2-uuid')
self.assertEqual(wb.workbook['B'].cell(7, 1).value, 'B-3-uuid')
self.assertEqual(wb.workbook['C'].cell(4, 1).value, 'c.uuid')
self.assertEqual(wb.workbook['C'].cell(5, 1).value, 'C-1-uuid')
self.assertEqual(wb.workbook['C'].cell(6, 1).value, 'C-2-uuid')
self.assertEqual(wb.workbook['C'].cell(7, 1).value, 'C-3-uuid')
self.assertFalse(wb.workbook['X'].cell(5, 1).value, None)
def test_importable_worksheets(self):
# given:
importable_names = ['Organ From Donor', 'Cell Suspension', 'Project']
workbook = create_test_workbook(*importable_names)
workbook.create_sheet('Schemas')
# and:
ingest_workbook = IngestWorkbook(workbook)
# when:
actual_worksheets = ingest_workbook.importable_worksheets()
# then:
actual_titles = [ingest_worksheet.title for ingest_worksheet in actual_worksheets]
self.assertEqual(importable_names, actual_titles)
def test_do_import_do_not_update_project_and_no_project_worksheet(self):
# given:
workbook = create_test_workbook('Users')
# when:
spreadsheet_json, errors = self.workbook_importer.do_import(
IngestWorkbook(workbook),
project_uuid='project-uuid',
update_project=False,
is_update=False)
# then:
self.assertEqual(len(errors), 0)
self.assertEqual(len(spreadsheet_json.get('project', {}).keys()), 1)
self.assertFalse(
spreadsheet_json['project']['project-uuid']['is_reference'])
self.assertTrue(spreadsheet_json['project']['project-uuid']
['is_linking_reference'])
def test_add_schemas_worksheet_existing_schemas(self):
# given
sheets = ['A', 'B', 'C']
workbook = create_test_workbook(*sheets)
wb = IngestWorkbook(workbook)
wb.workbook.create_sheet('Schemas')
wb.workbook['Schemas'].cell(1, 1).value = 'X'
wb.workbook['Schemas'].cell(2, 1).value = 'x'
wb.workbook['Schemas'].cell(3, 1).value = 'y'
wb.workbook['Schemas'].cell(4, 1).value = 'z'
# when
wb.add_schemas_worksheet(['schema1', 'schema2', 'schema3'])
# then
self.assertEqual(wb.workbook['Schemas'].cell(1, 1).value, 'X')
self.assertEqual(wb.workbook['Schemas'].cell(2, 1).value, 'x')
self.assertEqual(wb.workbook['Schemas'].cell(3, 1).value, 'y')
self.assertEqual(wb.workbook['Schemas'].cell(4, 1).value, 'z')
def test_do_import_multiple_projects(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
template_mgr.template.json_schemas = self.mock_json_schemas
template_mgr.get_concrete_type = MagicMock(return_value='project')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
expected_error = {
'location':
'sheet=Project',
'type':
'MultipleProjectsFound',
'detail':
'The spreadsheet should only be associated to a single project.'
}
# and:
project_1 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=1)
project_2 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=2)
worksheet_importer.do_import = MagicMock(
side_effect=[([project_1, project_2], no_errors)])
# and:
workbook = create_test_workbook('Project')
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json, errors = workbook_importer.do_import(
IngestWorkbook(workbook), is_update=False)
# then:
self.assertIn(
expected_error, errors,
f'Errors expected to contain {MultipleProjectsFound.__name__}.')
def test_do_import_with_module_tab(self, worksheet_importer_constructor):
# given:
concrete_type_map = {
'Project': 'project',
'User': '******',
'User - SN Profiles': 'users'
}
self.template_mgr.get_concrete_type = lambda key: concrete_type_map.get(
key)
project = MetadataEntity(concrete_type='Project',
domain_type='Project')
user = MetadataEntity(concrete_type='user',
domain_type='user',
object_id=773,
content={'user_name': 'janedoe'})
fb_profile = MetadataEntity(concrete_type='sn_profile',
domain_type='user',
object_id=773,
content={
'sn_profiles': {
'name': 'facebook',
'id': '392'
},
'description': 'extra fb field'
})
ig_profile = MetadataEntity(concrete_type='sn_profile',
domain_type='user',
object_id=773,
content={
'sn_profiles': {
'name': 'instagram',
'id': 'a92'
},
'description': 'extra ig field'
})
no_errors = []
self.worksheet_importer.do_import = MagicMock(side_effect=[(
[project],
no_errors), ([user],
no_errors), ([fb_profile, ig_profile], no_errors)])
self.workbook = create_test_workbook('Project', 'User',
'User - SN Profiles')
self.ingest_workbook = IngestWorkbook(self.workbook)
# when:
spreadsheet_json, errors = self.workbook_importer.do_import(
self.ingest_workbook, is_update=False)
# then:
expected_errors = [{
'key': 'description',
'value': 'extra fb field'
}, {
'key': 'description',
'value': 'extra ig field'
}]
self.assertIsNotNone(spreadsheet_json)
self.assertEqual(
errors,
self.workbook_importer.list_data_removal_errors(
'User - SN Profiles', expected_errors))
self.assertEqual(2, len(spreadsheet_json))
# and:
user_map = spreadsheet_json.get('user')
self.assertIsNotNone(user_map)
# and:
janedoe = user_map.get(773)
self.assertIsNotNone(janedoe)
content = janedoe.get('content')
self.assertEqual('janedoe', content.get('user_name'))
self.assertEqual(['user_name', 'sn_profiles'], list(content.keys()))
# and:
sn_profiles = content.get('sn_profiles')
self.assertIsNotNone(sn_profiles)
self.assertEqual(2, len(sn_profiles))
# and:
self.assertEqual({'name': 'facebook', 'id': '392'}, sn_profiles[0])
self.assertEqual({'name': 'instagram', 'id': 'a92'}, sn_profiles[1])
def test_create_row_template_for_module_worksheet(self,
determine_strategy):
# given:
template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='mock_ingest_api')
# and:
spec_map = {
'product': {
'schema': {
'domain_entity': 'merchandise/product'
}
}
}
template.lookup_property_from_template = lambda key: spec_map.get(
key, None)
domain_entity = "merchandise/product"
schema_url = "http://schema.sample.com/product"
template.get_tabs_config = MagicMock()
template.lookup_metadata_schema_name_given_title = MagicMock(
return_value="product_type")
template.get_latest_schema = MagicMock(return_value=schema_url)
schema = {
"schema": {
"domain_entity": domain_entity,
"url": schema_url
}
}
property_one_schema = {
"description": "Property one description",
"value_type": "string"
}
property_two_schema = {
"description": "Property two description",
"value_type": "string"
}
spec_map = {
"product_type": schema,
"product.info.id": property_one_schema,
"product.reviews.rating": property_two_schema
}
template.lookup_property_attributes_in_metadata = lambda key: spec_map.get(
key)
# and:
template_mgr = TemplateManager(template, ingest_api)
# and:
workbook = create_test_workbook('Product - Reviews')
reviews_worksheet = workbook['Product - Reviews']
reviews_worksheet['A4'] = 'product.info.id'
reviews_worksheet['B4'] = 'product.reviews.rating'
# and: set up strategies
id_strategy = MagicMock(name='id_strategy')
rating_strategy = MagicMock(name='rating_strategy')
determine_strategy.side_effect = [id_strategy, rating_strategy]
# when:
row_template = template_mgr.create_row_template(
IngestWorksheet(reviews_worksheet))
# and:
self.assertIsNotNone(row_template)
self.assertIn(id_strategy, row_template.cell_conversions)
self.assertIn(rating_strategy, row_template.cell_conversions)