def setup_workbook_with_project_worksheets(self,
worksheet_importer_constructor):
self.template_mgr.get_concrete_type = MagicMock(return_value='project')
worksheet_importer = WorkbookImporter(self.template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
# and:
project = MetadataEntity(domain_type='project',
concrete_type='project',
content={'description': 'test project'})
jsmith = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'John',
'email': '*****@*****.**'
}
})
ppan = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'Peter',
'email': '*****@*****.**'
}
})
worksheet_importer.do_import = MagicMock(
side_effect=[([project], no_errors), ([jsmith, ppan], no_errors)])
# and:
workbook = create_test_workbook('Project', 'Project - Contributors')
ingest_workbook = IngestWorkbook(workbook)
return ingest_workbook
def test_do_import_no_id_metadata(self):
# given:
row_template = MagicMock('row_template')
# and:
paper_metadata = MetadataEntity(content={'product_name': 'paper'},
links={'delivery': ['123', '456']})
pen_metadata = MetadataEntity(content={'product_name': 'pen'},
links={'delivery': ['789']})
row_template.do_import = MagicMock(side_effect=[paper_metadata, pen_metadata])
# and:
mock_template_manager = MagicMock('template_manager')
mock_template_manager.create_row_template = MagicMock(return_value=row_template)
mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2'])
mock_template_manager.get_concrete_entity_of_tab = MagicMock(return_value='concrete_entity')
# and:
workbook = Workbook()
worksheet = workbook.create_sheet('product')
worksheet['A6'] = 'paper'
worksheet['A7'] = 'pen'
# when:
worksheet_importer = WorksheetImporter()
result = worksheet_importer.do_import(worksheet, mock_template_manager)
# then:
self.assertEqual(2, len(result.keys()))
def test_apply_with_empty_cell_data(self):
# given:
cell_conversion = LinkedIdentityCellConversion('item.item_number',
'line_order')
# and:
metadata = MetadataEntity()
items = ['item_no_56', 'item_no_199']
metadata.add_links('line_order', items)
# when:
cell_conversion.apply(metadata, '')
# then:
actual_items = metadata.get_links('line_order')
self.assertEqual(2, len(actual_items))
# and:
expected_ids = [id for id in items]
for expected_id in expected_ids:
self.assertTrue(expected_id in actual_items,
f'[{expected_id}] not in list.')
self.assertFalse('' in actual_items, f" '' should not be in the list.")
def test_do_import(self):
# given:
row_template = MagicMock('row_template')
no_errors = []
# and:
john_doe = MetadataEntity(object_id='profile_1')
emma_jackson = MetadataEntity(object_id='profile_2')
row_template.do_import = MagicMock('import_row', side_effect=[(john_doe, no_errors), (emma_jackson, no_errors)])
# and:
mock_template_manager = MagicMock('template_manager')
mock_template_manager.create_row_template = MagicMock(return_value=row_template)
mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2'])
mock_template_manager.get_concrete_type = MagicMock(return_value='concrete_entity')
# and:
workbook = Workbook()
worksheet = workbook.create_sheet('user_profile')
worksheet['A4'] = 'header'
worksheet['A6'] = 'john'
worksheet['A7'] = 'emma'
# when:
worksheet_importer = WorksheetImporter(mock_template_manager)
profiles, errors = worksheet_importer.do_import(IngestWorksheet(worksheet))
# then:
self.assertEqual(2, len(profiles))
self.assertIn(john_doe, profiles)
self.assertIn(emma_jackson, profiles)
self.assertEqual(errors, [])
# and: domain and concrete type should be set
pass
def test_apply_multiple_with_previous_values(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = ListElementCellConversion('staff.developers.name',
converter)
# and:
metadata = MetadataEntity()
metadata.define_content('developers', [{'age': 32}, {'age': 23}])
# when:
cell_conversion.apply(metadata, 'John||Paul||George||Ringo')
# then:
developers = metadata.get_content('developers')
self.assertEqual(4, len(developers))
# and:
expected_names = [
'John - converted', 'Paul - converted', 'George - converted',
'Ringo - converted'
]
expected_ages = [32, 23, None, None]
for index, developer in enumerate(developers):
self.assertEqual(expected_names[index], developer.get('name'))
self.assertEqual(expected_ages[index], developer.get('age'))
def test_do_import_multiple_projects(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
project_1 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=1)
project_2 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=2)
worksheet_importer.do_import = MagicMock(
side_effect=[[project_1, project_2]])
# and:
workbook = create_test_workbook('Project')
workbook_importer = WorkbookImporter(template_mgr)
# when:
exception_thrown = False
try:
workbook_importer.do_import(IngestWorkbook(workbook))
except MultipleProjectsFound:
exception_thrown = True
# then:
self.assertTrue(
exception_thrown,
f'Expected to throw {MultipleProjectsFound.__name__}.')
def test_apply(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = LinkingDetailCellConversion('profile.user.name', converter)
metadata = MetadataEntity()
# when:
cell_conversion.apply(metadata, 'John Doe')
# then:
self.assertEqual('John Doe - converted', metadata.get_linking_detail('user.name'))
def add_project_reference(self, project_uuid, project: MetadataEntity = None):
if project:
project.object_id = project_uuid
project.is_linking_reference = True
project.is_reference = True
else:
project = MetadataEntity(domain_type=_PROJECT_TYPE,
concrete_type=_PROJECT_TYPE,
object_id=project_uuid,
is_linking_reference=True)
self.add_submittable(project)
def test_do_import_project_worksheet(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorkbookImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
project = MetadataEntity(domain_type='project',
concrete_type='project',
content={'description': 'test project'})
jsmith = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'John',
'email': '*****@*****.**'
}
})
ppan = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'Peter',
'email': '*****@*****.**'
}
})
worksheet_importer.do_import = MagicMock(
side_effect=[[project], [jsmith, ppan]])
# and:
workbook = create_test_workbook('Project', 'Project - Contributors')
ingest_workbook = IngestWorkbook(workbook)
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json = workbook_importer.do_import(ingest_workbook)
# then:
project_map = spreadsheet_json.get('project')
self.assertEqual(1, len(project_map))
project_content = list(project_map.values())[0].get('content')
self.assertEqual('test project', project_content.get('description'))
# and:
contributors = project_content.get('contributors')
self.assertEqual(2, len(contributors))
self.assertIn({
'name': 'John',
'email': '*****@*****.**'
}, contributors)
self.assertIn({
'name': 'Peter',
'email': '*****@*****.**'
}, contributors)
def test_apply_multiple_values(self):
# given:
cell_conversion = ExternalReferenceCellConversion('company.uuid', 'organisation')
# when:
metadata = MetadataEntity()
cell_conversion.apply(metadata, '7e56de9||2fe9eb0')
# then:
expected_ids = ['7e56de9', '2fe9eb0']
self.assertCountEqual(expected_ids, metadata.get_external_links('organisation'))
def test_add_links(self):
# given:
metadata = MetadataEntity()
# when:
new_links = ['73f909', '83fddf1', '9004811']
metadata.add_links('profile', new_links)
# then:
profile_links = metadata.get_links('profile')
self.assertIsNotNone(profile_links)
self.assertCountEqual(new_links, profile_links)
def test_add_external_links(self):
# given:
metadata = MetadataEntity()
# when:
new_links = ['77701ee', '254aefb']
metadata.add_external_links('file', new_links)
# then:
file_links = metadata.get_external_links('file')
self.assertIsNotNone(file_links)
self.assertCountEqual(new_links, file_links)
def test_apply(self):
# given:
cell_conversion = ExternalReferenceCellConversion('user.uuid', 'account')
# when:
metadata = MetadataEntity()
cell_conversion.apply(metadata, '621bfa0')
# then:
account_list = metadata.get_external_links('account')
self.assertIsNotNone(account_list, '[account] list in external links expected.')
self.assertEqual(1, len(account_list))
self.assertTrue('621bfa0' in account_list, 'Expected content not in list.')
def test_apply(self):
# given:
int_converter = MagicMock('int_converter')
int_converter.convert = lambda __: 27
# and:
cell_conversion = DirectCellConversion('profile.user.age', int_converter)
# when:
metadata = MetadataEntity()
cell_conversion.apply(metadata, '27')
# then:
self.assertEqual(27, metadata.get_content('user.age'))
def test_do_import(self):
# given:
row_template = MagicMock('row_template')
# and:
john_doe_content = {'name': 'John Doe'}
john_doe_links = {}
john_doe_external_links = {'organisations': ['org_88', 'org_110', 'org_452']}
john_doe_linking_details = {'position': 'manager'}
john_doe = MetadataEntity(object_id='profile_1', content=john_doe_content,
links=john_doe_links, external_links=john_doe_external_links,
linking_details=john_doe_linking_details)
# and:
emma_jackson_content = {'name': 'Emma Jackson'}
emma_jackson_links = {'friends': ['profile_19', 'profile_8']}
emma_jackson_external_links = {}
emma_jackson_linking_details = {}
emma_jackson = MetadataEntity(object_id='profile_2', content=emma_jackson_content,
links=emma_jackson_links,
external_links=emma_jackson_external_links,
linking_details=emma_jackson_linking_details)
# and:
row_template.do_import = MagicMock('import_row', side_effect=[john_doe, emma_jackson])
# and:
mock_template_manager = MagicMock('template_manager')
mock_template_manager.create_row_template = MagicMock(return_value=row_template)
mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2'])
mock_template_manager.get_concrete_entity_of_tab = MagicMock(return_value='concrete_entity')
# and:
workbook = Workbook()
worksheet = workbook.create_sheet('user_profile')
worksheet['A6'] = 'john'
worksheet['A7'] = 'emma'
# when:
worksheet_importer = WorksheetImporter()
profile = worksheet_importer.do_import(worksheet, mock_template_manager)
# then:
self.assertEqual(2, len(profile.keys()))
self._assert_correct_profile(profile, 'profile_1', john_doe_content, john_doe_links,
john_doe_external_links, john_doe_linking_details)
self._assert_correct_profile(profile, 'profile_2', emma_jackson_content,
emma_jackson_links, emma_jackson_external_links,
emma_jackson_linking_details)
def test_apply_none_data(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = ListElementCellConversion('user.name', converter)
# and:
metadata = MetadataEntity(content={'user': [{'id': '65fd8'}]})
# when:
cell_conversion.apply(metadata, None)
# then:
list_element = metadata.get_content('user')[0]
self.assertTrue('name' not in list_element.keys(),
'[name] should not be added to element.')
def test_apply_with_empty_cell_data(self):
# given:
cell_conversion = LinkedExternalReferenceCellConversion(
'user.uuid', 'account')
# and:
metadata = MetadataEntity()
# when:
cell_conversion.apply(metadata, '')
# then:
self.assertFalse(
metadata.get_external_links('account'),
'When cell is empty, no external link should be added')
def test_do_import_with_update_spreadsheet(self,
worksheet_importer_constructor):
# given:
concrete_type_map = {'project': 'project', 'users': 'users'}
self.template_mgr.get_concrete_type = lambda key: concrete_type_map.get(
key)
sheet_names = ['project', 'users']
workbook = create_ingest_workbook(sheet_names, ['uuid', 'description'])
workbook_importer = WorkbookImporter(self.template_mgr)
# and
worksheet_importer = WorksheetImporter(self.template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
expected_errors = []
for sheet_name in sheet_names:
expected_errors.append({
'location':
f'sheet={sheet_name}',
'type':
'UnexpectedEntityUUIDFound',
'detail':
f'The {sheet_name} entities in the spreadsheet shouldn’t have UUIDs.'
})
# and:
project = MetadataEntity(domain_type='project',
concrete_type='project',
object_id=910)
user1 = MetadataEntity(concrete_type='user',
domain_type='user',
object_id=1,
content={'user_name': 'jdelacruz'})
worksheet_importer.do_import = \
MagicMock(side_effect=[([project], no_errors), ([user1], no_errors)])
# when:
spreadsheet_json, errors = workbook_importer.do_import(workbook,
is_update=False)
# then:
self.assertTrue(
all(elem in errors for elem in expected_errors),
f'Errors expected to contain {UnexpectedEntityUUIDFound.__name__}.'
)
def test_apply_with_previous_entries(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = LinkingDetailCellConversion('product.item.name', converter)
# and:
metadata = MetadataEntity(linking_details={
'item': {'description': 'a jar of milk'}
})
# when:
cell_conversion.apply(metadata, 'milk')
# then:
self.assertEqual('milk - converted', metadata.get_linking_detail('item.name'))
self.assertEqual('a jar of milk', metadata.get_linking_detail('item.description'))
def test_do_import_no_projects(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
item = MetadataEntity(concrete_type='product',
domain_type='product',
object_id=910)
worksheet_importer.do_import = MagicMock(side_effect=[[item]])
# and:
workbook = create_test_workbook('Item')
workbook_importer = WorkbookImporter(template_mgr)
# when:
thrown_exception = False
try:
workbook_importer.do_import(IngestWorkbook(workbook))
except NoProjectFound:
thrown_exception = True
# then:
self.assertTrue(thrown_exception,
f'Expected to throw {NoProjectFound.__name__}.')
def test_do_import_no_project(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
expected_error = {
'location': 'File',
'type': 'NoProjectFound',
'detail': 'The spreadsheet should be associated to a project.'
}
# and:
item = MetadataEntity(concrete_type='product',
domain_type='product',
object_id=910)
worksheet_importer.do_import = MagicMock(side_effect=[([item],
no_errors)])
# and:
workbook = create_test_workbook('Item')
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json, errors = workbook_importer.do_import(
IngestWorkbook(workbook), is_update=False)
# then:
self.assertIn(
expected_error, errors,
f'Errors expected to contain {NoProjectFound.__name__}.')
def do_import(self, row: IngestRow, is_module=False):
row_errors = []
metadata = MetadataEntity(domain_type=self.domain_type, concrete_type=self.concrete_type,
content=self.default_values, row=row, is_module=is_module)
for index, cell in enumerate(row.values):
if cell.value is None:
continue
conversion: CellConversion = self.cell_conversions[index]
try:
conversion.apply(metadata, cell.value)
except ValueError as e:
error_detail = f'Could not convert to {conversion.converter.type().value} ' \
f'from: {type(cell.value).__name__} {cell.value}'
if type(cell.value) == float:
error_detail += " This may be because the default excel number format is float."
row_errors.append({
"location": f'column={index}, value={cell.value}',
"type": "Conversion Error",
"detail": error_detail
})
except Exception as e:
row_errors.append({
"location": f'column={index}, value={cell.value}',
"type": e.__class__.__name__,
"detail": str(e)
})
return metadata, row_errors
def test_add_links_multiple_times(self):
# given:
metadata = MetadataEntity()
# and:
batch_1 = ['78de112', '963fefed']
batch_2 = ['2daab01']
# when:
metadata.add_links('item', batch_1)
metadata.add_links('item', batch_2)
# then:
item_links = metadata.get_links('item')
self.assertIsNotNone(item_links)
self.assertCountEqual(batch_1 + batch_2, item_links)
def do_import(self, row):
metadata = MetadataEntity(content=self.default_values)
for index, cell in enumerate(row):
if cell.value is None:
continue
conversion: CellConversion = self.cell_conversions[index]
conversion.apply(metadata, cell.value)
return metadata
def test_apply(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = ListElementCellConversion('stuff.list_of_things.name', converter)
# when:
metadata = MetadataEntity()
cell_conversion.apply(metadata, 'sample')
# then:
list_of_things = metadata.get_content('list_of_things')
self.assertIsNotNone(list_of_things)
self.assertEqual(1, len(list_of_things))
# and:
thing = list_of_things[0]
self.assertEqual('sample - converted', thing.get('name'))
def test_apply(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = FieldOfSingleElementListCellConversion('library.books.title', converter)
# when:
metadata = MetadataEntity()
cell_conversion.apply(metadata, 'Harry Potter')
# then:
books = metadata.get_content('books')
self.assertIsNotNone(books)
self.assertEqual(1, len(books))
# and:
thing = books[0]
self.assertEqual('Harry Potter - converted', thing.get('title'))
def test_apply_skips_none_data(self):
# given:
string_converter = StringConverter()
cell_conversion = DirectCellConversion('product.id', string_converter)
# when:
metadata = MetadataEntity(
content={'product': {
'name': 'product name'
}})
cell_conversion.apply(metadata, None)
# then:
product = metadata.get_content('product')
self.assertIsNotNone(product)
self.assertTrue('id' not in product,
'[id] not expected to be in product field')
def test_retain_content_fields(self):
# given:
content = {
'user_name': 'jdelacruz',
'password': '******',
'description': 'temp'
}
metadata_entity = MetadataEntity(domain_type='user',
concrete_type='user',
object_id=1,
content=content)
# when:
metadata_entity.retain_fields('user_name')
# then:
self.assertEqual(['user_name'],
list(metadata_entity.content.as_dict().keys()))
def test_apply(self):
# given:
converter = _create_mock_string_converter()
cell_conversion = IdentityCellConversion('product.product_id', converter)
# and:
metadata = MetadataEntity()
# when:
cell_conversion.apply(metadata, 'product_no_144')
# then:
expected_id = 'product_no_144 - converted'
self.assertEqual(expected_id, metadata.object_id)
# and: identity value should be in content
self.assertEqual(expected_id, metadata.get_content('product_id'))
def test_apply_with_previous_entries(self):
# given:
metadata = MetadataEntity()
metadata.add_external_links('store_item', ['109bdd9', 'c3c35e6'])
# and:
cell_conversion = ExternalReferenceCellConversion('product.uuid', 'store_item')
# when:
cell_conversion.apply(metadata, '73de901')
# then:
store_item_list = metadata.get_external_links('store_item')
self.assertIsNotNone(store_item_list, '[store_item] list in external links expected.')
# and:
expected_ids = ['109bdd9', '73de901', 'c3c35e6']
self.assertCountEqual(expected_ids, store_item_list)
