def setup_workbook_with_project_worksheets(self, worksheet_importer_constructor): self.template_mgr.get_concrete_type = MagicMock(return_value='project') worksheet_importer = WorkbookImporter(self.template_mgr) worksheet_importer_constructor.return_value = worksheet_importer no_errors = [] # and: project = MetadataEntity(domain_type='project', concrete_type='project', content={'description': 'test project'}) jsmith = MetadataEntity(domain_type='project', concrete_type='contact', content={ 'contributors': { 'name': 'John', 'email': '*****@*****.**' } }) ppan = MetadataEntity(domain_type='project', concrete_type='contact', content={ 'contributors': { 'name': 'Peter', 'email': '*****@*****.**' } }) worksheet_importer.do_import = MagicMock( side_effect=[([project], no_errors), ([jsmith, ppan], no_errors)]) # and: workbook = create_test_workbook('Project', 'Project - Contributors') ingest_workbook = IngestWorkbook(workbook) return ingest_workbook
def test_do_import_no_id_metadata(self): # given: row_template = MagicMock('row_template') # and: paper_metadata = MetadataEntity(content={'product_name': 'paper'}, links={'delivery': ['123', '456']}) pen_metadata = MetadataEntity(content={'product_name': 'pen'}, links={'delivery': ['789']}) row_template.do_import = MagicMock(side_effect=[paper_metadata, pen_metadata]) # and: mock_template_manager = MagicMock('template_manager') mock_template_manager.create_row_template = MagicMock(return_value=row_template) mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2']) mock_template_manager.get_concrete_entity_of_tab = MagicMock(return_value='concrete_entity') # and: workbook = Workbook() worksheet = workbook.create_sheet('product') worksheet['A6'] = 'paper' worksheet['A7'] = 'pen' # when: worksheet_importer = WorksheetImporter() result = worksheet_importer.do_import(worksheet, mock_template_manager) # then: self.assertEqual(2, len(result.keys()))
def test_apply_with_empty_cell_data(self): # given: cell_conversion = LinkedIdentityCellConversion('item.item_number', 'line_order') # and: metadata = MetadataEntity() items = ['item_no_56', 'item_no_199'] metadata.add_links('line_order', items) # when: cell_conversion.apply(metadata, '') # then: actual_items = metadata.get_links('line_order') self.assertEqual(2, len(actual_items)) # and: expected_ids = [id for id in items] for expected_id in expected_ids: self.assertTrue(expected_id in actual_items, f'[{expected_id}] not in list.') self.assertFalse('' in actual_items, f" '' should not be in the list.")
def test_do_import(self): # given: row_template = MagicMock('row_template') no_errors = [] # and: john_doe = MetadataEntity(object_id='profile_1') emma_jackson = MetadataEntity(object_id='profile_2') row_template.do_import = MagicMock('import_row', side_effect=[(john_doe, no_errors), (emma_jackson, no_errors)]) # and: mock_template_manager = MagicMock('template_manager') mock_template_manager.create_row_template = MagicMock(return_value=row_template) mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2']) mock_template_manager.get_concrete_type = MagicMock(return_value='concrete_entity') # and: workbook = Workbook() worksheet = workbook.create_sheet('user_profile') worksheet['A4'] = 'header' worksheet['A6'] = 'john' worksheet['A7'] = 'emma' # when: worksheet_importer = WorksheetImporter(mock_template_manager) profiles, errors = worksheet_importer.do_import(IngestWorksheet(worksheet)) # then: self.assertEqual(2, len(profiles)) self.assertIn(john_doe, profiles) self.assertIn(emma_jackson, profiles) self.assertEqual(errors, []) # and: domain and concrete type should be set pass
def test_apply_multiple_with_previous_values(self): # given: converter = _create_mock_string_converter() cell_conversion = ListElementCellConversion('staff.developers.name', converter) # and: metadata = MetadataEntity() metadata.define_content('developers', [{'age': 32}, {'age': 23}]) # when: cell_conversion.apply(metadata, 'John||Paul||George||Ringo') # then: developers = metadata.get_content('developers') self.assertEqual(4, len(developers)) # and: expected_names = [ 'John - converted', 'Paul - converted', 'George - converted', 'Ringo - converted' ] expected_ages = [32, 23, None, None] for index, developer in enumerate(developers): self.assertEqual(expected_names[index], developer.get('name')) self.assertEqual(expected_ages[index], developer.get('age'))
def test_do_import_multiple_projects(self, worksheet_importer_constructor): # given: template_mgr = MagicMock(name='template_manager') worksheet_importer = WorksheetImporter(template_mgr) worksheet_importer_constructor.return_value = worksheet_importer # and: project_1 = MetadataEntity(concrete_type='project', domain_type='project', object_id=1) project_2 = MetadataEntity(concrete_type='project', domain_type='project', object_id=2) worksheet_importer.do_import = MagicMock( side_effect=[[project_1, project_2]]) # and: workbook = create_test_workbook('Project') workbook_importer = WorkbookImporter(template_mgr) # when: exception_thrown = False try: workbook_importer.do_import(IngestWorkbook(workbook)) except MultipleProjectsFound: exception_thrown = True # then: self.assertTrue( exception_thrown, f'Expected to throw {MultipleProjectsFound.__name__}.')
def test_apply(self): # given: converter = _create_mock_string_converter() cell_conversion = LinkingDetailCellConversion('profile.user.name', converter) metadata = MetadataEntity() # when: cell_conversion.apply(metadata, 'John Doe') # then: self.assertEqual('John Doe - converted', metadata.get_linking_detail('user.name'))
def add_project_reference(self, project_uuid, project: MetadataEntity = None): if project: project.object_id = project_uuid project.is_linking_reference = True project.is_reference = True else: project = MetadataEntity(domain_type=_PROJECT_TYPE, concrete_type=_PROJECT_TYPE, object_id=project_uuid, is_linking_reference=True) self.add_submittable(project)
def test_do_import_project_worksheet(self, worksheet_importer_constructor): # given: template_mgr = MagicMock(name='template_manager') worksheet_importer = WorkbookImporter(template_mgr) worksheet_importer_constructor.return_value = worksheet_importer # and: project = MetadataEntity(domain_type='project', concrete_type='project', content={'description': 'test project'}) jsmith = MetadataEntity(domain_type='project', concrete_type='contact', content={ 'contributors': { 'name': 'John', 'email': '*****@*****.**' } }) ppan = MetadataEntity(domain_type='project', concrete_type='contact', content={ 'contributors': { 'name': 'Peter', 'email': '*****@*****.**' } }) worksheet_importer.do_import = MagicMock( side_effect=[[project], [jsmith, ppan]]) # and: workbook = create_test_workbook('Project', 'Project - Contributors') ingest_workbook = IngestWorkbook(workbook) workbook_importer = WorkbookImporter(template_mgr) # when: spreadsheet_json = workbook_importer.do_import(ingest_workbook) # then: project_map = spreadsheet_json.get('project') self.assertEqual(1, len(project_map)) project_content = list(project_map.values())[0].get('content') self.assertEqual('test project', project_content.get('description')) # and: contributors = project_content.get('contributors') self.assertEqual(2, len(contributors)) self.assertIn({ 'name': 'John', 'email': '*****@*****.**' }, contributors) self.assertIn({ 'name': 'Peter', 'email': '*****@*****.**' }, contributors)
def test_apply_multiple_values(self): # given: cell_conversion = ExternalReferenceCellConversion('company.uuid', 'organisation') # when: metadata = MetadataEntity() cell_conversion.apply(metadata, '7e56de9||2fe9eb0') # then: expected_ids = ['7e56de9', '2fe9eb0'] self.assertCountEqual(expected_ids, metadata.get_external_links('organisation'))
def test_add_links(self): # given: metadata = MetadataEntity() # when: new_links = ['73f909', '83fddf1', '9004811'] metadata.add_links('profile', new_links) # then: profile_links = metadata.get_links('profile') self.assertIsNotNone(profile_links) self.assertCountEqual(new_links, profile_links)
def test_add_external_links(self): # given: metadata = MetadataEntity() # when: new_links = ['77701ee', '254aefb'] metadata.add_external_links('file', new_links) # then: file_links = metadata.get_external_links('file') self.assertIsNotNone(file_links) self.assertCountEqual(new_links, file_links)
def test_apply(self): # given: cell_conversion = ExternalReferenceCellConversion('user.uuid', 'account') # when: metadata = MetadataEntity() cell_conversion.apply(metadata, '621bfa0') # then: account_list = metadata.get_external_links('account') self.assertIsNotNone(account_list, '[account] list in external links expected.') self.assertEqual(1, len(account_list)) self.assertTrue('621bfa0' in account_list, 'Expected content not in list.')
def test_apply(self): # given: int_converter = MagicMock('int_converter') int_converter.convert = lambda __: 27 # and: cell_conversion = DirectCellConversion('profile.user.age', int_converter) # when: metadata = MetadataEntity() cell_conversion.apply(metadata, '27') # then: self.assertEqual(27, metadata.get_content('user.age'))
def test_do_import(self): # given: row_template = MagicMock('row_template') # and: john_doe_content = {'name': 'John Doe'} john_doe_links = {} john_doe_external_links = {'organisations': ['org_88', 'org_110', 'org_452']} john_doe_linking_details = {'position': 'manager'} john_doe = MetadataEntity(object_id='profile_1', content=john_doe_content, links=john_doe_links, external_links=john_doe_external_links, linking_details=john_doe_linking_details) # and: emma_jackson_content = {'name': 'Emma Jackson'} emma_jackson_links = {'friends': ['profile_19', 'profile_8']} emma_jackson_external_links = {} emma_jackson_linking_details = {} emma_jackson = MetadataEntity(object_id='profile_2', content=emma_jackson_content, links=emma_jackson_links, external_links=emma_jackson_external_links, linking_details=emma_jackson_linking_details) # and: row_template.do_import = MagicMock('import_row', side_effect=[john_doe, emma_jackson]) # and: mock_template_manager = MagicMock('template_manager') mock_template_manager.create_row_template = MagicMock(return_value=row_template) mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2']) mock_template_manager.get_concrete_entity_of_tab = MagicMock(return_value='concrete_entity') # and: workbook = Workbook() worksheet = workbook.create_sheet('user_profile') worksheet['A6'] = 'john' worksheet['A7'] = 'emma' # when: worksheet_importer = WorksheetImporter() profile = worksheet_importer.do_import(worksheet, mock_template_manager) # then: self.assertEqual(2, len(profile.keys())) self._assert_correct_profile(profile, 'profile_1', john_doe_content, john_doe_links, john_doe_external_links, john_doe_linking_details) self._assert_correct_profile(profile, 'profile_2', emma_jackson_content, emma_jackson_links, emma_jackson_external_links, emma_jackson_linking_details)
def test_apply_none_data(self): # given: converter = _create_mock_string_converter() cell_conversion = ListElementCellConversion('user.name', converter) # and: metadata = MetadataEntity(content={'user': [{'id': '65fd8'}]}) # when: cell_conversion.apply(metadata, None) # then: list_element = metadata.get_content('user')[0] self.assertTrue('name' not in list_element.keys(), '[name] should not be added to element.')
def test_apply_with_empty_cell_data(self): # given: cell_conversion = LinkedExternalReferenceCellConversion( 'user.uuid', 'account') # and: metadata = MetadataEntity() # when: cell_conversion.apply(metadata, '') # then: self.assertFalse( metadata.get_external_links('account'), 'When cell is empty, no external link should be added')
def test_do_import_with_update_spreadsheet(self, worksheet_importer_constructor): # given: concrete_type_map = {'project': 'project', 'users': 'users'} self.template_mgr.get_concrete_type = lambda key: concrete_type_map.get( key) sheet_names = ['project', 'users'] workbook = create_ingest_workbook(sheet_names, ['uuid', 'description']) workbook_importer = WorkbookImporter(self.template_mgr) # and worksheet_importer = WorksheetImporter(self.template_mgr) worksheet_importer_constructor.return_value = worksheet_importer no_errors = [] expected_errors = [] for sheet_name in sheet_names: expected_errors.append({ 'location': f'sheet={sheet_name}', 'type': 'UnexpectedEntityUUIDFound', 'detail': f'The {sheet_name} entities in the spreadsheet shouldn’t have UUIDs.' }) # and: project = MetadataEntity(domain_type='project', concrete_type='project', object_id=910) user1 = MetadataEntity(concrete_type='user', domain_type='user', object_id=1, content={'user_name': 'jdelacruz'}) worksheet_importer.do_import = \ MagicMock(side_effect=[([project], no_errors), ([user1], no_errors)]) # when: spreadsheet_json, errors = workbook_importer.do_import(workbook, is_update=False) # then: self.assertTrue( all(elem in errors for elem in expected_errors), f'Errors expected to contain {UnexpectedEntityUUIDFound.__name__}.' )
def test_apply_with_previous_entries(self): # given: converter = _create_mock_string_converter() cell_conversion = LinkingDetailCellConversion('product.item.name', converter) # and: metadata = MetadataEntity(linking_details={ 'item': {'description': 'a jar of milk'} }) # when: cell_conversion.apply(metadata, 'milk') # then: self.assertEqual('milk - converted', metadata.get_linking_detail('item.name')) self.assertEqual('a jar of milk', metadata.get_linking_detail('item.description'))
def test_do_import_no_projects(self, worksheet_importer_constructor): # given: template_mgr = MagicMock(name='template_manager') worksheet_importer = WorksheetImporter(template_mgr) worksheet_importer_constructor.return_value = worksheet_importer # and: item = MetadataEntity(concrete_type='product', domain_type='product', object_id=910) worksheet_importer.do_import = MagicMock(side_effect=[[item]]) # and: workbook = create_test_workbook('Item') workbook_importer = WorkbookImporter(template_mgr) # when: thrown_exception = False try: workbook_importer.do_import(IngestWorkbook(workbook)) except NoProjectFound: thrown_exception = True # then: self.assertTrue(thrown_exception, f'Expected to throw {NoProjectFound.__name__}.')
def test_do_import_no_project(self, worksheet_importer_constructor): # given: template_mgr = MagicMock(name='template_manager') worksheet_importer = WorksheetImporter(template_mgr) worksheet_importer_constructor.return_value = worksheet_importer no_errors = [] expected_error = { 'location': 'File', 'type': 'NoProjectFound', 'detail': 'The spreadsheet should be associated to a project.' } # and: item = MetadataEntity(concrete_type='product', domain_type='product', object_id=910) worksheet_importer.do_import = MagicMock(side_effect=[([item], no_errors)]) # and: workbook = create_test_workbook('Item') workbook_importer = WorkbookImporter(template_mgr) # when: spreadsheet_json, errors = workbook_importer.do_import( IngestWorkbook(workbook), is_update=False) # then: self.assertIn( expected_error, errors, f'Errors expected to contain {NoProjectFound.__name__}.')
def do_import(self, row: IngestRow, is_module=False): row_errors = [] metadata = MetadataEntity(domain_type=self.domain_type, concrete_type=self.concrete_type, content=self.default_values, row=row, is_module=is_module) for index, cell in enumerate(row.values): if cell.value is None: continue conversion: CellConversion = self.cell_conversions[index] try: conversion.apply(metadata, cell.value) except ValueError as e: error_detail = f'Could not convert to {conversion.converter.type().value} ' \ f'from: {type(cell.value).__name__} {cell.value}' if type(cell.value) == float: error_detail += " This may be because the default excel number format is float." row_errors.append({ "location": f'column={index}, value={cell.value}', "type": "Conversion Error", "detail": error_detail }) except Exception as e: row_errors.append({ "location": f'column={index}, value={cell.value}', "type": e.__class__.__name__, "detail": str(e) }) return metadata, row_errors
def test_add_links_multiple_times(self): # given: metadata = MetadataEntity() # and: batch_1 = ['78de112', '963fefed'] batch_2 = ['2daab01'] # when: metadata.add_links('item', batch_1) metadata.add_links('item', batch_2) # then: item_links = metadata.get_links('item') self.assertIsNotNone(item_links) self.assertCountEqual(batch_1 + batch_2, item_links)
def do_import(self, row): metadata = MetadataEntity(content=self.default_values) for index, cell in enumerate(row): if cell.value is None: continue conversion: CellConversion = self.cell_conversions[index] conversion.apply(metadata, cell.value) return metadata
def test_apply(self): # given: converter = _create_mock_string_converter() cell_conversion = ListElementCellConversion('stuff.list_of_things.name', converter) # when: metadata = MetadataEntity() cell_conversion.apply(metadata, 'sample') # then: list_of_things = metadata.get_content('list_of_things') self.assertIsNotNone(list_of_things) self.assertEqual(1, len(list_of_things)) # and: thing = list_of_things[0] self.assertEqual('sample - converted', thing.get('name'))
def test_apply(self): # given: converter = _create_mock_string_converter() cell_conversion = FieldOfSingleElementListCellConversion('library.books.title', converter) # when: metadata = MetadataEntity() cell_conversion.apply(metadata, 'Harry Potter') # then: books = metadata.get_content('books') self.assertIsNotNone(books) self.assertEqual(1, len(books)) # and: thing = books[0] self.assertEqual('Harry Potter - converted', thing.get('title'))
def test_apply_skips_none_data(self): # given: string_converter = StringConverter() cell_conversion = DirectCellConversion('product.id', string_converter) # when: metadata = MetadataEntity( content={'product': { 'name': 'product name' }}) cell_conversion.apply(metadata, None) # then: product = metadata.get_content('product') self.assertIsNotNone(product) self.assertTrue('id' not in product, '[id] not expected to be in product field')
def test_retain_content_fields(self): # given: content = { 'user_name': 'jdelacruz', 'password': '******', 'description': 'temp' } metadata_entity = MetadataEntity(domain_type='user', concrete_type='user', object_id=1, content=content) # when: metadata_entity.retain_fields('user_name') # then: self.assertEqual(['user_name'], list(metadata_entity.content.as_dict().keys()))
def test_apply(self): # given: converter = _create_mock_string_converter() cell_conversion = IdentityCellConversion('product.product_id', converter) # and: metadata = MetadataEntity() # when: cell_conversion.apply(metadata, 'product_no_144') # then: expected_id = 'product_no_144 - converted' self.assertEqual(expected_id, metadata.object_id) # and: identity value should be in content self.assertEqual(expected_id, metadata.get_content('product_id'))
def test_apply_with_previous_entries(self): # given: metadata = MetadataEntity() metadata.add_external_links('store_item', ['109bdd9', 'c3c35e6']) # and: cell_conversion = ExternalReferenceCellConversion('product.uuid', 'store_item') # when: cell_conversion.apply(metadata, '73de901') # then: store_item_list = metadata.get_external_links('store_item') self.assertIsNotNone(store_item_list, '[store_item] list in external links expected.') # and: expected_ids = ['109bdd9', '73de901', 'c3c35e6'] self.assertCountEqual(expected_ids, store_item_list)