def test_is_project__returns_false(self): # given worksheet = spreadsheet_utils.create_worksheet('not a Project', []) ingest_worksheet = IngestWorksheet(worksheet) # when and then self.assertFalse(ingest_worksheet.is_project())
def test_is_project__returns_true__when_case_is_lower(self): # given worksheet = spreadsheet_utils.create_worksheet('project', []) ingest_worksheet = IngestWorksheet(worksheet) # when and then self.assertTrue(ingest_worksheet.is_project())
def test_get_data_rows(self): # given: start_row_idx = 6 header_row_idx = 4 header_row = ['name', 'address', 'mobile', 'email'] expected_data_row = ['Jane Doe', 'Cambridge', '12-345-67', '*****@*****.**'] rows = [[], [], [], [], [], []] # initialise 6 rows rows[header_row_idx-1] = header_row rows[start_row_idx-1] = expected_data_row worksheet = spreadsheet_utils.create_worksheet('person', rows) # when ingest_worksheet = IngestWorksheet(worksheet, header_row_idx=header_row_idx) data_rows = ingest_worksheet.get_data_rows(start_row=start_row_idx) data_row_values = [] for row in data_rows: cell_values = [cell.value for cell in row] data_row_values.append(cell_values) # then: self.assertEqual(len(data_row_values), 1) self.assertEqual(data_row_values, [expected_data_row])
def test_is_project_module__returns_true(self): # given worksheet = spreadsheet_utils.create_worksheet( 'Project - Contributors', []) ingest_worksheet = IngestWorksheet(worksheet) # when and then self.assertTrue(ingest_worksheet.is_project_module())
def test_get_title(self): # given: workbook = create_test_workbook('User', 'User - SN Profiles') user_sheet = workbook.get_sheet_by_name('User') sn_profiles_sheet = workbook.get_sheet_by_name('User - SN Profiles') # and: user = IngestWorksheet(user_sheet) sn_profiles = IngestWorksheet(sn_profiles_sheet) # expect: self.assertEqual('User', user.title) self.assertEqual('User - SN Profiles', sn_profiles.title)
def test_get_column_headers_includes_blank_cells(self): # given: header_row = 4 rows = [['name', 'address', '', 'email']] worksheet = spreadsheet_utils.create_worksheet('person', rows, start_row=header_row) # when: ingest_worksheet = IngestWorksheet(worksheet, header_row_idx=header_row) column_headers = ingest_worksheet.get_column_headers() # then: self.assertEqual(len(column_headers), 4) self.assertEqual(column_headers, ['name', 'address', '', 'email'])
def test__has_column__returns_false__when_column_is_empty(self): # given: rows = [['name', 'address', 'mobile', 'email'], ['Jane Doe', 'Cambridge', '12-345-67', '*****@*****.**']] worksheet = spreadsheet_utils.create_worksheet('person', rows) ingest_worksheet = IngestWorksheet(worksheet, header_row_idx=1) # when: result = ingest_worksheet.has_column('') # then: self.assertFalse(result)
def test_do_import(self): # given: row_template = MagicMock('row_template') no_errors = [] # and: john_doe = MetadataEntity(object_id='profile_1') emma_jackson = MetadataEntity(object_id='profile_2') row_template.do_import = MagicMock('import_row', side_effect=[(john_doe, no_errors), (emma_jackson, no_errors)]) # and: mock_template_manager = MagicMock('template_manager') mock_template_manager.create_row_template = MagicMock(return_value=row_template) mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2']) mock_template_manager.get_concrete_type = MagicMock(return_value='concrete_entity') # and: workbook = Workbook() worksheet = workbook.create_sheet('user_profile') worksheet['A4'] = 'header' worksheet['A6'] = 'john' worksheet['A7'] = 'emma' # when: worksheet_importer = WorksheetImporter(mock_template_manager) profiles, errors = worksheet_importer.do_import(IngestWorksheet(worksheet)) # then: self.assertEqual(2, len(profiles)) self.assertIn(john_doe, profiles) self.assertIn(emma_jackson, profiles) self.assertEqual(errors, []) # and: domain and concrete type should be set pass
def test_create_row_template_with_default_values(self, determine_strategy, look_up): # given: schema_template = MagicMock('schema_template') ingest_api = MagicMock(name='ingest_api') # and: schema_url = 'http://schema.sample.com/profile' self._mock_schema_lookup(schema_template, schema_url=schema_url, main_category='profile', object_type='profile_type') # and: look_up.return_value = MagicMock('column_spec') determine_strategy.return_value = FakeConversion('') # and: workbook = Workbook() worksheet = workbook.create_sheet('profile') worksheet['A4'] = 'profile.name' ingest_worksheet = IngestWorksheet(worksheet) # when: template_manager = TemplateManager(schema_template, ingest_api) template_manager.get_schema_url = MagicMock(return_value=schema_url) row_template = template_manager.create_row_template(ingest_worksheet) # then: content_defaults = row_template.default_values self.assertIsNotNone(content_defaults) self.assertEqual(schema_url, content_defaults.get('describedBy')) self.assertEqual('profile', content_defaults.get('schema_type'))
def do_import(self, ingest_worksheet: IngestWorksheet): row_template = self.template.create_row_template(ingest_worksheet) records = [] rows = ingest_worksheet.get_data_rows() for index, row in enumerate(rows): metadata = row_template.do_import(row) if not metadata.object_id: metadata.object_id = self._generate_id() records.append(metadata) return records
def select_importable_worksheets(self, worksheet_titles: List[str]): worksheets = [] for title in worksheet_titles: worksheets.extend([ IngestWorksheet(worksheet) for worksheet in self.workbook.worksheets if (title not in SPECIAL_TABS) and ( title.lower() in worksheet.title.lower()) ]) return worksheets
def add_entity_uuids(self, submission: Submission): worksheets = {} col_idx = 1 entities = [ entity for entity in submission.get_entities() if entity.spreadsheet_location ] for entity in entities: worksheet_title = entity.spreadsheet_location.get( 'worksheet_title') row_index = entity.spreadsheet_location.get('row_index') if not worksheets.get(worksheet_title): worksheet = self.workbook[worksheet_title] ingest_worksheet = IngestWorksheet(worksheet=worksheet) worksheets[worksheet_title] = ingest_worksheet column_header = f'{entity.concrete_type}.uuid' if column_header not in ingest_worksheet.get_column_headers(): ingest_worksheet.insert_column_with_header( column_header, col_idx) ingest_worksheet = worksheets.get(worksheet_title) ingest_worksheet.cell(row=row_index, column=col_idx).value = entity.uuid
def do_import(self, ingest_worksheet: IngestWorksheet): records = [] worksheet_errors = [] try: row_template = self.template.create_row_template(ingest_worksheet) rows = ingest_worksheet.get_data_rows() for index, row in enumerate(rows): metadata, row_errors = row_template.do_import(row, ingest_worksheet.is_module_tab()) for error in row_errors: if 'location' in error: error["location"] = f'sheet={ingest_worksheet.title} row={index}, {error["location"]}' else: error["location"] = f'sheet={ingest_worksheet.title} row={index}' worksheet_errors.append(error) if not metadata.object_id: metadata.object_id = self._generate_id() records.append(metadata) except Exception as e: worksheet_errors.append({ "location": f'sheet={ingest_worksheet.title}', "type": e.__class__.__name__, "detail": str(e) }) return records, worksheet_errors
def create_row_template(self, ingest_worksheet: IngestWorksheet): concrete_type = self.get_concrete_type(ingest_worksheet.title) domain_type = self.get_domain_type(concrete_type) column_headers = ingest_worksheet.get_column_headers() cell_conversions = [] context = self._determine_context(concrete_type, ingest_worksheet) header_counter = {} for header in column_headers: if not header_counter.get(header): header_counter[header] = 0 header_counter[header] = header_counter[header] + 1 column_spec = ColumnSpecification(self.template, header, concrete_type, context=context, order_of_occurrence=header_counter[header]) strategy = conversion_strategy.determine_strategy(column_spec) cell_conversions.append(strategy) default_values = None if not ingest_worksheet.is_module_tab(): default_values = self._define_default_values(concrete_type) return RowTemplate(domain_type, concrete_type, cell_conversions, default_values=default_values)
def test_create_row_template_with_default_values(self, determine_strategy): # given: schema_template = MagicMock('schema_template') ingest_api = MagicMock(name='mock_ingest_api') # and: domain_entity = "profile/profile_type" schema_url = "http://schema.sample.com/profile" schema_template.get_tabs_config = MagicMock() schema_template.lookup_metadata_schema_name_given_title = MagicMock( return_value="profile_type") schema_template.get_latest_schema = MagicMock(return_value=schema_url) schema = { "schema": { "domain_entity": domain_entity, "url": schema_url } } property_schema = { "description": "Property description", "value_type": "string" } spec_map = {"profile_type": schema, "profile.name": property_schema} schema_template.lookup_property_from_template = lambda key: spec_map.get( key) # and: determine_strategy.return_value = FakeConversion('') # and: workbook = Workbook() worksheet = workbook.create_sheet('profile') worksheet['A4'] = 'profile.name' ingest_worksheet = IngestWorksheet(worksheet) # when: template_manager = TemplateManager(schema_template, ingest_api) template_manager.get_schema_url = MagicMock(return_value=schema_url) row_template = template_manager.create_row_template(ingest_worksheet) # then: content_defaults = row_template.default_values self.assertIsNotNone(content_defaults) self.assertEqual(schema_url, content_defaults.get('describedBy')) self.assertEqual('profile', content_defaults.get('schema_type'))
def test_is_module_tab(self): # given: workbook = create_test_workbook('Product', 'Product - History') product_sheet = workbook.get_sheet_by_name('Product') history_sheet = workbook.get_sheet_by_name('Product - History') # and: product = IngestWorksheet(product_sheet) history = IngestWorksheet(history_sheet) # expect: self.assertFalse(product.is_module_tab()) self.assertTrue(history.is_module_tab())
def test_do_import_no_id_metadata(self): # given: row_template = MagicMock('row_template') no_errors = [] # and: paper_metadata = MetadataEntity(content={'product_name': 'paper'}, links={'delivery': ['123', '456']}) pen_metadata = MetadataEntity(content={'product_name': 'pen'}, links={'delivery': ['789']}) row_template.do_import = MagicMock(side_effect=([(paper_metadata, no_errors), (pen_metadata, no_errors)])) # and: mock_template_manager = MagicMock('template_manager') mock_template_manager.create_row_template = MagicMock(return_value=row_template) mock_template_manager.get_header_row = MagicMock(return_value=['header1', 'header2']) mock_template_manager.get_concrete_type = MagicMock(return_value='concrete_entity') # and: workbook = Workbook() worksheet = workbook.create_sheet('product') worksheet['A6'] = 'paper' worksheet['A7'] = 'pen' # when: worksheet_importer = WorksheetImporter(mock_template_manager) results, errors = worksheet_importer.do_import(IngestWorksheet(worksheet)) # then: self.assertEqual(2, len(results)) self.assertIn(paper_metadata, results) self.assertIn(pen_metadata, results) self.assertEqual(errors, []) # and: object id should be assigned paper_id = paper_metadata.object_id self.assertIsNotNone(paper_id) pen_id = pen_metadata.object_id self.assertIsNotNone(pen_id) self.assertNotEqual(paper_id, pen_id)
def test_create_row_template_with_none_header(self, determine_strategy): # given: schema_template = MagicMock('schema_template') ingest_api = MagicMock(name='ingest_api') # and: do_nothing_strategy = FakeConversion('') determine_strategy.return_value = do_nothing_strategy # and: self._mock_schema_lookup(schema_template) # and: workbook = Workbook() worksheet = workbook.create_sheet('sample') worksheet['A4'] = None ingest_worksheet = IngestWorksheet(worksheet) # when: template_manager = TemplateManager(schema_template, ingest_api) row_template = template_manager.create_row_template(ingest_worksheet) # then: self.assertEqual(0, len(row_template.cell_conversions))
def test_create_row_template(self, determine_strategy, look_up): # given: template = MagicMock(name='schema_template') ingest_api = MagicMock(name='ingest_api') # and: concrete_type = 'user' template.get_tab_key = MagicMock(return_value=concrete_type) # and: spec_map = { 'user': { 'schema': { 'domain_entity': 'main_category/subdomain' } } } template.lookup = lambda key: spec_map.get(key, None) # and: set up column spec name_column_spec = MagicMock(name='name_column_spec') numbers_column_spec = MagicMock(name='numbers_column_spec') look_up.side_effect = [name_column_spec, numbers_column_spec] # and: name_strategy = MagicMock('name_strategy') numbers_strategy = MagicMock('numbers_strategy') determine_strategy.side_effect = [name_strategy, numbers_strategy] # and: prepare worksheet header_row_idx = 4 workbook = Workbook() worksheet = workbook.create_sheet('sample') worksheet[f'A{header_row_idx}'] = 'user.profile.first_name' worksheet[f'B{header_row_idx}'] = 'numbers' ingest_worksheet = IngestWorksheet(worksheet, header_row_idx=header_row_idx) # when: template_manager = TemplateManager(template, ingest_api) row_template: RowTemplate = template_manager.create_row_template( ingest_worksheet) # then: expected_calls = [ call(template, 'user.profile.first_name', concrete_type, context=concrete_type, order_of_occurrence=1), call(template, 'numbers', concrete_type, context=concrete_type, order_of_occurrence=1) ] look_up.assert_has_calls(expected_calls) determine_strategy.assert_has_calls( [call(name_column_spec), call(numbers_column_spec)]) # and: self.assertIsNotNone(row_template) self.assertEqual('main_category', row_template.domain_type) self.assertEqual(concrete_type, row_template.concrete_type) self.assertEqual(2, len(row_template.cell_conversions)) self.assertTrue(name_strategy in row_template.cell_conversions) self.assertTrue(numbers_strategy in row_template.cell_conversions)
def test_create_row_template_for_module_worksheet(self, determine_strategy, look_up): # given: template = MagicMock(name='schema_template') ingest_api = MagicMock(name='ingest_api') # TODO define method in SchemaTemplate that returns domain and concrete types #module-tabs # and: concrete_type = 'product' template.get_tab_key = MagicMock(return_value=concrete_type) # and: spec_map = { 'product': { 'schema': { 'domain_entity': 'merchandise/product' } } } template.lookup = lambda key: spec_map.get(key, None) # and: template_mgr = TemplateManager(template, ingest_api) # and: workbook = create_test_workbook('Product - Reviews') reviews_worksheet = workbook.get_sheet_by_name('Product - Reviews') reviews_worksheet['A4'] = 'product.info.id' reviews_worksheet['B4'] = 'product.reviews.rating' # and: set up dummy look up results id_spec = MagicMock(name='id_spec') rating_spec = MagicMock(name='rating_spec') look_up.side_effect = [id_spec, rating_spec] # and: set up strategies id_strategy = MagicMock(name='id_strategy') rating_strategy = MagicMock(name='rating_strategy') determine_strategy.side_effect = { id_spec: id_strategy, rating_spec: rating_strategy }.get # when: row_template = template_mgr.create_row_template( IngestWorksheet(reviews_worksheet)) # then: expected_calls = [ call(template, 'product.info.id', concrete_type, order_of_occurrence=1, context='product.reviews'), call(template, 'product.reviews.rating', concrete_type, order_of_occurrence=1, context='product.reviews') ] look_up.assert_has_calls(expected_calls) # and: self.assertIsNotNone(row_template) self.assertIn(id_strategy, row_template.cell_conversions) self.assertIn(rating_strategy, row_template.cell_conversions)
def importable_worksheets(self): return [ IngestWorksheet(worksheet) for worksheet in self.workbook.worksheets if worksheet.title not in SPECIAL_TABS ]
def test_create_row_template_for_module_worksheet(self, determine_strategy): # given: template = MagicMock(name='schema_template') ingest_api = MagicMock(name='mock_ingest_api') # and: spec_map = { 'product': { 'schema': { 'domain_entity': 'merchandise/product' } } } template.lookup_property_from_template = lambda key: spec_map.get( key, None) domain_entity = "merchandise/product" schema_url = "http://schema.sample.com/product" template.get_tabs_config = MagicMock() template.lookup_metadata_schema_name_given_title = MagicMock( return_value="product_type") template.get_latest_schema = MagicMock(return_value=schema_url) schema = { "schema": { "domain_entity": domain_entity, "url": schema_url } } property_one_schema = { "description": "Property one description", "value_type": "string" } property_two_schema = { "description": "Property two description", "value_type": "string" } spec_map = { "product_type": schema, "product.info.id": property_one_schema, "product.reviews.rating": property_two_schema } template.lookup_property_attributes_in_metadata = lambda key: spec_map.get( key) # and: template_mgr = TemplateManager(template, ingest_api) # and: workbook = create_test_workbook('Product - Reviews') reviews_worksheet = workbook['Product - Reviews'] reviews_worksheet['A4'] = 'product.info.id' reviews_worksheet['B4'] = 'product.reviews.rating' # and: set up strategies id_strategy = MagicMock(name='id_strategy') rating_strategy = MagicMock(name='rating_strategy') determine_strategy.side_effect = [id_strategy, rating_strategy] # when: row_template = template_mgr.create_row_template( IngestWorksheet(reviews_worksheet)) # and: self.assertIsNotNone(row_template) self.assertIn(id_strategy, row_template.cell_conversions) self.assertIn(rating_strategy, row_template.cell_conversions)
def test_get_module_field_name(self): # given: workbook = create_test_workbook('Product - Reviews', 'User - SN Profiles', 'Log - file-names', 'Account') # and: simple reviews_sheet = workbook.get_sheet_by_name('Product - Reviews') reviews = IngestWorksheet(reviews_sheet) # and: with space in between sn_profiles_sheet = workbook.get_sheet_by_name('User - SN Profiles') sn_profiles = IngestWorksheet(sn_profiles_sheet) # and: with hyphen file_names_sheet = workbook.get_sheet_by_name('Log - file-names') file_names = IngestWorksheet(file_names_sheet) # and: not module worksheet account_sheet = workbook.get_sheet_by_name('Account') account = IngestWorksheet(account_sheet) # expect: self.assertEqual('reviews', reviews.get_module_field_name()) self.assertEqual('sn_profiles', sn_profiles.get_module_field_name()) self.assertEqual('file_names', file_names.get_module_field_name()) self.assertIsNone(account.get_module_field_name())
def test_create_row_template(self, determine_strategy): # given: template = MagicMock(name='schema_template') ingest_api = MagicMock(name='mock_ingest_api') # and: concrete_type = "user" domain_entity = "main_category/subdomain" schema_url = "http://schema.sample.com/main_category" template.get_tabs_config = MagicMock() template.lookup_metadata_schema_name_given_title = MagicMock( return_value=concrete_type) template.get_latest_schema = MagicMock(return_value=schema_url) schema = { "schema": { "domain_entity": domain_entity, "url": schema_url } } property_one_schema = { "description": "Property one description", "value_type": "string" } property_two_schema = { "description": "Property two description", "value_type": "string" } spec_map = { concrete_type: schema, "user.profile.first_name": property_one_schema, "user.numbers": property_two_schema } template.lookup_property_from_template = lambda key: spec_map.get(key) # and: name_strategy = MagicMock('name_strategy') numbers_strategy = MagicMock('numbers_strategy') determine_strategy.side_effect = [name_strategy, numbers_strategy] # and: prepare worksheet header_row_idx = 4 workbook = Workbook() worksheet = workbook.create_sheet('sample') worksheet[f'A{header_row_idx}'] = 'user.profile.first_name' worksheet[f'B{header_row_idx}'] = 'user.numbers' ingest_worksheet = IngestWorksheet(worksheet, header_row_idx=header_row_idx) # when: template_manager = TemplateManager(template, ingest_api) row_template: RowTemplate = template_manager.create_row_template( ingest_worksheet) # then: self.assertEqual(determine_strategy.call_count, 2) # and: self.assertIsNotNone(row_template) self.assertEqual('main_category', row_template.domain_type) self.assertEqual(concrete_type, row_template.concrete_type) self.assertEqual(2, len(row_template.cell_conversions)) self.assertTrue(name_strategy in row_template.cell_conversions) self.assertTrue(numbers_strategy in row_template.cell_conversions)