def do_import(self, workbook: IngestWorkbook, is_update, project_uuid=None, update_project=False,
worksheet_titles: List[str] = None):
registry = _ImportRegistry(self.template_mgr)
if worksheet_titles:
importable_worksheets = workbook.select_importable_worksheets(worksheet_titles)
else:
importable_worksheets = workbook.importable_worksheets()
workbook_errors = self.validate_worksheets(is_update, importable_worksheets)
importable_worksheets = [ws for ws in importable_worksheets]
for worksheet in importable_worksheets:
try:
self._import_worksheet(project_uuid, registry, update_project, workbook_errors, worksheet)
except Exception as e:
workbook_errors.append(
{"location": f'sheet={worksheet.title}', "type": e.__class__.__name__, "detail": str(e)})
if not registry.has_project() and not project_uuid:
e = NoProjectFound()
workbook_errors.append({"location": "File", "type": e.__class__.__name__, "detail": str(e)})
if project_uuid and update_project and not registry.has_project():
workbook_errors.append({"location": "File", "type": "NoProjectWorksheet",
"detail": "The option to update the project was specified but there is no project "
"worksheet found."})
if project_uuid and not registry.has_project():
registry.add_project_reference(project_uuid)
self._import_modules(registry, workbook_errors)
return registry.flatten(), workbook_errors
def test_get_schemas(self):
# given:
workbook = Workbook()
# and:
schemas_sheet = workbook.create_sheet('Schemas')
schemas_sheet['A1'] = 'schema'
# and:
base_url = 'https://schema.humancellatlas.org'
expected_schemas = [
f'{base_url}/type/biomaterial/cell_suspension',
f'{base_url}type/biomaterial/organ_from_donor',
f'{base_url}/type/process/library_preparation'
]
header_offset = 2
for index, schema in enumerate(expected_schemas):
schemas_sheet[f'A{index + header_offset}'] = schema
# and:
ingest_workbook = IngestWorkbook(workbook)
# when:
actual_schemas = ingest_workbook.get_schemas()
# then:
self.assertEqual(expected_schemas, actual_schemas)
def test_add_schemas_worksheet(self):
# given
sheets = ['A', 'B', 'C', 'X']
workbook = create_test_workbook(*sheets)
wb = IngestWorkbook(workbook)
# when
wb.add_schemas_worksheet(['schema1', 'schema2', 'schema3'])
# then
self.assertEqual(wb.workbook['Schemas'].cell(1, 1).value, 'Schemas')
self.assertEqual(wb.workbook['Schemas'].cell(2, 1).value, 'schema1')
self.assertEqual(wb.workbook['Schemas'].cell(3, 1).value, 'schema2')
self.assertEqual(wb.workbook['Schemas'].cell(4, 1).value, 'schema3')
def test_add_entity_uuids(self):
# given:
entities = [Entity('type', 'AA', 'content', concrete_type='a', ingest_json={'uuid': {'uuid': 'A-1-uuid'}},
spreadsheet_location={'row_index': 5, 'worksheet_title': 'A'}),
Entity('type', 'AA', 'content', concrete_type='a', ingest_json={'uuid': {'uuid': 'A-2-uuid'}},
spreadsheet_location={'row_index': 6, 'worksheet_title': 'A'}),
Entity('type', 'AA', 'content', concrete_type='a', ingest_json={'uuid': {'uuid': 'A-3-uuid'}},
spreadsheet_location={'row_index': 7, 'worksheet_title': 'A'}),
Entity('type', 'BB', 'content', concrete_type='b', ingest_json={'uuid': {'uuid': 'B-1-uuid'}},
spreadsheet_location={'row_index': 5, 'worksheet_title': 'B'}),
Entity('type', 'BB', 'content', concrete_type='b', ingest_json={'uuid': {'uuid': 'B-2-uuid'}},
spreadsheet_location={'row_index': 6, 'worksheet_title': 'B'}),
Entity('type', 'BB', 'content', concrete_type='b', ingest_json={'uuid': {'uuid': 'B-3-uuid'}},
spreadsheet_location={'row_index': 7, 'worksheet_title': 'B'}),
Entity('type', 'CC', 'content', concrete_type='c', ingest_json={'uuid': {'uuid': 'C-1-uuid'}},
spreadsheet_location={'row_index': 5, 'worksheet_title': 'C'}),
Entity('type', 'CC', 'content', concrete_type='c', ingest_json={'uuid': {'uuid': 'C-2-uuid'}},
spreadsheet_location={'row_index': 6, 'worksheet_title': 'C'}),
Entity('type', 'CC', 'content', concrete_type='c', ingest_json={'uuid': {'uuid': 'C-3-uuid'}},
spreadsheet_location={'row_index': 7, 'worksheet_title': 'C'}),
Entity('type', 'XX', 'content', concrete_type='x', ingest_json={'uuid': {'uuid': 'X-3-uuid'}})]
mock_submission = Mock('submission')
mock_submission.get_entities = Mock(return_value=entities)
sheets = ['A', 'B', 'C', 'X']
workbook = create_test_workbook(*sheets)
wb = IngestWorkbook(workbook)
# when
wb.add_entity_uuids(mock_submission)
# then
self.assertEqual(wb.workbook['A'].cell(4, 1).value, 'a.uuid')
self.assertEqual(wb.workbook['A'].cell(5, 1).value, 'A-1-uuid')
self.assertEqual(wb.workbook['A'].cell(6, 1).value, 'A-2-uuid')
self.assertEqual(wb.workbook['A'].cell(7, 1).value, 'A-3-uuid')
self.assertEqual(wb.workbook['B'].cell(4, 1).value, 'b.uuid')
self.assertEqual(wb.workbook['B'].cell(5, 1).value, 'B-1-uuid')
self.assertEqual(wb.workbook['B'].cell(6, 1).value, 'B-2-uuid')
self.assertEqual(wb.workbook['B'].cell(7, 1).value, 'B-3-uuid')
self.assertEqual(wb.workbook['C'].cell(4, 1).value, 'c.uuid')
self.assertEqual(wb.workbook['C'].cell(5, 1).value, 'C-1-uuid')
self.assertEqual(wb.workbook['C'].cell(6, 1).value, 'C-2-uuid')
self.assertEqual(wb.workbook['C'].cell(7, 1).value, 'C-3-uuid')
self.assertFalse(wb.workbook['X'].cell(5, 1).value, None)
def test_importable_worksheets(self):
# given:
importable_names = ['Organ From Donor', 'Cell Suspension', 'Project']
workbook = create_test_workbook(*importable_names)
workbook.create_sheet('Schemas')
# and:
ingest_workbook = IngestWorkbook(workbook)
# when:
actual_worksheets = ingest_workbook.importable_worksheets()
# then:
actual_titles = [ingest_worksheet.title for ingest_worksheet in actual_worksheets]
self.assertEqual(importable_names, actual_titles)
def update_spreadsheet_with_uuids(submission: Submission, template_mgr: TemplateManager, file_path):
if not submission:
return
wb = IngestWorkbook.from_file(file_path, read_only=False)
wb.add_entity_uuids(submission)
wb.add_schemas_worksheet(template_mgr.get_schemas())
return wb.save(file_path)
def test_do_import_no_project(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
expected_error = {
'location': 'File',
'type': 'NoProjectFound',
'detail': 'The spreadsheet should be associated to a project.'
}
# and:
item = MetadataEntity(concrete_type='product',
domain_type='product',
object_id=910)
worksheet_importer.do_import = MagicMock(side_effect=[([item],
no_errors)])
# and:
workbook = create_test_workbook('Item')
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json, errors = workbook_importer.do_import(
IngestWorkbook(workbook), is_update=False)
# then:
self.assertIn(
expected_error, errors,
f'Errors expected to contain {NoProjectFound.__name__}.')
def setup_workbook_with_project_worksheets(self,
worksheet_importer_constructor):
self.template_mgr.get_concrete_type = MagicMock(return_value='project')
worksheet_importer = WorkbookImporter(self.template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
# and:
project = MetadataEntity(domain_type='project',
concrete_type='project',
content={'description': 'test project'})
jsmith = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'John',
'email': '*****@*****.**'
}
})
ppan = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'Peter',
'email': '*****@*****.**'
}
})
worksheet_importer.do_import = MagicMock(
side_effect=[([project], no_errors), ([jsmith, ppan], no_errors)])
# and:
workbook = create_test_workbook('Project', 'Project - Contributors')
ingest_workbook = IngestWorkbook(workbook)
return ingest_workbook
def test_do_import_multiple_projects(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
project_1 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=1)
project_2 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=2)
worksheet_importer.do_import = MagicMock(
side_effect=[[project_1, project_2]])
# and:
workbook = create_test_workbook('Project')
workbook_importer = WorkbookImporter(template_mgr)
# when:
exception_thrown = False
try:
workbook_importer.do_import(IngestWorkbook(workbook))
except MultipleProjectsFound:
exception_thrown = True
# then:
self.assertTrue(
exception_thrown,
f'Expected to throw {MultipleProjectsFound.__name__}.')
def test_do_import_no_projects(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
item = MetadataEntity(concrete_type='product',
domain_type='product',
object_id=910)
worksheet_importer.do_import = MagicMock(side_effect=[[item]])
# and:
workbook = create_test_workbook('Item')
workbook_importer = WorkbookImporter(template_mgr)
# when:
thrown_exception = False
try:
workbook_importer.do_import(IngestWorkbook(workbook))
except NoProjectFound:
thrown_exception = True
# then:
self.assertTrue(thrown_exception,
f'Expected to throw {NoProjectFound.__name__}.')
def setUp(self, worksheet_importer_constructor) -> None:
self.template_mgr = MagicMock(name='template_manager')
self.template_mgr.template.json_schemas = self.mock_json_schemas
self.concrete_type_map = {'Project': 'project', 'Users': 'users'}
self.template_mgr.get_concrete_type = lambda key: self.concrete_type_map.get(
key)
self.worksheet_importer = WorksheetImporter(self.template_mgr)
worksheet_importer_constructor.return_value = self.worksheet_importer
self.workbook_importer = WorkbookImporter(self.template_mgr)
self.workbook = create_test_workbook('Project', 'Users')
self.ingest_workbook = IngestWorkbook(self.workbook)
def test_add_schemas_worksheet_existing_schemas(self):
# given
sheets = ['A', 'B', 'C']
workbook = create_test_workbook(*sheets)
wb = IngestWorkbook(workbook)
wb.workbook.create_sheet('Schemas')
wb.workbook['Schemas'].cell(1, 1).value = 'X'
wb.workbook['Schemas'].cell(2, 1).value = 'x'
wb.workbook['Schemas'].cell(3, 1).value = 'y'
wb.workbook['Schemas'].cell(4, 1).value = 'z'
# when
wb.add_schemas_worksheet(['schema1', 'schema2', 'schema3'])
# then
self.assertEqual(wb.workbook['Schemas'].cell(1, 1).value, 'X')
self.assertEqual(wb.workbook['Schemas'].cell(2, 1).value, 'x')
self.assertEqual(wb.workbook['Schemas'].cell(3, 1).value, 'y')
self.assertEqual(wb.workbook['Schemas'].cell(4, 1).value, 'z')
def test_do_import_project_worksheet(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorkbookImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
project = MetadataEntity(domain_type='project',
concrete_type='project',
content={'description': 'test project'})
jsmith = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'John',
'email': '*****@*****.**'
}
})
ppan = MetadataEntity(domain_type='project',
concrete_type='contact',
content={
'contributors': {
'name': 'Peter',
'email': '*****@*****.**'
}
})
worksheet_importer.do_import = MagicMock(
side_effect=[[project], [jsmith, ppan]])
# and:
workbook = create_test_workbook('Project', 'Project - Contributors')
ingest_workbook = IngestWorkbook(workbook)
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json = workbook_importer.do_import(ingest_workbook)
# then:
project_map = spreadsheet_json.get('project')
self.assertEqual(1, len(project_map))
project_content = list(project_map.values())[0].get('content')
self.assertEqual('test project', project_content.get('description'))
# and:
contributors = project_content.get('contributors')
self.assertEqual(2, len(contributors))
self.assertIn({
'name': 'John',
'email': '*****@*****.**'
}, contributors)
self.assertIn({
'name': 'Peter',
'email': '*****@*****.**'
}, contributors)
def _generate_project_json_from_workbook(self, workbook):
ingest_workbook = IngestWorkbook(workbook)
template_mgr = self._setup_template_manager_for_project_import()
workbook_importer = WorkbookImporter(template_mgr)
spreadsheet_json, errors = workbook_importer.do_import(ingest_workbook, False, worksheet_titles=['Project'])
if errors:
return None, errors
else:
projects = list(spreadsheet_json.get('project').values())
project = projects[0] if projects else None
project_metadata = project.get('content')
return project_metadata, []
def generate_json(self, file_path, is_update, project_uuid=None, update_project=False):
ingest_workbook = IngestWorkbook.from_file(file_path)
try:
template_mgr = template_manager.build(ingest_workbook.get_schemas(), self.ingest_api)
except Exception as e:
raise SchemaRetrievalError(
f'There was an error retrieving the schema information to process the spreadsheet. {str(e)}')
workbook_importer = WorkbookImporter(template_mgr)
spreadsheet_json, errors = workbook_importer.do_import(ingest_workbook, is_update, project_uuid, update_project)
return spreadsheet_json, template_mgr, errors
def create_ingest_workbook(sheet_names,
column_names,
include_default_sheet=False) -> IngestWorkbook:
header_row_idx = 4
workbook = Workbook()
for sheet_name in sheet_names:
worksheet = workbook.create_sheet(sheet_name)
worksheet[f'A{header_row_idx}'] = f'{sheet_name}.{column_names[0]}'
worksheet[f'B{header_row_idx}'] = f'{sheet_name}.{column_names[1]}'
check_default_sheet_inclusion(include_default_sheet, workbook)
return IngestWorkbook(workbook)
def test_importable_worksheets(self):
# given:
workbook = Workbook()
# and:
importable_names = ['Organ From Donor', 'Cell Suspension']
expected_worksheets = [
workbook.create_sheet(name) for name in importable_names
]
workbook.create_sheet('Schemas')
# and:
default_worksheet = workbook.get_sheet_by_name('Sheet')
workbook.remove_sheet(default_worksheet)
# and:
ingest_workbook = IngestWorkbook(workbook)
# when:
actual_worksheets = ingest_workbook.importable_worksheets()
# then:
self.assertEqual(expected_worksheets, actual_worksheets)
def test_do_import_update_project_but_no_project_worksheet(self):
# given:
workbook = create_test_workbook('Users')
# when:
spreadsheet_json, errors = self.workbook_importer.do_import(
IngestWorkbook(workbook),
project_uuid='project-uuid',
update_project=True,
is_update=False)
# then:
self.assertEqual(len(errors), 1)
self.assertEqual('NoProjectWorksheet', errors[0].get('type'))
def test_do_import_do_not_update_project_and_no_project_worksheet(self):
# given:
workbook = create_test_workbook('Users')
# when:
spreadsheet_json, errors = self.workbook_importer.do_import(
IngestWorkbook(workbook),
project_uuid='project-uuid',
update_project=False,
is_update=False)
# then:
self.assertEqual(len(errors), 0)
self.assertEqual(len(spreadsheet_json.get('project', {}).keys()), 1)
self.assertFalse(
spreadsheet_json['project']['project-uuid']['is_reference'])
self.assertTrue(spreadsheet_json['project']['project-uuid']
['is_linking_reference'])
def do_import(self, workbook: IngestWorkbook, project_uuid=None):
registry = _ImportRegistry()
for worksheet in workbook.importable_worksheets():
metadata_entities = self.worksheet_importer.do_import(worksheet)
module_field_name = worksheet.get_module_field_name()
for entity in metadata_entities:
if worksheet.is_module_tab():
entity.retain_content_fields(module_field_name)
registry.add_module(entity)
else:
registry.add_submittable(entity)
if registry.has_project():
registry.import_modules()
else:
raise NoProjectFound()
return registry.flatten()
def test_do_import(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and:
project = MetadataEntity(concrete_type='project',
domain_type='project')
jdelacruz = MetadataEntity(concrete_type='user',
domain_type='user',
object_id=1,
content={'user_name': 'jdelacruz'})
setsuna_f_seiei = MetadataEntity(concrete_type='user',
domain_type='user',
object_id=96,
content={'user_name': 'sayyeah'})
worksheet_importer.do_import = MagicMock(
side_effect=[[project], [jdelacruz, setsuna_f_seiei]])
# and:
workbook = create_test_workbook('Project', 'Users')
ingest_workbook = IngestWorkbook(workbook)
workbook_importer = WorkbookImporter(template_mgr)
# when:
workbook_json = workbook_importer.do_import(ingest_workbook)
# then:
self.assertIsNotNone(workbook_json)
# and:
user_map = workbook_json.get('user')
self.assertIsNotNone(user_map)
self.assertEqual(2, len(user_map))
self.assertEqual([jdelacruz.object_id, setsuna_f_seiei.object_id],
list(user_map.keys()))
# and:
self.assertEqual({'user_name': 'jdelacruz'},
user_map.get(1)['content'])
self.assertEqual({'user_name': 'sayyeah'}, user_map.get(96)['content'])
def do_import(self, workbook: IngestWorkbook, project_uuid=None):
spreadsheet_json = {}
self.import_or_reference_project(project_uuid, spreadsheet_json,
workbook)
for worksheet in workbook.importable_worksheets():
concrete_entity = self.template_mgr.get_concrete_entity_of_tab(
worksheet.title)
domain_entity = self.template_mgr.get_domain_entity(
concrete_entity)
entities_dict = self.worksheet_importer.do_import(
worksheet, self.template_mgr)
if spreadsheet_json.get(domain_entity) is None:
spreadsheet_json[domain_entity] = {}
spreadsheet_json[domain_entity].update(entities_dict)
return spreadsheet_json
def test_do_import(self, worksheet_importer_constructor):
# given: set up template manager
key_label_map = {
'Project': 'project',
'Cell Suspension': 'cell_suspension'
}
domain_entity_map = {
'project': 'project',
'cell_suspension': 'biomaterial'
}
mock_template_manager = MagicMock()
mock_template_manager.get_concrete_entity_of_tab = lambda key: key_label_map.get(key)
mock_template_manager.get_domain_entity = lambda key: domain_entity_map.get(key)
# and: set up worksheet importer
worksheet_importer = IdentifiableWorksheetImporter()
expected_json = self._fake_worksheet_import(worksheet_importer, mock_template_manager)
# and: set up workbook
workbook = Workbook()
ingest_workbook = IngestWorkbook(workbook)
schema_list = self._mock_get_schemas(ingest_workbook)
self._mock_importable_worksheets(ingest_workbook, workbook)
# and: mock WorksheetImporter constructor
worksheet_importer_constructor.return_value = worksheet_importer
workbook_importer = WorkbookImporter(mock_template_manager)
workbook_importer.import_or_reference_project = MagicMock()
# when:
workbook_output = workbook_importer.do_import(ingest_workbook)
# then:
self.assertEqual(2, len(list(workbook_output.keys())))
self.assertEqual(['project', 'biomaterial'], list(workbook_output.keys()))
self.assertEqual(2, len(list(workbook_output['project'].keys())))
self.assertEqual(expected_json['project'], workbook_output['project'])
self.assertEqual(expected_json['biomaterial'], workbook_output['biomaterial'])
def test_do_import_multiple_projects(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
template_mgr.template.json_schemas = self.mock_json_schemas
template_mgr.get_concrete_type = MagicMock(return_value='project')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
no_errors = []
expected_error = {
'location':
'sheet=Project',
'type':
'MultipleProjectsFound',
'detail':
'The spreadsheet should only be associated to a single project.'
}
# and:
project_1 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=1)
project_2 = MetadataEntity(concrete_type='project',
domain_type='project',
object_id=2)
worksheet_importer.do_import = MagicMock(
side_effect=[([project_1, project_2], no_errors)])
# and:
workbook = create_test_workbook('Project')
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json, errors = workbook_importer.do_import(
IngestWorkbook(workbook), is_update=False)
# then:
self.assertIn(
expected_error, errors,
f'Errors expected to contain {MultipleProjectsFound.__name__}.')
def _create_ingest_workbook(file_path):
workbook = openpyxl.load_workbook(filename=file_path, read_only=True)
return IngestWorkbook(workbook)
def test_do_import_with_module_tab(self, worksheet_importer_constructor):
# given:
template_mgr = MagicMock(name='template_manager')
worksheet_importer = WorksheetImporter(template_mgr)
worksheet_importer_constructor.return_value = worksheet_importer
# and: stub worksheet importer
project = MetadataEntity(concrete_type='Project',
domain_type='Project')
user = MetadataEntity(concrete_type='user',
domain_type='user',
object_id=773,
content={'user_name': 'janedoe'})
fb_profile = MetadataEntity(concrete_type='sn_profile',
domain_type='user',
object_id=773,
content={
'sn_profiles': {
'name': 'facebook',
'id': '392'
},
'description': 'extra field'
})
ig_profile = MetadataEntity(concrete_type='sn_profile',
domain_type='user',
object_id=773,
content={
'sn_profiles': {
'name': 'instagram',
'id': 'a92'
},
'description': 'extra field'
})
worksheet_importer.do_import = MagicMock(
side_effect=[[project], [user], [fb_profile, ig_profile]])
# and: create test workbook
workbook = create_test_workbook('Project', 'User',
'User - SN Profiles')
ingest_workbook = IngestWorkbook(workbook)
workbook_importer = WorkbookImporter(template_mgr)
# when:
spreadsheet_json = workbook_importer.do_import(ingest_workbook)
# then:
self.assertIsNotNone(spreadsheet_json)
self.assertEqual(2, len(spreadsheet_json))
# and:
user_map = spreadsheet_json.get('user')
self.assertIsNotNone(user_map)
# and:
janedoe = user_map.get(773)
self.assertIsNotNone(janedoe)
content = janedoe.get('content')
self.assertEqual('janedoe', content.get('user_name'))
self.assertEqual(['user_name', 'sn_profiles'], list(content.keys()))
# and:
sn_profiles = content.get('sn_profiles')
self.assertIsNotNone(sn_profiles)
self.assertEqual(2, len(sn_profiles))
# and:
self.assertEqual({'name': 'facebook', 'id': '392'}, sn_profiles[0])
self.assertEqual({'name': 'instagram', 'id': 'a92'}, sn_profiles[1])
def test_do_import_with_module_tab(self, worksheet_importer_constructor):
# given:
concrete_type_map = {
'Project': 'project',
'User': '******',
'User - SN Profiles': 'users'
}
self.template_mgr.get_concrete_type = lambda key: concrete_type_map.get(
key)
project = MetadataEntity(concrete_type='Project',
domain_type='Project')
user = MetadataEntity(concrete_type='user',
domain_type='user',
object_id=773,
content={'user_name': 'janedoe'})
fb_profile = MetadataEntity(concrete_type='sn_profile',
domain_type='user',
object_id=773,
content={
'sn_profiles': {
'name': 'facebook',
'id': '392'
},
'description': 'extra fb field'
})
ig_profile = MetadataEntity(concrete_type='sn_profile',
domain_type='user',
object_id=773,
content={
'sn_profiles': {
'name': 'instagram',
'id': 'a92'
},
'description': 'extra ig field'
})
no_errors = []
self.worksheet_importer.do_import = MagicMock(side_effect=[(
[project],
no_errors), ([user],
no_errors), ([fb_profile, ig_profile], no_errors)])
self.workbook = create_test_workbook('Project', 'User',
'User - SN Profiles')
self.ingest_workbook = IngestWorkbook(self.workbook)
# when:
spreadsheet_json, errors = self.workbook_importer.do_import(
self.ingest_workbook, is_update=False)
# then:
expected_errors = [{
'key': 'description',
'value': 'extra fb field'
}, {
'key': 'description',
'value': 'extra ig field'
}]
self.assertIsNotNone(spreadsheet_json)
self.assertEqual(
errors,
self.workbook_importer.list_data_removal_errors(
'User - SN Profiles', expected_errors))
self.assertEqual(2, len(spreadsheet_json))
# and:
user_map = spreadsheet_json.get('user')
self.assertIsNotNone(user_map)
# and:
janedoe = user_map.get(773)
self.assertIsNotNone(janedoe)
content = janedoe.get('content')
self.assertEqual('janedoe', content.get('user_name'))
self.assertEqual(['user_name', 'sn_profiles'], list(content.keys()))
# and:
sn_profiles = content.get('sn_profiles')
self.assertIsNotNone(sn_profiles)
self.assertEqual(2, len(sn_profiles))
# and:
self.assertEqual({'name': 'facebook', 'id': '392'}, sn_profiles[0])
self.assertEqual({'name': 'instagram', 'id': 'a92'}, sn_profiles[1])
