def test_create_template_node(self):
# given:
schema_template = MagicMock(name='schema_template')
schema_template.get_tab_key = MagicMock(return_value='concrete_entity')
schema_url = 'https://schema.humancellatlas.org/type/biomaterial/5.1.0/donor_organsim'
lookup_map = {
'concrete_entity': {
'schema': {
'domain_entity': 'biomaterial',
'url': schema_url
}
}
}
schema_template.lookup = lambda key: lookup_map.get(key)
ingest_api = MagicMock(name='ingest_api')
# and:
template_manager = TemplateManager(schema_template, ingest_api)
# and:
workbook = Workbook()
donor_worksheet = workbook.create_sheet('Donor')
# when:
data_node: DataNode = template_manager.create_template_node(
donor_worksheet)
# then:
data = data_node.as_dict()
self.assertEqual(schema_url, data.get('describedBy'))
self.assertEqual('biomaterial', data.get('schema_type'))
def test_get_schema_url(self):
# given
schema_template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='ingest_api')
latest_url = 'https://schema.humancellatlas.org/type/biomaterial/5.0.0/donor_organism'
spec = {
'schema': {
'high_level_entity': 'type',
'domain_entity': 'biomaterial',
'module': 'donor_organism',
'url': latest_url
}
}
schema_template.lookup = MagicMock(name='lookup', return_value=spec)
template_manager = TemplateManager(schema_template, ingest_api)
template_manager.get_latest_schema_url = MagicMock(
return_value=latest_url)
# when:
url = template_manager.get_schema_url('cell_suspension')
# then:
self.assertEqual(
'https://schema.humancellatlas.org/type/biomaterial/5.0.0/donor_organism',
url)
def test_get_schema_type(self):
# given
schema_template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='mock_ingest_api')
spec = {
'schema': {
'high_level_entity':
'type',
'domain_entity':
'biomaterial',
'module':
'donor_organism',
'url':
'https://schema.humancellatlas.org/type/biomaterial/5.0.0/donor_organism'
}
}
schema_template.lookup_property_from_template = MagicMock(
name='lookup_property_from_template', return_value=spec)
template_manager = TemplateManager(schema_template, ingest_api)
# when:
domain_entity = template_manager.get_domain_type('cell_suspension')
# then:
self.assertEqual('biomaterial', domain_entity)
def test_create_row_template_with_default_values(self, determine_strategy, build_raw):
# given:
schema_template = MagicMock('schema_template')
ingest_api = MagicMock(name='ingest_api')
# and:
schema_url = 'http://schema.sample.com/profile'
self._mock_schema_lookup(schema_template, schema_url=schema_url, main_category='profile',
object_type='profile_type')
# and:
build_raw.return_value = MagicMock('column_spec')
determine_strategy.return_value = FakeConversion('')
# and:
workbook = Workbook()
worksheet = workbook.create_sheet('profile')
worksheet['A4'] = 'profile.name'
# when:
template_manager = TemplateManager(schema_template, ingest_api)
template_manager.get_schema_url = MagicMock(return_value=schema_url)
row_template = template_manager.create_row_template(worksheet)
# then:
content_defaults = row_template.default_values
self.assertIsNotNone(content_defaults)
self.assertEqual(schema_url, content_defaults.get('describedBy'))
self.assertEqual('profile', content_defaults.get('schema_type'))
def test_get_concrete_type_of_regular_worksheet(self):
# given
schema_template = MagicMock(name='schema_template')
schema_template.get_tab_key = MagicMock(return_value='user_profile')
manager = TemplateManager(schema_template,
MagicMock(name='ingest_api'))
# expect:
self.assertEqual('user_profile',
manager.get_concrete_type('User Profile'))
schema_template.get_tab_key.assert_called_with('User Profile')
def test_get_domain_type(self):
# given:
template = MagicMock(name='schema_template')
schema_spec = {'schema': {'domain_entity': 'user/profile'}}
template.lookup = MagicMock(return_value=schema_spec)
# and:
template_manager = TemplateManager(template,
MagicMock(name='ingest_api'))
# expect:
self.assertEqual('user', template_manager.get_domain_type('profile'))
def test_get_concrete_type_of_module_worksheet(self):
# given:
schema_template = MagicMock(name='schema_template')
schema_template.get_tab_key = MagicMock(return_value='product')
manager = TemplateManager(schema_template,
MagicMock(name='ingest_api'))
# expect:
self.assertEqual('product',
manager.get_concrete_type('Product - Barcodes'))
# and:
schema_template.get_tab_key.assert_called_with('Product')
def test_get_concrete_type_of_regular_worksheet(self):
# given
schema_template = MagicMock(name='schema_template')
schema_template.lookup_metadata_schema_name_given_title = MagicMock(
return_value='user_profile')
manager = TemplateManager(schema_template,
MagicMock(name='mock_ingest_api'))
# expect:
self.assertEqual('user_profile',
manager.get_concrete_type('User Profile'))
schema_template.lookup_metadata_schema_name_given_title.assert_called_with(
'User Profile')
def update_spreadsheet_with_uuids(submission: Submission, template_mgr: TemplateManager, file_path):
if not submission:
return
wb = IngestWorkbook.from_file(file_path, read_only=False)
wb.add_entity_uuids(submission)
wb.add_schemas_worksheet(template_mgr.get_schemas())
return wb.save(file_path)
def test_get_concrete_type_of_worksheet_invalid_format(self):
# given:
schema_template = MagicMock(name='schema_template')
manager = TemplateManager(schema_template,
MagicMock(name='ingest_api'))
# when:
raised_exception = None
try:
manager.get_concrete_type('- does not match format -')
except InvalidTabName as exception:
raised_exception = exception
# then:
self.assertIsNotNone(raised_exception)
self.assertEqual('- does not match format -',
raised_exception.tab_name)
def test_create_row_template_with_default_values(self, determine_strategy):
# given:
schema_template = MagicMock('schema_template')
ingest_api = MagicMock(name='mock_ingest_api')
# and:
domain_entity = "profile/profile_type"
schema_url = "http://schema.sample.com/profile"
schema_template.get_tabs_config = MagicMock()
schema_template.lookup_metadata_schema_name_given_title = MagicMock(
return_value="profile_type")
schema_template.get_latest_schema = MagicMock(return_value=schema_url)
schema = {
"schema": {
"domain_entity": domain_entity,
"url": schema_url
}
}
property_schema = {
"description": "Property description",
"value_type": "string"
}
spec_map = {"profile_type": schema, "profile.name": property_schema}
schema_template.lookup_property_from_template = lambda key: spec_map.get(
key)
# and:
determine_strategy.return_value = FakeConversion('')
# and:
workbook = Workbook()
worksheet = workbook.create_sheet('profile')
worksheet['A4'] = 'profile.name'
ingest_worksheet = IngestWorksheet(worksheet)
# when:
template_manager = TemplateManager(schema_template, ingest_api)
template_manager.get_schema_url = MagicMock(return_value=schema_url)
row_template = template_manager.create_row_template(ingest_worksheet)
# then:
content_defaults = row_template.default_values
self.assertIsNotNone(content_defaults)
self.assertEqual(schema_url, content_defaults.get('describedBy'))
self.assertEqual('profile', content_defaults.get('schema_type'))
def test_create_row_template_with_none_header(self, determine_strategy):
# given:
schema_template = MagicMock('schema_template')
ingest_api = MagicMock(name='ingest_api')
# and:
do_nothing_strategy = FakeConversion('')
determine_strategy.return_value = do_nothing_strategy
# and:
self._mock_schema_lookup(schema_template)
# and:
workbook = Workbook()
worksheet = workbook.create_sheet('sample')
worksheet['A4'] = None
# when:
template_manager = TemplateManager(schema_template, ingest_api)
row_template = template_manager.create_row_template(worksheet)
# then:
self.assertEqual(0, len(row_template.cell_conversions))
def test_get_entity_of_tab(self):
# given
key_label_map = {
'Project': 'project',
'Donor': 'donor',
'Specimen from organism': 'specimen_from_organism'
}
fake_tabs_config = MagicMock(name='tabs_config')
fake_tabs_config.get_key_for_label = lambda key: key_label_map.get(key)
schema_template = MagicMock(name='schema_template')
schema_template.get_tabs_config = MagicMock(return_value=fake_tabs_config)
ingest_api = MagicMock(name='ingest_api')
template_manager = TemplateManager(schema_template, ingest_api)
# when:
entity = template_manager.get_concrete_entity_of_tab('Specimen from organism')
# then:
self.assertEqual('specimen_from_organism', entity)
def test_create_row_template(self, determine_strategy, look_up):
# given:
template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='ingest_api')
# and:
concrete_type = 'user'
template.get_tab_key = MagicMock(return_value=concrete_type)
# and:
spec_map = {
'user': {
'schema': {
'domain_entity': 'main_category/subdomain'
}
}
}
template.lookup = lambda key: spec_map.get(key, None)
# and: set up column spec
name_column_spec = MagicMock(name='name_column_spec')
numbers_column_spec = MagicMock(name='numbers_column_spec')
look_up.side_effect = [name_column_spec, numbers_column_spec]
# and:
name_strategy = MagicMock('name_strategy')
numbers_strategy = MagicMock('numbers_strategy')
determine_strategy.side_effect = [name_strategy, numbers_strategy]
# and: prepare worksheet
header_row_idx = 4
workbook = Workbook()
worksheet = workbook.create_sheet('sample')
worksheet[f'A{header_row_idx}'] = 'user.profile.first_name'
worksheet[f'B{header_row_idx}'] = 'numbers'
ingest_worksheet = IngestWorksheet(worksheet,
header_row_idx=header_row_idx)
# when:
template_manager = TemplateManager(template, ingest_api)
row_template: RowTemplate = template_manager.create_row_template(
ingest_worksheet)
# then:
expected_calls = [
call(template,
'user.profile.first_name',
concrete_type,
context=concrete_type,
order_of_occurrence=1),
call(template,
'numbers',
concrete_type,
context=concrete_type,
order_of_occurrence=1)
]
look_up.assert_has_calls(expected_calls)
determine_strategy.assert_has_calls(
[call(name_column_spec),
call(numbers_column_spec)])
# and:
self.assertIsNotNone(row_template)
self.assertEqual('main_category', row_template.domain_type)
self.assertEqual(concrete_type, row_template.concrete_type)
self.assertEqual(2, len(row_template.cell_conversions))
self.assertTrue(name_strategy in row_template.cell_conversions)
self.assertTrue(numbers_strategy in row_template.cell_conversions)
def do_import(self, worksheet, template: TemplateManager):
row_template = template.create_simple_row_template(worksheet)
records = self._import_using_row_template(template, worksheet,
row_template)
return list(records.values())
def test_create_row_template_for_module_worksheet(self, determine_strategy,
look_up):
# given:
template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='ingest_api')
# TODO define method in SchemaTemplate that returns domain and concrete types #module-tabs
# and:
concrete_type = 'product'
template.get_tab_key = MagicMock(return_value=concrete_type)
# and:
spec_map = {
'product': {
'schema': {
'domain_entity': 'merchandise/product'
}
}
}
template.lookup = lambda key: spec_map.get(key, None)
# and:
template_mgr = TemplateManager(template, ingest_api)
# and:
workbook = create_test_workbook('Product - Reviews')
reviews_worksheet = workbook.get_sheet_by_name('Product - Reviews')
reviews_worksheet['A4'] = 'product.info.id'
reviews_worksheet['B4'] = 'product.reviews.rating'
# and: set up dummy look up results
id_spec = MagicMock(name='id_spec')
rating_spec = MagicMock(name='rating_spec')
look_up.side_effect = [id_spec, rating_spec]
# and: set up strategies
id_strategy = MagicMock(name='id_strategy')
rating_strategy = MagicMock(name='rating_strategy')
determine_strategy.side_effect = {
id_spec: id_strategy,
rating_spec: rating_strategy
}.get
# when:
row_template = template_mgr.create_row_template(
IngestWorksheet(reviews_worksheet))
# then:
expected_calls = [
call(template,
'product.info.id',
concrete_type,
order_of_occurrence=1,
context='product.reviews'),
call(template,
'product.reviews.rating',
concrete_type,
order_of_occurrence=1,
context='product.reviews')
]
look_up.assert_has_calls(expected_calls)
# and:
self.assertIsNotNone(row_template)
self.assertIn(id_strategy, row_template.cell_conversions)
self.assertIn(rating_strategy, row_template.cell_conversions)
def test_create_row_template(self, determine_strategy):
# given:
template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='mock_ingest_api')
# and:
concrete_type = "user"
domain_entity = "main_category/subdomain"
schema_url = "http://schema.sample.com/main_category"
template.get_tabs_config = MagicMock()
template.lookup_metadata_schema_name_given_title = MagicMock(
return_value=concrete_type)
template.get_latest_schema = MagicMock(return_value=schema_url)
schema = {
"schema": {
"domain_entity": domain_entity,
"url": schema_url
}
}
property_one_schema = {
"description": "Property one description",
"value_type": "string"
}
property_two_schema = {
"description": "Property two description",
"value_type": "string"
}
spec_map = {
concrete_type: schema,
"user.profile.first_name": property_one_schema,
"user.numbers": property_two_schema
}
template.lookup_property_from_template = lambda key: spec_map.get(key)
# and:
name_strategy = MagicMock('name_strategy')
numbers_strategy = MagicMock('numbers_strategy')
determine_strategy.side_effect = [name_strategy, numbers_strategy]
# and: prepare worksheet
header_row_idx = 4
workbook = Workbook()
worksheet = workbook.create_sheet('sample')
worksheet[f'A{header_row_idx}'] = 'user.profile.first_name'
worksheet[f'B{header_row_idx}'] = 'user.numbers'
ingest_worksheet = IngestWorksheet(worksheet,
header_row_idx=header_row_idx)
# when:
template_manager = TemplateManager(template, ingest_api)
row_template: RowTemplate = template_manager.create_row_template(
ingest_worksheet)
# then:
self.assertEqual(determine_strategy.call_count, 2)
# and:
self.assertIsNotNone(row_template)
self.assertEqual('main_category', row_template.domain_type)
self.assertEqual(concrete_type, row_template.concrete_type)
self.assertEqual(2, len(row_template.cell_conversions))
self.assertTrue(name_strategy in row_template.cell_conversions)
self.assertTrue(numbers_strategy in row_template.cell_conversions)
def test_create_row_template_for_module_worksheet(self,
determine_strategy):
# given:
template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='mock_ingest_api')
# and:
spec_map = {
'product': {
'schema': {
'domain_entity': 'merchandise/product'
}
}
}
template.lookup_property_from_template = lambda key: spec_map.get(
key, None)
domain_entity = "merchandise/product"
schema_url = "http://schema.sample.com/product"
template.get_tabs_config = MagicMock()
template.lookup_metadata_schema_name_given_title = MagicMock(
return_value="product_type")
template.get_latest_schema = MagicMock(return_value=schema_url)
schema = {
"schema": {
"domain_entity": domain_entity,
"url": schema_url
}
}
property_one_schema = {
"description": "Property one description",
"value_type": "string"
}
property_two_schema = {
"description": "Property two description",
"value_type": "string"
}
spec_map = {
"product_type": schema,
"product.info.id": property_one_schema,
"product.reviews.rating": property_two_schema
}
template.lookup_property_attributes_in_metadata = lambda key: spec_map.get(
key)
# and:
template_mgr = TemplateManager(template, ingest_api)
# and:
workbook = create_test_workbook('Product - Reviews')
reviews_worksheet = workbook['Product - Reviews']
reviews_worksheet['A4'] = 'product.info.id'
reviews_worksheet['B4'] = 'product.reviews.rating'
# and: set up strategies
id_strategy = MagicMock(name='id_strategy')
rating_strategy = MagicMock(name='rating_strategy')
determine_strategy.side_effect = [id_strategy, rating_strategy]
# when:
row_template = template_mgr.create_row_template(
IngestWorksheet(reviews_worksheet))
# and:
self.assertIsNotNone(row_template)
self.assertIn(id_strategy, row_template.cell_conversions)
self.assertIn(rating_strategy, row_template.cell_conversions)
def test_create_row_template(self, determine_strategy, build_raw):
# given:
schema_template = MagicMock(name='schema_template')
ingest_api = MagicMock(name='ingest_api')
# and:
tabs_config = MagicMock('tabs_config')
object_type = 'sample_object'
tabs_config.get_key_for_label = MagicMock(return_value=object_type)
schema_template.get_tabs_config = MagicMock(return_value=tabs_config)
# and: set up column spec
name_column_spec = MagicMock('name_column_spec')
numbers_column_spec = MagicMock('numbers_column_spec')
build_raw.side_effect = [name_column_spec, numbers_column_spec]
# and: set up raw spec
name_raw_spec = MagicMock('name_raw_spec')
name_raw_parent_spec = MagicMock('name_raw_parent_spec')
numbers_raw_spec = MagicMock('numbers_raw_spec')
# TODO move the logic of creating the column spec to SchemaTemplate
# and:
schema = {'schema': {'domain_entity': 'main_category/subdomain'}}
spec_map = {
'user.profile.first_name': name_raw_spec,
'user.profile': name_raw_parent_spec,
'numbers': numbers_raw_spec,
'user': schema
}
schema_template.lookup = lambda key: spec_map.get(key, None)
# and:
name_strategy = MagicMock('name_strategy')
numbers_strategy = MagicMock('numbers_strategy')
determine_strategy.side_effect = [name_strategy, numbers_strategy]
# and: prepare worksheet
workbook = Workbook()
worksheet = workbook.create_sheet('sample')
worksheet['A4'] = 'user.profile.first_name'
worksheet['B4'] = 'numbers'
# when:
template_manager = TemplateManager(schema_template, ingest_api)
row_template: RowTemplate = template_manager.create_row_template(worksheet)
# then:
expected_calls = [
call('user.profile.first_name', 'sample_object', 'main_category', name_raw_spec,
order_of_occurence=1, parent=name_raw_parent_spec),
call('numbers', 'sample_object', None, numbers_raw_spec, order_of_occurence=1, parent=None)
]
build_raw.assert_has_calls(expected_calls)
determine_strategy.assert_has_calls([call(name_column_spec), call(numbers_column_spec)])
# and:
self.assertIsNotNone(row_template)
self.assertEqual(2, len(row_template.cell_conversions))
self.assertTrue(name_strategy in row_template.cell_conversions)
self.assertTrue(numbers_strategy in row_template.cell_conversions)
def do_import(self, worksheet, template: TemplateManager):
row_template = template.create_row_template(worksheet)
self.concrete_entity = template.get_concrete_entity_of_tab(
worksheet.title)
return self._import_using_row_template(template, worksheet,
row_template)