async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddCountAttribute operation and a condition using the object model"""
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an AddCountAttribute operation
add_count_attr_op = corpus.make_object(CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF)
add_count_attr_op.count_attribute = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testCount')
add_count_attr_op.count_attribute.data_type = corpus.make_ref(CdmObjectType.DATA_TYPE_REF, 'integer', True)
projection.operations.append(add_count_attr_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddCountAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# Count attribute: "testCount"
self.assertEqual(5, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual('testCount', resolved_entity_with_reference_only.attributes[4].name)
self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4].applied_traits.item('is.linkedEntity.array.count'))
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddCountAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# No Count attribute added, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id', resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name', resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value', resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddAttributeGroup operation and a condition using the object model"""
test_name = 'test_conditional_proj_using_object_model'
corpus = ProjectionTestUtils.get_corpus(test_name, self.tests_subpath)
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity.
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'.
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'structured==true'
# Create an AddAttributeGroup operation
add_att_group_op = corpus.make_object(CdmObjectType.OPERATION_ADD_ATTRIBUTE_GROUP_DEF)
add_att_group_op.attribute_group_name = 'PersonAttributeGroup'
projection.operations.append(add_att_group_op)
# Create an entity reference to hold this projection.
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity.
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') # type: CdmEntityAttributeDefinition
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive.
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddAttributeGroup operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition not met, keep attributes in flat list
self.assertEqual(4, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddAttributeGroup operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition met, put all attributes in an attribute group
att_group_definition = self.validate_attribute_group(resolved_entity_with_structured.attributes, 'PersonAttributeGroup')
self.assertEqual(4, len(att_group_definition.members))
self.assertEqual('id', att_group_definition.members[0].name)
self.assertEqual('name', att_group_definition.members[1].name)
self.assertEqual('value', att_group_definition.members[2].name)
self.assertEqual('date', att_group_definition.members[3].name)
async def test_setting_traits_for_resolution_guidance_attributes(self):
'''
Test that "is.linkedEntity.name" and "is.linkedEntity.identifier" traits are set when "selectedTypeAttribute" and "foreignKeyAttribute"
are present in the entity's resolution guidance.
'''
corpus = TestHelper.get_local_corpus(
self.tests_subpath,
'test_setting_traits_for_resolution_guidance_attributes'
) # type: CdmCorpusDefinition
entity = await corpus.fetch_object_async(
'local:/Customer.cdm.json/Customer') # type: CdmEntityDefinition
# Resolve with default directives to get "is.linkedEntity.name" trait.
res_opt = ResolveOptions(wrt_doc=entity.in_document)
resolved_entity = await entity.create_resolved_entity_async(
'resolved', res_opt)
self.assertEqual(
'is.linkedEntity.name',
resolved_entity.attributes[1].applied_traits[6].named_reference)
# Resolve with referenceOnly directives to get "is.linkedEntity.identifier" trait.
res_opt = ResolveOptions(wrt_doc=entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
resolved_entity = await entity.create_resolved_entity_async(
'resolved2', res_opt)
self.assertEqual(
'is.linkedEntity.identifier',
resolved_entity.attributes[0].applied_traits[7].named_reference)
async def seek_entities(f: 'CdmManifestDefinition'):
if f.entities is not None:
spew.spew_line(f.folder_path)
for entity in f.entities:
ent = entity
current_file = f
while isinstance(ent, CdmReferencedEntityDeclarationDefinition):
corpus_path = corpus.storage.create_absolute_corpus_path(ent.entity_path, current_file)
ent = await corpus.fetch_object_async(corpus_path)
current_file = ent
corpus_path = corpus.storage.create_absolute_corpus_path(ent.entity_path, current_file)
res_opt = ResolveOptions()
res_opt.imports_load_strategy = ImportsLoadStrategy.LOAD
new_ent = await corpus.fetch_object_async(corpus_path, res_opt=res_opt)
res_opt.wrt_doc = new_ent.in_document
res_opt.directives = directives
res_ent = ResolvedEntity(res_opt, new_ent)
res_ent.spew(res_opt, spew, ' ', True)
if f.sub_manifests:
for sub_manifest in f.sub_manifests:
corpus_path = corpus.storage.create_absolute_corpus_path(sub_manifest.definition, f)
await seek_entities(await corpus.fetch_object_async(corpus_path))
def _copy_resolve_options(res_opt: 'ResolveOptions') -> 'ResolveOptions':
from cdm.utilities import ResolveOptions # pylint: disable=redefined-outer-name
res_opt_copy = ResolveOptions()
res_opt_copy.wrt_doc = res_opt.wrt_doc
res_opt_copy._relationship_depth = res_opt._relationship_depth
res_opt_copy._localize_references_for = res_opt._localize_references_for
res_opt_copy._indexing_doc = res_opt._indexing_doc
if res_opt.directives:
res_opt_copy.directives = res_opt.directives.copy()
return res_opt_copy
async def test_resolved_attribute_limit(self):
expected_log_codes = { CdmLogCode.ERR_REL_MAX_RESOLVED_ATTR_REACHED }
corpus = TestHelper.get_local_corpus(self.tests_sub_path, 'test_resolved_attribute_limit', expected_codes=expected_log_codes) # type: CdmCorpusDefinition
main_entity = await corpus.fetch_object_async('local:/mainEntity.cdm.json/mainEntity') # type: CdmEntityDefinition
res_opt = ResolveOptions(wrt_doc=main_entity.in_document)
# if attribute limit is reached, entity should be None
res_opt._resolved_attribute_limit = 4
resEnt = await main_entity.create_resolved_entity_async('{}_zeroAtts'.format(main_entity.entity_name), res_opt) # type: CdmEntityDefinition
self.assertIsNone(resEnt)
# when the attribute limit is set to null, there should not be a limit on the possible number of attributes
res_opt._resolved_attribute_limit = None
ras = main_entity._fetch_resolved_attributes(res_opt) # type: ResolvedAttributeSet
resEnt = await main_entity.create_resolved_entity_async('{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
self.assertEqual(5, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
# using the default limit number
res_opt = ResolveOptions(wrt_doc=main_entity.in_document)
ras = main_entity._fetch_resolved_attributes(res_opt)
resEnt = await main_entity.create_resolved_entity_async('{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
self.assertEqual(5, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'})
ras = main_entity._fetch_resolved_attributes(res_opt)
resEnt = await main_entity.create_resolved_entity_async('{}_normalized_structured'.format(main_entity.entity_name), res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
# the attribute count is different because one attribute is a group that contains two different attributes
self.assertEqual(4, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# again there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
async def test_missing_condition_in_json(self):
"""
Test case scenario for Bug #25 from the projections internal bug bash
Reference Link: https:#commondatamodel.visualstudio.com/CDM/_workitems/edit/25
"""
test_name = 'test_missing_condition_in_json'
corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
def callback(level, message):
self.fail(message)
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
manifest = await corpus.fetch_object_async('default.manifest.cdm.json')
entity_name = 'SalesNestedFK'
entity = await corpus.fetch_object_async('local:/{}.cdm.json/{}'.format(entity_name, entity_name), manifest)
self.assertIsNotNone(entity)
res_opt = ResolveOptions(entity.in_document)
# where, res_opts_combinations[1] == 'referenceOnly'
res_opt.directives = AttributeResolutionDirectiveSet(self.res_opts_combinations[1])
resolved_folder = corpus.storage.fetch_root_folder('output')
was_info_message_received = {}
was_info_message_received['infoMessagedReceived'] = False
def callback_2(level, message):
if 'CdmProjection | Optional expression missing. Implicit expression will automatically apply. | _construct_projection_context' in message:
was_info_message_received['infoMessagedReceived'] = True
corpus.set_event_callback(callback_2, CdmStatusLevel.INFO)
resolved_entity = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity_name), res_opt, resolved_folder)
self.assertIsNotNone(resolved_entity)
self.assertTrue(was_info_message_received['infoMessagedReceived'])
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an ExcludeAttributes operation and a condition using the object model"""
corpus = TestHelper.get_local_corpus(
self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount(
'local',
LocalAdapter(
TestHelper.get_actual_output_folder_path(
self.tests_subpath,
'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an ExcludeAttributes operation
exclude_attrs_op = corpus.make_object(
CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF)
exclude_attrs_op.exclude_attributes.append('id')
exclude_attrs_op.exclude_attributes.append('date')
projection.operations.append(exclude_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(
set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: ['id', 'date']
self.assertEqual(2,
len(resolved_entity_with_reference_only.attributes))
self.assertEqual(
'name', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual(
'value', resolved_entity_with_reference_only.attributes[1].name)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(
set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: none, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id',
resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name',
resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value',
resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date',
resolved_entity_with_structured.attributes[3].name)
async def logical_manipulation_using_projections():
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
A single logical definition can be resolved into multiple physical layouts. The directives are used to instruct the ObjectModel about how it should to
resolve the logical definition provided. To achieve this, we define projections that run conditionally, depending on the directives provided when
calling create_resolved_entity_async.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
# Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
corpus = CdmCorpusDefinition()
print('Configure storage adapters')
# Configure storage adapters to point at the target local manifest location and at the fake public standards
path_from_exe_to_example_root = '../../'
corpus.storage.mount(
'local',
LocalAdapter(path_from_exe_to_example_root +
'8-logical-manipulation-using-projections/sample-data'))
corpus.storage.default_namespace = 'local' # local is our default. so any paths that start out navigating without a device tag will assume local
# Fake cdm, normaly use the CDM Standards adapter
# Mount it as the 'cdm' device, not the default so must use 'cdm:/folder' to get there
corpus.storage.mount(
'cdm',
LocalAdapter(path_from_exe_to_example_root +
'example-public-standards'))
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async(
'local:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('Address.cdm.json')
entity = logical_doc.definitions.append(
'Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append(
'name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append(
'age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append(
'email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print('Get \'resolved\' folder where the resolved entities will be saved.')
resolved_folder = await corpus.fetch_object_async(
'local:/resolved/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
res_normalized_entity = await entity.create_resolved_entity_async(
f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(
f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
res_reference_only_entity = await entity.create_resolved_entity_async(
f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(
f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
res_structured_entity = await entity.create_resolved_entity_async(
f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(
f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({})
res_array_entity = await entity.create_resolved_entity_async(
f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(
f'{res_array_entity.entity_name}.cdm.json')
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddArtifactAttribute operation and a condition using the object model"""
test_name = 'test_conditional_proj_using_object_model'
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath,
test_name)
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity.
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'.
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'structured==true'
# Create an AddArtifactAttribute operation
add_artifact_attribute_op = corpus.make_object(
CdmObjectType.OPERATION_ADD_ARTIFACT_ATTRIBUTE_DEF)
add_artifact_attribute_op.new_attribute = corpus.make_object(
CdmObjectType.TYPE_ATTRIBUTE_DEF, "newName")
add_artifact_attribute_op.new_attribute.data_type = corpus.make_ref(
CdmObjectType.DATA_TYPE_REF, "string", True)
projection.operations.append(add_artifact_attribute_op)
# Create an entity reference to hold this projection.
projection_entity_ref = corpus.make_object(
CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity.
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF,
'TestEntityAttribute') # type: CdmEntityAttributeDefinition
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive.
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition not met, keep attributes in flat list
self.assertEqual(4,
len(resolved_entity_with_reference_only.attributes))
self.assertEqual(
'id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual(
'name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual(
'value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual(
'date', resolved_entity_with_reference_only.attributes[3].name)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition met, keep attributes in flat list and add the new attribute "newName" all attributes at the end
self.assertEqual(5, len(resolved_entity_with_structured.attributes))
self.assertEqual('id',
resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name',
resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value',
resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date',
resolved_entity_with_structured.attributes[3].name)
self.assertEqual('newName',
resolved_entity_with_structured.attributes[4].name)
async def create_resolved_manifest_async(
self, new_manifest_name: str, new_entity_document_name_format: str
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
if self.entities is None:
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
self.ctx.logger.error(
'New folder for manifest not found {}'.format(
new_folder_path), 'create_resolved_manifest_async')
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
self.ctx.logger.debug(
'resolving manifest {}'.format(source_manifest_path),
'create_resolved_manifest_async')
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name)
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
# mapping from entity path to resolved entity path for translating relationhsip paths
res_ent_map = {} # type: Dict[str, str]
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
self.ctx.logger.error('Unable to get entity from reference')
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
self.ctx.logger.error(
'New folder not found {}'.format(new_document_path))
return None
# next create the resolved entity.
res_opt = ResolveOptions()
res_opt.wrt_doc = ent_def.in_document
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
self.ctx.logger.debug(
' resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
# absolute path is needed for generating relationships
absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path(
result.entity_path, resolved_manifest)
res_ent_map[self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.at_corpus_path,
ent_def.in_document)] = absolute_ent_path
self.ctx.logger.debug(' calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus._calculate_entity_graph_async(
resolved_manifest, res_ent_map)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddArtifactAttribute operation and a condition using the object model"""
test_name = 'test_conditional_proj_using_object_model'
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath,
test_name)
local_root = corpus.storage.fetch_root_folder('local')
corpus.storage.mount('traitGroup',
LocalAdapter(self.trait_group_file_path))
# Create an entity.
entity = ProjectionTestUtils.create_entity(corpus, local_root)
entity.in_document.imports.append('traitGroup:/TraitGroup.cdm.json')
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'.
projection = ProjectionTestUtils.create_projection(
corpus, local_root) # type: CdmProjection
projection.condition = 'structured==true'
projection.run_sequentially = True
# Create an AlterTraits operation
alter_traits_op_1 = corpus.make_object(
CdmObjectType.OPERATION_ALTER_TRAITS_DEF
) # type: CdmOperationAlterTraits
alter_traits_op_1.traits_to_add = CdmCollection(
corpus.ctx, alter_traits_op_1, CdmObjectType.TRAIT_REF)
alter_traits_op_1.traits_to_add.append(
corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG100", True))
alter_traits_op_1.traits_to_add.append(
corpus.make_ref(CdmObjectType.TRAIT_GROUP_REF, "JobTitleBase",
True))
alter_traits_op_1.traits_to_remove = CdmCollection(
corpus.ctx, alter_traits_op_1, CdmObjectType.TRAIT_REF)
alter_traits_op_1.traits_to_remove.append(
corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG300", True))
projection.operations.append(alter_traits_op_1)
alter_traits_op_2 = corpus.make_object(
CdmObjectType.OPERATION_ALTER_TRAITS_DEF
) # type: CdmOperationAlterTraits
trait_g4 = corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG4",
True) # type: CdmTraitReference
trait_g4.arguments.append('precision', '5')
trait_g4.arguments.append('scale', '15')
alter_traits_op_2.traits_to_add = CdmCollection(
corpus.ctx, alter_traits_op_2, CdmObjectType.TRAIT_REF)
alter_traits_op_2.traits_to_add.append(trait_g4)
alter_traits_op_2.apply_to = []
alter_traits_op_2.apply_to.append('name')
projection.operations.append(alter_traits_op_2)
# Create an entity reference to hold this projection.
projection_entity_ref = corpus.make_object(
CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity.
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF,
'TestEntityAttribute') # type: CdmEntityAttributeDefinition
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive.
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition not met, no traits are added
self.assertEqual(4,
len(resolved_entity_with_reference_only.attributes))
self.validate_trait(resolved_entity_with_reference_only.attributes[0],
'id', False, True)
self.validate_trait(resolved_entity_with_reference_only.attributes[1],
'name', False, True)
self.validate_trait(resolved_entity_with_reference_only.attributes[2],
'value', False, True)
self.validate_trait(resolved_entity_with_reference_only.attributes[3],
'date', False, True)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition met, new traits are added
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.validate_trait(resolved_entity_with_structured.attributes[0],
'id')
self.validate_trait(resolved_entity_with_structured.attributes[1],
'name', True)
self.validate_trait(resolved_entity_with_structured.attributes[2],
'value')
self.validate_trait(resolved_entity_with_structured.attributes[3],
'date')
async def run_test_with_values(
self, test_name: str, source_entity_name: str,
expected_context_default: 'AttributeContextExpectedValue',
expected_context_normalized: 'AttributeContextExpectedValue',
expected_context_reference_only: 'AttributeContextExpectedValue',
expected_context_structured: 'AttributeContextExpectedValue',
expected_context_normalized_structured: 'AttributeContextExpectedValue',
expected_context_reference_only_normalized:
'AttributeContextExpectedValue',
expected_context_reference_only_structured:
'AttributeContextExpectedValue',
expected_context_reference_only_normalized_structured:
'AttributeContextExpectedValue',
expected_default: 'List[AttributeExpectedValue]',
expected_normalized: 'List[AttributeExpectedValue]',
expected_reference_only: 'List[AttributeExpectedValue]',
expected_structured: 'List[AttributeExpectedValue]',
expected_normalized_structured: 'List[AttributeExpectedValue]',
expected_reference_only_normalized: 'List[AttributeExpectedValue]',
expected_reference_only_structured: 'List[AttributeExpectedValue]',
expected_reference_only_normalized_structured:
'List[AttributeExpectedValue]'
) -> None:
"""This method runs the tests with a set expected attributes & attribute context values and validated the actual result."""
try:
test_input_path = TestHelper.get_input_folder_path(
self.tests_subpath, test_name)
test_actual_path = TestHelper.get_actual_output_folder_path(
self.tests_subpath, test_name)
test_expected_path = TestHelper.get_expected_output_folder_path(
self.tests_subpath, test_name)
corpus_path = test_input_path[0:(len(test_input_path) -
len('/Input'))]
test_actual_path = os.path.abspath(test_actual_path)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(corpus_path))
corpus.storage.mount('cdm', LocalAdapter(self.schema_docs_path))
corpus.storage.default_namespace = 'local'
out_folder_path = corpus.storage.adapter_path_to_corpus_path(
test_actual_path) + '/' # interesting bug
out_folder = await corpus.fetch_object_async(
out_folder_path) # type: CdmFolderDefinition
src_entity_def = await corpus.fetch_object_async(
'local:/Input/{}.cdm.json/{}'.format(source_entity_name,
source_entity_name)
) # type: CdmEntityDefinition
self.assertTrue(src_entity_def is not None)
res_opt = ResolveOptions(wrt_doc=src_entity_def.in_document)
resolved_entity_def = None # type: CdmEntityDefinition
output_entity_name = ''
output_entity_file_name = ''
entity_file_name = ''
if expected_context_default and expected_default:
entity_file_name = 'd'
res_opt.directives = AttributeResolutionDirectiveSet(set())
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_normalized and expected_normalized:
entity_file_name = 'n'
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only and expected_reference_only:
entity_file_name = 'ro'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_structured and expected_structured:
entity_file_name = 's'
res_opt.directives = AttributeResolutionDirectiveSet(
{'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_normalized_structured and expected_normalized_structured:
entity_file_name = 'n_s'
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only_normalized and expected_reference_only_normalized:
entity_file_name = 'ro_n'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'normalized'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only_structured and expected_reference_only_structured:
entity_file_name = 'ro_s'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only_normalized_structured and expected_reference_only_normalized_structured:
entity_file_name = 'ro_n_s'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'normalized', 'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
except Exception as e:
self.fail(e)
async def run_test_with_values(
self, test_name: str, source_entity_name: str,
expected_context_default: 'AttributeContextExpectedValue',
expected_context_normalized: 'AttributeContextExpectedValue',
expected_context_reference_only: 'AttributeContextExpectedValue',
expected_context_structured: 'AttributeContextExpectedValue',
expected_context_normalized_structured: 'AttributeContextExpectedValue',
expected_context_reference_only_normalized:
'AttributeContextExpectedValue',
expected_context_reference_only_structured:
'AttributeContextExpectedValue',
expected_context_reference_only_normalized_structured:
'AttributeContextExpectedValue',
expected_default: 'List[AttributeExpectedValue]',
expected_normalized: 'List[AttributeExpectedValue]',
expected_reference_only: 'List[AttributeExpectedValue]',
expected_structured: 'List[AttributeExpectedValue]',
expected_normalized_structured: 'List[AttributeExpectedValue]',
expected_reference_only_normalized: 'List[AttributeExpectedValue]',
expected_reference_only_structured: 'List[AttributeExpectedValue]',
expected_reference_only_normalized_structured:
'List[AttributeExpectedValue]'
) -> None:
"""This method runs the tests with a set expected attributes & attribute context values and validated the actual result."""
try:
test_input_path = TestHelper.get_input_folder_path(
self.tests_subpath, test_name)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('localInput', LocalAdapter(test_input_path))
corpus.storage.mount('cdm', LocalAdapter(self.schema_docs_path))
corpus.storage.default_namespace = 'localInput'
src_entity_def = await corpus.fetch_object_async(
'localInput:/{}.cdm.json/{}'.format(source_entity_name,
source_entity_name))
self.assertTrue(src_entity_def is not None)
res_opt = ResolveOptions(wrt_doc=src_entity_def.in_document)
resolved_entity_def = None # type: CdmEntityDefinition
output_entity_name = ''
output_entity_file_name = ''
entity_file_name = ''
if expected_context_default and expected_default:
entity_file_name = 'default'
res_opt.directives = AttributeResolutionDirectiveSet(set())
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(expected_context_default,
expected_default,
resolved_entity_def)
if expected_context_normalized and expected_normalized:
entity_file_name = 'normalized'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'normalized'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(expected_context_normalized,
expected_normalized,
resolved_entity_def)
if expected_context_reference_only and expected_reference_only:
entity_file_name = 'referenceOnly'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'referenceOnly'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(
expected_context_reference_only, expected_reference_only,
resolved_entity_def)
if expected_context_structured and expected_structured:
entity_file_name = 'structured'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'structured'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(expected_context_structured,
expected_structured,
resolved_entity_def)
if expected_context_normalized_structured and expected_normalized_structured:
entity_file_name = 'normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'normalized', 'structured'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(
expected_context_normalized_structured,
expected_normalized_structured, resolved_entity_def)
if expected_context_reference_only_normalized and expected_reference_only_normalized:
entity_file_name = 'referenceOnly_normalized'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'referenceOnly', 'normalized'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(
expected_context_reference_only_normalized,
expected_reference_only_normalized, resolved_entity_def)
if expected_context_reference_only_structured and expected_reference_only_structured:
entity_file_name = 'referenceOnly_structured'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'referenceOnly', 'structured'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(
expected_context_reference_only_structured,
expected_reference_only_structured, resolved_entity_def)
if expected_context_reference_only_normalized_structured and expected_reference_only_normalized_structured:
entity_file_name = 'referenceOnly_normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet(
set({'referenceOnly', 'normalized', 'structured'}))
output_entity_name = '{}_Resolved_{}'.format(
source_entity_name, entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt)
self.validate_output_with_values(
expected_context_reference_only_normalized_structured,
expected_reference_only_normalized_structured,
resolved_entity_def)
except Exception as e:
self.fail(e)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an ArrayExpansion operation and a condition using the object model"""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an ArrayExpansion operation
array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF)
array_expansion_op.start_ordinal = 1
array_expansion_op.end_ordinal = 2
projection.operations.append(array_expansion_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity
projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF)
projection2.source = projection_entity_ref
# Create a RenameAttributes operation
rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF)
rename_attrs_op.rename_format = '{m}{o}'
projection2.operations.append(rename_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref2.explicit_reference = projection2
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref2
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(8, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id1', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name1', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value1', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date1', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual('id2', resolved_entity_with_reference_only.attributes[4].name)
self.assertEqual('name2', resolved_entity_with_reference_only.attributes[5].name)
self.assertEqual('value2', resolved_entity_with_reference_only.attributes[6].name)
self.assertEqual('date2', resolved_entity_with_reference_only.attributes[7].name)
# Now resolve the entity with the default directives
res_opt.directives = AttributeResolutionDirectiveSet(set([]))
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id', resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name', resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value', resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def run_test(self, test_name: str, source_entity_name: str) -> None:
test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name)
test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name)
test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('localInput', LocalAdapter(test_input_path))
corpus.storage.mount('localExpectedOutput', LocalAdapter(test_expected_output_path))
corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path))
corpus.storage.mount('cdm', LocalAdapter(TestHelper.get_schema_docs_root()))
corpus.storage.default_namespace = 'localInput'
src_entity_def = await corpus.fetch_object_async('localInput:/{0}.cdm.json/{0}'.format(source_entity_name)) # type: CdmEntityDefinition
self.assertIsNotNone(src_entity_def)
res_opt = ResolveOptions(src_entity_def.in_document, directives=AttributeResolutionDirectiveSet(set()))
actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition
resolved_entity_def = None
output_entity_file_name = ''
entity_file_name = ''
entity_file_name = 'default'
res_opt.directives = AttributeResolutionDirectiveSet(set())
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'normalized'
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'structured'
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_normalized'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
async def logical_manipulation_using_projections(
self, corpus: CdmCorpusDefinition):
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async(
'output:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('local:/Address.cdm.json')
entity = logical_doc.definitions.append(
'Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append(
'name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append(
'age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append(
'email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print(
'Get \'resolved\' folder where the resolved entities will be saved.'
)
resolved_folder = await corpus.fetch_object_async(
'output:/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
res_normalized_entity = await entity.create_resolved_entity_async(
f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(
f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
res_reference_only_entity = await entity.create_resolved_entity_async(
f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(
f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
res_structured_entity = await entity.create_resolved_entity_async(
f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(
f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({})
res_array_entity = await entity.create_resolved_entity_async(
f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(
f'{res_array_entity.entity_name}.cdm.json')
