async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddCountAttribute operation and a condition using the object model"""
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an AddCountAttribute operation
add_count_attr_op = corpus.make_object(CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF)
add_count_attr_op.count_attribute = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testCount')
add_count_attr_op.count_attribute.data_type = corpus.make_ref(CdmObjectType.DATA_TYPE_REF, 'integer', True)
projection.operations.append(add_count_attr_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddCountAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# Count attribute: "testCount"
self.assertEqual(5, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual('testCount', resolved_entity_with_reference_only.attributes[4].name)
self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4].applied_traits.item('is.linkedEntity.array.count'))
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddCountAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# No Count attribute added, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id', resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name', resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value', resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddAttributeGroup operation and a condition using the object model"""
test_name = 'test_conditional_proj_using_object_model'
corpus = ProjectionTestUtils.get_corpus(test_name, self.tests_subpath)
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity.
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'.
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'structured==true'
# Create an AddAttributeGroup operation
add_att_group_op = corpus.make_object(CdmObjectType.OPERATION_ADD_ATTRIBUTE_GROUP_DEF)
add_att_group_op.attribute_group_name = 'PersonAttributeGroup'
projection.operations.append(add_att_group_op)
# Create an entity reference to hold this projection.
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity.
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') # type: CdmEntityAttributeDefinition
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive.
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddAttributeGroup operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition not met, keep attributes in flat list
self.assertEqual(4, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddAttributeGroup operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition met, put all attributes in an attribute group
att_group_definition = self.validate_attribute_group(resolved_entity_with_structured.attributes, 'PersonAttributeGroup')
self.assertEqual(4, len(att_group_definition.members))
self.assertEqual('id', att_group_definition.members[0].name)
self.assertEqual('name', att_group_definition.members[1].name)
self.assertEqual('value', att_group_definition.members[2].name)
self.assertEqual('date', att_group_definition.members[3].name)
async def test_setting_traits_for_resolution_guidance_attributes(self):
'''
Test that "is.linkedEntity.name" and "is.linkedEntity.identifier" traits are set when "selectedTypeAttribute" and "foreignKeyAttribute"
are present in the entity's resolution guidance.
'''
corpus = TestHelper.get_local_corpus(
self.tests_subpath,
'test_setting_traits_for_resolution_guidance_attributes'
) # type: CdmCorpusDefinition
entity = await corpus.fetch_object_async(
'local:/Customer.cdm.json/Customer') # type: CdmEntityDefinition
# Resolve with default directives to get "is.linkedEntity.name" trait.
res_opt = ResolveOptions(wrt_doc=entity.in_document)
resolved_entity = await entity.create_resolved_entity_async(
'resolved', res_opt)
self.assertEqual(
'is.linkedEntity.name',
resolved_entity.attributes[1].applied_traits[6].named_reference)
# Resolve with referenceOnly directives to get "is.linkedEntity.identifier" trait.
res_opt = ResolveOptions(wrt_doc=entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
resolved_entity = await entity.create_resolved_entity_async(
'resolved2', res_opt)
self.assertEqual(
'is.linkedEntity.identifier',
resolved_entity.attributes[0].applied_traits[7].named_reference)
async def get_resolved_entity(
corpus: 'CdmCorpusDefinition',
input_entity: 'CdmEntityDefinition',
resolution_options: List[str],
add_res_opt_to_name: Optional[bool] = False
) -> 'CdmEntityDefinition':
"""Resolves an entity"""
ro_hash_set = set()
for i in range(len(resolution_options)):
ro_hash_set.add(resolution_options[i])
resolved_entity_name = ''
if add_res_opt_to_name:
file_name_suffix = ProjectionTestUtils.get_resolution_option_name_suffix(
resolution_options)
resolved_entity_name = 'Resolved_{}{}'.format(
input_entity.entity_name, file_name_suffix)
else:
resolved_entity_name = 'Resolved_{}'.format(
input_entity.entity_name)
ro = ResolveOptions(
input_entity.in_document,
directives=AttributeResolutionDirectiveSet(ro_hash_set))
resolved_folder = corpus.storage.fetch_root_folder('output')
resolved_entity = await input_entity.create_resolved_entity_async(
resolved_entity_name, ro, resolved_folder)
return resolved_entity
async def test_resolve_test_corpus(self):
self.assertTrue(os.path.exists(TestHelper.get_schema_docs_root()))
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(TestHelper.get_schema_docs_root()))
manifest = await corpus.fetch_object_async(TestHelper.cdm_standards_schema_path) # type: CdmManifestDefinition
directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'})
all_resolved = await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
self.assertNotEqual(all_resolved, '')
def fetch_resolved_entity_references(
self, res_opt: 'ResolveOptions') -> 'ResolvedEntityReferenceSet':
# this whole resolved entity ref goo will go away when resolved documents are done.
# for now, it breaks if structured att sets get made.
from cdm.resolvedmodel import ResolvedEntityReferenceSet
from cdm.utilities import AttributeResolutionDirectiveSet
from .cdm_object import CdmObject
was_previously_resolving = self.ctx.corpus._is_currently_resolving
self.ctx.corpus._is_currently_resolving = True
if not res_opt:
res_opt = ResolveOptions(
self, self.ctx.corpus.default_resolution_directives)
res_opt = CdmObject._copy_resolve_options(res_opt)
res_opt.directives = AttributeResolutionDirectiveSet(
set(['normalized', 'referenceOnly']))
ctx = self.ctx
ent_ref_set_cache = ctx.fetch_cache(self, res_opt, 'entRefSet')
if not ent_ref_set_cache:
ent_ref_set_cache = ResolvedEntityReferenceSet(res_opt)
if not self._resolving_entity_references:
self._resolving_entity_references = True
# get from any base class and then 'fix' those to point here instead.
ext_ref = self.extends_entity
if ext_ref:
ext_def = cast('CdmEntityDefinition',
ext_ref.fetch_object_definition(res_opt))
if ext_def:
inherited = ext_def.fetch_resolved_entity_references(
res_opt)
if inherited:
for res in inherited.rer_set:
res = res.copy()
res.referencing.entity = self
ent_ref_set_cache.rer_set.append(res)
if self.attributes:
for attribute in self.attributes:
# if any refs come back from attributes, they don't know who we are, so they don't set the entity
sub = attribute.fetch_resolved_entity_references(
res_opt)
if sub:
for res in sub.rer_set:
res.referencing.entity = self
ent_ref_set_cache.add(sub)
ctx.update_cache(self, res_opt, 'entRefSet', ent_ref_set_cache)
self._resolving_entity_references = False
self.ctx.corpus._is_currently_resolving = was_previously_resolving
return ent_ref_set_cache
async def test_resolve_test_corpus(self):
# only using cdm namespace, which is set to schema docs
corpus = TestHelper.get_local_corpus('', '')
manifest = await corpus.fetch_object_async(
'cdm:/standards.manifest.cdm.json') # type: CdmManifestDefinition
directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'normalized'})
all_resolved = await self.list_all_resolved(corpus, directives,
manifest,
StringSpewCatcher())
self.assertNotEqual(all_resolved, '')
async def get_resolved_entity(corpus: 'CdmCorpusDefinition', input_entity: 'CdmEntityDefinition', directives: List[str]) -> 'CdmEntityDefinition':
"""Resolves an entity"""
ro_hash_set = set(directives)
resolved_entity_name = 'Resolved_{}'.format(input_entity.entity_name)
res_opt = ResolveOptions(input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set))
resolved_folder = corpus.storage.fetch_root_folder('output')
resolved_entity = await input_entity.create_resolved_entity_async(resolved_entity_name, res_opt, resolved_folder) # type: CdmEntityDefinition
return resolved_entity
async def test_resolve_symbol_reference(self):
"""Tests if a symbol imported with a moniker can be found as the last resource.
When resolving symbolEntity with respect to wrtEntity, the symbol fromEntity should be found correctly."""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_resolve_symbol_reference')
def callback(status_level: CdmStatusLevel, message: str):
self.fail(message)
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
wrt_entity = await corpus.fetch_object_async('local:/wrtEntity.cdm.json/wrtEntity') # type: CdmEntityDefinition
res_opt = ResolveOptions(wrt_entity, AttributeResolutionDirectiveSet())
await wrt_entity.create_resolved_entity_async('NewEntity', res_opt)
async def resolve_environment(self, test_name: str, environment: str) -> str:
test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
print('reading source files')
manifest = await corpus.fetch_object_async('local:/{}.manifest.cdm.json'.format(environment))
directives = AttributeResolutionDirectiveSet(set(['normalized', 'referenceOnly']))
return await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
async def test_resolved_attribute_limit(self):
expected_log_codes = { CdmLogCode.ERR_REL_MAX_RESOLVED_ATTR_REACHED }
corpus = TestHelper.get_local_corpus(self.tests_sub_path, 'test_resolved_attribute_limit', expected_codes=expected_log_codes) # type: CdmCorpusDefinition
main_entity = await corpus.fetch_object_async('local:/mainEntity.cdm.json/mainEntity') # type: CdmEntityDefinition
res_opt = ResolveOptions(wrt_doc=main_entity.in_document)
# if attribute limit is reached, entity should be None
res_opt._resolved_attribute_limit = 4
resEnt = await main_entity.create_resolved_entity_async('{}_zeroAtts'.format(main_entity.entity_name), res_opt) # type: CdmEntityDefinition
self.assertIsNone(resEnt)
# when the attribute limit is set to null, there should not be a limit on the possible number of attributes
res_opt._resolved_attribute_limit = None
ras = main_entity._fetch_resolved_attributes(res_opt) # type: ResolvedAttributeSet
resEnt = await main_entity.create_resolved_entity_async('{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
self.assertEqual(5, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
# using the default limit number
res_opt = ResolveOptions(wrt_doc=main_entity.in_document)
ras = main_entity._fetch_resolved_attributes(res_opt)
resEnt = await main_entity.create_resolved_entity_async('{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
self.assertEqual(5, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'})
ras = main_entity._fetch_resolved_attributes(res_opt)
resEnt = await main_entity.create_resolved_entity_async('{}_normalized_structured'.format(main_entity.entity_name), res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
# the attribute count is different because one attribute is a group that contains two different attributes
self.assertEqual(4, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# again there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
async def resolve_environment(self,
environment: str,
storage_adapter_root=TEST_ADAPTER_ROOT):
corpus = CdmCorpusDefinition()
adapter = LocalAdapter(root=storage_adapter_root)
corpus.storage.mount('local', adapter)
print('reading source files')
manifest = await corpus.fetch_object_async(
'local:/{}.manifest.cdm.json'.format(environment))
directives = AttributeResolutionDirectiveSet(
set(['normalized', 'referenceOnly']))
return await self.list_all_resolved(corpus, directives, manifest,
StringSpewCatcher())
async def resolve_environment(tests_sub_path: str, test_name: str,
manifest_name: str) -> str:
"""
Resolve the entities in the given manifest.
:param tests_sub_path: Tests sub-folder name
:param test_name: The name of the test. It is used to decide the path of input / output files.
:param manifest_name: The name of the manifest to be used.
:return:
"""
corpus = TestHelper.get_local_corpus(tests_sub_path, test_name)
manifest = await corpus.fetch_object_async(
'local:/{}.manifest.cdm.json'.format(manifest_name))
directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
return await list_all_resolved(corpus, directives, manifest,
StringSpewCatcher())
async def _get_resolved_entity(
corpus: 'CdmCorpusDefinition', input_entity: 'CdmEntityDefinition',
resolution_options: List[str]) -> 'CdmEntityDefinition':
"""A function to resolve an entity"""
ro_hash_set = set()
for i in range(len(resolution_options)):
ro_hash_set.add(resolution_options[i])
resolved_entity_name = 'Resolved_{}'.format(input_entity.entity_name)
ro = ResolveOptions(
input_entity.in_document,
directives=AttributeResolutionDirectiveSet(ro_hash_set))
resolved_folder = corpus.storage.fetch_root_folder('output')
resolved_entity = await input_entity.create_resolved_entity_async(
resolved_entity_name, ro, resolved_folder)
return resolved_entity
async def _save_resolved(
self,
corpus: 'CdmCorpusDefinition',
manifest: 'CdmManifestDefinition',
test_name: str,
input_entity: 'CdmEntityDefinition',
resolution_options: List[str]
) -> 'CdmEntityDefinition':
ro_hash_set = set()
for i in range(len(resolution_options)):
ro_hash_set.add(resolution_options[i])
file_name_suffix = TestUtils.get_resolution_option_name_suffix(resolution_options)
resolved_entity_name = 'Resolved_{}{}.cdm.json'.format(input_entity.entity_name, file_name_suffix)
ro = ResolveOptions(input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set))
resolved_folder = corpus.storage.fetch_root_folder('output')
resolved_entity = await input_entity.create_resolved_entity_async(resolved_entity_name, ro, resolved_folder)
return resolved_entity
async def resolve_environment(tests_sub_path: str, test_name: str,
manifest_name: str) -> str:
"""
Resolve the entities in the given manifest.
:param tests_sub_path: Tests sub-folder name
:param test_name: The name of the test. It is used to decide the path of input / output files.
:param manifest_name: The name of the manifest to be used.
:return:
"""
test_input_path = TestHelper.get_input_folder_path(tests_sub_path,
test_name)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
manifest = await corpus.fetch_object_async(
'local:/{}.manifest.cdm.json'.format(manifest_name))
directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
return await list_all_resolved(corpus, directives, manifest,
StringSpewCatcher())
async def test_missing_condition_in_json(self):
"""
Test case scenario for Bug #25 from the projections internal bug bash
Reference Link: https:#commondatamodel.visualstudio.com/CDM/_workitems/edit/25
"""
test_name = 'test_missing_condition_in_json'
corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
def callback(level, message):
self.fail(message)
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
manifest = await corpus.fetch_object_async('default.manifest.cdm.json')
entity_name = 'SalesNestedFK'
entity = await corpus.fetch_object_async('local:/{}.cdm.json/{}'.format(entity_name, entity_name), manifest)
self.assertIsNotNone(entity)
res_opt = ResolveOptions(entity.in_document)
# where, res_opts_combinations[1] == 'referenceOnly'
res_opt.directives = AttributeResolutionDirectiveSet(self.res_opts_combinations[1])
resolved_folder = corpus.storage.fetch_root_folder('output')
was_info_message_received = {}
was_info_message_received['infoMessagedReceived'] = False
def callback_2(level, message):
if 'CdmProjection | Optional expression missing. Implicit expression will automatically apply. | _construct_projection_context' in message:
was_info_message_received['infoMessagedReceived'] = True
corpus.set_event_callback(callback_2, CdmStatusLevel.INFO)
resolved_entity = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity_name), res_opt, resolved_folder)
self.assertIsNotNone(resolved_entity)
self.assertTrue(was_info_message_received['infoMessagedReceived'])
async def logical_manipulation_using_projections():
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
A single logical definition can be resolved into multiple physical layouts. The directives are used to instruct the ObjectModel about how it should to
resolve the logical definition provided. To achieve this, we define projections that run conditionally, depending on the directives provided when
calling create_resolved_entity_async.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
# Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
corpus = CdmCorpusDefinition()
print('Configure storage adapters')
# Configure storage adapters to point at the target local manifest location and at the fake public standards
path_from_exe_to_example_root = '../../'
corpus.storage.mount(
'local',
LocalAdapter(path_from_exe_to_example_root +
'8-logical-manipulation-using-projections/sample-data'))
corpus.storage.default_namespace = 'local' # local is our default. so any paths that start out navigating without a device tag will assume local
# Fake cdm, normaly use the CDM Standards adapter
# Mount it as the 'cdm' device, not the default so must use 'cdm:/folder' to get there
corpus.storage.mount(
'cdm',
LocalAdapter(path_from_exe_to_example_root +
'example-public-standards'))
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async(
'local:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('Address.cdm.json')
entity = logical_doc.definitions.append(
'Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append(
'name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append(
'age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append(
'email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print('Get \'resolved\' folder where the resolved entities will be saved.')
resolved_folder = await corpus.fetch_object_async(
'local:/resolved/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
res_normalized_entity = await entity.create_resolved_entity_async(
f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(
f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
res_reference_only_entity = await entity.create_resolved_entity_async(
f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(
f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
res_structured_entity = await entity.create_resolved_entity_async(
f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(
f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({})
res_array_entity = await entity.create_resolved_entity_async(
f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(
f'{res_array_entity.entity_name}.cdm.json')
def _construct_resolved_attributes(
self,
res_opt: 'ResolveOptions',
under: Optional['CdmAttributeContext'] = None
) -> 'ResolvedAttributeSetBuilder':
from cdm.resolvedmodel import AttributeResolutionContext, ResolvedAttribute, ResolvedAttributeSetBuilder, ResolvedTrait
from cdm.utilities import AttributeContextParameters, AttributeResolutionDirectiveSet
from .cdm_object import CdmObject
rasb = ResolvedAttributeSetBuilder()
ctx_ent = self.entity
under_att = under
acp_ent = None
if not res_opt._in_circular_reference:
arc = self._fetch_att_res_context(res_opt)
# complete cheating but is faster.
# this purpose will remove all of the attributes that get collected here, so dumb and slow to go get them
rel_info = self._get_relationship_info(arc.res_opt, arc)
res_opt._depth_info.current_depth = rel_info.next_depth
res_opt._depth_info.max_depth_exceeded = rel_info.max_depth_exceeded
res_opt._depth_info.max_depth = rel_info.max_depth
ctx_ent_obj_def = ctx_ent.fetch_object_definition(res_opt)
if ctx_ent_obj_def and ctx_ent_obj_def.object_type == CdmObjectType.PROJECTION_DEF:
# A Projection
# if the max depth is exceeded it should not try to execute the projection
if not res_opt._depth_info.max_depth_exceeded:
proj_directive = ProjectionDirective(
res_opt, self, ctx_ent)
proj_def = ctx_ent_obj_def
proj_ctx = proj_def._construct_projection_context(
proj_directive, under)
rasb._resolved_attribute_set = proj_def._extract_resolved_attributes(
proj_ctx, under_att)
else:
# An Entity Reference
if under_att:
# make a context for this attribute that holds the attributes that come up from the entity
acp_ent = AttributeContextParameters(
under=under_att,
type=CdmAttributeContextType.ENTITY,
name=ctx_ent.fetch_object_definition_name(),
regarding=ctx_ent,
include_traits=True)
if rel_info.is_by_ref:
# make the entity context that a real recursion would have give us
if under:
under = rasb._resolved_attribute_set.create_attribute_context(
res_opt, acp_ent)
# if selecting from one of many attributes, then make a context for each one
if under and rel_info.selects_one:
# the right way to do this is to get a resolved entity from the embedded entity and then
# look through the attribute context hierarchy for non-nested entityReferenceAsAttribute nodes
# that seems like a disaster waiting to happen given endless looping, etc.
# for now, just insist that only the top level entity attributes declared in the ref entity will work
ent_pick_from = self.entity.fetch_object_definition(
res_opt)
atts_pick = ent_pick_from.attributes
if ent_pick_from and atts_pick:
for attribute in atts_pick:
if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF:
# a table within a table. as expected with a selects_one attribute
# since this is by ref, we won't get the atts from the table, but we do need the traits that hold the key
# these are the same contexts that would get created if we recursed
# first this attribute
acp_ent_att = AttributeContextParameters(
under=under,
type=CdmAttributeContextType.
ATTRIBUTE_DEFINITION,
name=attribute.
fetch_object_definition_name(),
regarding=attribute,
include_traits=True)
pick_under = rasb._resolved_attribute_set.create_attribute_context(
res_opt, acp_ent_att)
# and the entity under that attribute
pick_ent = attribute.entity
pick_ent_type = CdmAttributeContextType.PROJECTION if pick_ent.fetch_object_definition(
res_opt
).object_type == CdmObjectType.PROJECTION_DEF else CdmAttributeContextType.ENTITY
acp_ent_att_ent = AttributeContextParameters(
under=pick_under,
type=pick_ent_type,
name=pick_ent.
fetch_object_definition_name(),
regarding=pick_ent,
include_traits=True)
rasb._resolved_attribute_set.create_attribute_context(
res_opt, acp_ent_att_ent)
# if we got here because of the max depth, need to impose the directives to make the trait work as expected
if rel_info.max_depth_exceeded:
if not arc.res_opt.directives:
arc.res_opt.directives = AttributeResolutionDirectiveSet(
)
arc.res_opt.directives.add('referenceOnly')
else:
res_link = res_opt.copy()
res_link._symbol_ref_set = res_opt._symbol_ref_set
rasb.merge_attributes(
self.entity._fetch_resolved_attributes(
res_link, acp_ent))
# need to pass up max_depth_exceeded if it was hit
if res_link._depth_info.max_depth_exceeded:
res_opt._depth_info = res_link._depth_info._copy()
# from the traits of purpose and applied here, see if new attributes get generated
rasb._resolved_attribute_set.attribute_context = under_att
rasb.apply_traits(arc)
rasb.generate_applier_attributes(
arc, True) # True = apply the prepared traits to new atts
# this may have added symbols to the dependencies, so merge them
res_opt._symbol_ref_set._merge(arc.res_opt._symbol_ref_set)
# use the traits for linked entity identifiers to record the actual foreign key links
if rasb._resolved_attribute_set and rasb._resolved_attribute_set._set and rel_info.is_by_ref:
for att in rasb._resolved_attribute_set._set:
reqd_trait = att.resolved_traits.find(
res_opt, 'is.linkedEntity.identifier')
if not reqd_trait:
continue
if not reqd_trait.parameter_values:
logger.warning(
self._ctx, self._TAG,
CdmEntityAttributeDefinition.
_construct_resolved_attributes.__name__,
self.at_corpus_path, CdmLogCode.
WARN_IDENTIFIER_ARGUMENTS_NOT_SUPPORTED)
continue
ent_references = []
att_references = []
def add_entity_reference(
entity_ref: 'CdmEntityReference',
namespace: str):
ent_def = entity_ref.fetch_object_definition(
res_opt)
required_trait = entity_ref._fetch_resolved_traits(
res_opt).find(res_opt, 'is.identifiedBy')
if required_trait and ent_def:
att_ref = required_trait.parameter_values.fetch_parameter_value(
'attribute').value
att_name = att_ref.named_reference.split(
'/')[-1]
absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.at_corpus_path,
ent_def.in_document)
ent_references.append(absolute_ent_path)
att_references.append(att_name)
if rel_info.selects_one:
ent_pick_from = self.entity.fetch_object_definition(
res_opt)
atts_pick = ent_pick_from.attributes if ent_pick_from else None
if atts_pick:
for attribute in atts_pick:
if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF:
add_entity_reference(
attribute.entity,
self.in_document.namespace)
else:
add_entity_reference(self.entity,
self.in_document.namespace)
c_ent = self.ctx.corpus.make_object(
CdmObjectType.CONSTANT_ENTITY_DEF)
c_ent.entity_shape = self.ctx.corpus.make_ref(
CdmObjectType.ENTITY_REF, 'entityGroupSet', True)
c_ent.constant_values = [[
entity_ref, att_references[idx]
] for idx, entity_ref in enumerate(ent_references)]
param = self.ctx.corpus.make_ref(
CdmObjectType.ENTITY_REF, c_ent, False)
reqd_trait.parameter_values.update_parameter_value(
res_opt, 'entityReferences', param)
# a 'structured' directive wants to keep all entity attributes together in a group
if arc and arc.res_opt.directives and arc.res_opt.directives.has(
'structured'):
# make one resolved attribute with a name from this entityAttribute that contains the set
# of atts we just put together.
ra_sub = ResolvedAttribute(arc.traits_to_apply.res_opt,
rasb._resolved_attribute_set,
self.name, under_att)
if rel_info.is_array:
# put a resolved trait on this att group
# hope I never need to do this again and then need to make a function for this
tr = self.ctx.corpus.make_object(
CdmObjectType.TRAIT_REF, 'is.linkedEntity.array',
True)
t = tr.fetch_object_definition(res_opt)
rt = ResolvedTrait(t, None, [], [])
ra_sub.resolved_traits = ra_sub.resolved_traits.merge(
rt, True)
depth = rasb._resolved_attribute_set._depth_traveled
rasb = ResolvedAttributeSetBuilder()
rasb._resolved_attribute_set.attribute_context = ra_sub.att_ctx # this got set to null with the new builder
rasb.own_one(ra_sub)
rasb._resolved_attribute_set._depth_traveled = depth
# how ever they got here, mark every attribute from this entity attribute as now being 'owned' by this entityAtt
rasb._resolved_attribute_set._set_attribute_ownership(self.name)
rasb._resolved_attribute_set._depth_traveled += 1
return rasb
async def run_test_with_values(
self, test_name: str, source_entity_name: str,
expected_context_default: 'AttributeContextExpectedValue',
expected_context_normalized: 'AttributeContextExpectedValue',
expected_context_reference_only: 'AttributeContextExpectedValue',
expected_context_structured: 'AttributeContextExpectedValue',
expected_context_normalized_structured: 'AttributeContextExpectedValue',
expected_context_reference_only_normalized:
'AttributeContextExpectedValue',
expected_context_reference_only_structured:
'AttributeContextExpectedValue',
expected_context_reference_only_normalized_structured:
'AttributeContextExpectedValue',
expected_default: 'List[AttributeExpectedValue]',
expected_normalized: 'List[AttributeExpectedValue]',
expected_reference_only: 'List[AttributeExpectedValue]',
expected_structured: 'List[AttributeExpectedValue]',
expected_normalized_structured: 'List[AttributeExpectedValue]',
expected_reference_only_normalized: 'List[AttributeExpectedValue]',
expected_reference_only_structured: 'List[AttributeExpectedValue]',
expected_reference_only_normalized_structured:
'List[AttributeExpectedValue]'
) -> None:
"""This method runs the tests with a set expected attributes & attribute context values and validated the actual result."""
try:
test_input_path = TestHelper.get_input_folder_path(
self.tests_subpath, test_name)
test_actual_path = TestHelper.get_actual_output_folder_path(
self.tests_subpath, test_name)
test_expected_path = TestHelper.get_expected_output_folder_path(
self.tests_subpath, test_name)
corpus_path = test_input_path[0:(len(test_input_path) -
len('/Input'))]
test_actual_path = os.path.abspath(test_actual_path)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(corpus_path))
corpus.storage.mount('cdm', LocalAdapter(self.schema_docs_path))
corpus.storage.default_namespace = 'local'
out_folder_path = corpus.storage.adapter_path_to_corpus_path(
test_actual_path) + '/' # interesting bug
out_folder = await corpus.fetch_object_async(
out_folder_path) # type: CdmFolderDefinition
src_entity_def = await corpus.fetch_object_async(
'local:/Input/{}.cdm.json/{}'.format(source_entity_name,
source_entity_name)
) # type: CdmEntityDefinition
self.assertTrue(src_entity_def is not None)
res_opt = ResolveOptions(wrt_doc=src_entity_def.in_document)
resolved_entity_def = None # type: CdmEntityDefinition
output_entity_name = ''
output_entity_file_name = ''
entity_file_name = ''
if expected_context_default and expected_default:
entity_file_name = 'd'
res_opt.directives = AttributeResolutionDirectiveSet(set())
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_normalized and expected_normalized:
entity_file_name = 'n'
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only and expected_reference_only:
entity_file_name = 'ro'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_structured and expected_structured:
entity_file_name = 's'
res_opt.directives = AttributeResolutionDirectiveSet(
{'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_normalized_structured and expected_normalized_structured:
entity_file_name = 'n_s'
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only_normalized and expected_reference_only_normalized:
entity_file_name = 'ro_n'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'normalized'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only_structured and expected_reference_only_structured:
entity_file_name = 'ro_s'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
if expected_context_reference_only_normalized_structured and expected_reference_only_normalized_structured:
entity_file_name = 'ro_n_s'
res_opt.directives = AttributeResolutionDirectiveSet(
{'referenceOnly', 'normalized', 'structured'})
output_entity_name = '{}_R_{}'.format(source_entity_name,
entity_file_name)
output_entity_file_name = '{}.cdm.json'.format(
output_entity_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(
output_entity_name, res_opt, out_folder)
await self.save_actual_entity_and_validate_with_expected(
os.path.join(test_expected_path, output_entity_file_name),
resolved_entity_def)
except Exception as e:
self.fail(e)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an ArrayExpansion operation and a condition using the object model"""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an ArrayExpansion operation
array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF)
array_expansion_op.start_ordinal = 1
array_expansion_op.end_ordinal = 2
projection.operations.append(array_expansion_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity
projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF)
projection2.source = projection_entity_ref
# Create a RenameAttributes operation
rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF)
rename_attrs_op.rename_format = '{m}{o}'
projection2.operations.append(rename_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref2.explicit_reference = projection2
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref2
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(8, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id1', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name1', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value1', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date1', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual('id2', resolved_entity_with_reference_only.attributes[4].name)
self.assertEqual('name2', resolved_entity_with_reference_only.attributes[5].name)
self.assertEqual('value2', resolved_entity_with_reference_only.attributes[6].name)
self.assertEqual('date2', resolved_entity_with_reference_only.attributes[7].name)
# Now resolve the entity with the default directives
res_opt.directives = AttributeResolutionDirectiveSet(set([]))
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id', resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name', resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value', resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def run_test(self, test_name: str, source_entity_name: str) -> None:
test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name)
test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name)
test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('localInput', LocalAdapter(test_input_path))
corpus.storage.mount('localExpectedOutput', LocalAdapter(test_expected_output_path))
corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path))
corpus.storage.mount('cdm', LocalAdapter(TestHelper.get_schema_docs_root()))
corpus.storage.default_namespace = 'localInput'
src_entity_def = await corpus.fetch_object_async('localInput:/{0}.cdm.json/{0}'.format(source_entity_name)) # type: CdmEntityDefinition
self.assertIsNotNone(src_entity_def)
res_opt = ResolveOptions(src_entity_def.in_document, directives=AttributeResolutionDirectiveSet(set()))
actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition
resolved_entity_def = None
output_entity_file_name = ''
entity_file_name = ''
entity_file_name = 'default'
res_opt.directives = AttributeResolutionDirectiveSet(set())
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'normalized'
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'structured'
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_normalized'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an ExcludeAttributes operation and a condition using the object model"""
corpus = TestHelper.get_local_corpus(
self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount(
'local',
LocalAdapter(
TestHelper.get_actual_output_folder_path(
self.tests_subpath,
'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an ExcludeAttributes operation
exclude_attrs_op = corpus.make_object(
CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF)
exclude_attrs_op.exclude_attributes.append('id')
exclude_attrs_op.exclude_attributes.append('date')
projection.operations.append(exclude_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(
set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: ['id', 'date']
self.assertEqual(2,
len(resolved_entity_with_reference_only.attributes))
self.assertEqual(
'name', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual(
'value', resolved_entity_with_reference_only.attributes[1].name)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(
set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: none, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id',
resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name',
resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value',
resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date',
resolved_entity_with_structured.attributes[3].name)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddArtifactAttribute operation and a condition using the object model"""
test_name = 'test_conditional_proj_using_object_model'
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath,
test_name)
local_root = corpus.storage.fetch_root_folder('local')
corpus.storage.mount('traitGroup',
LocalAdapter(self.trait_group_file_path))
# Create an entity.
entity = ProjectionTestUtils.create_entity(corpus, local_root)
entity.in_document.imports.append('traitGroup:/TraitGroup.cdm.json')
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'.
projection = ProjectionTestUtils.create_projection(
corpus, local_root) # type: CdmProjection
projection.condition = 'structured==true'
projection.run_sequentially = True
# Create an AlterTraits operation
alter_traits_op_1 = corpus.make_object(
CdmObjectType.OPERATION_ALTER_TRAITS_DEF
) # type: CdmOperationAlterTraits
alter_traits_op_1.traits_to_add = CdmCollection(
corpus.ctx, alter_traits_op_1, CdmObjectType.TRAIT_REF)
alter_traits_op_1.traits_to_add.append(
corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG100", True))
alter_traits_op_1.traits_to_add.append(
corpus.make_ref(CdmObjectType.TRAIT_GROUP_REF, "JobTitleBase",
True))
alter_traits_op_1.traits_to_remove = CdmCollection(
corpus.ctx, alter_traits_op_1, CdmObjectType.TRAIT_REF)
alter_traits_op_1.traits_to_remove.append(
corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG300", True))
projection.operations.append(alter_traits_op_1)
alter_traits_op_2 = corpus.make_object(
CdmObjectType.OPERATION_ALTER_TRAITS_DEF
) # type: CdmOperationAlterTraits
trait_g4 = corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG4",
True) # type: CdmTraitReference
trait_g4.arguments.append('precision', '5')
trait_g4.arguments.append('scale', '15')
alter_traits_op_2.traits_to_add = CdmCollection(
corpus.ctx, alter_traits_op_2, CdmObjectType.TRAIT_REF)
alter_traits_op_2.traits_to_add.append(trait_g4)
alter_traits_op_2.apply_to = []
alter_traits_op_2.apply_to.append('name')
projection.operations.append(alter_traits_op_2)
# Create an entity reference to hold this projection.
projection_entity_ref = corpus.make_object(
CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity.
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF,
'TestEntityAttribute') # type: CdmEntityAttributeDefinition
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive.
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition not met, no traits are added
self.assertEqual(4,
len(resolved_entity_with_reference_only.attributes))
self.validate_trait(resolved_entity_with_reference_only.attributes[0],
'id', False, True)
self.validate_trait(resolved_entity_with_reference_only.attributes[1],
'name', False, True)
self.validate_trait(resolved_entity_with_reference_only.attributes[2],
'value', False, True)
self.validate_trait(resolved_entity_with_reference_only.attributes[3],
'date', False, True)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Original set of attributes: ['id', 'name', 'value', 'date']
# Condition met, new traits are added
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.validate_trait(resolved_entity_with_structured.attributes[0],
'id')
self.validate_trait(resolved_entity_with_structured.attributes[1],
'name', True)
self.validate_trait(resolved_entity_with_structured.attributes[2],
'value')
self.validate_trait(resolved_entity_with_structured.attributes[3],
'date')
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition',
options: 'CopyOptions', new_name: str,
save_referenced: bool) -> bool:
"""a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work
because the corpus knows about persistence types and about the storage adapters
if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or
to a different name, then the source object is still 'dirty'
an option will cause us to also save any linked documents."""
# find out if the storage adapter is able to write.
namespace = doc.namespace
if namespace is None:
namespace = self._corpus.storage.default_namespace
adapter = self._corpus.storage.fetch_adapter(namespace)
if adapter is None:
logger.error(
self._ctx, self._TAG, self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_ADAPTER_NOT_FOUND_FOR_NAMESPACE,
namespace)
return False
if not adapter.can_write():
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_ADAPTER_WRITE_FAILURE,
namespace)
return False
if not new_name:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
# what kind of document is requested?
# check file extensions using a case-insensitive ordinal string comparison.
persistence_type = self.MODEL_JSON if new_name.lower().endswith(
self.MODEL_JSON_EXTENSION) else self.CDM_FOLDER
if persistence_type == self.MODEL_JSON and new_name.lower(
) != self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE,
new_name, self.MODEL_JSON_EXTENSION)
return False
# save the object into a json blob
res_opt = {
'wrt_doc': doc,
'directives': AttributeResolutionDirectiveSet()
}
persisted_doc = None
try:
if new_name.lower().endswith(
PersistenceLayer.MODEL_JSON_EXTENSION) or new_name.lower(
).endswith(
PersistenceLayer.MANIFEST_EXTENSION) or new_name.lower(
).endswith(PersistenceLayer.FOLIO_EXTENSION):
if persistence_type == self.CDM_FOLDER:
from cdm.persistence.cdmfolder import ManifestPersistence
persisted_doc = ManifestPersistence.to_data(
doc, res_opt, options)
else:
if new_name != self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FAILURE, new_name)
return False
from cdm.persistence.modeljson import ManifestPersistence
persisted_doc = await ManifestPersistence.to_data(
doc, res_opt, options)
elif new_name.lower().endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
persisted_doc = DocumentPersistence.to_data(
doc, res_opt, options)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_CLASS_MISSING, new_name)
return False
except Exception as e:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_PERSIST_ERROR, new_name,
e)
return False
if not persisted_doc:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_PERSIST_FAILED, new_name)
return False
# turn the name into a path
new_path = '{}{}'.format(doc.folder_path, new_name)
new_path = self._ctx.corpus.storage.create_absolute_corpus_path(
new_path, doc)
if new_path.startswith(namespace + ':'):
new_path = new_path[len(namespace) + 1:]
# ask the adapter to make it happen
try:
content = persisted_doc.encode()
await adapter.write_async(new_path, content)
doc._file_system_modified_time = await adapter.compute_last_modified_time_async(
new_path)
# Write the adapter's config.
if options._is_top_level_document:
await self._corpus.storage.save_adapters_config_async(
'/config.json', adapter)
# The next document won't be top level, so reset the flag.
options._is_top_level_document = False
except Exception as e:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_WRITE_FAILURE, new_name,
e)
return False
# if we also want to save referenced docs, then it depends on what kind of thing just got saved
# if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things
# definition will save imports, manifests will save imports, schemas, sub manifests
if save_referenced and persistence_type == self.CDM_FOLDER:
saved_linked_docs = await doc._save_linked_documents_async(options)
if not saved_linked_docs:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_SAVE_LINK_DOCS, new_name)
return False
return True
async def create_resolved_manifest_async(
self, new_manifest_name: str, new_entity_document_name_format: str
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
if self.entities is None:
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
self.ctx.logger.error(
'New folder for manifest not found {}'.format(
new_folder_path), 'create_resolved_manifest_async')
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
self.ctx.logger.debug(
'resolving manifest {}'.format(source_manifest_path),
'create_resolved_manifest_async')
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name)
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
# mapping from entity path to resolved entity path for translating relationhsip paths
res_ent_map = {} # type: Dict[str, str]
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
self.ctx.logger.error('Unable to get entity from reference')
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
self.ctx.logger.error(
'New folder not found {}'.format(new_document_path))
return None
# next create the resolved entity.
res_opt = ResolveOptions()
res_opt.wrt_doc = ent_def.in_document
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
self.ctx.logger.debug(
' resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
# absolute path is needed for generating relationships
absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path(
result.entity_path, resolved_manifest)
res_ent_map[self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.at_corpus_path,
ent_def.in_document)] = absolute_ent_path
self.ctx.logger.debug(' calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus._calculate_entity_graph_async(
resolved_manifest, res_ent_map)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
def _construct_resolved_attributes(
self,
res_opt: 'ResolveOptions',
under: Optional['CdmAttributeContext'] = None
) -> 'ResolvedAttributeSetBuilder':
from cdm.resolvedmodel import AttributeResolutionContext, ResolvedAttribute, ResolvedAttributeSetBuilder, ResolvedTrait
from cdm.utilities import AttributeContextParameters, AttributeResolutionDirectiveSet
from .cdm_attribute_resolution_guidance_def import CdmAttributeResolutionGuidanceDefinition
from .cdm_object import CdmObject
rasb = ResolvedAttributeSetBuilder()
ctx_ent = self.entity
under_att = under
acp_ent = None
if under_att:
# make a context for this attribute that holds the attributes that come up from the entity
acp_ent = AttributeContextParameters(
under=under_att,
type=CdmAttributeContextType.ENTITY,
name=ctx_ent.fetch_object_definition_name(),
regarding=ctx_ent,
include_traits=True)
rts_this_att = self._fetch_resolved_traits(res_opt)
# this context object holds all of the info about what needs to happen to resolve these attributes.
# make a copy and add defaults if missing
res_guide_with_default = None
if self.resolution_guidance is not None:
res_guide_with_default = self.resolution_guidance.copy(res_opt)
else:
res_guide_with_default = CdmAttributeResolutionGuidanceDefinition(
self.ctx)
res_guide_with_default._update_attribute_defaults(self.name)
arc = AttributeResolutionContext(res_opt, res_guide_with_default,
rts_this_att)
# complete cheating but is faster.
# this purpose will remove all of the attributes that get collected here, so dumb and slow to go get them
rel_info = self._get_relationship_info(arc.res_opt, arc)
if rel_info.is_by_ref:
# make the entity context that a real recursion would have give us
if under:
under = rasb.ras.create_attribute_context(res_opt, acp_ent)
# if selecting from one of many attributes, then make a context for each one
if under and rel_info.selects_one:
# the right way to do this is to get a resolved entity from the embedded entity and then
# look through the attribute context hierarchy for non-nested entityReferenceAsAttribute nodes
# that seems like a disaster waiting to happen given endless looping, etc.
# for now, just insist that only the top level entity attributes declared in the ref entity will work
ent_pick_from = self.entity.fetch_object_definition(res_opt)
atts_pick = ent_pick_from.attributes
if ent_pick_from and atts_pick:
for attribute in atts_pick:
if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF:
# a table within a table. as expected with a selects_one attribute
# since this is by ref, we won't get the atts from the table, but we do need the traits that hold the key
# these are the same contexts that would get created if we recursed
# first this attribute
acp_ent_att = AttributeContextParameters(
under=under,
type=CdmAttributeContextType.
ATTRIBUTE_DEFINITION,
name=attribute.fetch_object_definition_name(),
regarding=attribute,
include_traits=True)
pick_under = rasb.ras.create_attribute_context(
res_opt, acp_ent_att)
# and the entity under that attribute
pick_ent = attribute.entity
acp_ent_att_ent = AttributeContextParameters(
under=pick_under,
type=CdmAttributeContextType.ENTITY,
name=pick_ent.fetch_object_definition_name(),
regarding=pick_ent,
include_traits=True)
rasb.ras.create_attribute_context(
res_opt, acp_ent_att_ent)
# if we got here because of the max depth, need to impose the directives to make the trait work as expected
if rel_info.max_depth_exceeded:
if not arc.res_opt.directives:
arc.res_opt.directives = AttributeResolutionDirectiveSet()
arc.res_opt.directives.add('referenceOnly')
else:
res_link = CdmObject._copy_resolve_options(res_opt)
res_link.symbol_ref_set = res_opt.symbol_ref_set
res_link._relationship_depth = rel_info.next_depth
rasb.merge_attributes(
self.entity._fetch_resolved_attributes(res_link, acp_ent))
# from the traits of purpose and applied here, see if new attributes get generated
rasb.ras.attribute_context = under_att
rasb.apply_traits(arc)
rasb.generate_applier_attributes(
arc, True) # True = apply the prepared traits to new atts
# this may have added symbols to the dependencies, so merge them
res_opt.symbol_ref_set.merge(arc.res_opt.symbol_ref_set)
# use the traits for linked entity identifiers to record the actual foreign key links
if rasb.ras and rasb.ras.set and rel_info.is_by_ref:
for att in rasb.ras.set:
reqd_trait = att.resolved_traits.find(
res_opt, 'is.linkedEntity.identifier')
if not reqd_trait:
continue
if not reqd_trait.parameter_values:
logger.warning(
self._TAG, self.ctx,
'is.linkedEntity.identifier does not support arguments'
)
continue
ent_references = []
att_references = []
def add_entity_reference(entity_ref: 'CdmEntityReference',
namespace: str):
ent_def = entity_ref.fetch_object_definition(res_opt)
required_trait = entity_ref._fetch_resolved_traits(
res_opt).find(res_opt, 'is.identifiedBy')
if required_trait and ent_def:
att_ref = required_trait.parameter_values.fetch_parameter_value(
'attribute').value
att_name = att_ref.named_reference.split('/')[-1]
# path should be absolute and without a namespace
relative_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.at_corpus_path, ent_def.in_document)
if relative_ent_path.startswith(namespace + ':'):
relative_ent_path = relative_ent_path[len(namespace
) + 1:]
ent_references.append(relative_ent_path)
att_references.append(att_name)
if rel_info.selects_one:
ent_pick_from = self.entity.fetch_object_definition(
res_opt)
atts_pick = ent_pick_from.attributes if ent_pick_from else None
if atts_pick:
for attribute in atts_pick:
if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF:
add_entity_reference(
attribute.entity,
self.in_document.namespace)
else:
add_entity_reference(self.entity,
self.in_document.namespace)
c_ent = self.ctx.corpus.make_object(
CdmObjectType.CONSTANT_ENTITY_DEF)
c_ent.entity_shape = self.ctx.corpus.make_ref(
CdmObjectType.ENTITY_REF, 'entityGroupSet', True)
c_ent.constant_values = [[
entity_ref, att_references[idx]
] for idx, entity_ref in enumerate(ent_references)]
param = self.ctx.corpus.make_ref(CdmObjectType.ENTITY_REF,
c_ent, False)
reqd_trait.parameter_values.update_parameter_value(
res_opt, 'entityReferences', param)
# a 'structured' directive wants to keep all entity attributes together in a group
if arc and arc.res_opt.directives and arc.res_opt.directives.has(
'structured'):
ra_sub = ResolvedAttribute(rts_this_att.res_opt, rasb.ras,
self.name, rasb.ras.attribute_context)
if rel_info.is_array:
# put a resolved trait on this att group, yuck,
# hope I never need to do this again and then need to make a function for this
tr = self.ctx.corpus.make_object(CdmObjectType.TRAIT_REF,
'is.linkedEntity.array', True)
t = tr.fetch_object_definition(res_opt)
rt = ResolvedTrait(t, None, [], [])
ra_sub.resolved_traits = ra_sub.resolved_traits.merge(rt, True)
rasb = ResolvedAttributeSetBuilder()
rasb.own_one(ra_sub)
return rasb
async def logical_manipulation_using_projections(
self, corpus: CdmCorpusDefinition):
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async(
'output:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('local:/Address.cdm.json')
entity = logical_doc.definitions.append(
'Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append(
'name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append(
'age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append(
'email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print(
'Get \'resolved\' folder where the resolved entities will be saved.'
)
resolved_folder = await corpus.fetch_object_async(
'output:/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
res_normalized_entity = await entity.create_resolved_entity_async(
f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(
f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
res_reference_only_entity = await entity.create_resolved_entity_async(
f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(
f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
res_structured_entity = await entity.create_resolved_entity_async(
f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(
f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({})
res_array_entity = await entity.create_resolved_entity_async(
f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(
f'{res_array_entity.entity_name}.cdm.json')
async def test_circular_reference(self):
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, 'TestCircularReference')
customer = await cdm_corpus.fetch_object_async('local:/Customer.cdm.json/Customer')
res_customer_structured = await customer.create_resolved_entity_async('resCustomer', ResolveOptions(customer.in_document, AttributeResolutionDirectiveSet({'normalized', 'structured', 'noMaxDepth'})))
# check that the circular reference attribute has a single id attribute
store_group_att = res_customer_structured.attributes[1].explicit_reference
customer_group_att = store_group_att.members[1].explicit_reference
self.assertEqual(len(customer_group_att.members), 1)
self.assertEqual(customer_group_att.members[0].name, 'customerId')
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition', options: 'CopyOptions', new_name: str, save_referenced: bool) -> bool:
"""a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work
because the corpus knows about persistence types and about the storage adapters
if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or
to a different name, then the source object is still 'dirty'
an option will cause us to also save any linked documents."""
# find out if the storage adapter is able to write.
namespace = doc.namespace
if namespace is None:
namespace = self._corpus.storage.default_namespace
adapter = self._corpus.storage.fetch_adapter(namespace)
if adapter is None:
logger.error(self._TAG, self._ctx, 'Couldn\'t find a storage adapter registered for the namespace \'{}\''.format(
namespace), self._save_document_as_async.__name__)
return False
if not adapter.can_write():
logger.error(self._TAG, self._ctx, 'The storage adapter \'{}\' claims it is unable to write files.'.format(
namespace), self._save_document_as_async.__name__)
return False
if not new_name:
logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._save_document_as_async.__name__)
return None
# what kind of document is requested?
# check file extensions using a case-insensitive ordinal string comparison.
persistence_type = self.MODEL_JSON if new_name.lower().endswith(self.MODEL_JSON_EXTENSION) else \
(self.ODI if new_name.lower().endswith(self.ODI_EXTENSION) else self.CDM_FOLDER)
if persistence_type == self.ODI and new_name.lower() != self.ODI_EXTENSION:
logger.error(self._TAG, self._ctx, 'Failed to persist \'{}\', as it\'s not an acceptable filename. It must be {}'.format(
new_name, self.ODI_EXTENSION), self._save_document_as_async.__name__)
return False
if persistence_type == self.MODEL_JSON and new_name.lower() != self.MODEL_JSON_EXTENSION:
logger.error(self._TAG, self._ctx, 'Failed to persist \'{}\', as it\'s not an acceptable filename. It must be {}'.format(
new_name, self.MODEL_JSON_EXTENSION), self._save_document_as_async.__name__)
return False
# save the object into a json blob
res_opt = {'wrt_doc': doc, 'directives': AttributeResolutionDirectiveSet()}
persisted_doc = None
persistence_class = self._fetch_registered_persistence_format(new_name)
if persistence_class:
try:
method = persistence_class.to_data
parameters = [doc, res_opt, options]
# Check if to_data() is asynchronous for this persistence class.
if not persistence_class in self._is_registered_persistence_async:
# Cache whether this persistence class has async methods.
self._is_registered_persistence_async[persistence_class] = persistence_class.is_persistence_async
if self._is_registered_persistence_async[persistence_class]:
# We don't know what the return type of ToData() is going to be and Task<T> is not covariant,
# so we can't use Task<dynamic> here. Instead, we just await on a Task object without a return value
# and fetch the Result property from it, which will have the result of ToData().
persisted_doc = await method(*parameters)
else:
persisted_doc = method(*parameters)
except Exception as e:
logger.error(self._TAG, self._ctx, 'Could not persist file \'{}\'. Reason \'{}\'.'.format(new_name, e), self._save_document_as_async.__name__)
return False
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\'.'.format(
new_name), self._save_document_as_async.__name__)
return False
if not persisted_doc:
logger.error(self._TAG, self._ctx, 'Failed to persist \'{}\''.format(new_name), self._save_document_as_async.__name__)
return False
if persistence_type == self.ODI:
await self._save_odi_documents(persisted_doc, adapter, new_name)
return True
# turn the name into a path
new_path = '{}{}'.format(doc.folder_path, new_name)
new_path = self._ctx.corpus.storage.create_absolute_corpus_path(new_path, doc)
if new_path.startswith(namespace + ':'):
new_path = new_path[len(namespace)+1:]
# ask the adapter to make it happen
try:
content = persisted_doc.encode()
await adapter.write_async(new_path, content)
doc._file_system_modified_time = await adapter.compute_last_modified_time_async(new_path)
# Write the adapter's config.
if options._is_top_level_document:
await self._corpus.storage.save_adapters_config_async('/config.json', adapter)
# The next document won't be top level, so reset the flag.
options._is_top_level_document = False
except Exception as e:
logger.error(self._TAG, self._ctx, 'Failed to write to the file \'{}\' for reason \'{}\''.format(new_name, e),
self._save_document_as_async.__name__)
return False
# if we also want to save referenced docs, then it depends on what kind of thing just got saved
# if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things
# definition will save imports, manifests will save imports, schemas, sub manifests
if save_referenced and persistence_type == self.CDM_FOLDER:
saved_linked_docs = await doc._save_linked_documents_async(options)
if not saved_linked_docs:
logger.error(self._TAG, self._ctx, 'Failed to save linked documents for file \'{}\''.format(new_name), self._save_document_as_async.__name__)
return False
return True
