async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an AddCountAttribute operation and a condition using the object model""" corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an AddCountAttribute operation add_count_attr_op = corpus.make_object(CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF) add_count_attr_op.count_attribute = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testCount') add_count_attr_op.count_attribute.data_type = corpus.make_ref(CdmObjectType.DATA_TYPE_REF, 'integer', True) projection.operations.append(add_count_attr_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddCountAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # Count attribute: "testCount" self.assertEqual(5, len(resolved_entity_with_reference_only.attributes)) self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name) self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name) self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name) self.assertEqual('testCount', resolved_entity_with_reference_only.attributes[4].name) self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4].applied_traits.item('is.linkedEntity.array.count')) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet(set(['structured'])) resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddCountAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # No Count attribute added, condition was false self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an AddAttributeGroup operation and a condition using the object model""" test_name = 'test_conditional_proj_using_object_model' corpus = ProjectionTestUtils.get_corpus(test_name, self.tests_subpath) local_root = corpus.storage.fetch_root_folder('local') # Create an entity. entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'. projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'structured==true' # Create an AddAttributeGroup operation add_att_group_op = corpus.make_object(CdmObjectType.OPERATION_ADD_ATTRIBUTE_GROUP_DEF) add_att_group_op.attribute_group_name = 'PersonAttributeGroup' projection.operations.append(add_att_group_op) # Create an entity reference to hold this projection. projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity. entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') # type: CdmEntityAttributeDefinition entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive. res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'}) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddAttributeGroup operation # Original set of attributes: ['id', 'name', 'value', 'date'] # Condition not met, keep attributes in flat list self.assertEqual(4, len(resolved_entity_with_reference_only.attributes)) self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name) self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name) self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet({'structured'}) resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddAttributeGroup operation # Original set of attributes: ['id', 'name', 'value', 'date'] # Condition met, put all attributes in an attribute group att_group_definition = self.validate_attribute_group(resolved_entity_with_structured.attributes, 'PersonAttributeGroup') self.assertEqual(4, len(att_group_definition.members)) self.assertEqual('id', att_group_definition.members[0].name) self.assertEqual('name', att_group_definition.members[1].name) self.assertEqual('value', att_group_definition.members[2].name) self.assertEqual('date', att_group_definition.members[3].name)
async def test_setting_traits_for_resolution_guidance_attributes(self): ''' Test that "is.linkedEntity.name" and "is.linkedEntity.identifier" traits are set when "selectedTypeAttribute" and "foreignKeyAttribute" are present in the entity's resolution guidance. ''' corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_setting_traits_for_resolution_guidance_attributes' ) # type: CdmCorpusDefinition entity = await corpus.fetch_object_async( 'local:/Customer.cdm.json/Customer') # type: CdmEntityDefinition # Resolve with default directives to get "is.linkedEntity.name" trait. res_opt = ResolveOptions(wrt_doc=entity.in_document) resolved_entity = await entity.create_resolved_entity_async( 'resolved', res_opt) self.assertEqual( 'is.linkedEntity.name', resolved_entity.attributes[1].applied_traits[6].named_reference) # Resolve with referenceOnly directives to get "is.linkedEntity.identifier" trait. res_opt = ResolveOptions(wrt_doc=entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'}) resolved_entity = await entity.create_resolved_entity_async( 'resolved2', res_opt) self.assertEqual( 'is.linkedEntity.identifier', resolved_entity.attributes[0].applied_traits[7].named_reference)
async def get_resolved_entity( corpus: 'CdmCorpusDefinition', input_entity: 'CdmEntityDefinition', resolution_options: List[str], add_res_opt_to_name: Optional[bool] = False ) -> 'CdmEntityDefinition': """Resolves an entity""" ro_hash_set = set() for i in range(len(resolution_options)): ro_hash_set.add(resolution_options[i]) resolved_entity_name = '' if add_res_opt_to_name: file_name_suffix = ProjectionTestUtils.get_resolution_option_name_suffix( resolution_options) resolved_entity_name = 'Resolved_{}{}'.format( input_entity.entity_name, file_name_suffix) else: resolved_entity_name = 'Resolved_{}'.format( input_entity.entity_name) ro = ResolveOptions( input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set)) resolved_folder = corpus.storage.fetch_root_folder('output') resolved_entity = await input_entity.create_resolved_entity_async( resolved_entity_name, ro, resolved_folder) return resolved_entity
async def test_resolve_test_corpus(self): self.assertTrue(os.path.exists(TestHelper.get_schema_docs_root())) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(TestHelper.get_schema_docs_root())) manifest = await corpus.fetch_object_async(TestHelper.cdm_standards_schema_path) # type: CdmManifestDefinition directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'}) all_resolved = await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher()) self.assertNotEqual(all_resolved, '')
def fetch_resolved_entity_references( self, res_opt: 'ResolveOptions') -> 'ResolvedEntityReferenceSet': # this whole resolved entity ref goo will go away when resolved documents are done. # for now, it breaks if structured att sets get made. from cdm.resolvedmodel import ResolvedEntityReferenceSet from cdm.utilities import AttributeResolutionDirectiveSet from .cdm_object import CdmObject was_previously_resolving = self.ctx.corpus._is_currently_resolving self.ctx.corpus._is_currently_resolving = True if not res_opt: res_opt = ResolveOptions( self, self.ctx.corpus.default_resolution_directives) res_opt = CdmObject._copy_resolve_options(res_opt) res_opt.directives = AttributeResolutionDirectiveSet( set(['normalized', 'referenceOnly'])) ctx = self.ctx ent_ref_set_cache = ctx.fetch_cache(self, res_opt, 'entRefSet') if not ent_ref_set_cache: ent_ref_set_cache = ResolvedEntityReferenceSet(res_opt) if not self._resolving_entity_references: self._resolving_entity_references = True # get from any base class and then 'fix' those to point here instead. ext_ref = self.extends_entity if ext_ref: ext_def = cast('CdmEntityDefinition', ext_ref.fetch_object_definition(res_opt)) if ext_def: inherited = ext_def.fetch_resolved_entity_references( res_opt) if inherited: for res in inherited.rer_set: res = res.copy() res.referencing.entity = self ent_ref_set_cache.rer_set.append(res) if self.attributes: for attribute in self.attributes: # if any refs come back from attributes, they don't know who we are, so they don't set the entity sub = attribute.fetch_resolved_entity_references( res_opt) if sub: for res in sub.rer_set: res.referencing.entity = self ent_ref_set_cache.add(sub) ctx.update_cache(self, res_opt, 'entRefSet', ent_ref_set_cache) self._resolving_entity_references = False self.ctx.corpus._is_currently_resolving = was_previously_resolving return ent_ref_set_cache
async def test_resolve_test_corpus(self): # only using cdm namespace, which is set to schema docs corpus = TestHelper.get_local_corpus('', '') manifest = await corpus.fetch_object_async( 'cdm:/standards.manifest.cdm.json') # type: CdmManifestDefinition directives = AttributeResolutionDirectiveSet( {'referenceOnly', 'normalized'}) all_resolved = await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher()) self.assertNotEqual(all_resolved, '')
async def get_resolved_entity(corpus: 'CdmCorpusDefinition', input_entity: 'CdmEntityDefinition', directives: List[str]) -> 'CdmEntityDefinition': """Resolves an entity""" ro_hash_set = set(directives) resolved_entity_name = 'Resolved_{}'.format(input_entity.entity_name) res_opt = ResolveOptions(input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set)) resolved_folder = corpus.storage.fetch_root_folder('output') resolved_entity = await input_entity.create_resolved_entity_async(resolved_entity_name, res_opt, resolved_folder) # type: CdmEntityDefinition return resolved_entity
async def test_resolve_symbol_reference(self): """Tests if a symbol imported with a moniker can be found as the last resource. When resolving symbolEntity with respect to wrtEntity, the symbol fromEntity should be found correctly.""" corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_resolve_symbol_reference') def callback(status_level: CdmStatusLevel, message: str): self.fail(message) corpus.set_event_callback(callback, CdmStatusLevel.WARNING) wrt_entity = await corpus.fetch_object_async('local:/wrtEntity.cdm.json/wrtEntity') # type: CdmEntityDefinition res_opt = ResolveOptions(wrt_entity, AttributeResolutionDirectiveSet()) await wrt_entity.create_resolved_entity_async('NewEntity', res_opt)
async def resolve_environment(self, test_name: str, environment: str) -> str: test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' print('reading source files') manifest = await corpus.fetch_object_async('local:/{}.manifest.cdm.json'.format(environment)) directives = AttributeResolutionDirectiveSet(set(['normalized', 'referenceOnly'])) return await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
async def test_resolved_attribute_limit(self): expected_log_codes = { CdmLogCode.ERR_REL_MAX_RESOLVED_ATTR_REACHED } corpus = TestHelper.get_local_corpus(self.tests_sub_path, 'test_resolved_attribute_limit', expected_codes=expected_log_codes) # type: CdmCorpusDefinition main_entity = await corpus.fetch_object_async('local:/mainEntity.cdm.json/mainEntity') # type: CdmEntityDefinition res_opt = ResolveOptions(wrt_doc=main_entity.in_document) # if attribute limit is reached, entity should be None res_opt._resolved_attribute_limit = 4 resEnt = await main_entity.create_resolved_entity_async('{}_zeroAtts'.format(main_entity.entity_name), res_opt) # type: CdmEntityDefinition self.assertIsNone(resEnt) # when the attribute limit is set to null, there should not be a limit on the possible number of attributes res_opt._resolved_attribute_limit = None ras = main_entity._fetch_resolved_attributes(res_opt) # type: ResolvedAttributeSet resEnt = await main_entity.create_resolved_entity_async('{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt) # there are 5 total attributes self.assertEqual(5, ras._resolved_attribute_count) self.assertEqual(5, len(ras._set)) self.assertEqual(3, len(main_entity.attributes)) # there are 2 attributes grouped in an entity attribute # and 2 attributes grouped in an attribute group self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members)) # using the default limit number res_opt = ResolveOptions(wrt_doc=main_entity.in_document) ras = main_entity._fetch_resolved_attributes(res_opt) resEnt = await main_entity.create_resolved_entity_async('{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt) # there are 5 total attributes self.assertEqual(5, ras._resolved_attribute_count) self.assertEqual(5, len(ras._set)) self.assertEqual(3, len(main_entity.attributes)) # there are 2 attributes grouped in an entity attribute # and 2 attributes grouped in an attribute group self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members)) res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'}) ras = main_entity._fetch_resolved_attributes(res_opt) resEnt = await main_entity.create_resolved_entity_async('{}_normalized_structured'.format(main_entity.entity_name), res_opt) # there are 5 total attributes self.assertEqual(5, ras._resolved_attribute_count) # the attribute count is different because one attribute is a group that contains two different attributes self.assertEqual(4, len(ras._set)) self.assertEqual(3, len(main_entity.attributes)) # again there are 2 attributes grouped in an entity attribute # and 2 attributes grouped in an attribute group self.assertEqual(2, len(main_entity.attributes[2].explicit_reference.members))
async def resolve_environment(self, environment: str, storage_adapter_root=TEST_ADAPTER_ROOT): corpus = CdmCorpusDefinition() adapter = LocalAdapter(root=storage_adapter_root) corpus.storage.mount('local', adapter) print('reading source files') manifest = await corpus.fetch_object_async( 'local:/{}.manifest.cdm.json'.format(environment)) directives = AttributeResolutionDirectiveSet( set(['normalized', 'referenceOnly'])) return await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
async def resolve_environment(tests_sub_path: str, test_name: str, manifest_name: str) -> str: """ Resolve the entities in the given manifest. :param tests_sub_path: Tests sub-folder name :param test_name: The name of the test. It is used to decide the path of input / output files. :param manifest_name: The name of the manifest to be used. :return: """ corpus = TestHelper.get_local_corpus(tests_sub_path, test_name) manifest = await corpus.fetch_object_async( 'local:/{}.manifest.cdm.json'.format(manifest_name)) directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) return await list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
async def _get_resolved_entity( corpus: 'CdmCorpusDefinition', input_entity: 'CdmEntityDefinition', resolution_options: List[str]) -> 'CdmEntityDefinition': """A function to resolve an entity""" ro_hash_set = set() for i in range(len(resolution_options)): ro_hash_set.add(resolution_options[i]) resolved_entity_name = 'Resolved_{}'.format(input_entity.entity_name) ro = ResolveOptions( input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set)) resolved_folder = corpus.storage.fetch_root_folder('output') resolved_entity = await input_entity.create_resolved_entity_async( resolved_entity_name, ro, resolved_folder) return resolved_entity
async def _save_resolved( self, corpus: 'CdmCorpusDefinition', manifest: 'CdmManifestDefinition', test_name: str, input_entity: 'CdmEntityDefinition', resolution_options: List[str] ) -> 'CdmEntityDefinition': ro_hash_set = set() for i in range(len(resolution_options)): ro_hash_set.add(resolution_options[i]) file_name_suffix = TestUtils.get_resolution_option_name_suffix(resolution_options) resolved_entity_name = 'Resolved_{}{}.cdm.json'.format(input_entity.entity_name, file_name_suffix) ro = ResolveOptions(input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set)) resolved_folder = corpus.storage.fetch_root_folder('output') resolved_entity = await input_entity.create_resolved_entity_async(resolved_entity_name, ro, resolved_folder) return resolved_entity
async def resolve_environment(tests_sub_path: str, test_name: str, manifest_name: str) -> str: """ Resolve the entities in the given manifest. :param tests_sub_path: Tests sub-folder name :param test_name: The name of the test. It is used to decide the path of input / output files. :param manifest_name: The name of the manifest to be used. :return: """ test_input_path = TestHelper.get_input_folder_path(tests_sub_path, test_name) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' manifest = await corpus.fetch_object_async( 'local:/{}.manifest.cdm.json'.format(manifest_name)) directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) return await list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
async def test_missing_condition_in_json(self): """ Test case scenario for Bug #25 from the projections internal bug bash Reference Link: https:#commondatamodel.visualstudio.com/CDM/_workitems/edit/25 """ test_name = 'test_missing_condition_in_json' corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) def callback(level, message): self.fail(message) corpus.set_event_callback(callback, CdmStatusLevel.WARNING) manifest = await corpus.fetch_object_async('default.manifest.cdm.json') entity_name = 'SalesNestedFK' entity = await corpus.fetch_object_async('local:/{}.cdm.json/{}'.format(entity_name, entity_name), manifest) self.assertIsNotNone(entity) res_opt = ResolveOptions(entity.in_document) # where, res_opts_combinations[1] == 'referenceOnly' res_opt.directives = AttributeResolutionDirectiveSet(self.res_opts_combinations[1]) resolved_folder = corpus.storage.fetch_root_folder('output') was_info_message_received = {} was_info_message_received['infoMessagedReceived'] = False def callback_2(level, message): if 'CdmProjection | Optional expression missing. Implicit expression will automatically apply. | _construct_projection_context' in message: was_info_message_received['infoMessagedReceived'] = True corpus.set_event_callback(callback_2, CdmStatusLevel.INFO) resolved_entity = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity_name), res_opt, resolved_folder) self.assertIsNotNone(resolved_entity) self.assertTrue(was_info_message_received['infoMessagedReceived'])
async def logical_manipulation_using_projections(): '''This sample demonstrates how to model a set of common scenarios using projections. The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model. Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'. A single logical definition can be resolved into multiple physical layouts. The directives are used to instruct the ObjectModel about how it should to resolve the logical definition provided. To achieve this, we define projections that run conditionally, depending on the directives provided when calling create_resolved_entity_async. To get an overview of the projections feature as well as all of the supported operations refer to the link below. https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview ''' # Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths corpus = CdmCorpusDefinition() print('Configure storage adapters') # Configure storage adapters to point at the target local manifest location and at the fake public standards path_from_exe_to_example_root = '../../' corpus.storage.mount( 'local', LocalAdapter(path_from_exe_to_example_root + '8-logical-manipulation-using-projections/sample-data')) corpus.storage.default_namespace = 'local' # local is our default. so any paths that start out navigating without a device tag will assume local # Fake cdm, normaly use the CDM Standards adapter # Mount it as the 'cdm' device, not the default so must use 'cdm:/folder' to get there corpus.storage.mount( 'cdm', LocalAdapter(path_from_exe_to_example_root + 'example-public-standards')) print('Create logical entity definition.') logical_folder = await corpus.fetch_object_async( 'local:/') # type: CdmFolderDefinition logical_doc = logical_folder.documents.append('Person.cdm.json') logical_doc.imports.append('Address.cdm.json') entity = logical_doc.definitions.append( 'Person') # type: CdmEntityDefinition # Add 'name' data typed attribute. name_attr = entity.attributes.append( 'name') # type: CdmTypeAttributeDefinition name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'age' data typed attribute. age_attr = entity.attributes.append( 'age') # type: CdmTypeAttributeDefinition age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'address' entity typed attribute. entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address') entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True) apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute') apply_default_behavior(entity_attr, 'addressFK', 'address') entity.attributes.append(entity_attr) # Add 'email' data typed attribute. email_attr = entity.attributes.append( 'email') # type: CdmTypeAttributeDefinition email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Save the logical definition of Person. await entity.in_document.save_as_async('Person.cdm.json') print('Get \'resolved\' folder where the resolved entities will be saved.') resolved_folder = await corpus.fetch_object_async( 'local:/resolved/') # type: CdmFolderDefinition res_opt = ResolveOptions(entity) # To get more information about directives and their meaning refer to # https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes # We will start by resolving this entity with the 'normalized' direcitve. # This directive will be used on this and the next two examples so we can analize the resolved entity # without the array expansion. print('Resolving logical entity with normalized directive.') res_opt.directives = AttributeResolutionDirectiveSet({'normalized'}) res_normalized_entity = await entity.create_resolved_entity_async( f'normalized_{entity.entity_name}', res_opt, resolved_folder) await res_normalized_entity.in_document.save_as_async( f'{res_normalized_entity.entity_name}.cdm.json') # Another common scenario is to resolve an entity using the 'referenceOnly' directive. # This directives is used to replace the relationships with a foreign key. print('Resolving logical entity with referenceOnly directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) res_reference_only_entity = await entity.create_resolved_entity_async( f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder) await res_reference_only_entity.in_document.save_as_async( f'{res_reference_only_entity.entity_name}.cdm.json') # When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that # a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the # entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group. print('Resolving logical entity with structured directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'structured'}) res_structured_entity = await entity.create_resolved_entity_async( f'structured_{entity.entity_name}', res_opt, resolved_folder) await res_structured_entity.in_document.save_as_async( f'{res_structured_entity.entity_name}.cdm.json') # Now let us remove the 'normalized' directive so the array expansion operation can run. print('Resolving logical entity without directives (array expansion).') res_opt.directives = AttributeResolutionDirectiveSet({}) res_array_entity = await entity.create_resolved_entity_async( f'array_expansion_{entity.entity_name}', res_opt, resolved_folder) await res_array_entity.in_document.save_as_async( f'{res_array_entity.entity_name}.cdm.json')
def _construct_resolved_attributes( self, res_opt: 'ResolveOptions', under: Optional['CdmAttributeContext'] = None ) -> 'ResolvedAttributeSetBuilder': from cdm.resolvedmodel import AttributeResolutionContext, ResolvedAttribute, ResolvedAttributeSetBuilder, ResolvedTrait from cdm.utilities import AttributeContextParameters, AttributeResolutionDirectiveSet from .cdm_object import CdmObject rasb = ResolvedAttributeSetBuilder() ctx_ent = self.entity under_att = under acp_ent = None if not res_opt._in_circular_reference: arc = self._fetch_att_res_context(res_opt) # complete cheating but is faster. # this purpose will remove all of the attributes that get collected here, so dumb and slow to go get them rel_info = self._get_relationship_info(arc.res_opt, arc) res_opt._depth_info.current_depth = rel_info.next_depth res_opt._depth_info.max_depth_exceeded = rel_info.max_depth_exceeded res_opt._depth_info.max_depth = rel_info.max_depth ctx_ent_obj_def = ctx_ent.fetch_object_definition(res_opt) if ctx_ent_obj_def and ctx_ent_obj_def.object_type == CdmObjectType.PROJECTION_DEF: # A Projection # if the max depth is exceeded it should not try to execute the projection if not res_opt._depth_info.max_depth_exceeded: proj_directive = ProjectionDirective( res_opt, self, ctx_ent) proj_def = ctx_ent_obj_def proj_ctx = proj_def._construct_projection_context( proj_directive, under) rasb._resolved_attribute_set = proj_def._extract_resolved_attributes( proj_ctx, under_att) else: # An Entity Reference if under_att: # make a context for this attribute that holds the attributes that come up from the entity acp_ent = AttributeContextParameters( under=under_att, type=CdmAttributeContextType.ENTITY, name=ctx_ent.fetch_object_definition_name(), regarding=ctx_ent, include_traits=True) if rel_info.is_by_ref: # make the entity context that a real recursion would have give us if under: under = rasb._resolved_attribute_set.create_attribute_context( res_opt, acp_ent) # if selecting from one of many attributes, then make a context for each one if under and rel_info.selects_one: # the right way to do this is to get a resolved entity from the embedded entity and then # look through the attribute context hierarchy for non-nested entityReferenceAsAttribute nodes # that seems like a disaster waiting to happen given endless looping, etc. # for now, just insist that only the top level entity attributes declared in the ref entity will work ent_pick_from = self.entity.fetch_object_definition( res_opt) atts_pick = ent_pick_from.attributes if ent_pick_from and atts_pick: for attribute in atts_pick: if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF: # a table within a table. as expected with a selects_one attribute # since this is by ref, we won't get the atts from the table, but we do need the traits that hold the key # these are the same contexts that would get created if we recursed # first this attribute acp_ent_att = AttributeContextParameters( under=under, type=CdmAttributeContextType. ATTRIBUTE_DEFINITION, name=attribute. fetch_object_definition_name(), regarding=attribute, include_traits=True) pick_under = rasb._resolved_attribute_set.create_attribute_context( res_opt, acp_ent_att) # and the entity under that attribute pick_ent = attribute.entity pick_ent_type = CdmAttributeContextType.PROJECTION if pick_ent.fetch_object_definition( res_opt ).object_type == CdmObjectType.PROJECTION_DEF else CdmAttributeContextType.ENTITY acp_ent_att_ent = AttributeContextParameters( under=pick_under, type=pick_ent_type, name=pick_ent. fetch_object_definition_name(), regarding=pick_ent, include_traits=True) rasb._resolved_attribute_set.create_attribute_context( res_opt, acp_ent_att_ent) # if we got here because of the max depth, need to impose the directives to make the trait work as expected if rel_info.max_depth_exceeded: if not arc.res_opt.directives: arc.res_opt.directives = AttributeResolutionDirectiveSet( ) arc.res_opt.directives.add('referenceOnly') else: res_link = res_opt.copy() res_link._symbol_ref_set = res_opt._symbol_ref_set rasb.merge_attributes( self.entity._fetch_resolved_attributes( res_link, acp_ent)) # need to pass up max_depth_exceeded if it was hit if res_link._depth_info.max_depth_exceeded: res_opt._depth_info = res_link._depth_info._copy() # from the traits of purpose and applied here, see if new attributes get generated rasb._resolved_attribute_set.attribute_context = under_att rasb.apply_traits(arc) rasb.generate_applier_attributes( arc, True) # True = apply the prepared traits to new atts # this may have added symbols to the dependencies, so merge them res_opt._symbol_ref_set._merge(arc.res_opt._symbol_ref_set) # use the traits for linked entity identifiers to record the actual foreign key links if rasb._resolved_attribute_set and rasb._resolved_attribute_set._set and rel_info.is_by_ref: for att in rasb._resolved_attribute_set._set: reqd_trait = att.resolved_traits.find( res_opt, 'is.linkedEntity.identifier') if not reqd_trait: continue if not reqd_trait.parameter_values: logger.warning( self._ctx, self._TAG, CdmEntityAttributeDefinition. _construct_resolved_attributes.__name__, self.at_corpus_path, CdmLogCode. WARN_IDENTIFIER_ARGUMENTS_NOT_SUPPORTED) continue ent_references = [] att_references = [] def add_entity_reference( entity_ref: 'CdmEntityReference', namespace: str): ent_def = entity_ref.fetch_object_definition( res_opt) required_trait = entity_ref._fetch_resolved_traits( res_opt).find(res_opt, 'is.identifiedBy') if required_trait and ent_def: att_ref = required_trait.parameter_values.fetch_parameter_value( 'attribute').value att_name = att_ref.named_reference.split( '/')[-1] absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.at_corpus_path, ent_def.in_document) ent_references.append(absolute_ent_path) att_references.append(att_name) if rel_info.selects_one: ent_pick_from = self.entity.fetch_object_definition( res_opt) atts_pick = ent_pick_from.attributes if ent_pick_from else None if atts_pick: for attribute in atts_pick: if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF: add_entity_reference( attribute.entity, self.in_document.namespace) else: add_entity_reference(self.entity, self.in_document.namespace) c_ent = self.ctx.corpus.make_object( CdmObjectType.CONSTANT_ENTITY_DEF) c_ent.entity_shape = self.ctx.corpus.make_ref( CdmObjectType.ENTITY_REF, 'entityGroupSet', True) c_ent.constant_values = [[ entity_ref, att_references[idx] ] for idx, entity_ref in enumerate(ent_references)] param = self.ctx.corpus.make_ref( CdmObjectType.ENTITY_REF, c_ent, False) reqd_trait.parameter_values.update_parameter_value( res_opt, 'entityReferences', param) # a 'structured' directive wants to keep all entity attributes together in a group if arc and arc.res_opt.directives and arc.res_opt.directives.has( 'structured'): # make one resolved attribute with a name from this entityAttribute that contains the set # of atts we just put together. ra_sub = ResolvedAttribute(arc.traits_to_apply.res_opt, rasb._resolved_attribute_set, self.name, under_att) if rel_info.is_array: # put a resolved trait on this att group # hope I never need to do this again and then need to make a function for this tr = self.ctx.corpus.make_object( CdmObjectType.TRAIT_REF, 'is.linkedEntity.array', True) t = tr.fetch_object_definition(res_opt) rt = ResolvedTrait(t, None, [], []) ra_sub.resolved_traits = ra_sub.resolved_traits.merge( rt, True) depth = rasb._resolved_attribute_set._depth_traveled rasb = ResolvedAttributeSetBuilder() rasb._resolved_attribute_set.attribute_context = ra_sub.att_ctx # this got set to null with the new builder rasb.own_one(ra_sub) rasb._resolved_attribute_set._depth_traveled = depth # how ever they got here, mark every attribute from this entity attribute as now being 'owned' by this entityAtt rasb._resolved_attribute_set._set_attribute_ownership(self.name) rasb._resolved_attribute_set._depth_traveled += 1 return rasb
async def run_test_with_values( self, test_name: str, source_entity_name: str, expected_context_default: 'AttributeContextExpectedValue', expected_context_normalized: 'AttributeContextExpectedValue', expected_context_reference_only: 'AttributeContextExpectedValue', expected_context_structured: 'AttributeContextExpectedValue', expected_context_normalized_structured: 'AttributeContextExpectedValue', expected_context_reference_only_normalized: 'AttributeContextExpectedValue', expected_context_reference_only_structured: 'AttributeContextExpectedValue', expected_context_reference_only_normalized_structured: 'AttributeContextExpectedValue', expected_default: 'List[AttributeExpectedValue]', expected_normalized: 'List[AttributeExpectedValue]', expected_reference_only: 'List[AttributeExpectedValue]', expected_structured: 'List[AttributeExpectedValue]', expected_normalized_structured: 'List[AttributeExpectedValue]', expected_reference_only_normalized: 'List[AttributeExpectedValue]', expected_reference_only_structured: 'List[AttributeExpectedValue]', expected_reference_only_normalized_structured: 'List[AttributeExpectedValue]' ) -> None: """This method runs the tests with a set expected attributes & attribute context values and validated the actual result.""" try: test_input_path = TestHelper.get_input_folder_path( self.tests_subpath, test_name) test_actual_path = TestHelper.get_actual_output_folder_path( self.tests_subpath, test_name) test_expected_path = TestHelper.get_expected_output_folder_path( self.tests_subpath, test_name) corpus_path = test_input_path[0:(len(test_input_path) - len('/Input'))] test_actual_path = os.path.abspath(test_actual_path) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(corpus_path)) corpus.storage.mount('cdm', LocalAdapter(self.schema_docs_path)) corpus.storage.default_namespace = 'local' out_folder_path = corpus.storage.adapter_path_to_corpus_path( test_actual_path) + '/' # interesting bug out_folder = await corpus.fetch_object_async( out_folder_path) # type: CdmFolderDefinition src_entity_def = await corpus.fetch_object_async( 'local:/Input/{}.cdm.json/{}'.format(source_entity_name, source_entity_name) ) # type: CdmEntityDefinition self.assertTrue(src_entity_def is not None) res_opt = ResolveOptions(wrt_doc=src_entity_def.in_document) resolved_entity_def = None # type: CdmEntityDefinition output_entity_name = '' output_entity_file_name = '' entity_file_name = '' if expected_context_default and expected_default: entity_file_name = 'd' res_opt.directives = AttributeResolutionDirectiveSet(set()) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_normalized and expected_normalized: entity_file_name = 'n' res_opt.directives = AttributeResolutionDirectiveSet( {'normalized'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_reference_only and expected_reference_only: entity_file_name = 'ro' res_opt.directives = AttributeResolutionDirectiveSet( {'referenceOnly'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_structured and expected_structured: entity_file_name = 's' res_opt.directives = AttributeResolutionDirectiveSet( {'structured'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_normalized_structured and expected_normalized_structured: entity_file_name = 'n_s' res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'structured'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_reference_only_normalized and expected_reference_only_normalized: entity_file_name = 'ro_n' res_opt.directives = AttributeResolutionDirectiveSet( {'referenceOnly', 'normalized'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_reference_only_structured and expected_reference_only_structured: entity_file_name = 'ro_s' res_opt.directives = AttributeResolutionDirectiveSet( {'referenceOnly', 'structured'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) if expected_context_reference_only_normalized_structured and expected_reference_only_normalized_structured: entity_file_name = 'ro_n_s' res_opt.directives = AttributeResolutionDirectiveSet( {'referenceOnly', 'normalized', 'structured'}) output_entity_name = '{}_R_{}'.format(source_entity_name, entity_file_name) output_entity_file_name = '{}.cdm.json'.format( output_entity_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async( output_entity_name, res_opt, out_folder) await self.save_actual_entity_and_validate_with_expected( os.path.join(test_expected_path, output_entity_file_name), resolved_entity_def) except Exception as e: self.fail(e)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an ArrayExpansion operation and a condition using the object model""" corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an ArrayExpansion operation array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF) array_expansion_op.start_ordinal = 1 array_expansion_op.end_ordinal = 2 projection.operations.append(array_expansion_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF) projection2.source = projection_entity_ref # Create a RenameAttributes operation rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF) rename_attrs_op.rename_format = '{m}{o}' projection2.operations.append(rename_attrs_op) # Create an entity reference to hold this projection projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref2.explicit_reference = projection2 # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref2 entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the projections # Original set of attributes: ["id", "name", "value", "date"] # Expand 1...2, renameFormat = {m}{o} self.assertEqual(8, len(resolved_entity_with_reference_only.attributes)) self.assertEqual('id1', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual('name1', resolved_entity_with_reference_only.attributes[1].name) self.assertEqual('value1', resolved_entity_with_reference_only.attributes[2].name) self.assertEqual('date1', resolved_entity_with_reference_only.attributes[3].name) self.assertEqual('id2', resolved_entity_with_reference_only.attributes[4].name) self.assertEqual('name2', resolved_entity_with_reference_only.attributes[5].name) self.assertEqual('value2', resolved_entity_with_reference_only.attributes[6].name) self.assertEqual('date2', resolved_entity_with_reference_only.attributes[7].name) # Now resolve the entity with the default directives res_opt.directives = AttributeResolutionDirectiveSet(set([])) resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the projections # Original set of attributes: ["id", "name", "value", "date"] # Expand 1...2, renameFormat = {m}{o} self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def run_test(self, test_name: str, source_entity_name: str) -> None: test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name) test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name) test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('localInput', LocalAdapter(test_input_path)) corpus.storage.mount('localExpectedOutput', LocalAdapter(test_expected_output_path)) corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path)) corpus.storage.mount('cdm', LocalAdapter(TestHelper.get_schema_docs_root())) corpus.storage.default_namespace = 'localInput' src_entity_def = await corpus.fetch_object_async('localInput:/{0}.cdm.json/{0}'.format(source_entity_name)) # type: CdmEntityDefinition self.assertIsNotNone(src_entity_def) res_opt = ResolveOptions(src_entity_def.in_document, directives=AttributeResolutionDirectiveSet(set())) actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition resolved_entity_def = None output_entity_file_name = '' entity_file_name = '' entity_file_name = 'default' res_opt.directives = AttributeResolutionDirectiveSet(set()) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'normalized' res_opt.directives = AttributeResolutionDirectiveSet({'normalized'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'structured' res_opt.directives = AttributeResolutionDirectiveSet({'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly_normalized' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly_structured' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'normalized_structured' res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly_normalized_structured' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized', 'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an ExcludeAttributes operation and a condition using the object model""" corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount( 'local', LocalAdapter( TestHelper.get_actual_output_folder_path( self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an ExcludeAttributes operation exclude_attrs_op = corpus.make_object( CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF) exclude_attrs_op.exclude_attributes.append('id') exclude_attrs_op.exclude_attributes.append('date') projection.operations.append(exclude_attrs_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object( CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet( set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the ExcludeAttributes operation # Original set of attributes: ['id', 'name', 'value', 'date'] # Excluded attributes: ['id', 'date'] self.assertEqual(2, len(resolved_entity_with_reference_only.attributes)) self.assertEqual( 'name', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual( 'value', resolved_entity_with_reference_only.attributes[1].name) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet( set(['structured'])) resolved_entity_with_structured = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the ExcludeAttributes operation # Original set of attributes: ['id', 'name', 'value', 'date'] # Excluded attributes: none, condition was false self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an AddArtifactAttribute operation and a condition using the object model""" test_name = 'test_conditional_proj_using_object_model' corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, test_name) local_root = corpus.storage.fetch_root_folder('local') corpus.storage.mount('traitGroup', LocalAdapter(self.trait_group_file_path)) # Create an entity. entity = ProjectionTestUtils.create_entity(corpus, local_root) entity.in_document.imports.append('traitGroup:/TraitGroup.cdm.json') # Create a projection with a condition that states the operation should only execute when the resolution directive is 'structured'. projection = ProjectionTestUtils.create_projection( corpus, local_root) # type: CdmProjection projection.condition = 'structured==true' projection.run_sequentially = True # Create an AlterTraits operation alter_traits_op_1 = corpus.make_object( CdmObjectType.OPERATION_ALTER_TRAITS_DEF ) # type: CdmOperationAlterTraits alter_traits_op_1.traits_to_add = CdmCollection( corpus.ctx, alter_traits_op_1, CdmObjectType.TRAIT_REF) alter_traits_op_1.traits_to_add.append( corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG100", True)) alter_traits_op_1.traits_to_add.append( corpus.make_ref(CdmObjectType.TRAIT_GROUP_REF, "JobTitleBase", True)) alter_traits_op_1.traits_to_remove = CdmCollection( corpus.ctx, alter_traits_op_1, CdmObjectType.TRAIT_REF) alter_traits_op_1.traits_to_remove.append( corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG300", True)) projection.operations.append(alter_traits_op_1) alter_traits_op_2 = corpus.make_object( CdmObjectType.OPERATION_ALTER_TRAITS_DEF ) # type: CdmOperationAlterTraits trait_g4 = corpus.make_ref(CdmObjectType.TRAIT_REF, "means.TraitG4", True) # type: CdmTraitReference trait_g4.arguments.append('precision', '5') trait_g4.arguments.append('scale', '15') alter_traits_op_2.traits_to_add = CdmCollection( corpus.ctx, alter_traits_op_2, CdmObjectType.TRAIT_REF) alter_traits_op_2.traits_to_add.append(trait_g4) alter_traits_op_2.apply_to = [] alter_traits_op_2.apply_to.append('name') projection.operations.append(alter_traits_op_2) # Create an entity reference to hold this projection. projection_entity_ref = corpus.make_object( CdmObjectType.ENTITY_REF, None) # type: CdmEntityReference projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity. entity_attribute = corpus.make_object( CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') # type: CdmEntityAttributeDefinition entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive. res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'}) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Original set of attributes: ['id', 'name', 'value', 'date'] # Condition not met, no traits are added self.assertEqual(4, len(resolved_entity_with_reference_only.attributes)) self.validate_trait(resolved_entity_with_reference_only.attributes[0], 'id', False, True) self.validate_trait(resolved_entity_with_reference_only.attributes[1], 'name', False, True) self.validate_trait(resolved_entity_with_reference_only.attributes[2], 'value', False, True) self.validate_trait(resolved_entity_with_reference_only.attributes[3], 'date', False, True) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet({'structured'}) resolved_entity_with_structured = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Original set of attributes: ['id', 'name', 'value', 'date'] # Condition met, new traits are added self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.validate_trait(resolved_entity_with_structured.attributes[0], 'id') self.validate_trait(resolved_entity_with_structured.attributes[1], 'name', True) self.validate_trait(resolved_entity_with_structured.attributes[2], 'value') self.validate_trait(resolved_entity_with_structured.attributes[3], 'date')
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition', options: 'CopyOptions', new_name: str, save_referenced: bool) -> bool: """a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work because the corpus knows about persistence types and about the storage adapters if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or to a different name, then the source object is still 'dirty' an option will cause us to also save any linked documents.""" # find out if the storage adapter is able to write. namespace = doc.namespace if namespace is None: namespace = self._corpus.storage.default_namespace adapter = self._corpus.storage.fetch_adapter(namespace) if adapter is None: logger.error( self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_ADAPTER_NOT_FOUND_FOR_NAMESPACE, namespace) return False if not adapter.can_write(): logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_ADAPTER_WRITE_FAILURE, namespace) return False if not new_name: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME) return None # what kind of document is requested? # check file extensions using a case-insensitive ordinal string comparison. persistence_type = self.MODEL_JSON if new_name.lower().endswith( self.MODEL_JSON_EXTENSION) else self.CDM_FOLDER if persistence_type == self.MODEL_JSON and new_name.lower( ) != self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE, new_name, self.MODEL_JSON_EXTENSION) return False # save the object into a json blob res_opt = { 'wrt_doc': doc, 'directives': AttributeResolutionDirectiveSet() } persisted_doc = None try: if new_name.lower().endswith( PersistenceLayer.MODEL_JSON_EXTENSION) or new_name.lower( ).endswith( PersistenceLayer.MANIFEST_EXTENSION) or new_name.lower( ).endswith(PersistenceLayer.FOLIO_EXTENSION): if persistence_type == self.CDM_FOLDER: from cdm.persistence.cdmfolder import ManifestPersistence persisted_doc = ManifestPersistence.to_data( doc, res_opt, options) else: if new_name != self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE, new_name) return False from cdm.persistence.modeljson import ManifestPersistence persisted_doc = await ManifestPersistence.to_data( doc, res_opt, options) elif new_name.lower().endswith(PersistenceLayer.CDM_EXTENSION): from cdm.persistence.cdmfolder import DocumentPersistence persisted_doc = DocumentPersistence.to_data( doc, res_opt, options) else: # Could not find a registered persistence class to handle this document type. logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING, new_name) return False except Exception as e: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FILE_PERSIST_ERROR, new_name, e) return False if not persisted_doc: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FILE_PERSIST_FAILED, new_name) return False # turn the name into a path new_path = '{}{}'.format(doc.folder_path, new_name) new_path = self._ctx.corpus.storage.create_absolute_corpus_path( new_path, doc) if new_path.startswith(namespace + ':'): new_path = new_path[len(namespace) + 1:] # ask the adapter to make it happen try: content = persisted_doc.encode() await adapter.write_async(new_path, content) doc._file_system_modified_time = await adapter.compute_last_modified_time_async( new_path) # Write the adapter's config. if options._is_top_level_document: await self._corpus.storage.save_adapters_config_async( '/config.json', adapter) # The next document won't be top level, so reset the flag. options._is_top_level_document = False except Exception as e: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FILE_WRITE_FAILURE, new_name, e) return False # if we also want to save referenced docs, then it depends on what kind of thing just got saved # if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things # definition will save imports, manifests will save imports, schemas, sub manifests if save_referenced and persistence_type == self.CDM_FOLDER: saved_linked_docs = await doc._save_linked_documents_async(options) if not saved_linked_docs: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_SAVE_LINK_DOCS, new_name) return False return True
async def create_resolved_manifest_async( self, new_manifest_name: str, new_entity_document_name_format: str ) -> Optional['CdmManifestDefinition']: """Creates a resolved copy of the manifest. new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities. The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any sub-folders described by the pattern should exist in the corpus prior to calling this function. """ if self.entities is None: return None if new_entity_document_name_format is None: new_entity_document_name_format = '{f}resolved/{n}.cdm.json' elif new_entity_document_name_format == '': # for back compat new_entity_document_name_format = '{n}.cdm.json' elif '{n}' not in new_entity_document_name_format: # for back compat new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json' source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.at_corpus_path, self) source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.folder.at_corpus_path, self) resolved_manifest_path_split = new_manifest_name.rfind('/') + 1 resolved_manifest_folder = None if resolved_manifest_path_split > 0: resolved_manifest_path = new_manifest_name[ 0:resolved_manifest_path_split] new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( resolved_manifest_path, self) resolved_manifest_folder = await self.ctx.corpus.fetch_object_async( new_folder_path) # type: CdmFolderDefinition if resolved_manifest_folder is None: self.ctx.logger.error( 'New folder for manifest not found {}'.format( new_folder_path), 'create_resolved_manifest_async') return None new_manifest_name = new_manifest_name[ resolved_manifest_path_split:] else: resolved_manifest_folder = self.owner self.ctx.logger.debug( 'resolving manifest {}'.format(source_manifest_path), 'create_resolved_manifest_async') # using the references present in the resolved entities, get an entity # create an imports doc with all the necessary resolved entity references and then resolve it resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name) # add the new document to the folder if resolved_manifest_folder.documents.append( resolved_manifest) is None: # when would this happen? return None # mapping from entity path to resolved entity path for translating relationhsip paths res_ent_map = {} # type: Dict[str, str] for entity in self.entities: ent_def = await self._get_entity_from_reference(entity, self) if not ent_def: self.ctx.logger.error('Unable to get entity from reference') return None # get the path from this manifest to the source entity. this will be the {f} replacement value source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.in_document.folder.at_corpus_path, self) f = '' if source_entity_full_path.startswith(source_manifest_folder_path): f = source_entity_full_path[len(source_manifest_folder_path):] new_document_full_path = new_entity_document_name_format.replace( '{n}', ent_def.entity_name).replace('{f}', f) new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( new_document_full_path, self) new_document_path_split = new_document_full_path.rfind('/') + 1 new_document_path = new_document_full_path[ 0:new_document_path_split] new_document_name = new_document_full_path[ new_document_path_split:] # make sure the new folder exists folder = await self.ctx.corpus.fetch_object_async( new_document_path) # type: CdmFolderDefinition if not folder: self.ctx.logger.error( 'New folder not found {}'.format(new_document_path)) return None # next create the resolved entity. res_opt = ResolveOptions() res_opt.wrt_doc = ent_def.in_document res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) self.ctx.logger.debug( ' resolving entity {} to document {}'.format( source_entity_full_path, new_document_full_path)) resolved_entity = await ent_def.create_resolved_entity_async( ent_def.entity_name, res_opt, folder, new_document_name) if not resolved_entity: # fail all resolution, if any one entity resolution fails return None result = entity.copy(res_opt) if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path( resolved_entity.at_corpus_path, resolved_manifest) result.entity_path = relative_entity_path or result.at_corpus_path resolved_manifest.entities.append(result) # absolute path is needed for generating relationships absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path( result.entity_path, resolved_manifest) res_ent_map[self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.at_corpus_path, ent_def.in_document)] = absolute_ent_path self.ctx.logger.debug(' calculating relationships') # Calculate the entity graph for just this manifest. await self.ctx.corpus._calculate_entity_graph_async( resolved_manifest, res_ent_map) # Stick results into the relationships list for the manifest. await resolved_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) # needed until Matt's changes with collections where I can propigate resolved_manifest._is_dirty = True return resolved_manifest
def _construct_resolved_attributes( self, res_opt: 'ResolveOptions', under: Optional['CdmAttributeContext'] = None ) -> 'ResolvedAttributeSetBuilder': from cdm.resolvedmodel import AttributeResolutionContext, ResolvedAttribute, ResolvedAttributeSetBuilder, ResolvedTrait from cdm.utilities import AttributeContextParameters, AttributeResolutionDirectiveSet from .cdm_attribute_resolution_guidance_def import CdmAttributeResolutionGuidanceDefinition from .cdm_object import CdmObject rasb = ResolvedAttributeSetBuilder() ctx_ent = self.entity under_att = under acp_ent = None if under_att: # make a context for this attribute that holds the attributes that come up from the entity acp_ent = AttributeContextParameters( under=under_att, type=CdmAttributeContextType.ENTITY, name=ctx_ent.fetch_object_definition_name(), regarding=ctx_ent, include_traits=True) rts_this_att = self._fetch_resolved_traits(res_opt) # this context object holds all of the info about what needs to happen to resolve these attributes. # make a copy and add defaults if missing res_guide_with_default = None if self.resolution_guidance is not None: res_guide_with_default = self.resolution_guidance.copy(res_opt) else: res_guide_with_default = CdmAttributeResolutionGuidanceDefinition( self.ctx) res_guide_with_default._update_attribute_defaults(self.name) arc = AttributeResolutionContext(res_opt, res_guide_with_default, rts_this_att) # complete cheating but is faster. # this purpose will remove all of the attributes that get collected here, so dumb and slow to go get them rel_info = self._get_relationship_info(arc.res_opt, arc) if rel_info.is_by_ref: # make the entity context that a real recursion would have give us if under: under = rasb.ras.create_attribute_context(res_opt, acp_ent) # if selecting from one of many attributes, then make a context for each one if under and rel_info.selects_one: # the right way to do this is to get a resolved entity from the embedded entity and then # look through the attribute context hierarchy for non-nested entityReferenceAsAttribute nodes # that seems like a disaster waiting to happen given endless looping, etc. # for now, just insist that only the top level entity attributes declared in the ref entity will work ent_pick_from = self.entity.fetch_object_definition(res_opt) atts_pick = ent_pick_from.attributes if ent_pick_from and atts_pick: for attribute in atts_pick: if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF: # a table within a table. as expected with a selects_one attribute # since this is by ref, we won't get the atts from the table, but we do need the traits that hold the key # these are the same contexts that would get created if we recursed # first this attribute acp_ent_att = AttributeContextParameters( under=under, type=CdmAttributeContextType. ATTRIBUTE_DEFINITION, name=attribute.fetch_object_definition_name(), regarding=attribute, include_traits=True) pick_under = rasb.ras.create_attribute_context( res_opt, acp_ent_att) # and the entity under that attribute pick_ent = attribute.entity acp_ent_att_ent = AttributeContextParameters( under=pick_under, type=CdmAttributeContextType.ENTITY, name=pick_ent.fetch_object_definition_name(), regarding=pick_ent, include_traits=True) rasb.ras.create_attribute_context( res_opt, acp_ent_att_ent) # if we got here because of the max depth, need to impose the directives to make the trait work as expected if rel_info.max_depth_exceeded: if not arc.res_opt.directives: arc.res_opt.directives = AttributeResolutionDirectiveSet() arc.res_opt.directives.add('referenceOnly') else: res_link = CdmObject._copy_resolve_options(res_opt) res_link.symbol_ref_set = res_opt.symbol_ref_set res_link._relationship_depth = rel_info.next_depth rasb.merge_attributes( self.entity._fetch_resolved_attributes(res_link, acp_ent)) # from the traits of purpose and applied here, see if new attributes get generated rasb.ras.attribute_context = under_att rasb.apply_traits(arc) rasb.generate_applier_attributes( arc, True) # True = apply the prepared traits to new atts # this may have added symbols to the dependencies, so merge them res_opt.symbol_ref_set.merge(arc.res_opt.symbol_ref_set) # use the traits for linked entity identifiers to record the actual foreign key links if rasb.ras and rasb.ras.set and rel_info.is_by_ref: for att in rasb.ras.set: reqd_trait = att.resolved_traits.find( res_opt, 'is.linkedEntity.identifier') if not reqd_trait: continue if not reqd_trait.parameter_values: logger.warning( self._TAG, self.ctx, 'is.linkedEntity.identifier does not support arguments' ) continue ent_references = [] att_references = [] def add_entity_reference(entity_ref: 'CdmEntityReference', namespace: str): ent_def = entity_ref.fetch_object_definition(res_opt) required_trait = entity_ref._fetch_resolved_traits( res_opt).find(res_opt, 'is.identifiedBy') if required_trait and ent_def: att_ref = required_trait.parameter_values.fetch_parameter_value( 'attribute').value att_name = att_ref.named_reference.split('/')[-1] # path should be absolute and without a namespace relative_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.at_corpus_path, ent_def.in_document) if relative_ent_path.startswith(namespace + ':'): relative_ent_path = relative_ent_path[len(namespace ) + 1:] ent_references.append(relative_ent_path) att_references.append(att_name) if rel_info.selects_one: ent_pick_from = self.entity.fetch_object_definition( res_opt) atts_pick = ent_pick_from.attributes if ent_pick_from else None if atts_pick: for attribute in atts_pick: if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF: add_entity_reference( attribute.entity, self.in_document.namespace) else: add_entity_reference(self.entity, self.in_document.namespace) c_ent = self.ctx.corpus.make_object( CdmObjectType.CONSTANT_ENTITY_DEF) c_ent.entity_shape = self.ctx.corpus.make_ref( CdmObjectType.ENTITY_REF, 'entityGroupSet', True) c_ent.constant_values = [[ entity_ref, att_references[idx] ] for idx, entity_ref in enumerate(ent_references)] param = self.ctx.corpus.make_ref(CdmObjectType.ENTITY_REF, c_ent, False) reqd_trait.parameter_values.update_parameter_value( res_opt, 'entityReferences', param) # a 'structured' directive wants to keep all entity attributes together in a group if arc and arc.res_opt.directives and arc.res_opt.directives.has( 'structured'): ra_sub = ResolvedAttribute(rts_this_att.res_opt, rasb.ras, self.name, rasb.ras.attribute_context) if rel_info.is_array: # put a resolved trait on this att group, yuck, # hope I never need to do this again and then need to make a function for this tr = self.ctx.corpus.make_object(CdmObjectType.TRAIT_REF, 'is.linkedEntity.array', True) t = tr.fetch_object_definition(res_opt) rt = ResolvedTrait(t, None, [], []) ra_sub.resolved_traits = ra_sub.resolved_traits.merge(rt, True) rasb = ResolvedAttributeSetBuilder() rasb.own_one(ra_sub) return rasb
async def logical_manipulation_using_projections( self, corpus: CdmCorpusDefinition): '''This sample demonstrates how to model a set of common scenarios using projections. The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model. Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'. To get an overview of the projections feature as well as all of the supported operations refer to the link below. https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview ''' print('Create logical entity definition.') logical_folder = await corpus.fetch_object_async( 'output:/') # type: CdmFolderDefinition logical_doc = logical_folder.documents.append('Person.cdm.json') logical_doc.imports.append('local:/Address.cdm.json') entity = logical_doc.definitions.append( 'Person') # type: CdmEntityDefinition # Add 'name' data typed attribute. name_attr = entity.attributes.append( 'name') # type: CdmTypeAttributeDefinition name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'age' data typed attribute. age_attr = entity.attributes.append( 'age') # type: CdmTypeAttributeDefinition age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'address' entity typed attribute. entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address') entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True) apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute') apply_default_behavior(entity_attr, 'addressFK', 'address') entity.attributes.append(entity_attr) # Add 'email' data typed attribute. email_attr = entity.attributes.append( 'email') # type: CdmTypeAttributeDefinition email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Save the logical definition of Person. await entity.in_document.save_as_async('Person.cdm.json') print( 'Get \'resolved\' folder where the resolved entities will be saved.' ) resolved_folder = await corpus.fetch_object_async( 'output:/') # type: CdmFolderDefinition res_opt = ResolveOptions(entity) # To get more information about directives and their meaning refer to # https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes # We will start by resolving this entity with the 'normalized' direcitve. # This directive will be used on this and the next two examples so we can analize the resolved entity # without the array expansion. print('Resolving logical entity with normalized directive.') res_opt.directives = AttributeResolutionDirectiveSet({'normalized'}) res_normalized_entity = await entity.create_resolved_entity_async( f'normalized_{entity.entity_name}', res_opt, resolved_folder) await res_normalized_entity.in_document.save_as_async( f'{res_normalized_entity.entity_name}.cdm.json') # Another common scenario is to resolve an entity using the 'referenceOnly' directive. # This directives is used to replace the relationships with a foreign key. print('Resolving logical entity with referenceOnly directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) res_reference_only_entity = await entity.create_resolved_entity_async( f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder) await res_reference_only_entity.in_document.save_as_async( f'{res_reference_only_entity.entity_name}.cdm.json') # When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that # a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the # entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group. print('Resolving logical entity with structured directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'structured'}) res_structured_entity = await entity.create_resolved_entity_async( f'structured_{entity.entity_name}', res_opt, resolved_folder) await res_structured_entity.in_document.save_as_async( f'{res_structured_entity.entity_name}.cdm.json') # Now let us remove the 'normalized' directive so the array expansion operation can run. print('Resolving logical entity without directives (array expansion).') res_opt.directives = AttributeResolutionDirectiveSet({}) res_array_entity = await entity.create_resolved_entity_async( f'array_expansion_{entity.entity_name}', res_opt, resolved_folder) await res_array_entity.in_document.save_as_async( f'{res_array_entity.entity_name}.cdm.json')
async def test_circular_reference(self): cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, 'TestCircularReference') customer = await cdm_corpus.fetch_object_async('local:/Customer.cdm.json/Customer') res_customer_structured = await customer.create_resolved_entity_async('resCustomer', ResolveOptions(customer.in_document, AttributeResolutionDirectiveSet({'normalized', 'structured', 'noMaxDepth'}))) # check that the circular reference attribute has a single id attribute store_group_att = res_customer_structured.attributes[1].explicit_reference customer_group_att = store_group_att.members[1].explicit_reference self.assertEqual(len(customer_group_att.members), 1) self.assertEqual(customer_group_att.members[0].name, 'customerId')
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition', options: 'CopyOptions', new_name: str, save_referenced: bool) -> bool: """a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work because the corpus knows about persistence types and about the storage adapters if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or to a different name, then the source object is still 'dirty' an option will cause us to also save any linked documents.""" # find out if the storage adapter is able to write. namespace = doc.namespace if namespace is None: namespace = self._corpus.storage.default_namespace adapter = self._corpus.storage.fetch_adapter(namespace) if adapter is None: logger.error(self._TAG, self._ctx, 'Couldn\'t find a storage adapter registered for the namespace \'{}\''.format( namespace), self._save_document_as_async.__name__) return False if not adapter.can_write(): logger.error(self._TAG, self._ctx, 'The storage adapter \'{}\' claims it is unable to write files.'.format( namespace), self._save_document_as_async.__name__) return False if not new_name: logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._save_document_as_async.__name__) return None # what kind of document is requested? # check file extensions using a case-insensitive ordinal string comparison. persistence_type = self.MODEL_JSON if new_name.lower().endswith(self.MODEL_JSON_EXTENSION) else \ (self.ODI if new_name.lower().endswith(self.ODI_EXTENSION) else self.CDM_FOLDER) if persistence_type == self.ODI and new_name.lower() != self.ODI_EXTENSION: logger.error(self._TAG, self._ctx, 'Failed to persist \'{}\', as it\'s not an acceptable filename. It must be {}'.format( new_name, self.ODI_EXTENSION), self._save_document_as_async.__name__) return False if persistence_type == self.MODEL_JSON and new_name.lower() != self.MODEL_JSON_EXTENSION: logger.error(self._TAG, self._ctx, 'Failed to persist \'{}\', as it\'s not an acceptable filename. It must be {}'.format( new_name, self.MODEL_JSON_EXTENSION), self._save_document_as_async.__name__) return False # save the object into a json blob res_opt = {'wrt_doc': doc, 'directives': AttributeResolutionDirectiveSet()} persisted_doc = None persistence_class = self._fetch_registered_persistence_format(new_name) if persistence_class: try: method = persistence_class.to_data parameters = [doc, res_opt, options] # Check if to_data() is asynchronous for this persistence class. if not persistence_class in self._is_registered_persistence_async: # Cache whether this persistence class has async methods. self._is_registered_persistence_async[persistence_class] = persistence_class.is_persistence_async if self._is_registered_persistence_async[persistence_class]: # We don't know what the return type of ToData() is going to be and Task<T> is not covariant, # so we can't use Task<dynamic> here. Instead, we just await on a Task object without a return value # and fetch the Result property from it, which will have the result of ToData(). persisted_doc = await method(*parameters) else: persisted_doc = method(*parameters) except Exception as e: logger.error(self._TAG, self._ctx, 'Could not persist file \'{}\'. Reason \'{}\'.'.format(new_name, e), self._save_document_as_async.__name__) return False else: # Could not find a registered persistence class to handle this document type. logger.error(self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\'.'.format( new_name), self._save_document_as_async.__name__) return False if not persisted_doc: logger.error(self._TAG, self._ctx, 'Failed to persist \'{}\''.format(new_name), self._save_document_as_async.__name__) return False if persistence_type == self.ODI: await self._save_odi_documents(persisted_doc, adapter, new_name) return True # turn the name into a path new_path = '{}{}'.format(doc.folder_path, new_name) new_path = self._ctx.corpus.storage.create_absolute_corpus_path(new_path, doc) if new_path.startswith(namespace + ':'): new_path = new_path[len(namespace)+1:] # ask the adapter to make it happen try: content = persisted_doc.encode() await adapter.write_async(new_path, content) doc._file_system_modified_time = await adapter.compute_last_modified_time_async(new_path) # Write the adapter's config. if options._is_top_level_document: await self._corpus.storage.save_adapters_config_async('/config.json', adapter) # The next document won't be top level, so reset the flag. options._is_top_level_document = False except Exception as e: logger.error(self._TAG, self._ctx, 'Failed to write to the file \'{}\' for reason \'{}\''.format(new_name, e), self._save_document_as_async.__name__) return False # if we also want to save referenced docs, then it depends on what kind of thing just got saved # if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things # definition will save imports, manifests will save imports, schemas, sub manifests if save_referenced and persistence_type == self.CDM_FOLDER: saved_linked_docs = await doc._save_linked_documents_async(options) if not saved_linked_docs: logger.error(self._TAG, self._ctx, 'Failed to save linked documents for file \'{}\''.format(new_name), self._save_document_as_async.__name__) return False return True