def copy( self, res_opt: Optional['ResolveOptions'] = None, host: Optional['CdmDocumentDefinition'] = None ) -> 'CdmDocumentDefinition': res_opt = res_opt if res_opt is not None else ResolveOptions( wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives) if host is None: copy = CdmDocumentDefinition(self.ctx, self.name) else: copy = host copy.ctx = self.ctx copy.name = self.name copy.definitions.clear() copy._declarations_indexed = False copy.internal_declarations = {} copy._needs_indexing = True copy.imports.clear() copy._imports_indexed = False copy.in_document = copy copy._is_dirty = True copy._folder_path = self._folder_path copy.schema = self.schema copy.json_schema_semantic_version = self.json_schema_semantic_version copy.document_version = self.document_version for definition in self.definitions: copy.definitions.append(definition) for imp in self.imports: copy.imports.append(imp) return copy
async def _save_resolved( self, corpus: 'CdmCorpusDefinition', manifest: 'CdmManifestDefinition', test_name: str, input_entity: 'CdmEntityDefinition', resolution_options: List[str]) -> 'CdmEntityDefinition': ro_hash_set = set() for i in range(len(resolution_options)): ro_hash_set.add(resolution_options[i]) file_name_suffix = self._get_resolution_option_name_suffix( resolution_options) resolved_entity_name = 'Resolved_{}{}.cdm.json'.format( input_entity.entity_name, file_name_suffix) ro = ResolveOptions( input_entity.in_document, directives=AttributeResolutionDirectiveSet(ro_hash_set)) resolved_folder = corpus.storage.fetch_root_folder('output') resolved_entity = await input_entity.create_resolved_entity_async( resolved_entity_name, ro, resolved_folder) return resolved_entity
def copy( self, res_opt: Optional['ResolveOptions'] = None, host: Optional['CdmManifestDeclarationDefinition'] = None ) -> 'CdmManifestDeclarationDefinition': if not res_opt: res_opt = ResolveOptions( wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives) if not host: copy = CdmManifestDeclarationDefinition(self.ctx, self.manifest_name) else: copy = host copy.ctx = self.ctx copy.manifest_name = self.get_name() copy.definition = self.definition copy.last_file_status_check_time = self.last_file_status_check_time copy.last_file_modified_time = self.last_file_modified_time self._copy_def(res_opt, copy) return copy
def copy( self, res_opt: Optional['ResolveOptions'] = None, host: Optional['CdmE2ERelationship'] = None ) -> 'CdmE2ERelationship': if not res_opt: res_opt = ResolveOptions( wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives) if not host: copy = CdmE2ERelationship(self.ctx, self.get_name()) else: copy = host copy.ctx = self.ctx copy.relationship_name = self.get_name() copy.from_entity = self.from_entity copy.from_entity_attribute = self.from_entity_attribute copy.to_entity = self.to_entity copy.to_entity_attribute = self.to_entity_attribute self._copy_def(res_opt, copy) return copy
async def test_cardinality_persistence(self): ''' Testing that cardinality settings are loaded and saved correctly ''' corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_cardinality_persistence') # test from_data entity = await corpus.fetch_object_async( 'local:/someEntity.cdm.json/someEntity' ) # type: CdmEntityDefinition attribute = entity.attributes[0] # type: CdmTypeAttributeDefinition self.assertIsNotNone(attribute.cardinality) self.assertEqual(attribute.cardinality.minimum, '0') self.assertEqual(attribute.cardinality.maximum, '1') # test to_data attribute_data = TypeAttributePersistence.to_data( attribute, ResolveOptions(entity.in_document), CopyOptions()) self.assertIsNotNone(attribute_data.cardinality) self.assertEqual(attribute_data.cardinality.minimum, '0') self.assertEqual(attribute_data.cardinality.maximum, '1')
async def test_circular_reference(self): cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, 'TestCircularReference') customer = await cdm_corpus.fetch_object_async('local:/Customer.cdm.json/Customer') res_customer_structured = await customer.create_resolved_entity_async('resCustomer', ResolveOptions(customer.in_document, AttributeResolutionDirectiveSet({'normalized', 'structured', 'noMaxDepth'}))) # check that the circular reference attribute has a single id attribute store_group_att = res_customer_structured.attributes[1].explicit_reference customer_group_att = store_group_att.members[1].explicit_reference self.assertEqual(len(customer_group_att.members), 1) self.assertEqual(customer_group_att.members[0].name, 'customerId')
async def run_test(self, test_name: str, source_entity_name: str) -> None: test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name) test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name) test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('localInput', LocalAdapter(test_input_path)) corpus.storage.mount('localExpectedOutput', LocalAdapter(test_expected_output_path)) corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path)) corpus.storage.mount('cdm', LocalAdapter(TestHelper.get_schema_docs_root())) corpus.storage.default_namespace = 'localInput' src_entity_def = await corpus.fetch_object_async('localInput:/{0}.cdm.json/{0}'.format(source_entity_name)) # type: CdmEntityDefinition self.assertIsNotNone(src_entity_def) res_opt = ResolveOptions(src_entity_def.in_document, directives=AttributeResolutionDirectiveSet(set())) actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition resolved_entity_def = None output_entity_file_name = '' entity_file_name = '' entity_file_name = 'default' res_opt.directives = AttributeResolutionDirectiveSet(set()) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'normalized' res_opt.directives = AttributeResolutionDirectiveSet({'normalized'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'structured' res_opt.directives = AttributeResolutionDirectiveSet({'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly_normalized' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly_structured' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'normalized_structured' res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path) entity_file_name = 'referenceOnly_normalized_structured' res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized', 'structured'}) output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name) resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder) if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True): self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
async def create_resolved_manifest_async( self, new_manifest_name: str, new_entity_document_name_format: Optional[str], directives: Optional[AttributeResolutionDirectiveSet] = None ) -> Optional['CdmManifestDefinition']: """Creates a resolved copy of the manifest. new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities. The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any sub-folders described by the pattern should exist in the corpus prior to calling this function. """ with logger._enter_scope(self._TAG, self.ctx, self.create_resolved_manifest_async.__name__): if self.entities is None: return None if not self.folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_MANIFEST_FAILED, self.manifest_name) return None if new_entity_document_name_format is None: new_entity_document_name_format = '{f}resolved/{n}.cdm.json' elif new_entity_document_name_format == '': # for back compat new_entity_document_name_format = '{n}.cdm.json' elif '{n}' not in new_entity_document_name_format: # for back compat new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json' source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.at_corpus_path, self) source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.folder.at_corpus_path, self) resolved_manifest_path_split = new_manifest_name.rfind('/') + 1 resolved_manifest_folder = None if resolved_manifest_path_split > 0: resolved_manifest_path = new_manifest_name[ 0:resolved_manifest_path_split] new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( resolved_manifest_path, self) resolved_manifest_folder = await self.ctx.corpus.fetch_object_async( new_folder_path) # type: CdmFolderDefinition if resolved_manifest_folder is None: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND, new_folder_path) return None new_manifest_name = new_manifest_name[ resolved_manifest_path_split:] else: resolved_manifest_folder = self.owner logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'resolving manifest {}'.format(source_manifest_path)) # using the references present in the resolved entities, get an entity # create an imports doc with all the necessary resolved entity references and then resolve it # sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name if new_manifest_name.lower().endswith('.manifest.cdm.json'): new_manifest_name = new_manifest_name[0:( len(new_manifest_name) - len('.manifest.cdm.json'))] resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name) # bring over any imports in this document or other bobbles resolved_manifest.schema = self.schema resolved_manifest.explanation = self.explanation resolved_manifest.document_version = self.document_version for imp in self.imports: resolved_manifest.imports.append(imp.copy()) # add the new document to the folder if resolved_manifest_folder.documents.append( resolved_manifest) is None: # when would this happen? return None for entity in self.entities: ent_def = await self._get_entity_from_reference(entity, self) if not ent_def: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_ENTITY_REF_ERROR) return None if not ent_def.in_document.folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_IS_NOT_FOLDERformat, ent_def.entity_name) return None # get the path from this manifest to the source entity. this will be the {f} replacement value source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.in_document.folder.at_corpus_path, self) f = '' if source_entity_full_path.startswith( source_manifest_folder_path): f = source_entity_full_path[len(source_manifest_folder_path ):] new_document_full_path = new_entity_document_name_format.replace( '{n}', ent_def.entity_name).replace('{f}', f) new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( new_document_full_path, self) new_document_path_split = new_document_full_path.rfind('/') + 1 new_document_path = new_document_full_path[ 0:new_document_path_split] new_document_name = new_document_full_path[ new_document_path_split:] # make sure the new folder exists folder = await self.ctx.corpus.fetch_object_async( new_document_path) # type: CdmFolderDefinition if not folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND, new_document_path) return None # next create the resolved entity. with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives res_opt = ResolveOptions(ent_def.in_document, with_directives.copy()) logger.debug( self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'resolving entity {} to document {}'.format( source_entity_full_path, new_document_full_path)) resolved_entity = await ent_def.create_resolved_entity_async( ent_def.entity_name, res_opt, folder, new_document_name) if not resolved_entity: # fail all resolution, if any one entity resolution fails return None result = entity.copy(res_opt) if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path( resolved_entity.at_corpus_path, resolved_manifest) result.entity_path = relative_entity_path or result.at_corpus_path resolved_manifest.entities.append(result) logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'calculating relationships') # Calculate the entity graph for just this manifest. await self.ctx.corpus.calculate_entity_graph_async( resolved_manifest) # Stick results into the relationships list for the manifest. await resolved_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) # needed until Matt's changes with collections where I can propigate resolved_manifest._is_dirty = True return resolved_manifest
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an ArrayExpansion operation and a condition using the object model""" corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an ArrayExpansion operation array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF) array_expansion_op.start_ordinal = 1 array_expansion_op.end_ordinal = 2 projection.operations.append(array_expansion_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF) projection2.source = projection_entity_ref # Create a RenameAttributes operation rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF) rename_attrs_op.rename_format = '{m}{o}' projection2.operations.append(rename_attrs_op) # Create an entity reference to hold this projection projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref2.explicit_reference = projection2 # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref2 entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the projections # Original set of attributes: ["id", "name", "value", "date"] # Expand 1...2, renameFormat = {m}{o} self.assertEqual(8, len(resolved_entity_with_reference_only.attributes)) self.assertEqual('id1', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual('name1', resolved_entity_with_reference_only.attributes[1].name) self.assertEqual('value1', resolved_entity_with_reference_only.attributes[2].name) self.assertEqual('date1', resolved_entity_with_reference_only.attributes[3].name) self.assertEqual('id2', resolved_entity_with_reference_only.attributes[4].name) self.assertEqual('name2', resolved_entity_with_reference_only.attributes[5].name) self.assertEqual('value2', resolved_entity_with_reference_only.attributes[6].name) self.assertEqual('date2', resolved_entity_with_reference_only.attributes[7].name) # Now resolve the entity with the default directives res_opt.directives = AttributeResolutionDirectiveSet(set([])) resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the projections # Original set of attributes: ["id", "name", "value", "date"] # Expand 1...2, renameFormat = {m}{o} self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder.folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter( folder.namespace) # type: StorageAdapter try: if adapter.can_read(): # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'request file: {}'.format(doc_path)) json_data = await adapter.read_async(doc_path) # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'received file: {}'.format(doc_path)) else: raise Exception('Storage Adapter is not enabled to read.') except Exception as e: # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'fail file: {}'.format(doc_path)) # when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if res_opt and res_opt.shallow_validation: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE, doc_path, folder.Namespace, e) else: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE, doc_path, str(folder.namespace), e) return None try: fs_modified_time = await adapter.compute_last_modified_time_async( doc_path) except Exception as e: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED, e.Message) if not doc_name: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME) return None doc_name_lower = doc_name.lower() # If loading an model.json file, check that it is named correctly. if doc_name_lower.endswith( self.MODEL_JSON_EXTENSION ) and not doc_name.lower() == self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None try: if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION ) or doc_name_lower.endswith( PersistenceLayer.FOLIO_EXTENSION): from cdm.persistence.cdmfolder import ManifestPersistence from cdm.persistence.cdmfolder.types import ManifestContent manifest = ManifestContent() manifest.decode(json_data) doc_content = ManifestPersistence.from_object( self._ctx, doc_name, folder.namespace, folder.folder_path, manifest) elif doc_name_lower.endswith( PersistenceLayer.MODEL_JSON_EXTENSION): if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None from cdm.persistence.modeljson import ManifestPersistence from cdm.persistence.modeljson.types import Model model = Model() model.decode(json_data) doc_content = await ManifestPersistence.from_object( self._ctx, model, folder) elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION): from cdm.persistence.cdmfolder import DocumentPersistence from cdm.persistence.cdmfolder.types import DocumentContent document = DocumentContent() document.decode(json_data) doc_content = DocumentPersistence.from_object( self._ctx, doc_name, folder.namespace, folder.folder_path, document) else: # Could not find a registered persistence class to handle this document type. logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING, doc_name) return None except Exception as e: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE, doc_path, e) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy( ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus. default_resolution_directives), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content
async def create_resolved_entity_async(self, new_ent_name: str, res_opt: Optional['ResolveOptions'] = None, folder: 'CdmFolderDefinition' = None, new_doc_name: str = None) -> 'CdmEntityDefinition': """Create a resolved copy of the entity.""" if not res_opt: res_opt = ResolveOptions(self, self.ctx.corpus.default_resolution_directives) if not res_opt.wrt_doc: logger.error(self._TAG, self.ctx, 'No WRT document was supplied.', self.create_resolved_entity_async.__name__) return None if not new_ent_name: logger.error(self._TAG, self.ctx, 'No Entity Name provided.', self.create_resolved_entity_async.__name__) return None folder = folder or self.in_document.folder file_name = new_doc_name or new_ent_name + PersistenceLayer.CDM_EXTENSION orig_doc = self.in_document.at_corpus_path # Don't overwite the source document. target_at_corpus_path = self.ctx.corpus.storage.create_absolute_corpus_path(folder.at_corpus_path, folder) + file_name if StringUtils.equals_with_ignore_case(target_at_corpus_path, orig_doc): logger.error(self._TAG, self.ctx, 'Attempting to replace source entity\'s document \'{}\''.format( target_at_corpus_path), self.create_resolved_entity_async.__name__) return None if not await res_opt.wrt_doc._index_if_needed(res_opt, True): logger.error(self._TAG, self.ctx, 'Couldn\'t index source document.', self.create_resolved_entity_async.__name__) return None # Make the top level attribute context for this entity. # for this whole section where we generate the attribute context tree and get resolved attributes # set the flag that keeps all of the parent changes and document dirty from from happening was_resolving = self.ctx.corpus._is_currently_resolving self.ctx.corpus._is_currently_resolving = True ent_name = new_ent_name ctx = self.ctx att_ctx_ent = ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_CONTEXT_DEF, ent_name, True) # type: CdmAttributeContext att_ctx_ent.ctx = ctx att_ctx_ent.in_document = self.in_document # Cheating a bit to put the paths in the right place. acp = AttributeContextParameters() acp._under = att_ctx_ent acp._type = CdmAttributeContextType.ATTRIBUTE_GROUP acp._name = 'attributeContext' att_ctx_ac = CdmAttributeContext._create_child_under(res_opt, acp) acp_ent = AttributeContextParameters() acp_ent._under = att_ctx_ac acp_ent._type = CdmAttributeContextType.ENTITY acp_ent._name = ent_name acp_ent._regarding = ctx.corpus.make_ref(CdmObjectType.ENTITY_REF, self.get_name(), True) # Use this whenever we need to keep references pointing at things that were already found. # Used when 'fixing' references by localizing to a new document. res_opt_copy = CdmObject._copy_resolve_options(res_opt) res_opt_copy._save_resolutions_on_copy = True # Resolve attributes with this context. the end result is that each resolved attribute points to the level of # the context where it was created. ras = self._fetch_resolved_attributes(res_opt_copy, acp_ent) if ras is None: self._resolving_attributes = False return None # Create a new copy of the attribute context for this entity. # TODO: all_att_ctx type ideally should be ordered set all_att_ctx = [] # type: List[CdmAttributeContext] new_node = cast('CdmAttributeContext', att_ctx_ent.copy_node(res_opt)) att_ctx_ent = att_ctx_ent._copy_attribute_context_tree(res_opt, new_node, ras, all_att_ctx, 'resolvedFrom') att_ctx = cast('CdmAttributeContext', cast('CdmAttributeContext', att_ctx_ent.contents[0]).contents[0]) self.ctx.corpus._is_currently_resolving = was_resolving # make a new document in given folder if provided or the same folder as the source entity folder.documents.remove(file_name) doc_res = folder.documents.append(file_name) # add a import of the source document orig_doc = self.ctx.corpus.storage.create_relative_corpus_path(orig_doc, doc_res) # just in case we missed the prefix doc_res.imports.append(orig_doc, "resolvedFrom") # make the empty entity ent_resolved = doc_res.definitions.append(ent_name) # set the context to the copy of the tree. fix the docs on the context nodes ent_resolved.attribute_context = att_ctx def visit_callback(obj: 'CdmObject', path: str) -> bool: obj.in_document = doc_res return False att_ctx.visit('{}/attributeContext/'.format(ent_name), visit_callback, None) # add the traits of the entity rts_ent = self._fetch_resolved_traits(res_opt) for rt in rts_ent.rt_set: trait_ref = CdmObject._resolved_trait_to_trait_ref(res_opt_copy, rt) ent_resolved.exhibits_traits.append(trait_ref) # The attributes have been named, shaped, etc. for this entity so now it is safe to go and make each attribute # context level point at these final versions of attributes. att_path_to_order = {} # type: Dict[str, int] def point_context_at_resolved_atts(ras_sub: 'ResolvedAttributeSet', path: str) -> None: for ra in ras_sub._set: ra_ctx_in_ent = [] # type: List[CdmAttributeContext] ra_ctx_set = ras_sub.rattr_to_attctxset.get(ra) # find the correct attctx for this copy, intersect these two sets # (interate over the shortest list) if len(all_att_ctx) < len(ra_ctx_set): for curr_att_ctx in all_att_ctx: if curr_att_ctx in ra_ctx_set: ra_ctx_in_ent.append(curr_att_ctx) else: for curr_att_ctx in ra_ctx_set: if curr_att_ctx in all_att_ctx: ra_ctx_in_ent.append(curr_att_ctx) for ra_ctx in ra_ctx_in_ent: refs = ra_ctx.contents # there might be more than one explanation for where and attribute came from when things get merges as they do # This won't work when I add the structured attributes to avoid name collisions. att_ref_path = path + ra.resolved_name if isinstance(ra.target, CdmObject): if not att_ref_path in att_path_to_order: att_ref = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_REF, att_ref_path, True) # type: CdmObjectReference # only need one explanation for this path to the insert order att_path_to_order[att_ref.named_reference] = ra.insert_order refs.append(att_ref) else: att_ref_path += '/members/' point_context_at_resolved_atts(cast('ResolvedAttributeSet', ra.target), att_ref_path) point_context_at_resolved_atts(ras, ent_name + '/hasAttributes/') # generated attribute structures may end up with 0 attributes after that. prune them def clean_sub_group(sub_item, under_generated) -> bool: if sub_item.object_type == CdmObjectType.ATTRIBUTE_REF: return True # not empty ac = sub_item # type: CdmAttributeContext if ac.type == CdmAttributeContextType.GENERATED_SET: under_generated = True if not ac.contents: return False # empty # look at all children, make a set to remove to_remove = [] # type: List[CdmAttributeContext] for sub_sub in ac.contents: if not clean_sub_group(sub_sub, under_generated): potential_target = under_generated if not potential_target: # cast is safe because we returned false meaning empty and not a attribute ref # so is this the set holder itself? potential_target = sub_sub.type == CdmAttributeContextType.GENERATED_SET if potential_target: to_remove.append(sub_sub) for to_die in to_remove: ac.contents.remove(to_die) return bool(ac.contents) clean_sub_group(att_ctx, False) # Create an all-up ordering of attributes at the leaves of this tree based on insert order. Sort the attributes # in each context by their creation order and mix that with the other sub-contexts that have been sorted. def get_order_num(item: 'CdmObject') -> int: if item.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF: return order_contents(item) if item.object_type == CdmObjectType.ATTRIBUTE_REF: return att_path_to_order[item.named_reference] # put the mystery item on top. return -1 def order_contents(under: 'CdmAttributeContext') -> int: if under._lowest_order is None: under._lowest_order = -1 if len(under.contents) == 1: under._lowest_order = get_order_num(under.contents[0]) else: def get_and_update_order(item1: 'CdmObject', item2: 'CdmObject'): left_order = get_order_num(item1) right_order = get_order_num(item2) if left_order != -1 and (under._lowest_order == -1 or left_order < under._lowest_order): under._lowest_order = left_order if right_order != -1 and (under._lowest_order == -1 or right_order < under._lowest_order): under._lowest_order = right_order return left_order - right_order under.contents.sort(key=functools.cmp_to_key(get_and_update_order)) return under._lowest_order order_contents(att_ctx) # Resolved attributes can gain traits that are applied to an entity when referenced since these traits are # described in the context, it is redundant and messy to list them in the attribute. So, remove them. Create and # cache a set of names to look for per context. There is actually a hierarchy to this. All attributes from the # base entity should have all traits applied independently of the sub-context they come from. Same is true of # attribute entities. So do this recursively top down. ctx_to_trait_names = {} # type: Dict[CdmAttributeContext, Set[str]] def collect_context_traits(sub_att_ctx: 'CdmAttributeContext', inherited_trait_names: Set[str]) -> None: trait_names_here = set(inherited_trait_names) traits_here = sub_att_ctx.exhibits_traits if traits_here: for tat in traits_here: trait_names_here.add(tat.named_reference) ctx_to_trait_names[sub_att_ctx] = trait_names_here for cr in sub_att_ctx.contents: if cr.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF: # do this for all types? collect_context_traits(cast('CdmAttributeContext', cr), trait_names_here) collect_context_traits(att_ctx, set()) # add the attributes, put them in attribute groups if structure needed. res_att_to_ref_path = {} # type: Dict[ResolvedAttribute, str] def add_attributes(ras_sub: 'ResolvedAttributeSet', container: 'Union[CdmEntityDefinition, CdmAttributeGroupDefinition]', path: str) -> None: for ra in ras_sub._set: att_path = path + ra.resolved_name # use the path of the context associated with this attribute to find the new context that matches on path. ra_ctx_set = ras_sub.rattr_to_attctxset[ra] # find the correct att_ctx for this copy. # ra_ctx = next((ac for ac in all_att_ctx if ac in ra_ctx_set), None) # type: CdmAttributeContext if len(all_att_ctx) < len(ra_ctx_set): for curr_att_ctx in all_att_ctx: if curr_att_ctx in ra_ctx_set: ra_ctx = curr_att_ctx break else: for curr_att_ctx in ra_ctx_set: if curr_att_ctx in all_att_ctx: ra_ctx = curr_att_ctx break if isinstance(ra.target, ResolvedAttributeSet) and cast('ResolvedAttributeSet', ra.target)._set: # this is a set of attributes. Make an attribute group to hold them. att_grp = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_GROUP_DEF, ra.resolved_name) # type: CdmAttributeGroupDefinition att_grp.attribute_context = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_CONTEXT_REF, ra_ctx.at_corpus_path, True) # take any traits from the set and make them look like traits exhibited by the group. avoid_set = ctx_to_trait_names[ra_ctx] rts_att = ra.resolved_traits for rt in rts_att.rt_set: if not rt.trait.ugly: # Don't mention your ugly traits. if not avoid_set or rt.trait_name not in avoid_set: # Avoid the ones from the context. trait_ref = CdmObject._resolved_trait_to_trait_ref(res_opt_copy, rt) cast('CdmObjectDefinition', att_grp).exhibits_traits.append(trait_ref, isinstance(trait_ref, str)) # wrap it in a reference and then recurse with this as the new container. att_grp_ref = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_GROUP_REF, None) # type: CdmAttributeGroupReference att_grp_ref.explicit_reference = att_grp container._add_attribute_def(att_grp_ref) # isn't this where ... add_attributes(cast('ResolvedAttributeSet', ra.target), att_grp, att_path + '/(object)/members/') else: att = self.ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, ra.resolved_name) # type: CdmTypeAttributeDefinition att.attribute_context = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_CONTEXT_REF, ra_ctx.at_corpus_path, True) avoid_set = ctx_to_trait_names[ra_ctx] rts_att = ra.resolved_traits for rt in rts_att.rt_set: if not rt.trait.ugly: # Don't mention your ugly traits. if not avoid_set or rt.trait_name not in avoid_set: # Avoid the ones from the context. trait_ref = CdmObject._resolved_trait_to_trait_ref(res_opt_copy, rt) cast('CdmTypeAttributeDefinition', att).applied_traits.append(trait_ref, isinstance(trait_ref, str)) # none of the dataformat traits have the bit set that will make them turn into a property # this is intentional so that the format traits make it into the resolved object # but, we still want a guess as the data format, so get it and set it. implied_data_format = att.data_format if implied_data_format: att.data_format = implied_data_format container._add_attribute_def(att) res_att_to_ref_path[ra] = att_path add_attributes(ras, ent_resolved, ent_name + '/hasAttributes/') # Any resolved traits that hold arguments with attribute refs should get 'fixed' here. def replace_trait_att_ref(tr: 'CdmTraitReference', entity_hint: str) -> None: if tr.arguments: for arg in tr.arguments: v = arg._unresolved_value or arg.value # Is this an attribute reference? if v and isinstance(v, CdmObject) and cast('CdmObject', v).object_type == CdmObjectType.ATTRIBUTE_REF: # Only try this if the reference has no path to it (only happens with intra-entity att refs). att_ref = cast('CdmAttributeReference', v) if att_ref.named_reference and att_ref.named_reference.find('/') == -1: if not arg._unresolved_value: arg._unresolved_value = arg.value # Give a promise that can be worked out later. # Assumption is that the attribute must come from this entity. new_att_ref = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_REF, entity_hint + '/(resolvedAttributes)/' + att_ref.named_reference, True) # inDocument is not propagated during resolution, so set it here new_att_ref.in_document = arg.in_document arg.set_value(new_att_ref) # Fix entity traits. if ent_resolved.exhibits_traits: for et in ent_resolved.exhibits_traits: replace_trait_att_ref(et, new_ent_name) # Fix context traits. def fix_context_traits(sub_att_ctx: 'CdmAttributeContext', entity_hint: str) -> None: traits_here = sub_att_ctx.exhibits_traits if traits_here: for tr in traits_here: replace_trait_att_ref(tr, entity_hint) for cr in sub_att_ctx.contents: if cr.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF: # If this is a new entity context, get the name to pass along. sub_sub_att_ctx = cast('CdmAttributeContext', cr) sub_entity_hint = entity_hint if sub_sub_att_ctx.type == CdmAttributeContextType.ENTITY: sub_entity_hint = sub_sub_att_ctx.definition.named_reference # Do this for all types. fix_context_traits(sub_sub_att_ctx, sub_entity_hint) fix_context_traits(att_ctx, new_ent_name) # And the attribute traits. ent_atts = ent_resolved.attributes if ent_atts: for attribute in ent_atts: att_traits = attribute.applied_traits if att_traits: for tr in att_traits: replace_trait_att_ref(tr, new_ent_name) # We are about to put this content created in the context of various documents (like references to attributes # from base entities, etc.) into one specific document. All of the borrowed refs need to work. so, re-write all # string references to work from this new document. The catch-22 is that the new document needs these fixes done # before it can be used to make these fixes. The fix needs to happen in the middle of the refresh trigger the # document to refresh current content into the resolved OM. cast('CdmAttributeContext', att_ctx).parent = None # Remove the fake parent that made the paths work. res_opt_new = CdmObject._copy_resolve_options(res_opt) res_opt_new.wrt_doc = doc_res res_opt_new._localize_references_for = doc_res if not await doc_res.refresh_async(res_opt_new): logger.error(self._TAG, self.ctx, 'Failed to index the resolved document.', self.create_resolved_entity_async.__name__) return None # Get a fresh ref. ent_resolved = cast('CdmEntityDefinition', doc_res._fetch_object_from_document_path(ent_name, res_opt_new)) self.ctx.corpus._res_ent_map[self.at_corpus_path] = ent_resolved.at_corpus_path return ent_resolved
def _fetch_resolved_attributes( self, res_opt: 'ResolveOptions', acp_in_context: Optional['AttributeContextParameters'] = None ) -> 'ResolvedAttributeSet': from cdm.resolvedmodel import ResolvedAttributeSet from cdm.utilities import SymbolSet from .cdm_attribute_context import CdmAttributeContext from .cdm_corpus_def import CdmCorpusDefinition was_previously_resolving = self.ctx.corpus.is_currently_resolving self.ctx.corpus.is_currently_resolving = True if not res_opt: res_opt = ResolveOptions(self) kind = 'rasb' ctx = self.ctx cache_tag = ctx.corpus._fetch_definition_cache_tag( res_opt, self, kind, 'ctx' if acp_in_context else '') rasb_cache = ctx.cache.get(cache_tag) if cache_tag else None under_ctx = None # store the previous symbol set, we will need to add it with # children found from the constructResolvedTraits call curr_sym_ref_set = res_opt.symbol_ref_set or SymbolSet() res_opt.symbol_ref_set = SymbolSet() # get the moniker that was found and needs to be appended to all # refs in the children attribute context nodes from_moniker = res_opt._from_moniker res_opt._from_moniker = None if not rasb_cache: if self._resolving_attributes: # re-entered self attribute through some kind of self or looping reference. self.ctx.corpus.is_currently_resolving = was_previously_resolving return ResolvedAttributeSet() self._resolving_attributes = True # if a new context node is needed for these attributes, make it now if acp_in_context: under_ctx = CdmAttributeContext._create_child_under( res_opt, acp_in_context) rasb_cache = self._construct_resolved_attributes( res_opt, under_ctx) self._resolving_attributes = False # register set of possible docs odef = self.fetch_object_definition(res_opt) if odef is not None: ctx.corpus._register_definition_reference_symbols( odef, kind, res_opt.symbol_ref_set) # get the new cache tag now that we have the list of symbols cache_tag = ctx.corpus._fetch_definition_cache_tag( res_opt, self, kind, 'ctx' if acp_in_context else '') # save this as the cached version if cache_tag: ctx.cache[cache_tag] = rasb_cache if from_moniker and acp_in_context and cast( 'CdmObjectReference', self).named_reference: # create a fresh context old_context = acp_in_context.under.contents[-1] acp_in_context.under.contents.pop() under_ctx = CdmAttributeContext._create_child_under( res_opt, acp_in_context) old_context._copy_attribute_context_tree( res_opt, under_ctx, rasb_cache.ras, None, from_moniker) else: # get the SymbolSet for this cached object and pass that back key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind) res_opt.symbol_ref_set = ctx.corpus._definition_reference_symbols.get( key) # cache found. if we are building a context, then fix what we got instead of making a new one if acp_in_context: # make the new context under_ctx = CdmAttributeContext._create_child_under( res_opt, acp_in_context) rasb_cache.ras.attribute_context._copy_attribute_context_tree( res_opt, under_ctx, rasb_cache.ras, None, from_moniker) # merge child reference symbols set with current curr_sym_ref_set.merge(res_opt.symbol_ref_set) res_opt.symbol_ref_set = curr_sym_ref_set self.ctx.corpus.is_currently_resolving = was_previously_resolving return rasb_cache.ras
async def create_resolved_manifest_async( self, new_manifest_name: str, new_entity_document_name_format: str ) -> Optional['CdmManifestDefinition']: """Creates a resolved copy of the manifest. new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities. The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any sub-folders described by the pattern should exist in the corpus prior to calling this function. """ if self.entities is None: return None if new_entity_document_name_format is None: new_entity_document_name_format = '{f}resolved/{n}.cdm.json' elif new_entity_document_name_format == '': # for back compat new_entity_document_name_format = '{n}.cdm.json' elif '{n}' not in new_entity_document_name_format: # for back compat new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json' source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.at_corpus_path, self) source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.folder.at_corpus_path, self) resolved_manifest_path_split = new_manifest_name.rfind('/') + 1 resolved_manifest_folder = None if resolved_manifest_path_split > 0: resolved_manifest_path = new_manifest_name[ 0:resolved_manifest_path_split] new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( resolved_manifest_path, self) resolved_manifest_folder = await self.ctx.corpus.fetch_object_async( new_folder_path) # type: CdmFolderDefinition if resolved_manifest_folder is None: self.ctx.logger.error( 'New folder for manifest not found {}'.format( new_folder_path), 'create_resolved_manifest_async') return None new_manifest_name = new_manifest_name[ resolved_manifest_path_split:] else: resolved_manifest_folder = self.owner self.ctx.logger.debug( 'resolving manifest {}'.format(source_manifest_path), 'create_resolved_manifest_async') # using the references present in the resolved entities, get an entity # create an imports doc with all the necessary resolved entity references and then resolve it resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name) # add the new document to the folder if resolved_manifest_folder.documents.append( resolved_manifest) is None: # when would this happen? return None # mapping from entity path to resolved entity path for translating relationhsip paths res_ent_map = {} # type: Dict[str, str] for entity in self.entities: ent_def = await self._get_entity_from_reference(entity, self) if not ent_def: self.ctx.logger.error('Unable to get entity from reference') return None # get the path from this manifest to the source entity. this will be the {f} replacement value source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.in_document.folder.at_corpus_path, self) f = '' if source_entity_full_path.startswith(source_manifest_folder_path): f = source_entity_full_path[len(source_manifest_folder_path):] new_document_full_path = new_entity_document_name_format.replace( '{n}', ent_def.entity_name).replace('{f}', f) new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( new_document_full_path, self) new_document_path_split = new_document_full_path.rfind('/') + 1 new_document_path = new_document_full_path[ 0:new_document_path_split] new_document_name = new_document_full_path[ new_document_path_split:] # make sure the new folder exists folder = await self.ctx.corpus.fetch_object_async( new_document_path) # type: CdmFolderDefinition if not folder: self.ctx.logger.error( 'New folder not found {}'.format(new_document_path)) return None # next create the resolved entity. res_opt = ResolveOptions() res_opt.wrt_doc = ent_def.in_document res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) self.ctx.logger.debug( ' resolving entity {} to document {}'.format( source_entity_full_path, new_document_full_path)) resolved_entity = await ent_def.create_resolved_entity_async( ent_def.entity_name, res_opt, folder, new_document_name) if not resolved_entity: # fail all resolution, if any one entity resolution fails return None result = entity.copy(res_opt) if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path( resolved_entity.at_corpus_path, resolved_manifest) result.entity_path = relative_entity_path or result.at_corpus_path resolved_manifest.entities.append(result) # absolute path is needed for generating relationships absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path( result.entity_path, resolved_manifest) res_ent_map[self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.at_corpus_path, ent_def.in_document)] = absolute_ent_path self.ctx.logger.debug(' calculating relationships') # Calculate the entity graph for just this manifest. await self.ctx.corpus._calculate_entity_graph_async( resolved_manifest, res_ent_map) # Stick results into the relationships list for the manifest. await resolved_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) # needed until Matt's changes with collections where I can propigate resolved_manifest._is_dirty = True return resolved_manifest
def fetch_object_definition(self, res_opt: 'ResolveOptions') -> 'CdmObjectDefinition': if res_opt is None: res_opt = ResolveOptions(self) return self._fetch_resolved_reference(res_opt)
def create_simple_reference(self, res_opt: Optional['ResolveOptions'] = None) -> 'CdmObjectReference': if not res_opt: res_opt = ResolveOptions(self) if self.named_reference: return self._copy_ref_object(res_opt, self.named_reference, True) return self._copy_ref_object(res_opt, self._declared_path + self.explicit_reference.get_name(), True)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an ExcludeAttributes operation and a condition using the object model""" corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount( 'local', LocalAdapter( TestHelper.get_actual_output_folder_path( self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an ExcludeAttributes operation exclude_attrs_op = corpus.make_object( CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF) exclude_attrs_op.exclude_attributes.append('id') exclude_attrs_op.exclude_attributes.append('date') projection.operations.append(exclude_attrs_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object( CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet( set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the ExcludeAttributes operation # Original set of attributes: ['id', 'name', 'value', 'date'] # Excluded attributes: ['id', 'date'] self.assertEqual(2, len(resolved_entity_with_reference_only.attributes)) self.assertEqual( 'name', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual( 'value', resolved_entity_with_reference_only.attributes[1].name) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet( set(['structured'])) resolved_entity_with_structured = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the ExcludeAttributes operation # Original set of attributes: ['id', 'name', 'value', 'date'] # Excluded attributes: none, condition was false self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_resolved_attribute_limit(self): corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_resolved_attribute_limit') # type: CdmCorpusDefinition main_entity = await corpus.fetch_object_async( 'local:/mainEntity.cdm.json/mainEntity' ) # type: CdmEntityDefinition res_opt = ResolveOptions(wrt_doc=main_entity.in_document) # if attribute limit is reached, entity should be None res_opt._resolved_attribute_limit = 4 resEnt = await main_entity.create_resolved_entity_async( '{}_zeroAtts'.format(main_entity.entity_name), res_opt) # type: CdmEntityDefinition self.assertIsNone(resEnt) # when the attribute limit is set to null, there should not be a limit on the possible number of attributes res_opt._resolved_attribute_limit = None ras = main_entity._fetch_resolved_attributes( res_opt) # type: ResolvedAttributeSet resEnt = await main_entity.create_resolved_entity_async( '{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt) # there are 5 total attributes self.assertEqual(5, ras._resolved_attribute_count) self.assertEqual(5, len(ras._set)) self.assertEqual(3, len(main_entity.attributes)) # there are 2 attributes grouped in an entity attribute # and 2 attributes grouped in an attribute group self.assertEqual( 2, len(main_entity.attributes[2].explicit_reference.members)) # using the default limit number res_opt = ResolveOptions(wrt_doc=main_entity.in_document) ras = main_entity._fetch_resolved_attributes(res_opt) resEnt = await main_entity.create_resolved_entity_async( '{}_normalized_referenceOnly'.format(main_entity.entity_name), res_opt) # there are 5 total attributes self.assertEqual(5, ras._resolved_attribute_count) self.assertEqual(5, len(ras._set)) self.assertEqual(3, len(main_entity.attributes)) # there are 2 attributes grouped in an entity attribute # and 2 attributes grouped in an attribute group self.assertEqual( 2, len(main_entity.attributes[2].explicit_reference.members)) res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'structured'}) ras = main_entity._fetch_resolved_attributes(res_opt) resEnt = await main_entity.create_resolved_entity_async( '{}_normalized_structured'.format(main_entity.entity_name), res_opt) # there are 5 total attributes self.assertEqual(5, ras._resolved_attribute_count) # the attribute count is different because one attribute is a group that contains two different attributes self.assertEqual(4, len(ras._set)) self.assertEqual(3, len(main_entity.attributes)) # again there are 2 attributes grouped in an entity attribute # and 2 attributes grouped in an attribute group self.assertEqual( 2, len(main_entity.attributes[2].explicit_reference.members))
def _fetch_resolved_traits( self, res_opt: 'ResolveOptions') -> 'ResolvedTraitSet': from cdm.resolvedmodel import ResolvedTraitSet, ResolvedTraitSetBuilder from cdm.utilities import SymbolSet was_previously_resolving = self.ctx.corpus.is_currently_resolving self.ctx.corpus.is_currently_resolving = True if not res_opt: res_opt = ResolveOptions(self) kind = 'rtsb' ctx = self.ctx cache_tag_a = ctx.corpus._fetch_definition_cache_tag( res_opt, self, kind) rtsb_all = None # type: ResolvedTraitSetBuilder if self._trait_cache is None: self._trait_cache = {} elif cache_tag_a: rtsb_all = self._trait_cache.get(cache_tag_a) # store the previous document set, we will need to add it with # children found from the constructResolvedTraits call curr_doc_ref_set = res_opt.symbol_ref_set if curr_doc_ref_set is None: curr_doc_ref_set = SymbolSet() res_opt.symbol_ref_set = SymbolSet() if rtsb_all is None: rtsb_all = ResolvedTraitSetBuilder() if not self._resolving_traits: self._resolving_traits = True self._construct_resolved_traits(rtsb_all, res_opt) self._resolving_traits = False obj_def = self.fetch_object_definition(res_opt) if obj_def: # register set of possible docs ctx.corpus._register_definition_reference_symbols( obj_def, kind, res_opt.symbol_ref_set) if rtsb_all.resolved_trait_set is None: # nothing came back, but others will assume there is a set in this builder rtsb_all.resolved_trait_set = ResolvedTraitSet(res_opt) # get the new cache tag now that we have the list of docs cache_tag_a = ctx.corpus._fetch_definition_cache_tag( res_opt, self, kind) if cache_tag_a: self._trait_cache[cache_tag_a] = rtsb_all else: # cache was found # get the SymbolSet for this cached object from .cdm_corpus_def import CdmCorpusDefinition key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind) temp_doc_ref_set = ctx.corpus._definition_reference_symbols.get( key) res_opt.symbol_ref_set = temp_doc_ref_set # merge child document set with current curr_doc_ref_set.merge(res_opt.symbol_ref_set) res_opt.symbol_ref_set = curr_doc_ref_set self.ctx.corpus.is_currently_resolving = was_previously_resolving return rtsb_all.resolved_trait_set
def is_derived_from(self, base: str, res_opt: Optional['ResolveOptions'] = None) -> bool: res_opt = res_opt if res_opt is not None else ResolveOptions(wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives) return self._is_derived_from_def(res_opt, self.extends_entity, self.get_name(), base)
def _fetch_resolved_traits( self, res_opt: 'ResolveOptions') -> 'ResolvedTraitSet': from cdm.resolvedmodel import ResolvedTrait, ResolvedTraitSet from cdm.utilities import SymbolSet from .cdm_corpus_def import CdmCorpusDefinition res_opt = res_opt if res_opt is not None else ResolveOptions( self, self.ctx.corpus.default_resolution_directives) kind = 'rtsb' ctx = self.ctx # this may happen 0, 1 or 2 times. so make it fast base_trait = None # type: CdmTraitDefinition base_values = None # type: List[CdmArgumentValue] def get_base_info(base_trait, base_values): if self.extends_trait: base_trait = self.extends_trait.fetch_object_definition( res_opt) if base_trait: base_rts = self.extends_trait._fetch_resolved_traits( res_opt) if base_rts and base_rts.size == 1: base_pv = base_rts.get( base_trait).parameter_values if base_rts.get( base_trait) else None if base_pv: base_values = base_pv.values return base_trait, base_values # see if one is already cached # if this trait has parameters, then the base trait found through the reference might be a different reference # because trait references are unique per argument value set. so use the base as a part of the cache tag # since it is expensive to figure out the extra tag, cache that too! if self._base_is_known_to_have_parameters is None: base_trait, base_values = get_base_info(base_trait, base_values) # is a cache tag needed? then make one self._base_is_known_to_have_parameters = False if base_values: self._base_is_known_to_have_parameters = True cache_tag_extra = '' if self._base_is_known_to_have_parameters: cache_tag_extra = str(self.extends_trait.id) cache_tag = ctx.corpus._create_definition_cache_tag( res_opt, self, kind, cache_tag_extra) rts_result = ctx._cache.get(cache_tag) if cache_tag else None # store the previous reference symbol set, we will need to add it with # children found from the _construct_resolved_traits call curr_sym_ref_set = res_opt._symbol_ref_set or SymbolSet() res_opt._symbol_ref_set = SymbolSet() # if not, then make one and save it if not rts_result: base_trait, base_values = get_base_info(base_trait, base_values) if base_trait: # get the resolution of the base class and use the values as a starting point for this trait's values if not self._has_set_flags: # inherit these flags if self.elevated is None: self.elevated = base_trait.elevated if self.ugly is None: self.ugly = base_trait.ugly if self.associated_properties is None: self.associated_properties = base_trait.associated_properties self._has_set_flags = True pc = self._fetch_all_parameters(res_opt) av = [] # type: List[CdmArgumentValue] was_set = [] # type: List[bool] self._this_is_known_to_have_parameters = bool(pc.sequence) for i in range(len(pc.sequence)): # either use the default value or (higher precidence) the value taken from the base reference value = pc.sequence[i].default_value if base_values and i < len(base_values): base_value = base_values[i] if base_value: value = base_value av.append(value) was_set.append(False) # save it res_trait = ResolvedTrait(self, pc, av, was_set) rts_result = ResolvedTraitSet(res_opt) rts_result.merge(res_trait, False) # register set of possible symbols ctx.corpus._register_definition_reference_symbols( self.fetch_object_definition(res_opt), kind, res_opt._symbol_ref_set) # get the new cache tag now that we have the list of docs cache_tag = ctx.corpus._create_definition_cache_tag( res_opt, self, kind, cache_tag_extra) if cache_tag: ctx._cache[cache_tag] = rts_result else: # cache found # get the SymbolSet for this cached object key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind) res_opt._symbol_ref_set = ctx.corpus._definition_reference_symbols.get( key) # merge child document set with current curr_sym_ref_set._merge(res_opt._symbol_ref_set) res_opt._symbol_ref_set = curr_sym_ref_set return rts_result
def _fetch_resolved_traits( self, res_opt: 'ResolveOptions') -> 'ResolvedTraitSet': from cdm.utilities import SymbolSet from .cdm_corpus_def import CdmCorpusDefinition res_opt = res_opt if res_opt is not None else ResolveOptions( self, self.ctx.corpus.default_resolution_directives) kind = 'rtsb' ctx = self.ctx # get referenced trait trait = self.fetch_object_definition(res_opt) rts_trait = None if not trait: return ctx.corpus._fetch_empty_resolved_trait_set(res_opt) # see if one is already cached # cache by name unless there are parameter if trait._this_is_known_to_have_parameters is None: # never been resolved, it will happen soon, so why not now? rts_trait = trait._fetch_resolved_traits(res_opt) cache_by_path = True if trait._this_is_known_to_have_parameters is not None: cache_by_path = not trait._this_is_known_to_have_parameters cache_tag = ctx.corpus._create_definition_cache_tag( res_opt, self, kind, '', cache_by_path, trait.at_corpus_path) rts_result = ctx._cache.get(cache_tag) if cache_tag else None # store the previous reference symbol set, we will need to add it with # children found from the _construct_resolved_traits call curr_sym_ref_set = res_opt._symbol_ref_set or SymbolSet() res_opt._symbol_ref_set = SymbolSet() # if not, then make one and save it if not rts_result: # get the set of resolutions, should just be this one trait if not rts_trait: # store current doc ref set new_doc_ref_set = res_opt._symbol_ref_set res_opt._symbol_ref_set = SymbolSet() rts_trait = trait._fetch_resolved_traits(res_opt) # bubble up symbol reference set from children if new_doc_ref_set: new_doc_ref_set._merge(res_opt._symbol_ref_set) res_opt._symbol_ref_set = new_doc_ref_set if rts_trait: rts_result = rts_trait.deep_copy() # now if there are argument for this application, set the values in the array if self.arguments and rts_result: # if never tried to line up arguments with parameters, do that if not self._resolved_arguments: self._resolved_arguments = True params = trait._fetch_all_parameters(res_opt) param_found = None a_value = None for index, argument in enumerate(self.arguments): param_found = params.resolve_parameter( index, argument.get_name()) argument._resolved_parameter = param_found a_value = argument.value a_value = ctx.corpus._const_type_check( res_opt, self.in_document, param_found, a_value) argument.value = a_value for argument in self.arguments: rts_result.set_parameter_value_from_argument( trait, argument) # register set of possible symbols ctx.corpus._register_definition_reference_symbols( self.fetch_object_definition(res_opt), kind, res_opt._symbol_ref_set) # get the new cache tag now that we have the list of docs cache_tag = ctx.corpus._create_definition_cache_tag( res_opt, self, kind, '', cache_by_path, trait.at_corpus_path) if cache_tag: ctx._cache[cache_tag] = rts_result else: # cache was found # get the SymbolSet for this cached object key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind) res_opt._symbol_ref_set = ctx.corpus._definition_reference_symbols.get( key) # merge child document set with current curr_sym_ref_set._merge(res_opt._symbol_ref_set) res_opt._symbol_ref_set = curr_sym_ref_set return rts_result
async def logical_manipulation_using_projections( self, corpus: CdmCorpusDefinition): '''This sample demonstrates how to model a set of common scenarios using projections. The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model. Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'. To get an overview of the projections feature as well as all of the supported operations refer to the link below. https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview ''' print('Create logical entity definition.') logical_folder = await corpus.fetch_object_async( 'output:/') # type: CdmFolderDefinition logical_doc = logical_folder.documents.append('Person.cdm.json') logical_doc.imports.append('local:/Address.cdm.json') entity = logical_doc.definitions.append( 'Person') # type: CdmEntityDefinition # Add 'name' data typed attribute. name_attr = entity.attributes.append( 'name') # type: CdmTypeAttributeDefinition name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'age' data typed attribute. age_attr = entity.attributes.append( 'age') # type: CdmTypeAttributeDefinition age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'address' entity typed attribute. entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address') entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True) apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute') apply_default_behavior(entity_attr, 'addressFK', 'address') entity.attributes.append(entity_attr) # Add 'email' data typed attribute. email_attr = entity.attributes.append( 'email') # type: CdmTypeAttributeDefinition email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Save the logical definition of Person. await entity.in_document.save_as_async('Person.cdm.json') print( 'Get \'resolved\' folder where the resolved entities will be saved.' ) resolved_folder = await corpus.fetch_object_async( 'output:/') # type: CdmFolderDefinition res_opt = ResolveOptions(entity) # To get more information about directives and their meaning refer to # https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes # We will start by resolving this entity with the 'normalized' direcitve. # This directive will be used on this and the next two examples so we can analize the resolved entity # without the array expansion. print('Resolving logical entity with normalized directive.') res_opt.directives = AttributeResolutionDirectiveSet({'normalized'}) res_normalized_entity = await entity.create_resolved_entity_async( f'normalized_{entity.entity_name}', res_opt, resolved_folder) await res_normalized_entity.in_document.save_as_async( f'{res_normalized_entity.entity_name}.cdm.json') # Another common scenario is to resolve an entity using the 'referenceOnly' directive. # This directives is used to replace the relationships with a foreign key. print('Resolving logical entity with referenceOnly directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) res_reference_only_entity = await entity.create_resolved_entity_async( f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder) await res_reference_only_entity.in_document.save_as_async( f'{res_reference_only_entity.entity_name}.cdm.json') # When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that # a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the # entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group. print('Resolving logical entity with structured directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'structured'}) res_structured_entity = await entity.create_resolved_entity_async( f'structured_{entity.entity_name}', res_opt, resolved_folder) await res_structured_entity.in_document.save_as_async( f'{res_structured_entity.entity_name}.cdm.json') # Now let us remove the 'normalized' directive so the array expansion operation can run. print('Resolving logical entity without directives (array expansion).') res_opt.directives = AttributeResolutionDirectiveSet({}) res_array_entity = await entity.create_resolved_entity_async( f'array_expansion_{entity.entity_name}', res_opt, resolved_folder) await res_array_entity.in_document.save_as_async( f'{res_array_entity.entity_name}.cdm.json')
def fetch_resolved_entity_references( self, res_opt: Optional['ResolveOptions'] = None ) -> 'ResolvedEntityReferenceSet': res_opt = res_opt if res_opt is not None else ResolveOptions( wrt_doc=self) from cdm.resolvedmodel import AttributeResolutionContext, ResolvedEntityReference, ResolvedEntityReferenceSide, ResolvedEntityReferenceSet from cdm.utilities import ResolveOptions from .cdm_object import CdmObject rts_this_att = self._fetch_resolved_traits(res_opt) res_guide = self.resolution_guidance # this context object holds all of the info about what needs to happen to resolve these attributes arc = AttributeResolutionContext(res_opt, res_guide, rts_this_att) rel_info = self._get_relationship_info(res_opt, arc) if not rel_info.is_by_ref or rel_info.is_array: return None # only place this is used, so logic here instead of encapsulated. # make a set and the one ref it will hold rers = ResolvedEntityReferenceSet(res_opt) rer = ResolvedEntityReference() # referencing attribute(s) come from this attribute rer.referencing._rasb.merge_attributes( self._fetch_resolved_attributes(res_opt)) def resolve_side( ent_ref: 'CdmEntityReference') -> ResolvedEntityReferenceSide: side_other = ResolvedEntityReferenceSide() if ent_ref: # reference to the other entity, hard part is the attribue name. # by convention, this is held in a trait that identifies the key side_other.entity = ent_ref.fetch_object_definition(res_opt) if side_other.entity: other_attribute = None other_opts = ResolveOptions(wrt_doc=res_opt.wrt_doc, directives=res_opt.directives) trait = ent_ref._fetch_resolved_traits(other_opts).find( other_opts, 'is.identifiedBy') if trait and trait.parameter_values and trait.parameter_values.length: other_ref = trait.parameter_values.fetch_parameter_value( 'attribute').value if isinstance(other_ref, CdmObject): other_attribute = other_ref.fetch_object_definition( other_opts) if other_attribute: side_other._rasb.own_one( side_other.entity. _fetch_resolved_attributes(other_opts).get( other_attribute.get_name()).copy()) return side_other # either several or one entity # for now, a sub for the 'select one' idea if self.entity.explicit_reference: ent_pick_from = self.entity.fetch_object_definition(res_opt) atts_pick = ent_pick_from.attributes if ent_pick_from and atts_pick: for attribute in atts_pick: if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF: entity = attribute.entity rer.referenced.append(resolve_side(entity)) else: rer.referenced.append(resolve_side(self.entity)) rers.rer_set.append(rer) return rers
def fetch_replacement_value(cls, res_opt: 'ResolveOptions', old_value: 'CdmArgumentValue', new_value: 'CdmArgumentValue', was_set: bool) -> 'CdmArgumentValue': from cdm.objectmodel import CdmObject if old_value is None: return new_value if not was_set: # Must explicitly set a value to override if a new value is not set, then new_value holds nothing or the # default. In this case, if there was already a value in this argument then just keep using it. return old_value if not isinstance(old_value, CdmObject): return new_value ov = old_value # type: CdmObject nv = new_value # type: CdmObject # Replace an old table with a new table? Actually just mash them together. if (ov is not None and ov.object_type == CdmObjectType.ENTITY_REF and nv is not None and not isinstance(nv, str) and nv.object_type == CdmObjectType.ENTITY_REF): old_ent = ov.fetch_object_definition( res_opt) # type: CdmConstantEntityDefinition new_ent = nv.fetch_object_definition( res_opt) # type: CdmConstantEntityDefinition # Check that the entities are the same shape. if new_ent is None: return ov # BUG ent_def_shape = old_ent.entity_shape.fetch_object_definition( res_opt) if (old_ent is None or (ent_def_shape != new_ent.entity_shape.fetch_object_definition(res_opt))): return nv old_cv = old_ent.constant_values new_cv = new_ent.constant_values # Rows in old? if not old_cv: return nv # Rows in new? if not new_cv: return ov # Make a set of rows in the old one and add the new ones. This will union the two find rows in the new # one that are not in the old one. Slow, but these are small usually. unioned_rows = OrderedDict() # see if any of the entity atts are the primary key, meaning, the only thing that causes us to merge dups unique. # i know this makes you think about a snake eating its own tail, but fetch the resolved attributes of the constant shape pk_att = -1 if ent_def_shape: from cdm.utilities import ResolveOptions res_opt_shape = ResolveOptions(ent_def_shape.in_document) res_atts_shape = ent_def_shape._fetch_resolved_attributes( res_opt_shape) # type: ResolvedAttributeSet if res_atts_shape: tmp_item = next( filter( lambda ra: (ra.resolved_traits.find( res_opt_shape, 'is.identifiedBy') is not None), res_atts_shape._set), None) pk_att = res_atts_shape._set.index( tmp_item) if tmp_item else -1 for i in range(len(old_cv)): row = old_cv[i] # the entity might have a PK, if so, only look at that values as the key if pk_att != -1: key = row[pk_att] else: key = '::'.join(row) unioned_rows[key] = row for i in range(len(new_cv)): row = new_cv[i] # the entity might have a PK, if so, only look at that values as the key if pk_att != -1: key = row[pk_att] else: key = '::'.join(row) unioned_rows[key] = row if len(unioned_rows) == len(old_cv): return ov replacement_ent = old_ent.copy( res_opt) # type: CdmConstantEntityDefinition replacement_ent.constant_values = list(unioned_rows.values()) return res_opt.wrt_doc.ctx.corpus.make_ref( CdmObjectType.ENTITY_REF, replacement_ent, False) return new_value
async def populate_manifest_relationships_async( self, option: CdmRelationshipDiscoveryStyle = CdmRelationshipDiscoveryStyle. ALL ) -> None: """Populate the relationships that the entities in the current manifest are involved in.""" with logger._enter_scope( self._TAG, self.ctx, self.populate_manifest_relationships_async.__name__): self.relationships.clear() rel_cache = set() # Set[str] for ent_dec in self.entities: ent_path = await self._get_entity_path_from_declaration( ent_dec, self) curr_entity = await self.ctx.corpus.fetch_object_async( ent_path) # type: Optional[CdmEntityDefinition] if curr_entity is None: continue # handle the outgoing relationships outgoing_rels = self.ctx.corpus.fetch_outgoing_relationships( curr_entity) # List[CdmE2ERelationship] if outgoing_rels: for rel in outgoing_rels: cache_key = rel2_cache_key(rel) if cache_key not in rel_cache and self._is_rel_allowed( rel, option): self.relationships.append( self._localize_rel_to_manifest(rel)) rel_cache.add(cache_key) incoming_rels = self.ctx.corpus.fetch_incoming_relationships( curr_entity) # type: List[CdmE2ERelationship] if incoming_rels: for in_rel in incoming_rels: # get entity object for current toEntity current_in_base = await self.ctx.corpus.fetch_object_async( in_rel.to_entity, self) # type: Optional[CdmEntityDefinition] if not current_in_base: continue # create graph of inheritance for to current_in_base # graph represented by an array where entity at i extends entity at i+1 to_inheritance_graph = [ ] # type: List[CdmEntityDefinition] while current_in_base: res_opt = ResolveOptions( wrt_doc=current_in_base.in_document) current_in_base = current_in_base.extends_entity.fetch_object_definition( res_opt ) if current_in_base.extends_entity else None if current_in_base: to_inheritance_graph.append(current_in_base) # add current incoming relationship cache_key = rel2_cache_key(in_rel) if cache_key not in rel_cache and self._is_rel_allowed( in_rel, option): self.relationships.append( self._localize_rel_to_manifest(in_rel)) rel_cache.add(cache_key) # if A points at B, A's base classes must point at B as well for base_entity in to_inheritance_graph: incoming_rels_for_base = self.ctx.corpus.fetch_incoming_relationships( base_entity) # type: List[CdmE2ERelationship] if incoming_rels_for_base: for in_rel_base in incoming_rels_for_base: new_rel = self.ctx.corpus.make_object( CdmObjectType.E2E_RELATIONSHIP_DEF, '') new_rel.from_entity = in_rel_base.from_entity new_rel.from_entity_attribute = in_rel_base.from_entity_attribute new_rel.to_entity = in_rel.to_entity new_rel.to_entity_attribute = in_rel.to_entity_attribute base_rel_cache_key = rel2_cache_key( new_rel) if base_rel_cache_key not in rel_cache and self._is_rel_allowed( new_rel, option): self.relationships.append( self._localize_rel_to_manifest( new_rel)) rel_cache.add(base_rel_cache_key) if self.sub_manifests: for sub_manifest_def in self.sub_manifests: corpus_path = self.ctx.corpus.storage.create_absolute_corpus_path( sub_manifest_def.definition, self) sub_manifest = await self.ctx.corpus.fetch_object_async( corpus_path) # type: Optional[CdmManifestDefinition] await sub_manifest.populate_manifest_relationships_async( option)
async def refresh_async(self, res_opt: Optional['ResolveOptions'] = None) -> bool: """updates indexes for document content, call this after modifying objects in the document""" res_opt = res_opt if res_opt is not None else ResolveOptions(wrt_doc=self) self._needs_indexing = True return await self._index_if_needed(res_opt)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an AddTypeAttribute operation and a condition using the object model""" corpus = ProjectionTestUtils.get_local_corpus( self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount( 'local', LocalAdapter( TestHelper.get_actual_output_folder_path( self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an AddTypeAttribute operation add_type_attr_op = corpus.make_object( CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF) add_type_attr_op.type_attribute = corpus.make_object( CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testType') add_type_attr_op.type_attribute.data_type = corpus.make_ref( CdmObjectType.DATA_TYPE_REF, 'entityName', True) projection.operations.append(add_type_attr_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object( CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet( set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddTypeAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # Type attribute: "testType" self.assertEqual(5, len(resolved_entity_with_reference_only.attributes)) self.assertEqual( 'id', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual( 'name', resolved_entity_with_reference_only.attributes[1].name) self.assertEqual( 'value', resolved_entity_with_reference_only.attributes[2].name) self.assertEqual( 'date', resolved_entity_with_reference_only.attributes[3].name) self.assertEqual( 'testType', resolved_entity_with_reference_only.attributes[4].name) self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4]. applied_traits.item('is.linkedEntity.name')) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet( set(['structured'])) resolved_entity_with_structured = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddTypeAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # No Type attribute added, condition was false self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder.folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter(folder.namespace) # type: StorageAdapter try: if adapter.can_read(): # log message used by navigator, do not change or remove logger.debug(self._TAG, self._ctx, 'request file: {}'.format(doc_path), self._load_document_from_path_async.__name__) json_data = await adapter.read_async(doc_path) # log message used by navigator, do not change or remove logger.debug(self._TAG, self._ctx, 'received file: {}'.format(doc_path), self._load_document_from_path_async.__name__) else: raise Exception('Storage Adapter is not enabled to read.') except Exception as e: # log message used by navigator, do not change or remove logger.debug(self._TAG, self._ctx, 'fail file: {}'.format(doc_path), self._load_document_from_path_async.__name__) message = 'Could not read {} from the \'{}\' namespace.\n Reason: {}'.format(doc_path, folder.namespace, e) # when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if res_opt and res_opt.shallow_validation: logger.warning(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__) else: logger.error(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__) return None try: fs_modified_time = await adapter.compute_last_modified_time_async(doc_path) except Exception as e: logger.warning(self._TAG, self._ctx, 'Failed to compute file last modified time. Reason {}'.format(e), self._load_document_from_path_async.__name__) if not doc_name: logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._load_document_from_path_async.__name__) return None # If loading an odi.json/model.json file, check that it is named correctly. if doc_name.lower().endswith(self.ODI_EXTENSION) and not doc_name.lower() == self.ODI_EXTENSION: logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format( doc_name, self.ODI_EXTENSION), self._load_document_from_path_async.__name__) return None if doc_name.lower().endswith(self.MODEL_JSON_EXTENSION) and not doc_name.lower() == self.MODEL_JSON_EXTENSION: logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format( doc_name, self.MODEL_JSON_EXTENSION), self._load_document_from_path_async.__name__) return None # Fetch the correct persistence class to use. persistence_class = self._fetch_registered_persistence_format(doc_name) if persistence_class: try: method = persistence_class.from_data parameters = [self._ctx, doc_name, json_data, folder] # check if from_data() is asynchronous for this persistence class. if persistence_class not in self._is_registered_persistence_async: # Cache whether this persistence class has async methods. self._is_registered_persistence_async[persistence_class] = persistence_class.is_persistence_async if self._is_registered_persistence_async[persistence_class]: doc_content = await method(*parameters) else: doc_content = method(*parameters) except Exception as e: logger.error(self._TAG, self._ctx, 'Could not convert \'{}\'. Reason \'{}\''.format(doc_name, e), self._load_document_from_path_async.__name__) return None else: # could not find a registered persistence class to handle this document type. logger.error(self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\''.format( doc_name), self._load_document_from_path_async.__name__) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy(ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus.default_resolution_directives), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content
def _finalize_attribute_context(self, res_opt: ResolveOptions, path_start: str, doc_home: CdmDocumentDefinition, doc_from: CdmDocumentDefinition, moniker_for_doc_from: str, finished: bool): # run over the attCtx tree again and 'fix it' fix means replace the parent and lineage reference path strings with # final values from new home and set the inDocument and fix any references to definitions # keep track of the paths to documents for fixing symbol refs. expensive to compute found_doc_paths = OrderedDict() if moniker_for_doc_from and not moniker_for_doc_from.isspace(): moniker_for_doc_from = '{}/'.format(moniker_for_doc_from) # first step makes sure every node in the tree has a good path for itself and a good document # second pass uses the paths from nodes to fix references to other nodes def _fix_att_ctx_node_paths(sub_item: CdmObject, path_from: str) -> None: ac = cast(CdmAttributeContext, sub_item) if ac is None: return ac.in_document = doc_home # fix up the reference to definition. need to get path from this document to the # add moniker if this is a reference if ac.definition: ac.definition.in_document = doc_home if ac.definition and ac.definition.named_reference: # need the real path to this thing from the explicitRef held in the portable reference # the real path is {monikerFrom/}{path from 'from' document to document holding the explicit ref/{declaredPath of explicitRef}} # if we have never looked up the path between docs, do that now doc_from_def = ac.definition.explicit_reference.in_document # if all parts not set, this is a broken portal ref! path_between_docs = found_doc_paths[ doc_from_def] if doc_from_def in found_doc_paths else None if path_between_docs is None: path_between_docs = doc_from._import_path_to_doc( doc_from_def) if path_between_docs is None: # hmm. hmm. path_between_docs = '' found_doc_paths[doc_from] = path_between_docs cast('CdmObjectReference', ac.definition)._localize_portable_reference( res_opt, '{}{}'.format(moniker_for_doc_from, path_between_docs)) # doc of parent ref if ac.parent: ac.parent.in_document = doc_home # doc of lineage refs if ac.lineage: for lin in ac.lineage: lin.in_document = doc_home divider = '/' if not ac.at_corpus_path or not path_from.endswith( '/') else '' ac.at_corpus_path = '{}{}{}'.format(path_from, divider, ac.name) if not ac.contents: return # look at all children for sub_sub in ac.contents: if sub_sub.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF: _fix_att_ctx_node_paths(sub_sub, ac.at_corpus_path) _fix_att_ctx_node_paths(self, path_start) # now fix any lineage and parent references def _fix_att_ctx_node_lineage(sub_item) -> None: ac = cast(CdmAttributeContext, sub_item) # type: CdmAttributeContext if not ac: return # for debugLindeage, write id # ac.name = '{}({})'.format(ac.name, ac.id) # parent ref if ac.parent and ac.parent.explicit_reference: ac.parent.named_reference = cast( CdmAttributeContext, ac.parent.explicit_reference).at_corpus_path # for debugLindeage, write id # ac.parent.named_reference = cast(CdmAttributeContext, ac.parent.explicit_reference).at_corpus_path + '(' + ac.parent.explicit_reference.id + ')' # fix lineage if ac.lineage: for lin in ac.lineage: if lin.explicit_reference: # use the new path as the ref lin.named_reference = cast( CdmAttributeContext, lin.explicit_reference).at_corpus_path # for debugLindeage, write id # lin.named_reference = cast(CdmAttributeContext, lin.explicit_reference).at_corpus_path + '(' + lin.explicit_reference.id + ')' if not ac.contents: return # look at all children for sub_sub in ac.contents: _fix_att_ctx_node_lineage(sub_sub) _fix_att_ctx_node_lineage(self) if finished: res_opt._save_resolutions_on_copy = False res_opt._map_old_ctx_to_new_ctx = None return True
async def logical_manipulation_using_projections(): '''This sample demonstrates how to model a set of common scenarios using projections. The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model. Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'. A single logical definition can be resolved into multiple physical layouts. The directives are used to instruct the ObjectModel about how it should to resolve the logical definition provided. To achieve this, we define projections that run conditionally, depending on the directives provided when calling create_resolved_entity_async. To get an overview of the projections feature as well as all of the supported operations refer to the link below. https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview ''' # Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths corpus = CdmCorpusDefinition() print('Configure storage adapters') # Configure storage adapters to point at the target local manifest location and at the fake public standards path_from_exe_to_example_root = '../../' corpus.storage.mount( 'local', LocalAdapter(path_from_exe_to_example_root + '8-logical-manipulation-using-projections/sample-data')) corpus.storage.default_namespace = 'local' # local is our default. so any paths that start out navigating without a device tag will assume local # Fake cdm, normaly use the CDM Standards adapter # Mount it as the 'cdm' device, not the default so must use 'cdm:/folder' to get there corpus.storage.mount( 'cdm', LocalAdapter(path_from_exe_to_example_root + 'example-public-standards')) print('Create logical entity definition.') logical_folder = await corpus.fetch_object_async( 'local:/') # type: CdmFolderDefinition logical_doc = logical_folder.documents.append('Person.cdm.json') logical_doc.imports.append('Address.cdm.json') entity = logical_doc.definitions.append( 'Person') # type: CdmEntityDefinition # Add 'name' data typed attribute. name_attr = entity.attributes.append( 'name') # type: CdmTypeAttributeDefinition name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'age' data typed attribute. age_attr = entity.attributes.append( 'age') # type: CdmTypeAttributeDefinition age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Add 'address' entity typed attribute. entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address') entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True) apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute') apply_default_behavior(entity_attr, 'addressFK', 'address') entity.attributes.append(entity_attr) # Add 'email' data typed attribute. email_attr = entity.attributes.append( 'email') # type: CdmTypeAttributeDefinition email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True) # Save the logical definition of Person. await entity.in_document.save_as_async('Person.cdm.json') print('Get \'resolved\' folder where the resolved entities will be saved.') resolved_folder = await corpus.fetch_object_async( 'local:/resolved/') # type: CdmFolderDefinition res_opt = ResolveOptions(entity) # To get more information about directives and their meaning refer to # https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes # We will start by resolving this entity with the 'normalized' direcitve. # This directive will be used on this and the next two examples so we can analize the resolved entity # without the array expansion. print('Resolving logical entity with normalized directive.') res_opt.directives = AttributeResolutionDirectiveSet({'normalized'}) res_normalized_entity = await entity.create_resolved_entity_async( f'normalized_{entity.entity_name}', res_opt, resolved_folder) await res_normalized_entity.in_document.save_as_async( f'{res_normalized_entity.entity_name}.cdm.json') # Another common scenario is to resolve an entity using the 'referenceOnly' directive. # This directives is used to replace the relationships with a foreign key. print('Resolving logical entity with referenceOnly directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) res_reference_only_entity = await entity.create_resolved_entity_async( f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder) await res_reference_only_entity.in_document.save_as_async( f'{res_reference_only_entity.entity_name}.cdm.json') # When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that # a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the # entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group. print('Resolving logical entity with structured directive.') res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'structured'}) res_structured_entity = await entity.create_resolved_entity_async( f'structured_{entity.entity_name}', res_opt, resolved_folder) await res_structured_entity.in_document.save_as_async( f'{res_structured_entity.entity_name}.cdm.json') # Now let us remove the 'normalized' directive so the array expansion operation can run. print('Resolving logical entity without directives (array expansion).') res_opt.directives = AttributeResolutionDirectiveSet({}) res_array_entity = await entity.create_resolved_entity_async( f'array_expansion_{entity.entity_name}', res_opt, resolved_folder) await res_array_entity.in_document.save_as_async( f'{res_array_entity.entity_name}.cdm.json')