async def test_ref_entity_with_slash_path(self): test_name = 'test_ref_entity_with_slash_path' slash_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) slash_local_path = ( slash_corpus.storage.namespace_adapters.get('local')).root slash_adapter = LocalAdapterWithSlashPath(slash_local_path, '/') slash_corpus.storage.mount('slash', slash_adapter) slash_corpus.storage.defaultNamespace = 'slash' # load model.json files with paths generated using both '/' and '\' slash_manifest = await slash_corpus.fetch_object_async( 'slash:/model.json') # manually add the reference model location, path will vary on each machine ref_model_trait = slash_manifest.exhibits_traits.item( 'is.modelConversion.referenceModelMap') entity_path = slash_manifest.entities[0].entity_path ref_model_trait.arguments[0].value[ 0].location = slash_adapter.create_adapter_path( entity_path[0:entity_path.rindex('/')]) slash_model = await ManifestPersistence.to_data( slash_manifest, ResolveOptions(), CopyOptions()) self.assertIsNotNone(slash_model) self.assertEqual(1, len(slash_model.entities)) back_slash_corpus = TestHelper.get_local_corpus( self.tests_subpath, test_name) back_slash_local_path = ( back_slash_corpus.storage.namespace_adapters.get('local')).root back_slash_adapter = LocalAdapterWithSlashPath(back_slash_local_path, '\\') back_slash_corpus.storage.mount('backslash', back_slash_adapter) back_slash_corpus.storage.default_namespace = 'backslash' back_slash_manifest = await back_slash_corpus.fetch_object_async( 'backslash:/model.json') # manually add the reference model location, path will vary on each machine back_slash_ref_model_trait = back_slash_manifest.exhibits_traits.item( 'is.modelConversion.referenceModelMap') back_slash_entity_path = back_slash_manifest.entities[0].entity_path back_slash_ref_model_trait.arguments[0].value[ 0].location = back_slash_adapter.create_adapter_path( back_slash_entity_path[0:back_slash_entity_path. rindex('/')]).replace('/', '\\\\') back_slash_model = await ManifestPersistence.to_data( back_slash_manifest, ResolveOptions(), CopyOptions()) self.assertIsNotNone(back_slash_model) self.assertEqual(1, len(back_slash_model.entities))
async def save_as_async(self, new_name: str, save_referenced: bool = False, options: Optional['CopyOptions'] = None) -> bool: """saves the document back through the adapter in the requested format format is specified via document name/extension based on conventions: 'model.json' for back compat model, '*.manifest.json' for manifest, '*.json' for cdm defs save_referenced (default False) when true will also save any schema defintion documents that are linked from the source doc and that have been modified. existing document names are used for those.""" options = options if options is not None else CopyOptions() index_if_needed = await self._index_if_needed( ResolveOptions( wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives)) if not index_if_needed: logger.error( self._TAG, self.ctx, 'Failed to index document prior to save {}.'.format(self.name), self.save_as_async.__name__) return False if new_name == self.name: self._is_dirty = False return await self.ctx.corpus.persistence._save_document_as_async( self, options, new_name, save_referenced)
async def test_not_saving_config_file(self): """Test setting SaveConfigFile to false and checking if the file is not saved.""" test_name = 'test_not_saving_config_file' corpus = TestHelper.get_local_corpus(self.test_subpath, test_name) # Load manifest from input folder. manifest = await corpus.fetch_object_async('default.manifest.cdm.json') # Move manifest to output folder. output_folder = corpus.storage.fetch_root_folder('output') for entity_dec in manifest.entities: entity = await corpus.fetch_object_async(entity_dec.entity_path, manifest) output_folder.documents.append(entity.in_document) output_folder.documents.append(manifest) # Make sure the output folder is empty. TestHelper.delete_files_from_actual_output(TestHelper.get_actual_output_folder_path(self.test_subpath, test_name)) # Save manifest to output folder. copy_options = CopyOptions() copy_options.save_config_file = False await manifest.save_as_async("default.manifest.cdm.json", False, copy_options) # Compare the result. TestHelper.compare_folder_files_equality( TestHelper.get_expected_output_folder_path(self.test_subpath, test_name), TestHelper.get_actual_output_folder_path(self.test_subpath, test_name))
async def _save_linked_documents_async(self, options: 'CopyOptions') -> bool: docs = [] if not options: options = CopyOptions() # the only linked documents would be the imports if self.imports: for imp in self.imports: # get the document object from the import doc_path = self.ctx.corpus.storage.create_absolute_corpus_path( imp.corpus_path, self) if not doc_path: logger.error(self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_VALDN_INVALID_CORPUS_PATH, imp.corpus_path) return False try: obj_at = await self.ctx.corpus.fetch_object_async(doc_path) if not isinstance(obj_at, CdmDocumentDefinition): logger.error( self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_INVALID_CAST, doc_path, 'CdmDocumentDefinition') return False elif not obj_at: logger.error( self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_PERSIST_OBJECT_NOT_FOUND, imp.corpus_path) return False doc_imp = obj_at.in_document if doc_imp is not None and doc_imp._is_dirty: docs.append(doc_imp) except Exception as e: logger.error(self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_PERSIST_OBJECT_NOT_FOUND, imp.corpus_path + ' ' + str(e)) return False for doc_imp in docs: # save it with the same name if not await doc_imp.save_as_async(doc_imp.name, True, options): logger.error(self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_IMPORT_SAVING_FAILURE, doc_imp.name) return False return True
async def save_as_async(self, new_name: str, save_referenced: bool = False, options: Optional['CopyOptions'] = None) -> bool: """saves the document back through the adapter in the requested format format is specified via document name/extension based on conventions: 'model.json' for back compat model, '*.manifest.json' for manifest, '*.json' for cdm defs save_referenced (default False) when true will also save any schema defintion documents that are linked from the source doc and that have been modified. existing document names are used for those.""" with logger._enter_scope(self._TAG, self.ctx, self.save_as_async.__name__): options = options if options is not None else CopyOptions() index_if_needed = await self._index_if_needed( ResolveOptions( wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives)) if not index_if_needed: logger.error(self.ctx, self._TAG, self.save_as_async.__name__, self.at_corpus_path, CdmLogCode.ERR_INDEX_FAILED, self.name) return False if new_name == self.name: self._is_dirty = False # Import here to avoid circular import from .cdm_entity_def import CdmEntityDefinition from .cdm_manifest_def import CdmManifestDefinition if not await self.ctx.corpus.persistence._save_document_as_async( self, options, new_name, save_referenced): return False # Log the telemetry if the document is a manifest if isinstance(self, CdmManifestDefinition): for entity in self.entities: if isinstance(entity, CdmLocalEntityDeclarationDefinition): entity.reset_last_file_modified_old_time() for relationship in self.relationships: relationship.reset_last_file_modified_old_time() logger._ingest_manifest_telemetry( self, self.ctx, CdmDocumentDefinition.__name__, self.save_as_async.__name__, self.at_corpus_path) # Log the telemetry of all entities contained in the document else: for obj in self.definitions: if isinstance(obj, CdmEntityDefinition): logger._ingest_entity_telemetry( obj, self.ctx, CdmDocumentDefinition.__name__, self.save_as_async.__name__, obj.at_corpus_path) return True
def fetch_value_string(self, res_opt: 'ResolveOptions') -> str: from cdm.objectmodel import CdmObject if self.value is None: return '' if isinstance(self.value, str): return self.value elif isinstance(self.value, CdmObject): # If this is a constant table, then expand into an HTML table. object_def = self.value.fetch_object_definition( res_opt) # type: CdmConstantEntityDefinition if self.value.object_type == CdmObjectType.ENTITY_REF and object_def is not None and object_def.object_type == CdmObjectType.CONSTANT_ENTITY_DEF: ent_shape = object_def.entity_shape ent_values = object_def.constant_values if not ent_values: return '' rows = [] shape_atts = ent_shape._fetch_resolved_attributes(res_opt) if shape_atts is not None and shape_atts._set is not None: for row_data in ent_values: if not row_data: continue row = OrderedDict() for (c, tvalue) in enumerate(row_data): col_att = shape_atts._set[c] if col_att is not None and tvalue is not None: row[col_att.resolved_name] = tvalue rows.append(row) rows_string = [self._spew_dict(row) for row in rows] return '[' + ','.join(rows_string) + ']' # Should be a reference to an object. from cdm.persistence import PersistenceLayer from cdm.utilities import CopyOptions data = PersistenceLayer.to_data(self.value, res_opt, CopyOptions(string_refs=False), PersistenceLayer.CDM_FOLDER) if isinstance(data, str): return data # TODO: the line bellow won't work, the result is going to be the address of the object. return str(data) else: return str(self.value) return ''
async def save_as_async(self, new_name: str, save_referenced: bool = False, options: Optional['CopyOptions'] = None) -> bool: """saves the document back through the adapter in the requested format format is specified via document name/extension based on conventions: 'model.json' for back compat model, '*.manifest.json' for manifest, '*.json' for cdm defs save_referenced (default false) when true will also save any schema defintion documents that are linked from the source doc and that have been modified. existing document names are used for those.""" options = options if options is not None else CopyOptions() if new_name == self.name: self._is_dirty = False return await self.ctx.corpus._save_document_as(self, options, new_name, save_referenced)
async def test_loading_and_saving_csv_partition_traits(self): """Tests that the trait is.partition.format.CSV is saved when contains arguments not supported by fileFormatSettings.""" cdm_corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_loading_and_saving_csv_partition_traits') manifest = await cdm_corpus.fetch_object_async( 'model.json') # type: CdmManifestDefinition # If the data partition trait is.partition.format.CSV being saved has arguments that are not supported by fileFormatSettings # the trait should also be persisted. manifestData = await ManifestPersistence.to_data( manifest, ResolveOptions(manifest.in_document), CopyOptions()) localEntity = manifestData.entities[0] self.assertEqual(len(localEntity.partitions[0].traits), 1) # Remove the argument that is not supported by fileFormatSettings and check if the trait is removed after that. csv_trait = manifest.entities[0].data_partitions[0].exhibits_traits[0] csv_trait.arguments.remove(csv_trait.arguments.item('newline')) manifestData = await ManifestPersistence.to_data( manifest, ResolveOptions(manifest.in_document), CopyOptions()) localEntity = manifestData.entities[0] self.assertIsNone(localEntity.partitions[0].traits)
async def test_from_and_to_data_with_elevated_traits(self): corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_from_and_to_data_with_elevated_traits') # need to set schema docs to the cdm namespace instead of using resources corpus.storage.mount("cdm", LocalAdapter(TestHelper.get_schema_docs_root())) def callback(level, message): self.assertTrue('unable to resolve an entity' not in message) corpus.set_event_callback(callback, CdmStatusLevel.WARNING) entity = await corpus.fetch_object_async( 'local:/Account.cdm.json/Account') # type: CdmEntityDefinition res_entity = await entity.create_resolved_entity_async( '{}_'.format(entity.entity_name)) # type: CdmEntityDefinition PersistenceLayer.to_data( res_entity, ResolveOptions(wrt_doc=res_entity.in_document), CopyOptions(string_refs=True), PersistenceLayer.CDM_FOLDER)
async def test_imports_for_rel_elevated_purpose_traits(self): """ Testing that import for elevated purpose traits for relationships are added. """ test_name = 'test_imports_for_rel_elevated_purpose_traits' corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) root_manifest = await corpus.fetch_object_async( 'local:/default.manifest.cdm.json' ) # type: 'CdmManifestDefinition' sub_manifest = await corpus.fetch_object_async( root_manifest.sub_manifests[0].definition) await corpus.calculate_entity_graph_async(root_manifest) await root_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) self.assertEqual('specialized/Gold.cdm.json', root_manifest.imports[0].corpus_path) self.assertEqual('/Lead.cdm.json', sub_manifest.imports[0].corpus_path) corpus.storage.fetch_root_folder('output').documents.append( root_manifest) corpus.storage.fetch_root_folder('output').documents.append( sub_manifest) copy_options = CopyOptions() copy_options.save_config_file = False await root_manifest.save_as_async('output:/default.manifest.cdm.json', False, copy_options) # "acct.trait" in Acct.cdm.json. relationships in the manifests contain these 2 traits, # so the manifest should import these two entity documents, but Lead.cdm.json imports Acct.cdm.json. # Thus, the manifest can only import Lead.cdm.json await sub_manifest.save_as_async( 'output:/default-submanifest.manifest.cdm.json', False, copy_options) TestHelper.compare_folder_files_equality( TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name), TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name))
async def save_actual_entity_and_validate_with_expected( self, expected_path: str, actual_resolved_entity_def: CdmEntityDefinition, update_expected_output: bool = False) -> None: """Runs validation to test actual output vs expected output for attributes collection vs attribute context.""" co = CopyOptions() co._is_top_level_document = False await actual_resolved_entity_def.in_document.save_as_async( actual_resolved_entity_def.in_document.name, options=co) actual_path = actual_resolved_entity_def.ctx.corpus.storage.corpus_path_to_adapter_path( actual_resolved_entity_def.in_document.at_corpus_path) with open(actual_path, 'r', encoding='utf-8') as actual_file: actual_ctx = actual_file.read() if update_expected_output: with open(expected_path, 'w', encoding='utf-8') as expected_file: expected_file.write(actual_ctx) with open(expected_path, 'r', encoding='utf-8') as expected_file: expected_ctx = expected_file.read() self.assertEqual(expected_ctx, actual_ctx)
async def test_cardinality_persistence(self): ''' Testing that cardinality settings are loaded and saved correctly ''' corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_cardinality_persistence') # test from_data entity = await corpus.fetch_object_async( 'local:/someEntity.cdm.json/someEntity' ) # type: CdmEntityDefinition attribute = entity.attributes[0] # type: CdmTypeAttributeDefinition self.assertIsNotNone(attribute.cardinality) self.assertEqual(attribute.cardinality.minimum, '0') self.assertEqual(attribute.cardinality.maximum, '1') # test to_data attribute_data = TypeAttributePersistence.to_data( attribute, ResolveOptions(entity.in_document), CopyOptions()) self.assertIsNotNone(attribute_data.cardinality) self.assertEqual(attribute_data.cardinality.minimum, '0') self.assertEqual(attribute_data.cardinality.maximum, '1')
async def test_update_relationships(self): test_name = 'test_update_relationships' expected_rels = TestHelper.get_expected_output_data( self.tests_subpath, test_name, 'expectedRels.json') temp_from_file_path = 'fromEntTemp.cdm.json' temp_from_entity_path = 'local:/fromEntTemp.cdm.json/fromEnt' temp_to_entity_path = 'local:/toEnt.cdm.json/toEnt' corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) manifest = await corpus.fetch_object_async( 'local:/main.manifest.cdm.json') # type: CdmManifestDefinition manifest_no_to_ent = await corpus.fetch_object_async( 'local:/mainNoToEnt.manifest.cdm.json' ) # type: CdmManifestDefinition from_ent = await corpus.fetch_object_async( 'local:/fromEnt.cdm.json/fromEnt') # type: CdmEntityDefinition co = CopyOptions() co._is_top_level_document = False await from_ent.in_document.save_as_async(temp_from_file_path, options=co) async def reload_from_entity(): await from_ent.in_document.save_as_async(temp_from_file_path, options=co) # fetch again to reset the cache await corpus.fetch_object_async(temp_from_entity_path, None, shallow_validation=False, force_reload=True) try: # 1. test when entity attribute is removed await corpus.calculate_entity_graph_async(manifest) await manifest.populate_manifest_relationships_async() # check that the relationship has been created correctly self.verify_relationships(manifest, expected_rels) # now remove the entity attribute, which removes the relationship removed_attribute = from_ent.attributes[ 0] # type: CdmAttributeItem from_ent.attributes.pop(0) await reload_from_entity() await corpus.calculate_entity_graph_async(manifest) await manifest.populate_manifest_relationships_async() # check that the relationship has been removed self.verify_relationships(manifest, []) # 2. test when the to entity is removed # restore the entity to the original state from_ent.attributes.append(removed_attribute) await reload_from_entity() await corpus.calculate_entity_graph_async(manifest) await manifest.populate_manifest_relationships_async() # check that the relationship has been created correctly self.verify_relationships(manifest, expected_rels) # remove the to entity from_ent.attributes.pop(0) await reload_from_entity() # fetch again to reset the cache await corpus.fetch_object_async(temp_to_entity_path, None, False, True) await corpus.calculate_entity_graph_async(manifest_no_to_ent) await manifest_no_to_ent.populate_manifest_relationships_async() # check that the relationship has been removed self.verify_relationships(manifest_no_to_ent, []) finally: # clean up the file created from_path = corpus.storage.corpus_path_to_adapter_path( 'local:/' + temp_from_file_path) if os.path.exists(from_path): os.remove(from_path)
def fetch_value_string(self, res_opt: 'ResolveOptions') -> str: from cdm.objectmodel import CdmObject if self.value is None: return '' if isinstance(self.value, str): return self.value elif isinstance(self.value, CdmObject): # If this is a constant table, then expand into an HTML table. object_def = self.value.fetch_object_definition( res_opt) # type: CdmConstantEntityDefinition if self.value.object_type == CdmObjectType.ENTITY_REF and object_def is not None and object_def.object_type == CdmObjectType.CONSTANT_ENTITY_DEF: ent_shape = object_def.entity_shape ent_values = object_def.constant_values if not ent_values: return '' rows = [] shape_atts = ent_shape._fetch_resolved_attributes(res_opt) for row_data in ent_values: if not row_data: continue row = {} for (c, tvalue) in enumerate(row_data): col_att = shape_atts.set[c] if col_att is not None and tvalue is not None: row[col_att.resolved_name] = tvalue rows.append(row) if rows: keys = list(rows[0].keys()) keys.sort() first_key = keys[0] second_key = keys[1] if len(keys) > 1 else keys[0] rows.sort(key=lambda row: (row[first_key].lower(), row[ second_key].lower())) rows_string = [self._spew_dict(row) for row in rows] return '[' + ','.join(rows_string) + ']' # Should be a reference to an object. from cdm.persistence import persistence_layer from cdm.utilities import CopyOptions data = persistence_layer.to_data(self.value, res_opt, 'CdmFolder', CopyOptions(string_refs=False)) if isinstance(data, str): return data # TODO: the line bellow won't work, the result is going to be the address of the object. return str(data) else: return str(self.value) return ''
def test_persist_attribute_group_definition(self): corpus = CdmCorpusDefinition() att_group = CdmAttributeGroupDefinition(corpus.ctx, 'attGroup') persisted_group = AttributeGroupPersistence.to_data( att_group, ResolveOptions(att_group.in_document), CopyOptions()) self.assertIsNotNone(persisted_group.members)