def test_manifest_for_copy_data(self): """Test for copy data.""" test_name = 'test_manifest_for_copy_data' corpus = self.get_corpus() content = TestHelper.get_input_file_content( self.tests_subpath, test_name, 'complete.manifest.cdm.json') cdm_manifest = ManifestPersistence.from_object( corpus.ctx, 'docName', 'someNamespace', '/', ManifestContent().decode(content)) manifest_object = ManifestPersistence.to_data(cdm_manifest, None, None) self.assertEqual(manifest_object.schema, 'CdmManifestDefinition.cdm.json') self.assertEqual(manifest_object.jsonSchemaSemanticVersion, '0.9.0') self.assertEqual(manifest_object.manifestName, 'cdmTest') self.assertEqual(manifest_object.explanation, 'test cdm folder for cdm version 0.9+') self.assertEqual(1, len(manifest_object.imports)) self.assertEqual(manifest_object.imports[0].corpusPath, '/primitives.cdm.json') self.assertEqual(1, len(manifest_object.exhibitsTraits)) self.assertEqual(2, len(manifest_object.entities)) self.assertEqual(manifest_object.entities[0]['entityName'], 'testEntity') self.assertEqual(1, len(manifest_object.subManifests)) self.assertEqual(manifest_object.subManifests[0].definition, 'test definition') self.assertEqual(manifest_object.lastFileModifiedTime, None)
def test_load_local_entity_with_data_partition_pattern(self): content = TestHelper.get_input_file_content( self.test_subpath, 'test_load_local_entity_with_data_partition_pattern', 'entities.manifest.cdm.json') manifest_content = ManifestContent() manifest_content.decode(content) cdm_manifest = ManifestPersistence.from_object( CdmCorpusContext(CdmCorpusDefinition(), None), 'entities', 'testNamespace', '/', manifest_content) self.assertEqual(len(cdm_manifest.entities), 2) entity1 = cdm_manifest.entities[0] self.assertEqual(entity1.object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) self.assertEqual(len(entity1.data_partition_patterns), 1) pattern1 = entity1.data_partition_patterns[0] self.assertEqual(pattern1.name, 'testPattern') self.assertEqual(pattern1.explanation, 'test explanation') self.assertEqual(pattern1.root_location, 'test location') self.assertEqual(pattern1.regular_expression, '\\s*') self.assertEqual(len(pattern1.parameters), 2) self.assertEqual(pattern1.parameters[0], 'testParam1') self.assertEqual(pattern1.parameters[1], 'testParam2') self.assertEqual(pattern1.specialized_schema, 'test special schema') self.assertEqual(len(pattern1.exhibits_traits), 1) entity2 = cdm_manifest.entities[1] self.assertEqual(entity2.object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) self.assertEqual(len(entity2.data_partition_patterns), 1) pattern2 = entity2.data_partition_patterns[0] self.assertEqual(pattern2.name, 'testPattern2') self.assertEqual(pattern2.root_location, 'test location2') self.assertEqual(pattern2.glob_pattern, '/*.csv') manifest_data = ManifestPersistence.to_data(cdm_manifest, None, None) self.assertEqual(len(manifest_data.entities), 2) entity_data1 = manifest_data.entities[0] self.assertEqual(len(entity_data1.dataPartitionPatterns), 1) pattern_data1 = entity_data1.dataPartitionPatterns[0] self.assertEqual(pattern_data1.name, 'testPattern') self.assertEqual(pattern_data1.explanation, 'test explanation') self.assertEqual(pattern_data1.rootLocation, 'test location') self.assertEqual(pattern_data1.regularExpression, '\\s*') self.assertEqual(len(pattern_data1.parameters), 2) self.assertEqual(pattern_data1.parameters[0], 'testParam1') self.assertEqual(pattern_data1.parameters[1], 'testParam2') self.assertEqual(pattern_data1.specializedSchema, 'test special schema') self.assertEqual(len(pattern_data1.exhibitsTraits), 1) pattern_data2 = manifest_data.entities[1].dataPartitionPatterns[0] self.assertEqual(pattern_data2.name, 'testPattern2') self.assertEqual(pattern_data2.rootLocation, 'test location2') self.assertEqual(pattern_data2.globPattern, '/*.csv')
def test_folio_with_everything(self): """Testing for back-comp folio loading.""" test_name = 'test_folio_with_everything' corpus = self.get_corpus() content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'complete.folio.cdm.json') cdm_manifest = ManifestPersistence.from_object(corpus.ctx, 'docName', 'someNamespace', '/', ManifestContent().decode(content)) self.assertEqual(1, len(cdm_manifest.sub_manifests)) self.assertEqual(2, len(cdm_manifest.entities)) self.assertEqual('cdmTest', cdm_manifest.manifest_name) content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'noname.folio.cdm.json') cdm_manifest = ManifestPersistence.from_object(corpus.ctx, 'docName.folio.cdm.json', 'someNamespace', '/', ManifestContent().decode(content)) self.assertEqual(1, len(cdm_manifest.sub_manifests)) self.assertEqual(2, len(cdm_manifest.entities)) self.assertEqual('docName', cdm_manifest.manifest_name)
def test_load_local_entity_with_data_partition_pattern(self): content = TestHelper.get_input_file_content( self.test_subpath, "test_load_local_entity_with_data_partition_pattern", "entities.manifest.cdm.json") manifest_content = ManifestContent() manifest_content.decode(content) cdmManifest = ManifestPersistence.from_object( CdmCorpusContext(CdmCorpusDefinition(), None), "entities", "testNamespace", "/", manifest_content) self.assertEqual(len(cdmManifest.entities), 1) self.assertEqual(cdmManifest.entities[0].object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity = cdmManifest.entities[0] self.assertEqual(len(entity.data_partition_patterns), 1) pattern = entity.data_partition_patterns[0] self.assertEqual(pattern.name, "testPattern") self.assertEqual(pattern.explanation, "test explanation") self.assertEqual(pattern.root_location, "test location") self.assertEqual(pattern.regular_expression, "\\s*") self.assertEqual(len(pattern.parameters), 2) self.assertEqual(pattern.parameters[0], "testParam1") self.assertEqual(pattern.parameters[1], "testParam2") self.assertEqual(pattern.specialized_schema, "test special schema") self.assertEqual(len(pattern.exhibits_traits), 1)
def test_load_folder_with_no_entity_folders(self): """Testing for manifest impl instance with no entities and no sub manifests.""" test_name = 'test_load_folder_with_no_entity_folders' corpus = self.get_corpus() content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'empty.manifest.cdm.json') cdm_manifest = ManifestPersistence.from_object( corpus.ctx, 'cdmTest', 'someNamespace', '/', ManifestContent().decode(content)) self.assertEqual(cdm_manifest.schema, 'CdmManifestDefinition.cdm.json') self.assertEqual(cdm_manifest.manifest_name, 'cdmTest') self.assertEqual(cdm_manifest.json_schema_semantic_version, '0.9.0') self.assertEqual( time_utils.get_formatted_date_string( cdm_manifest.last_file_modified_time), '2008-09-15T23:53:23.000Z') self.assertEqual(cdm_manifest.explanation, 'test cdm folder for cdm version 0.9+') self.assertEqual(1, len(cdm_manifest.imports)) self.assertEqual(cdm_manifest.imports[0].corpus_path, '/primitives.cdm.json') self.assertEqual(0, len(cdm_manifest.entities)) self.assertEqual(1, len(cdm_manifest.exhibits_traits)) self.assertEqual(0, len(cdm_manifest.sub_manifests))
async def test_programmatically_create_partitions(self): corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter()) manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest') entity = manifest.entities.append('entity') relative_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'relative partition') relative_partition.location = 'relative/path' absolute_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'absolute partition') absolute_partition.location = 'local:/absolute/path' entity.data_partitions.append(relative_partition) entity.data_partitions.append(absolute_partition) manifest_data = ManifestPersistence.to_data(manifest, None, None) self.assertEqual(len(manifest_data.entities), 1) entityData = manifest_data.entities[0] partitions_list = entityData.dataPartitions self.assertEqual(len(partitions_list), 2) relative_partition_data = partitions_list[0] absolute_partition_data = partitions_list[-1] self.assertEqual(relative_partition_data.location, relative_partition.location) self.assertEqual(absolute_partition_data.location, absolute_partition.location)
async def test_pattern_with_non_existing_folder(self): corpus = TestHelper.get_local_corpus( self.test_subpath, "test_pattern_with_non_existing_folder") content = TestHelper.get_input_file_content( self.test_subpath, "test_pattern_with_non_existing_folder", "entities.manifest.cdm.json") manifest_content = ManifestContent() manifest_content.decode(content) cdmManifest = ManifestPersistence.from_object( CdmCorpusContext(corpus, None), "entities", "local", "/", manifest_content) error_logged = 0 def callback(level: CdmStatusLevel, message: str): if 'Failed to fetch all files in the folder location \'local:/testLocation\' described by a partition pattern. Exception:' in message: nonlocal error_logged error_logged += 1 corpus.set_event_callback(callback, CdmStatusLevel.WARNING) await cdmManifest.file_status_check_async() self.assertEqual(1, error_logged) self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0) # make sure the last check time is still being set self.assertIsNotNone( cdmManifest.entities[0].data_partition_patterns[0]. last_file_status_check_time)
async def test_loading_model_json_result_and_cdm_folder_to_data(self): test_name = 'test_loading_model_json_result_and_cdm_folder_to_data' cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) manifest = await cdm_corpus.fetch_object_async(PersistenceLayer.MODEL_JSON_EXTENSION, cdm_corpus.storage.fetch_root_folder('local')) expected_data = TestHelper.get_expected_output_data(self.tests_subpath, test_name, 'cdmFolder{}'.format(PersistenceLayer.CDM_EXTENSION)) actual_data = json.loads((CdmManifestPersistence.to_data(manifest, None, None)).encode()) error_msg = TestHelper.compare_same_object(expected_data, actual_data) self.assertEqual('', error_msg, error_msg)
async def test_to_incremental_partition_without_trait(self): """ Testing saving manifest with local entity declaration having an incremental partition without incremental trait. """ test_name = 'test_to_incremental_partition_without_trait' corpus = TestHelper.get_local_corpus(self.test_subpath, test_name) error_message_verified = False # not checking the CdmLogCode here as we want to check if this error message constructed correctly for the partition (it shares the same CdmLogCode with partition pattern) def callback(level, message): if 'Failed to persist object \'DeletePartition\'. This object does not contain the trait \'is.partition.incremental\', so it should not be in the collection \'incremental_partitions\'. | to_data' in message: nonlocal error_message_verified error_message_verified = True else: self.fail('Some unexpected failure - {}!'.format(message)) corpus.set_event_callback(callback, CdmStatusLevel.WARNING) manifest = CdmManifestDefinition(corpus.ctx, 'manifest') corpus.storage.fetch_root_folder('local').documents.append(manifest) entity = CdmEntityDefinition(corpus.ctx, 'entityName', None) create_document_for_entity(corpus, entity) localized_entity_declaration = manifest.entities.append(entity) upsert_incremental_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'UpsertPartition', False) upsert_incremental_partition.location = '/IncrementalData' upsert_incremental_partition.specialized_schema = 'csv' upsert_incremental_partition.exhibits_traits.append( Constants._INCREMENTAL_TRAIT_NAME, [['type', CdmIncrementalPartitionType.UPSERT.value]]) delete_partition = corpus.make_object(CdmObjectType.DATA_PARTITION_DEF, 'DeletePartition', False) delete_partition.location = '/IncrementalData' delete_partition.specialized_schema = 'csv' localized_entity_declaration.incremental_partitions.append( upsert_incremental_partition) localized_entity_declaration.incremental_partitions.append( delete_partition) with logger._enter_scope(DataPartitionTest.__name__, corpus.ctx, test_name): manifest_data = ManifestPersistence.to_data(manifest, None, None) self.assertEqual(1, len(manifest_data.entities)) entity_data = manifest_data.entities[0] self.assertEqual(1, len(entity_data.incrementalPartitions)) partition_data = entity_data.incrementalPartitions[0] self.assertEqual('UpsertPartition', partition_data.name) self.assertEqual(1, len(partition_data.exhibitsTraits)) self.assertEqual(Constants._INCREMENTAL_TRAIT_NAME, partition_data.exhibitsTraits[0].traitReference) self.assertTrue(error_message_verified)
async def test_programmatically_create_partitions(self): corpus = TestHelper.get_local_corpus( self.test_subpath, 'test_programmatically_create_partitions', no_input_and_output_folder=True) manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest') entity = manifest.entities.append('entity') relative_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'relative partition') relative_partition.location = 'relative/path' relative_partition.arguments['test1'] = ['argument1'] relative_partition.arguments['test2'] = ['argument2', 'argument3'] absolute_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'absolute partition') absolute_partition.location = 'local:/absolute/path' # add an empty arguments list to test empty list should not be displayed in ToData json. absolute_partition.arguments['test'] = [] entity.data_partitions.append(relative_partition) entity.data_partitions.append(absolute_partition) manifest_data = ManifestPersistence.to_data(manifest, None, None) self.assertEqual(len(manifest_data.entities), 1) entityData = manifest_data.entities[0] partitions_list = entityData.dataPartitions self.assertEqual(len(partitions_list), 2) relative_partition_data = partitions_list[0] absolute_partition_data = partitions_list[-1] self.assertEqual(relative_partition_data.location, relative_partition.location) arguments_list = relative_partition_data.arguments self.assertEqual(3, len(arguments_list)) checked_arguments = [] for argument in arguments_list: self.assertEqual(3, len(argument)) checked_arguments.append(argument.value) if argument.value == 'argument1': self.assertEqual('test1', argument.name) elif argument.value == 'argument2': self.assertEqual('test2', argument.name) elif argument.value == 'argument3': self.assertEqual('test2', argument.name) else: raise Exception('unexpected argument in data partitions') self.assertTrue('argument1' in checked_arguments) self.assertTrue('argument2' in checked_arguments) self.assertTrue('argument3' in checked_arguments) self.assertEqual(absolute_partition_data.location, absolute_partition.location) # test if empty argument list is set to null self.assertEqual(absolute_partition_data.arguments, None)
async def test_pattern_with_non_existing_folder(self): corpus = TestHelper.get_local_corpus(self.test_subpath, "test_pattern_with_non_existing_folder") content = TestHelper.get_input_file_content(self.test_subpath, "test_pattern_with_non_existing_folder", "entities.manifest.cdm.json") manifest_content = ManifestContent() manifest_content.decode(content) cdmManifest = ManifestPersistence.from_object(CdmCorpusContext(corpus, None), "entities", "local", "/", manifest_content) await cdmManifest.file_status_check_async() self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0) # make sure the last check time is still being set self.assertIsNotNone(cdmManifest.entities[0].data_partition_patterns[0].last_file_status_check_time)
async def test_loading_model_json_and_cdm_folder_to_data(self): test_name = 'test_loading_model_json_and_cdm_folder_to_data' cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) manifest = await cdm_corpus.fetch_object_async( PersistenceLayer.MODEL_JSON_EXTENSION, cdm_corpus.storage.fetch_root_folder('local')) actual_data = json.loads( (CdmManifestPersistence.to_data(manifest, None, None)).encode()) self._validate_output( test_name, 'cdmFolder{}'.format(PersistenceLayer.CDM_EXTENSION), actual_data)
async def test_loading_model_json_and_cdm_folder_to_data(self): test_name = 'test_loading_model_json_and_cdm_folder_to_data' cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) manifest = await cdm_corpus.fetch_object_async('model.json', cdm_corpus.storage.fetch_root_folder('local')) cdm_corpus.storage.fetch_root_folder('output').documents.append(manifest) await manifest.save_as_async('cdm.json', save_referenced=True) expected_data = TestHelper.get_expected_output_data(self.tests_subpath, test_name, 'cdmFolder.cdm.json') actual_data = json.loads((CdmManifestPersistence.to_data(manifest, None, None)).encode()) error_msg = TestHelper.compare_same_object(expected_data, actual_data) self.assertEqual('', error_msg, error_msg)
async def test_load_local_entitiy_with_data_partition(self): content = TestHelper.get_input_file_content( self.test_subpath, 'test_load_local_entity_with_data_partition', 'entities.manifest.cdm.json') manifest_content = ManifestContent() manifest_content.decode(content) cdm_manifest = ManifestPersistence.from_object( CdmCorpusContext(CdmCorpusDefinition(), None), 'entities', 'testNamespace', '/', manifest_content) self.assertEqual(len(cdm_manifest.entities), 1) self.assertEqual(cdm_manifest.entities[0].object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity = cdm_manifest.entities[0] self.assertEqual(len(entity.data_partitions), 2) relative_partition = entity.data_partitions[0] self.assertEqual(relative_partition.name, 'Sample data partition') self.assertEqual(relative_partition.location, 'test/location') # self.assertEqual(TimeUtils.GetFormattedDateString(relative_partition.LastFileModifiedTime), '2008-09-15T23:53:23.000Z') self.assertEqual(len(relative_partition.exhibits_traits), 1) self.assertEqual(relative_partition.specialized_schema, 'teststring') test_list = relative_partition.arguments['test'] self.assertEqual(len(test_list), 3) self.assertEqual(test_list[0], 'something') self.assertEqual(test_list[1], 'somethingelse') self.assertEqual(test_list[2], 'anotherthing') key_list = relative_partition.arguments['KEY'] self.assertEqual(len(key_list), 1) self.assertEqual(key_list[0], 'VALUE') self.assertFalse('wrong' in relative_partition.arguments) absolute_partition = entity.data_partitions[1] self.assertEqual(absolute_partition.location, "local:/some/test/location")
def test_manifest_with_blank_fields(self): """Testing for manifest impl instance with blank or empty values for manifest schema, name etc.""" test_name = 'test_manifest_with_blank_fields' corpus = self.get_corpus() content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'blank.manifest.cdm.json') cdm_manifest = ManifestPersistence.from_object( corpus.ctx, 'cdmTest', 'someNamespace', '/', ManifestContent().decode(content)) self.assertIsNone(cdm_manifest.schema) self.assertIsNone(cdm_manifest.document_version) self.assertEqual( time_utils._get_formatted_date_string( cdm_manifest.last_file_modified_time), '2008-09-15T23:53:23.000Z') self.assertEqual(cdm_manifest.explanation, 'test cdm folder for cdm version 1.0+') self.assertEqual(1, len(cdm_manifest.imports)) self.assertEqual(cdm_manifest.imports[0].corpus_path, '/primitives.cdm.json') self.assertEqual(0, len(cdm_manifest.entities)) self.assertEqual(1, len(cdm_manifest.exhibits_traits)) self.assertEqual(0, len(cdm_manifest.sub_manifests))
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder.folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter( folder.namespace) # type: StorageAdapter try: if adapter.can_read(): # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'request file: {}'.format(doc_path)) json_data = await adapter.read_async(doc_path) # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'received file: {}'.format(doc_path)) else: raise Exception('Storage Adapter is not enabled to read.') except Exception as e: # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'fail file: {}'.format(doc_path)) # when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if res_opt and res_opt.shallow_validation: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE, doc_path, folder.Namespace, e) else: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE, doc_path, str(folder.namespace), e) return None try: fs_modified_time = await adapter.compute_last_modified_time_async( doc_path) except Exception as e: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED, e.Message) if not doc_name: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME) return None doc_name_lower = doc_name.lower() # If loading an model.json file, check that it is named correctly. if doc_name_lower.endswith( self.MODEL_JSON_EXTENSION ) and not doc_name.lower() == self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None try: if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION ) or doc_name_lower.endswith( PersistenceLayer.FOLIO_EXTENSION): from cdm.persistence.cdmfolder import ManifestPersistence from cdm.persistence.cdmfolder.types import ManifestContent manifest = ManifestContent() manifest.decode(json_data) doc_content = ManifestPersistence.from_object( self._ctx, doc_name, folder.namespace, folder.folder_path, manifest) elif doc_name_lower.endswith( PersistenceLayer.MODEL_JSON_EXTENSION): if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None from cdm.persistence.modeljson import ManifestPersistence from cdm.persistence.modeljson.types import Model model = Model() model.decode(json_data) doc_content = await ManifestPersistence.from_object( self._ctx, model, folder) elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION): from cdm.persistence.cdmfolder import DocumentPersistence from cdm.persistence.cdmfolder.types import DocumentContent document = DocumentContent() document.decode(json_data) doc_content = DocumentPersistence.from_object( self._ctx, doc_name, folder.namespace, folder.folder_path, document) else: # Could not find a registered persistence class to handle this document type. logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING, doc_name) return None except Exception as e: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE, doc_path, e) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy( ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus. default_resolution_directives), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition', options: 'CopyOptions', new_name: str, save_referenced: bool) -> bool: """a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work because the corpus knows about persistence types and about the storage adapters if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or to a different name, then the source object is still 'dirty' an option will cause us to also save any linked documents.""" # find out if the storage adapter is able to write. namespace = doc.namespace if namespace is None: namespace = self._corpus.storage.default_namespace adapter = self._corpus.storage.fetch_adapter(namespace) if adapter is None: logger.error( self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_ADAPTER_NOT_FOUND_FOR_NAMESPACE, namespace) return False if not adapter.can_write(): logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_ADAPTER_WRITE_FAILURE, namespace) return False if not new_name: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME) return None # what kind of document is requested? # check file extensions using a case-insensitive ordinal string comparison. persistence_type = self.MODEL_JSON if new_name.lower().endswith( self.MODEL_JSON_EXTENSION) else self.CDM_FOLDER if persistence_type == self.MODEL_JSON and new_name.lower( ) != self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE, new_name, self.MODEL_JSON_EXTENSION) return False # save the object into a json blob res_opt = { 'wrt_doc': doc, 'directives': AttributeResolutionDirectiveSet() } persisted_doc = None try: if new_name.lower().endswith( PersistenceLayer.MODEL_JSON_EXTENSION) or new_name.lower( ).endswith( PersistenceLayer.MANIFEST_EXTENSION) or new_name.lower( ).endswith(PersistenceLayer.FOLIO_EXTENSION): if persistence_type == self.CDM_FOLDER: from cdm.persistence.cdmfolder import ManifestPersistence persisted_doc = ManifestPersistence.to_data( doc, res_opt, options) else: if new_name != self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE, new_name) return False from cdm.persistence.modeljson import ManifestPersistence persisted_doc = await ManifestPersistence.to_data( doc, res_opt, options) elif new_name.lower().endswith(PersistenceLayer.CDM_EXTENSION): from cdm.persistence.cdmfolder import DocumentPersistence persisted_doc = DocumentPersistence.to_data( doc, res_opt, options) else: # Could not find a registered persistence class to handle this document type. logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING, new_name) return False except Exception as e: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FILE_PERSIST_ERROR, new_name, e) return False if not persisted_doc: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FILE_PERSIST_FAILED, new_name) return False # turn the name into a path new_path = '{}{}'.format(doc.folder_path, new_name) new_path = self._ctx.corpus.storage.create_absolute_corpus_path( new_path, doc) if new_path.startswith(namespace + ':'): new_path = new_path[len(namespace) + 1:] # ask the adapter to make it happen try: content = persisted_doc.encode() await adapter.write_async(new_path, content) doc._file_system_modified_time = await adapter.compute_last_modified_time_async( new_path) # Write the adapter's config. if options._is_top_level_document: await self._corpus.storage.save_adapters_config_async( '/config.json', adapter) # The next document won't be top level, so reset the flag. options._is_top_level_document = False except Exception as e: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FILE_WRITE_FAILURE, new_name, e) return False # if we also want to save referenced docs, then it depends on what kind of thing just got saved # if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things # definition will save imports, manifests will save imports, schemas, sub manifests if save_referenced and persistence_type == self.CDM_FOLDER: saved_linked_docs = await doc._save_linked_documents_async(options) if not saved_linked_docs: logger.error(self._ctx, self._TAG, self._save_document_as_async.__name__, doc.at_corpus_path, CdmLogCode.ERR_PERSIST_SAVE_LINK_DOCS, new_name) return False return True