def test_manifest_for_copy_data(self):
"""Test for copy data."""
test_name = 'test_manifest_for_copy_data'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(
self.tests_subpath, test_name, 'complete.manifest.cdm.json')
cdm_manifest = ManifestPersistence.from_object(
corpus.ctx, 'docName', 'someNamespace', '/',
ManifestContent().decode(content))
manifest_object = ManifestPersistence.to_data(cdm_manifest, None, None)
self.assertEqual(manifest_object.schema,
'CdmManifestDefinition.cdm.json')
self.assertEqual(manifest_object.jsonSchemaSemanticVersion, '0.9.0')
self.assertEqual(manifest_object.manifestName, 'cdmTest')
self.assertEqual(manifest_object.explanation,
'test cdm folder for cdm version 0.9+')
self.assertEqual(1, len(manifest_object.imports))
self.assertEqual(manifest_object.imports[0].corpusPath,
'/primitives.cdm.json')
self.assertEqual(1, len(manifest_object.exhibitsTraits))
self.assertEqual(2, len(manifest_object.entities))
self.assertEqual(manifest_object.entities[0]['entityName'],
'testEntity')
self.assertEqual(1, len(manifest_object.subManifests))
self.assertEqual(manifest_object.subManifests[0].definition,
'test definition')
self.assertEqual(manifest_object.lastFileModifiedTime, None)
def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
'test_load_local_entity_with_data_partition_pattern',
'entities.manifest.cdm.json')
manifest_content = ManifestContent()
manifest_content.decode(content)
cdm_manifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), 'entities',
'testNamespace', '/', manifest_content)
self.assertEqual(len(cdm_manifest.entities), 2)
entity1 = cdm_manifest.entities[0]
self.assertEqual(entity1.object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
self.assertEqual(len(entity1.data_partition_patterns), 1)
pattern1 = entity1.data_partition_patterns[0]
self.assertEqual(pattern1.name, 'testPattern')
self.assertEqual(pattern1.explanation, 'test explanation')
self.assertEqual(pattern1.root_location, 'test location')
self.assertEqual(pattern1.regular_expression, '\\s*')
self.assertEqual(len(pattern1.parameters), 2)
self.assertEqual(pattern1.parameters[0], 'testParam1')
self.assertEqual(pattern1.parameters[1], 'testParam2')
self.assertEqual(pattern1.specialized_schema, 'test special schema')
self.assertEqual(len(pattern1.exhibits_traits), 1)
entity2 = cdm_manifest.entities[1]
self.assertEqual(entity2.object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
self.assertEqual(len(entity2.data_partition_patterns), 1)
pattern2 = entity2.data_partition_patterns[0]
self.assertEqual(pattern2.name, 'testPattern2')
self.assertEqual(pattern2.root_location, 'test location2')
self.assertEqual(pattern2.glob_pattern, '/*.csv')
manifest_data = ManifestPersistence.to_data(cdm_manifest, None, None)
self.assertEqual(len(manifest_data.entities), 2)
entity_data1 = manifest_data.entities[0]
self.assertEqual(len(entity_data1.dataPartitionPatterns), 1)
pattern_data1 = entity_data1.dataPartitionPatterns[0]
self.assertEqual(pattern_data1.name, 'testPattern')
self.assertEqual(pattern_data1.explanation, 'test explanation')
self.assertEqual(pattern_data1.rootLocation, 'test location')
self.assertEqual(pattern_data1.regularExpression, '\\s*')
self.assertEqual(len(pattern_data1.parameters), 2)
self.assertEqual(pattern_data1.parameters[0], 'testParam1')
self.assertEqual(pattern_data1.parameters[1], 'testParam2')
self.assertEqual(pattern_data1.specializedSchema,
'test special schema')
self.assertEqual(len(pattern_data1.exhibitsTraits), 1)
pattern_data2 = manifest_data.entities[1].dataPartitionPatterns[0]
self.assertEqual(pattern_data2.name, 'testPattern2')
self.assertEqual(pattern_data2.rootLocation, 'test location2')
self.assertEqual(pattern_data2.globPattern, '/*.csv')
def test_folio_with_everything(self):
"""Testing for back-comp folio loading."""
test_name = 'test_folio_with_everything'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'complete.folio.cdm.json')
cdm_manifest = ManifestPersistence.from_object(corpus.ctx, 'docName', 'someNamespace', '/', ManifestContent().decode(content))
self.assertEqual(1, len(cdm_manifest.sub_manifests))
self.assertEqual(2, len(cdm_manifest.entities))
self.assertEqual('cdmTest', cdm_manifest.manifest_name)
content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'noname.folio.cdm.json')
cdm_manifest = ManifestPersistence.from_object(corpus.ctx, 'docName.folio.cdm.json', 'someNamespace', '/', ManifestContent().decode(content))
self.assertEqual(1, len(cdm_manifest.sub_manifests))
self.assertEqual(2, len(cdm_manifest.entities))
self.assertEqual('docName', cdm_manifest.manifest_name)
def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
"test_load_local_entity_with_data_partition_pattern",
"entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), "entities",
"testNamespace", "/", manifest_content)
self.assertEqual(len(cdmManifest.entities), 1)
self.assertEqual(cdmManifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdmManifest.entities[0]
self.assertEqual(len(entity.data_partition_patterns), 1)
pattern = entity.data_partition_patterns[0]
self.assertEqual(pattern.name, "testPattern")
self.assertEqual(pattern.explanation, "test explanation")
self.assertEqual(pattern.root_location, "test location")
self.assertEqual(pattern.regular_expression, "\\s*")
self.assertEqual(len(pattern.parameters), 2)
self.assertEqual(pattern.parameters[0], "testParam1")
self.assertEqual(pattern.parameters[1], "testParam2")
self.assertEqual(pattern.specialized_schema, "test special schema")
self.assertEqual(len(pattern.exhibits_traits), 1)
def test_load_folder_with_no_entity_folders(self):
"""Testing for manifest impl instance with no entities and no sub manifests."""
test_name = 'test_load_folder_with_no_entity_folders'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(self.tests_subpath,
test_name,
'empty.manifest.cdm.json')
cdm_manifest = ManifestPersistence.from_object(
corpus.ctx, 'cdmTest', 'someNamespace', '/',
ManifestContent().decode(content))
self.assertEqual(cdm_manifest.schema, 'CdmManifestDefinition.cdm.json')
self.assertEqual(cdm_manifest.manifest_name, 'cdmTest')
self.assertEqual(cdm_manifest.json_schema_semantic_version, '0.9.0')
self.assertEqual(
time_utils.get_formatted_date_string(
cdm_manifest.last_file_modified_time),
'2008-09-15T23:53:23.000Z')
self.assertEqual(cdm_manifest.explanation,
'test cdm folder for cdm version 0.9+')
self.assertEqual(1, len(cdm_manifest.imports))
self.assertEqual(cdm_manifest.imports[0].corpus_path,
'/primitives.cdm.json')
self.assertEqual(0, len(cdm_manifest.entities))
self.assertEqual(1, len(cdm_manifest.exhibits_traits))
self.assertEqual(0, len(cdm_manifest.sub_manifests))
async def test_programmatically_create_partitions(self):
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter())
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest')
entity = manifest.entities.append('entity')
relative_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'relative partition')
relative_partition.location = 'relative/path'
absolute_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'absolute partition')
absolute_partition.location = 'local:/absolute/path'
entity.data_partitions.append(relative_partition)
entity.data_partitions.append(absolute_partition)
manifest_data = ManifestPersistence.to_data(manifest, None, None)
self.assertEqual(len(manifest_data.entities), 1)
entityData = manifest_data.entities[0]
partitions_list = entityData.dataPartitions
self.assertEqual(len(partitions_list), 2)
relative_partition_data = partitions_list[0]
absolute_partition_data = partitions_list[-1]
self.assertEqual(relative_partition_data.location,
relative_partition.location)
self.assertEqual(absolute_partition_data.location,
absolute_partition.location)
async def test_pattern_with_non_existing_folder(self):
corpus = TestHelper.get_local_corpus(
self.test_subpath, "test_pattern_with_non_existing_folder")
content = TestHelper.get_input_file_content(
self.test_subpath, "test_pattern_with_non_existing_folder",
"entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(
CdmCorpusContext(corpus, None), "entities", "local", "/",
manifest_content)
error_logged = 0
def callback(level: CdmStatusLevel, message: str):
if 'Failed to fetch all files in the folder location \'local:/testLocation\' described by a partition pattern. Exception:' in message:
nonlocal error_logged
error_logged += 1
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
await cdmManifest.file_status_check_async()
self.assertEqual(1, error_logged)
self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0)
# make sure the last check time is still being set
self.assertIsNotNone(
cdmManifest.entities[0].data_partition_patterns[0].
last_file_status_check_time)
async def test_loading_model_json_result_and_cdm_folder_to_data(self):
test_name = 'test_loading_model_json_result_and_cdm_folder_to_data'
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
manifest = await cdm_corpus.fetch_object_async(PersistenceLayer.MODEL_JSON_EXTENSION, cdm_corpus.storage.fetch_root_folder('local'))
expected_data = TestHelper.get_expected_output_data(self.tests_subpath, test_name, 'cdmFolder{}'.format(PersistenceLayer.CDM_EXTENSION))
actual_data = json.loads((CdmManifestPersistence.to_data(manifest, None, None)).encode())
error_msg = TestHelper.compare_same_object(expected_data, actual_data)
self.assertEqual('', error_msg, error_msg)
async def test_to_incremental_partition_without_trait(self):
"""
Testing saving manifest with local entity declaration having an incremental partition without incremental trait.
"""
test_name = 'test_to_incremental_partition_without_trait'
corpus = TestHelper.get_local_corpus(self.test_subpath, test_name)
error_message_verified = False
# not checking the CdmLogCode here as we want to check if this error message constructed correctly for the partition (it shares the same CdmLogCode with partition pattern)
def callback(level, message):
if 'Failed to persist object \'DeletePartition\'. This object does not contain the trait \'is.partition.incremental\', so it should not be in the collection \'incremental_partitions\'. | to_data' in message:
nonlocal error_message_verified
error_message_verified = True
else:
self.fail('Some unexpected failure - {}!'.format(message))
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
manifest = CdmManifestDefinition(corpus.ctx, 'manifest')
corpus.storage.fetch_root_folder('local').documents.append(manifest)
entity = CdmEntityDefinition(corpus.ctx, 'entityName', None)
create_document_for_entity(corpus, entity)
localized_entity_declaration = manifest.entities.append(entity)
upsert_incremental_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'UpsertPartition', False)
upsert_incremental_partition.location = '/IncrementalData'
upsert_incremental_partition.specialized_schema = 'csv'
upsert_incremental_partition.exhibits_traits.append(
Constants._INCREMENTAL_TRAIT_NAME,
[['type', CdmIncrementalPartitionType.UPSERT.value]])
delete_partition = corpus.make_object(CdmObjectType.DATA_PARTITION_DEF,
'DeletePartition', False)
delete_partition.location = '/IncrementalData'
delete_partition.specialized_schema = 'csv'
localized_entity_declaration.incremental_partitions.append(
upsert_incremental_partition)
localized_entity_declaration.incremental_partitions.append(
delete_partition)
with logger._enter_scope(DataPartitionTest.__name__, corpus.ctx,
test_name):
manifest_data = ManifestPersistence.to_data(manifest, None, None)
self.assertEqual(1, len(manifest_data.entities))
entity_data = manifest_data.entities[0]
self.assertEqual(1, len(entity_data.incrementalPartitions))
partition_data = entity_data.incrementalPartitions[0]
self.assertEqual('UpsertPartition', partition_data.name)
self.assertEqual(1, len(partition_data.exhibitsTraits))
self.assertEqual(Constants._INCREMENTAL_TRAIT_NAME,
partition_data.exhibitsTraits[0].traitReference)
self.assertTrue(error_message_verified)
async def test_programmatically_create_partitions(self):
corpus = TestHelper.get_local_corpus(
self.test_subpath,
'test_programmatically_create_partitions',
no_input_and_output_folder=True)
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest')
entity = manifest.entities.append('entity')
relative_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'relative partition')
relative_partition.location = 'relative/path'
relative_partition.arguments['test1'] = ['argument1']
relative_partition.arguments['test2'] = ['argument2', 'argument3']
absolute_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'absolute partition')
absolute_partition.location = 'local:/absolute/path'
# add an empty arguments list to test empty list should not be displayed in ToData json.
absolute_partition.arguments['test'] = []
entity.data_partitions.append(relative_partition)
entity.data_partitions.append(absolute_partition)
manifest_data = ManifestPersistence.to_data(manifest, None, None)
self.assertEqual(len(manifest_data.entities), 1)
entityData = manifest_data.entities[0]
partitions_list = entityData.dataPartitions
self.assertEqual(len(partitions_list), 2)
relative_partition_data = partitions_list[0]
absolute_partition_data = partitions_list[-1]
self.assertEqual(relative_partition_data.location,
relative_partition.location)
arguments_list = relative_partition_data.arguments
self.assertEqual(3, len(arguments_list))
checked_arguments = []
for argument in arguments_list:
self.assertEqual(3, len(argument))
checked_arguments.append(argument.value)
if argument.value == 'argument1':
self.assertEqual('test1', argument.name)
elif argument.value == 'argument2':
self.assertEqual('test2', argument.name)
elif argument.value == 'argument3':
self.assertEqual('test2', argument.name)
else:
raise Exception('unexpected argument in data partitions')
self.assertTrue('argument1' in checked_arguments)
self.assertTrue('argument2' in checked_arguments)
self.assertTrue('argument3' in checked_arguments)
self.assertEqual(absolute_partition_data.location,
absolute_partition.location)
# test if empty argument list is set to null
self.assertEqual(absolute_partition_data.arguments, None)
async def test_pattern_with_non_existing_folder(self):
corpus = TestHelper.get_local_corpus(self.test_subpath, "test_pattern_with_non_existing_folder")
content = TestHelper.get_input_file_content(self.test_subpath, "test_pattern_with_non_existing_folder", "entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(CdmCorpusContext(corpus, None), "entities", "local", "/", manifest_content)
await cdmManifest.file_status_check_async()
self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0)
# make sure the last check time is still being set
self.assertIsNotNone(cdmManifest.entities[0].data_partition_patterns[0].last_file_status_check_time)
async def test_loading_model_json_and_cdm_folder_to_data(self):
test_name = 'test_loading_model_json_and_cdm_folder_to_data'
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
manifest = await cdm_corpus.fetch_object_async(
PersistenceLayer.MODEL_JSON_EXTENSION,
cdm_corpus.storage.fetch_root_folder('local'))
actual_data = json.loads(
(CdmManifestPersistence.to_data(manifest, None, None)).encode())
self._validate_output(
test_name, 'cdmFolder{}'.format(PersistenceLayer.CDM_EXTENSION),
actual_data)
async def test_loading_model_json_and_cdm_folder_to_data(self):
test_name = 'test_loading_model_json_and_cdm_folder_to_data'
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
manifest = await cdm_corpus.fetch_object_async('model.json', cdm_corpus.storage.fetch_root_folder('local'))
cdm_corpus.storage.fetch_root_folder('output').documents.append(manifest)
await manifest.save_as_async('cdm.json', save_referenced=True)
expected_data = TestHelper.get_expected_output_data(self.tests_subpath, test_name, 'cdmFolder.cdm.json')
actual_data = json.loads((CdmManifestPersistence.to_data(manifest, None, None)).encode())
error_msg = TestHelper.compare_same_object(expected_data, actual_data)
self.assertEqual('', error_msg, error_msg)
async def test_load_local_entitiy_with_data_partition(self):
content = TestHelper.get_input_file_content(
self.test_subpath, 'test_load_local_entity_with_data_partition',
'entities.manifest.cdm.json')
manifest_content = ManifestContent()
manifest_content.decode(content)
cdm_manifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), 'entities',
'testNamespace', '/', manifest_content)
self.assertEqual(len(cdm_manifest.entities), 1)
self.assertEqual(cdm_manifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdm_manifest.entities[0]
self.assertEqual(len(entity.data_partitions), 2)
relative_partition = entity.data_partitions[0]
self.assertEqual(relative_partition.name, 'Sample data partition')
self.assertEqual(relative_partition.location, 'test/location')
# self.assertEqual(TimeUtils.GetFormattedDateString(relative_partition.LastFileModifiedTime), '2008-09-15T23:53:23.000Z')
self.assertEqual(len(relative_partition.exhibits_traits), 1)
self.assertEqual(relative_partition.specialized_schema, 'teststring')
test_list = relative_partition.arguments['test']
self.assertEqual(len(test_list), 3)
self.assertEqual(test_list[0], 'something')
self.assertEqual(test_list[1], 'somethingelse')
self.assertEqual(test_list[2], 'anotherthing')
key_list = relative_partition.arguments['KEY']
self.assertEqual(len(key_list), 1)
self.assertEqual(key_list[0], 'VALUE')
self.assertFalse('wrong' in relative_partition.arguments)
absolute_partition = entity.data_partitions[1]
self.assertEqual(absolute_partition.location,
"local:/some/test/location")
def test_manifest_with_blank_fields(self):
"""Testing for manifest impl instance with blank or empty values for manifest schema, name etc."""
test_name = 'test_manifest_with_blank_fields'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(self.tests_subpath,
test_name,
'blank.manifest.cdm.json')
cdm_manifest = ManifestPersistence.from_object(
corpus.ctx, 'cdmTest', 'someNamespace', '/',
ManifestContent().decode(content))
self.assertIsNone(cdm_manifest.schema)
self.assertIsNone(cdm_manifest.document_version)
self.assertEqual(
time_utils._get_formatted_date_string(
cdm_manifest.last_file_modified_time),
'2008-09-15T23:53:23.000Z')
self.assertEqual(cdm_manifest.explanation,
'test cdm folder for cdm version 1.0+')
self.assertEqual(1, len(cdm_manifest.imports))
self.assertEqual(cdm_manifest.imports[0].corpus_path,
'/primitives.cdm.json')
self.assertEqual(0, len(cdm_manifest.entities))
self.assertEqual(1, len(cdm_manifest.exhibits_traits))
self.assertEqual(0, len(cdm_manifest.sub_manifests))
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder.Namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, str(folder.namespace), e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if not doc_name:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, e)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition',
options: 'CopyOptions', new_name: str,
save_referenced: bool) -> bool:
"""a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work
because the corpus knows about persistence types and about the storage adapters
if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or
to a different name, then the source object is still 'dirty'
an option will cause us to also save any linked documents."""
# find out if the storage adapter is able to write.
namespace = doc.namespace
if namespace is None:
namespace = self._corpus.storage.default_namespace
adapter = self._corpus.storage.fetch_adapter(namespace)
if adapter is None:
logger.error(
self._ctx, self._TAG, self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_ADAPTER_NOT_FOUND_FOR_NAMESPACE,
namespace)
return False
if not adapter.can_write():
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_ADAPTER_WRITE_FAILURE,
namespace)
return False
if not new_name:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
# what kind of document is requested?
# check file extensions using a case-insensitive ordinal string comparison.
persistence_type = self.MODEL_JSON if new_name.lower().endswith(
self.MODEL_JSON_EXTENSION) else self.CDM_FOLDER
if persistence_type == self.MODEL_JSON and new_name.lower(
) != self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE,
new_name, self.MODEL_JSON_EXTENSION)
return False
# save the object into a json blob
res_opt = {
'wrt_doc': doc,
'directives': AttributeResolutionDirectiveSet()
}
persisted_doc = None
try:
if new_name.lower().endswith(
PersistenceLayer.MODEL_JSON_EXTENSION) or new_name.lower(
).endswith(
PersistenceLayer.MANIFEST_EXTENSION) or new_name.lower(
).endswith(PersistenceLayer.FOLIO_EXTENSION):
if persistence_type == self.CDM_FOLDER:
from cdm.persistence.cdmfolder import ManifestPersistence
persisted_doc = ManifestPersistence.to_data(
doc, res_opt, options)
else:
if new_name != self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FAILURE, new_name)
return False
from cdm.persistence.modeljson import ManifestPersistence
persisted_doc = await ManifestPersistence.to_data(
doc, res_opt, options)
elif new_name.lower().endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
persisted_doc = DocumentPersistence.to_data(
doc, res_opt, options)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_CLASS_MISSING, new_name)
return False
except Exception as e:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_PERSIST_ERROR, new_name,
e)
return False
if not persisted_doc:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_PERSIST_FAILED, new_name)
return False
# turn the name into a path
new_path = '{}{}'.format(doc.folder_path, new_name)
new_path = self._ctx.corpus.storage.create_absolute_corpus_path(
new_path, doc)
if new_path.startswith(namespace + ':'):
new_path = new_path[len(namespace) + 1:]
# ask the adapter to make it happen
try:
content = persisted_doc.encode()
await adapter.write_async(new_path, content)
doc._file_system_modified_time = await adapter.compute_last_modified_time_async(
new_path)
# Write the adapter's config.
if options._is_top_level_document:
await self._corpus.storage.save_adapters_config_async(
'/config.json', adapter)
# The next document won't be top level, so reset the flag.
options._is_top_level_document = False
except Exception as e:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_WRITE_FAILURE, new_name,
e)
return False
# if we also want to save referenced docs, then it depends on what kind of thing just got saved
# if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things
# definition will save imports, manifests will save imports, schemas, sub manifests
if save_referenced and persistence_type == self.CDM_FOLDER:
saved_linked_docs = await doc._save_linked_documents_async(options)
if not saved_linked_docs:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_SAVE_LINK_DOCS, new_name)
return False
return True
