def test_folio_with_everything(self):
"""Testing for back-comp folio loading."""
test_name = 'test_folio_with_everything'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'complete.folio.cdm.json')
cdm_manifest = ManifestPersistence.from_object(corpus.ctx, 'docName', 'someNamespace', '/', ManifestContent().decode(content))
self.assertEqual(1, len(cdm_manifest.sub_manifests))
self.assertEqual(2, len(cdm_manifest.entities))
self.assertEqual('cdmTest', cdm_manifest.manifest_name)
content = TestHelper.get_input_file_content(self.tests_subpath, test_name, 'noname.folio.cdm.json')
cdm_manifest = ManifestPersistence.from_object(corpus.ctx, 'docName.folio.cdm.json', 'someNamespace', '/', ManifestContent().decode(content))
self.assertEqual(1, len(cdm_manifest.sub_manifests))
self.assertEqual(2, len(cdm_manifest.entities))
self.assertEqual('docName', cdm_manifest.manifest_name)
def test_manifest_for_copy_data(self):
"""Test for copy data."""
test_name = 'test_manifest_for_copy_data'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(
self.tests_subpath, test_name, 'complete.manifest.cdm.json')
cdm_manifest = ManifestPersistence.from_object(
corpus.ctx, 'docName', 'someNamespace', '/',
ManifestContent().decode(content))
manifest_object = ManifestPersistence.to_data(cdm_manifest, None, None)
self.assertEqual(manifest_object.schema,
'CdmManifestDefinition.cdm.json')
self.assertEqual(manifest_object.jsonSchemaSemanticVersion, '0.9.0')
self.assertEqual(manifest_object.manifestName, 'cdmTest')
self.assertEqual(manifest_object.explanation,
'test cdm folder for cdm version 0.9+')
self.assertEqual(1, len(manifest_object.imports))
self.assertEqual(manifest_object.imports[0].corpusPath,
'/primitives.cdm.json')
self.assertEqual(1, len(manifest_object.exhibitsTraits))
self.assertEqual(2, len(manifest_object.entities))
self.assertEqual(manifest_object.entities[0]['entityName'],
'testEntity')
self.assertEqual(1, len(manifest_object.subManifests))
self.assertEqual(manifest_object.subManifests[0].definition,
'test definition')
self.assertEqual(manifest_object.lastFileModifiedTime, None)
def test_load_folder_with_no_entity_folders(self):
"""Testing for manifest impl instance with no entities and no sub manifests."""
test_name = 'test_load_folder_with_no_entity_folders'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(self.tests_subpath,
test_name,
'empty.manifest.cdm.json')
cdm_manifest = ManifestPersistence.from_object(
corpus.ctx, 'cdmTest', 'someNamespace', '/',
ManifestContent().decode(content))
self.assertEqual(cdm_manifest.schema, 'CdmManifestDefinition.cdm.json')
self.assertEqual(cdm_manifest.manifest_name, 'cdmTest')
self.assertEqual(cdm_manifest.json_schema_semantic_version, '0.9.0')
self.assertEqual(
time_utils.get_formatted_date_string(
cdm_manifest.last_file_modified_time),
'2008-09-15T23:53:23.000Z')
self.assertEqual(cdm_manifest.explanation,
'test cdm folder for cdm version 0.9+')
self.assertEqual(1, len(cdm_manifest.imports))
self.assertEqual(cdm_manifest.imports[0].corpus_path,
'/primitives.cdm.json')
self.assertEqual(0, len(cdm_manifest.entities))
self.assertEqual(1, len(cdm_manifest.exhibits_traits))
self.assertEqual(0, len(cdm_manifest.sub_manifests))
async def test_pattern_with_non_existing_folder(self):
corpus = TestHelper.get_local_corpus(
self.test_subpath, "test_pattern_with_non_existing_folder")
content = TestHelper.get_input_file_content(
self.test_subpath, "test_pattern_with_non_existing_folder",
"entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(
CdmCorpusContext(corpus, None), "entities", "local", "/",
manifest_content)
error_logged = 0
def callback(level: CdmStatusLevel, message: str):
if 'Failed to fetch all files in the folder location \'local:/testLocation\' described by a partition pattern. Exception:' in message:
nonlocal error_logged
error_logged += 1
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
await cdmManifest.file_status_check_async()
self.assertEqual(1, error_logged)
self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0)
# make sure the last check time is still being set
self.assertIsNotNone(
cdmManifest.entities[0].data_partition_patterns[0].
last_file_status_check_time)
def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
"test_load_local_entity_with_data_partition_pattern",
"entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), "entities",
"testNamespace", "/", manifest_content)
self.assertEqual(len(cdmManifest.entities), 1)
self.assertEqual(cdmManifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdmManifest.entities[0]
self.assertEqual(len(entity.data_partition_patterns), 1)
pattern = entity.data_partition_patterns[0]
self.assertEqual(pattern.name, "testPattern")
self.assertEqual(pattern.explanation, "test explanation")
self.assertEqual(pattern.root_location, "test location")
self.assertEqual(pattern.regular_expression, "\\s*")
self.assertEqual(len(pattern.parameters), 2)
self.assertEqual(pattern.parameters[0], "testParam1")
self.assertEqual(pattern.parameters[1], "testParam2")
self.assertEqual(pattern.specialized_schema, "test special schema")
self.assertEqual(len(pattern.exhibits_traits), 1)
def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
'test_load_local_entity_with_data_partition_pattern',
'entities.manifest.cdm.json')
manifest_content = ManifestContent()
manifest_content.decode(content)
cdm_manifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), 'entities',
'testNamespace', '/', manifest_content)
self.assertEqual(len(cdm_manifest.entities), 2)
entity1 = cdm_manifest.entities[0]
self.assertEqual(entity1.object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
self.assertEqual(len(entity1.data_partition_patterns), 1)
pattern1 = entity1.data_partition_patterns[0]
self.assertEqual(pattern1.name, 'testPattern')
self.assertEqual(pattern1.explanation, 'test explanation')
self.assertEqual(pattern1.root_location, 'test location')
self.assertEqual(pattern1.regular_expression, '\\s*')
self.assertEqual(len(pattern1.parameters), 2)
self.assertEqual(pattern1.parameters[0], 'testParam1')
self.assertEqual(pattern1.parameters[1], 'testParam2')
self.assertEqual(pattern1.specialized_schema, 'test special schema')
self.assertEqual(len(pattern1.exhibits_traits), 1)
entity2 = cdm_manifest.entities[1]
self.assertEqual(entity2.object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
self.assertEqual(len(entity2.data_partition_patterns), 1)
pattern2 = entity2.data_partition_patterns[0]
self.assertEqual(pattern2.name, 'testPattern2')
self.assertEqual(pattern2.root_location, 'test location2')
self.assertEqual(pattern2.glob_pattern, '/*.csv')
manifest_data = ManifestPersistence.to_data(cdm_manifest, None, None)
self.assertEqual(len(manifest_data.entities), 2)
entity_data1 = manifest_data.entities[0]
self.assertEqual(len(entity_data1.dataPartitionPatterns), 1)
pattern_data1 = entity_data1.dataPartitionPatterns[0]
self.assertEqual(pattern_data1.name, 'testPattern')
self.assertEqual(pattern_data1.explanation, 'test explanation')
self.assertEqual(pattern_data1.rootLocation, 'test location')
self.assertEqual(pattern_data1.regularExpression, '\\s*')
self.assertEqual(len(pattern_data1.parameters), 2)
self.assertEqual(pattern_data1.parameters[0], 'testParam1')
self.assertEqual(pattern_data1.parameters[1], 'testParam2')
self.assertEqual(pattern_data1.specializedSchema,
'test special schema')
self.assertEqual(len(pattern_data1.exhibitsTraits), 1)
pattern_data2 = manifest_data.entities[1].dataPartitionPatterns[0]
self.assertEqual(pattern_data2.name, 'testPattern2')
self.assertEqual(pattern_data2.rootLocation, 'test location2')
self.assertEqual(pattern_data2.globPattern, '/*.csv')
async def test_pattern_with_non_existing_folder(self):
corpus = TestHelper.get_local_corpus(self.test_subpath, "test_pattern_with_non_existing_folder")
content = TestHelper.get_input_file_content(self.test_subpath, "test_pattern_with_non_existing_folder", "entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(CdmCorpusContext(corpus, None), "entities", "local", "/", manifest_content)
await cdmManifest.file_status_check_async()
self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0)
# make sure the last check time is still being set
self.assertIsNotNone(cdmManifest.entities[0].data_partition_patterns[0].last_file_status_check_time)
async def test_load_local_entitiy_with_data_partition(self):
content = TestHelper.get_input_file_content(
self.test_subpath, 'test_load_local_entity_with_data_partition',
'entities.manifest.cdm.json')
manifest_content = ManifestContent()
manifest_content.decode(content)
cdm_manifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), 'entities',
'testNamespace', '/', manifest_content)
self.assertEqual(len(cdm_manifest.entities), 1)
self.assertEqual(cdm_manifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdm_manifest.entities[0]
self.assertEqual(len(entity.data_partitions), 2)
relative_partition = entity.data_partitions[0]
self.assertEqual(relative_partition.name, 'Sample data partition')
self.assertEqual(relative_partition.location, 'test/location')
# self.assertEqual(TimeUtils.GetFormattedDateString(relative_partition.LastFileModifiedTime), '2008-09-15T23:53:23.000Z')
self.assertEqual(len(relative_partition.exhibits_traits), 1)
self.assertEqual(relative_partition.specialized_schema, 'teststring')
test_list = relative_partition.arguments['test']
self.assertEqual(len(test_list), 3)
self.assertEqual(test_list[0], 'something')
self.assertEqual(test_list[1], 'somethingelse')
self.assertEqual(test_list[2], 'anotherthing')
key_list = relative_partition.arguments['KEY']
self.assertEqual(len(key_list), 1)
self.assertEqual(key_list[0], 'VALUE')
self.assertFalse('wrong' in relative_partition.arguments)
absolute_partition = entity.data_partitions[1]
self.assertEqual(absolute_partition.location,
"local:/some/test/location")
def test_manifest_with_blank_fields(self):
"""Testing for manifest impl instance with blank or empty values for manifest schema, name etc."""
test_name = 'test_manifest_with_blank_fields'
corpus = self.get_corpus()
content = TestHelper.get_input_file_content(self.tests_subpath,
test_name,
'blank.manifest.cdm.json')
cdm_manifest = ManifestPersistence.from_object(
corpus.ctx, 'cdmTest', 'someNamespace', '/',
ManifestContent().decode(content))
self.assertIsNone(cdm_manifest.schema)
self.assertIsNone(cdm_manifest.document_version)
self.assertEqual(
time_utils._get_formatted_date_string(
cdm_manifest.last_file_modified_time),
'2008-09-15T23:53:23.000Z')
self.assertEqual(cdm_manifest.explanation,
'test cdm folder for cdm version 1.0+')
self.assertEqual(1, len(cdm_manifest.imports))
self.assertEqual(cdm_manifest.imports[0].corpus_path,
'/primitives.cdm.json')
self.assertEqual(0, len(cdm_manifest.entities))
self.assertEqual(1, len(cdm_manifest.exhibits_traits))
self.assertEqual(0, len(cdm_manifest.sub_manifests))
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder.Namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, str(folder.namespace), e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if not doc_name:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, e)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content