async def convert_manifest_to_syms(doc: 'CdmManifestDefinition', adapter: 'StorageAdapter', path: str, res_opt: 'ResolveOptions', options: 'CopyOptions')-> 'SymsManifestContent':
database_entity = None
is_delta_sync = True
existing_syms_tables = None
existing_syms_relationship_entities = None
try:
req = await adapter.read_async(path)
database_entity = DatabaseEntity().deserialize(json.loads(req))
except Exception as e:
if e.code == 404:
is_delta_sync = False
else:
raise
if is_delta_sync:
json_data = await adapter.read_async("/" + database_entity.name + "/" + database_entity.name + ".manifest.cdm.json/entitydefinition")
from cdm.persistence.syms.models.query_artifacts_response import QueryArtifactsResponse
entities = QueryArtifactsResponse().deserialize(json.loads(json_data))
existing_syms_tables = entities
json_data = await adapter.read_async("/" + database_entity.name + "/" + database_entity.name+ ".manifest.cdm.json/relationships")
realtionships = QueryArtifactsResponse().deserialize(json.loads(json_data))
existing_syms_relationship_entities = realtionships
return await ManifestPersistence.to_data_async(doc, res_opt, options, is_delta_sync, existing_syms_tables, existing_syms_relationship_entities)
async def get_syms_model(adapter: 'StorageAdapter', database_response: str, doc_path: str)-> 'SymsManifestContent':
from cdm.persistence.syms.models.query_artifacts_response import QueryArtifactsResponse
db = DatabaseEntity().deserialize(json.loads(database_response))
entities = await adapter.read_async('/' + db.name + '/' + db.name + '.manifest.cdm.json/entitydefinition')
relationships = await adapter.read_async(doc_path + '/relationships')
return SymsManifestContent(
database = db,
entities = QueryArtifactsResponse().deserialize(json.loads(entities)),
relationships = QueryArtifactsResponse().deserialize(json.loads(relationships)),
intial_sync = False,
removed_entities = None,
removed_relationships = None)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder._folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder._namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if not doc_name:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
from cdm.persistence.syms import utils
if utils.check_if_syms_adapter(adapter):
from cdm.persistence.syms import ManifestDatabasesPersistence
from cdm.persistence.syms.types import SymsDatabasesResponse
if doc_name_lower == self.SYMS_DATABASES:
from cdm.persistence.syms.models.query_artifacts_response import QueryArtifactsResponse
databases = QueryArtifactsResponse()
databases = databases.deserialize(json.loads(json_data))
doc_content = ManifestDatabasesPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, databases)
elif self.MANIFEST_EXTENSION in doc_name_lower:
from cdm.persistence.syms import ManifestPersistence
manifest_content = await utils.get_syms_model(
adapter, json_data, doc_path)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, manifest_content)
elif self.CDM_EXTENSION in doc_name_lower:
from cdm.persistence.syms.models import TableEntity
from cdm.persistence.syms import DocumentPersistence
table = TableEntity(None, None).deserialize(
json.loads(json_data))
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, table)
elif doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, e)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content