def to_data(instance: CdmEntityDefinition, ctx: CdmCorpusContext,
res_opt: ResolveOptions, options: CopyOptions) -> TableEntity:
properties = EntityPersistence.create_table_propertybags(
instance, res_opt, options)
columns = []
for attribute in instance.attributes:
from cdm.persistence.syms import TypeAttributePersistence
col = TypeAttributePersistence.to_data(attribute, ctx, res_opt,
options)
columns.append(col)
storage_descriptor = StorageDescriptor(source=None,
format=None,
columns=columns)
partitioning = TablePartitioning(None)
teProperties = TableProperties(namespace=None,
table_type=TableType.managed,
properties=properties,
partitioning=partitioning,
storage_descriptor=storage_descriptor)
table_entity = TableEntity(name=instance.entity_name,
type=SASEntityType.table,
properties=teProperties)
return table_entity
async def to_data_async(instance: CdmManifestDefinition, res_opt: ResolveOptions, options: CopyOptions, is_delta_sync: bool = False, existing_syms_tables: List[TableEntity] = None, existing_syms_relationships: List[RelationshipEntity] = None) -> SymsManifestContent:
source = ManifestPersistence.create_data_source(instance)
if source == None:
return None
properties = ManifestPersistence.create_database_propertybags(instance, res_opt, options)
existing_table_entities = {}
removed_syms_table = []
if existing_syms_tables is not None and len(existing_syms_tables.items) > 0 :
# convert to dictionary for better searching
ents = []
for ent in instance.entities:
ents.append(ent.entity_name)
for item in existing_syms_tables.items:
entity_obj = TableEntity(None, None).deserialize(item)
if entity_obj.type == SASEntityType.table:
existing_table_entities[entity_obj.name] = entity_obj
if entity_obj.name not in ents:
removed_syms_table.append(entity_obj.name)
added_or_modified_syms_tables = await ManifestPersistence.create_syms_tables_objects(instance, res_opt, options, source.location,
existing_table_entities)
# TODO: Submanifest
existing_relationship_entities = {}
removed_syms_relationships = []
if existing_syms_relationships is not None and len(existing_syms_relationships.items) > 0:
rels = []
for rel in instance.relationships:
rels.append(rel.name)
for item in existing_syms_relationships.items:
rel_obj = RelationshipEntity(None, None).deserialize(item)
existing_relationship_entities[rel_obj.name] = rel_obj
if rel_obj.name not in rels:
removed_syms_relationships.append(rel_obj.name)
added_or_modified_syms_relationships = ManifestPersistence.create_relationships(instance, existing_relationship_entities, res_opt, options)
dbproperties = DatabaseProperties(source=source)
if instance.explanation is not None:
dbproperties.description = instance.explanation
else:
dbproperties.description = instance.manifest_name + " syms database"
dbproperties.properties = properties
database_entity = DatabaseEntity(name = instance.manifest_name,
properties = dbproperties,
type = SASEntityType.database)
syms_content = SymsManifestContent()
syms_content.database = database_entity
syms_content.entities = added_or_modified_syms_tables
syms_content.relationships = added_or_modified_syms_relationships
syms_content.intial_sync = not is_delta_sync
syms_content.removed_entities = removed_syms_table
syms_content.removed_relationships = removed_syms_relationships
return syms_content
async def convert_doc_to_syms_table(
ctx: 'CdmCorpusContext', doc: 'CdmDocumentDefinition',
adapter: 'StorageAdapte', name: str, res_opt: 'ResolveOptions',
options: 'CopyOptions') -> 'TableEntity':
existing_table_entity = TableEntity().decode(await
adapter.read_async(name))
return DocumentPersistence.to_data(ctx, doc,
existing_table_entity.Properties,
res_opt, options)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder._folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder._namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if not doc_name:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
from cdm.persistence.syms import utils
if utils.check_if_syms_adapter(adapter):
from cdm.persistence.syms import ManifestDatabasesPersistence
from cdm.persistence.syms.types import SymsDatabasesResponse
if doc_name_lower == self.SYMS_DATABASES:
from cdm.persistence.syms.models.query_artifacts_response import QueryArtifactsResponse
databases = QueryArtifactsResponse()
databases = databases.deserialize(json.loads(json_data))
doc_content = ManifestDatabasesPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, databases)
elif self.MANIFEST_EXTENSION in doc_name_lower:
from cdm.persistence.syms import ManifestPersistence
manifest_content = await utils.get_syms_model(
adapter, json_data, doc_path)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, manifest_content)
elif self.CDM_EXTENSION in doc_name_lower:
from cdm.persistence.syms.models import TableEntity
from cdm.persistence.syms import DocumentPersistence
table = TableEntity(None, None).deserialize(
json.loads(json_data))
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, table)
elif doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, e)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
def from_object(ctx: CdmCorpusContext, docname: str, namespace: str, path: str,
data_obj: 'SymsManifestContent') -> 'CdmManifestDefinition':
database = data_obj.database
if database == None or database.type != SASEntityType.database:
logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_INVALID_DB_OBJECT)
return None
database_properties = DatabaseProperties.deserialize(database.properties)
source = DataSource(None).deserialize(database_properties.source)
if database_properties == None or source == None:
logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_INVALID_DB_PROP_OBJECT)
return None
properties = database_properties.properties
manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF)
manifest._folder_path = path
manifest._namespace = namespace
manifest.manifest_name = data_obj.database.name
manifest.name = docname
manifest.explanation = database_properties.description
if properties is not None:
if "cdm:schema" in properties:
manifest.schema = properties["cdm:schema"]
if "cdm:jsonSchemaSemanticVersion" in properties:
manifest.json_schema_semantic_version = properties["cdm:jsonSchemaSemanticVersion"]
if "cdm:documentVersion" in properties:
manifest.DocumentVersion = properties["cdm:documentVersion"]
if "cdm:traits" in properties:
utils.add_list_to_cdm_collection(manifest.exhibits_traits,
utils.create_trait_reference_array(ctx, properties["cdm:traits"]))
if "cdm:imports" in properties:
for import_obj in properties["cdm:imports"]:
manifest.imports.append(ImportPersistence.from_data(ctx, json.loads(import_obj)))
if "cdm:lastFileStatusCheckTime" in properties:
manifest.last_file_status_check_time = dateutil.parser.parse(properties["cdm:lastFileStatusCheckTime"])
if "cdm:lastFileModifiedTime" in properties:
manifest.last_file_modified_time = dateutil.parser.parse(properties["cdm:lastFileModifiedTime"])
if "cdm:lastChildFileModifiedTime" in properties:
manifest.last_child_file_modified_time = dateutil.parser.parse(properties["cdm:lastChildFileModifiedTime"])
t2pm = TraitToPropertyMap(manifest)
source_trait = t2pm._fetch_trait_reference(ManifestPersistence.db_location_trait)
if source_trait == None:
source_trait = utils.create_source_trait(ctx, ManifestPersistence.db_location_trait, ManifestPersistence.db_location_trait_arg_name)
manifest.exhibits_traits.append(source_trait)
adls_path = utils.syms_path_to_adls_adapter_path(source.location)
adls_corpus_path = ctx.corpus.storage.adapter_path_to_corpus_path(adls_path)
if not adls_corpus_path:
path_tuple = StorageUtils.split_namespace_path(source_trait.arguments[0].value)
obj = utils.create_and_mount_adls_adapter_from_adls_path(ctx.corpus.storage, adls_path, path_tuple[0])
if obj == None:
logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_ADLS_ADAPTER_NOT_MOUNTED, adls_path)
return None
if data_obj.entities is not None:
for item in data_obj.entities.items:
entity_obj = TableEntity(None, None).deserialize(item)
if entity_obj.type == SASEntityType.table:
entity = LocalEntityDeclarationPersistence.from_data(ctx, entity_obj, manifest, database_properties.source.location)
if entity is not None:
manifest.entities.append(entity)
else:
logger.warning(ctx, _TAG, 'from_object', None, CdmLogCode.WARN_PERSIST_SYMS_ENTITY_SKIPPED, entity_obj.name)
if not (x for x in manifest.imports if x.corpus_path == Constants._FOUNDATIONS_CORPUS_PATH) or len(manifest.imports) == 0:
manifest.imports.append(Constants._FOUNDATIONS_CORPUS_PATH)
if data_obj.relationships is not None:
for item in data_obj.relationships.items:
relationship_entity = RelationshipEntity(None, None).deserialize(item)
manifest.relationships.extend(E2ERelationshipPersistence.from_data(ctx, relationship_entity))
# TODO: Submanifest
return manifest