def to_data(instance: CdmEntityDefinition, ctx: CdmCorpusContext, res_opt: ResolveOptions, options: CopyOptions) -> TableEntity: properties = EntityPersistence.create_table_propertybags( instance, res_opt, options) columns = [] for attribute in instance.attributes: from cdm.persistence.syms import TypeAttributePersistence col = TypeAttributePersistence.to_data(attribute, ctx, res_opt, options) columns.append(col) storage_descriptor = StorageDescriptor(source=None, format=None, columns=columns) partitioning = TablePartitioning(None) teProperties = TableProperties(namespace=None, table_type=TableType.managed, properties=properties, partitioning=partitioning, storage_descriptor=storage_descriptor) table_entity = TableEntity(name=instance.entity_name, type=SASEntityType.table, properties=teProperties) return table_entity
async def to_data_async(instance: CdmManifestDefinition, res_opt: ResolveOptions, options: CopyOptions, is_delta_sync: bool = False, existing_syms_tables: List[TableEntity] = None, existing_syms_relationships: List[RelationshipEntity] = None) -> SymsManifestContent: source = ManifestPersistence.create_data_source(instance) if source == None: return None properties = ManifestPersistence.create_database_propertybags(instance, res_opt, options) existing_table_entities = {} removed_syms_table = [] if existing_syms_tables is not None and len(existing_syms_tables.items) > 0 : # convert to dictionary for better searching ents = [] for ent in instance.entities: ents.append(ent.entity_name) for item in existing_syms_tables.items: entity_obj = TableEntity(None, None).deserialize(item) if entity_obj.type == SASEntityType.table: existing_table_entities[entity_obj.name] = entity_obj if entity_obj.name not in ents: removed_syms_table.append(entity_obj.name) added_or_modified_syms_tables = await ManifestPersistence.create_syms_tables_objects(instance, res_opt, options, source.location, existing_table_entities) # TODO: Submanifest existing_relationship_entities = {} removed_syms_relationships = [] if existing_syms_relationships is not None and len(existing_syms_relationships.items) > 0: rels = [] for rel in instance.relationships: rels.append(rel.name) for item in existing_syms_relationships.items: rel_obj = RelationshipEntity(None, None).deserialize(item) existing_relationship_entities[rel_obj.name] = rel_obj if rel_obj.name not in rels: removed_syms_relationships.append(rel_obj.name) added_or_modified_syms_relationships = ManifestPersistence.create_relationships(instance, existing_relationship_entities, res_opt, options) dbproperties = DatabaseProperties(source=source) if instance.explanation is not None: dbproperties.description = instance.explanation else: dbproperties.description = instance.manifest_name + " syms database" dbproperties.properties = properties database_entity = DatabaseEntity(name = instance.manifest_name, properties = dbproperties, type = SASEntityType.database) syms_content = SymsManifestContent() syms_content.database = database_entity syms_content.entities = added_or_modified_syms_tables syms_content.relationships = added_or_modified_syms_relationships syms_content.intial_sync = not is_delta_sync syms_content.removed_entities = removed_syms_table syms_content.removed_relationships = removed_syms_relationships return syms_content
async def convert_doc_to_syms_table( ctx: 'CdmCorpusContext', doc: 'CdmDocumentDefinition', adapter: 'StorageAdapte', name: str, res_opt: 'ResolveOptions', options: 'CopyOptions') -> 'TableEntity': existing_table_entity = TableEntity().decode(await adapter.read_async(name)) return DocumentPersistence.to_data(ctx, doc, existing_table_entity.Properties, res_opt, options)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder._folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter( folder._namespace) # type: StorageAdapter try: if adapter.can_read(): # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'request file: {}'.format(doc_path)) json_data = await adapter.read_async(doc_path) # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'received file: {}'.format(doc_path)) else: raise Exception('Storage Adapter is not enabled to read.') except Exception as e: # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'fail file: {}'.format(doc_path)) # when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if res_opt and res_opt.shallow_validation: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE, doc_path, folder._namespace, e) else: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE, doc_path, folder._namespace, e) return None try: fs_modified_time = await adapter.compute_last_modified_time_async( doc_path) except Exception as e: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED, e.Message) if not doc_name: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME) return None doc_name_lower = doc_name.lower() # If loading an model.json file, check that it is named correctly. if doc_name_lower.endswith( self.MODEL_JSON_EXTENSION ) and not doc_name.lower() == self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None try: from cdm.persistence.syms import utils if utils.check_if_syms_adapter(adapter): from cdm.persistence.syms import ManifestDatabasesPersistence from cdm.persistence.syms.types import SymsDatabasesResponse if doc_name_lower == self.SYMS_DATABASES: from cdm.persistence.syms.models.query_artifacts_response import QueryArtifactsResponse databases = QueryArtifactsResponse() databases = databases.deserialize(json.loads(json_data)) doc_content = ManifestDatabasesPersistence.from_object( self._ctx, doc_name, folder._namespace, folder._folder_path, databases) elif self.MANIFEST_EXTENSION in doc_name_lower: from cdm.persistence.syms import ManifestPersistence manifest_content = await utils.get_syms_model( adapter, json_data, doc_path) doc_content = ManifestPersistence.from_object( self._ctx, doc_name, folder._namespace, folder._folder_path, manifest_content) elif self.CDM_EXTENSION in doc_name_lower: from cdm.persistence.syms.models import TableEntity from cdm.persistence.syms import DocumentPersistence table = TableEntity(None, None).deserialize( json.loads(json_data)) doc_content = DocumentPersistence.from_object( self._ctx, doc_name, folder._namespace, folder._folder_path, table) elif doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION ) or doc_name_lower.endswith( PersistenceLayer.FOLIO_EXTENSION): from cdm.persistence.cdmfolder import ManifestPersistence from cdm.persistence.cdmfolder.types import ManifestContent manifest = ManifestContent() manifest.decode(json_data) doc_content = ManifestPersistence.from_object( self._ctx, doc_name, folder._namespace, folder._folder_path, manifest) elif doc_name_lower.endswith( PersistenceLayer.MODEL_JSON_EXTENSION): from cdm.persistence.modeljson import ManifestPersistence from cdm.persistence.modeljson.types import Model model = Model() model.decode(json_data) doc_content = await ManifestPersistence.from_object( self._ctx, model, folder) elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION): from cdm.persistence.cdmfolder import DocumentPersistence from cdm.persistence.cdmfolder.types import DocumentContent document = DocumentContent() document.decode(json_data) doc_content = DocumentPersistence.from_object( self._ctx, doc_name, folder._namespace, folder._folder_path, document) else: # Could not find a registered persistence class to handle this document type. logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING, doc_name) return None except Exception as e: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE, doc_path, e) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy( ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus. default_resolution_directives), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content
def from_object(ctx: CdmCorpusContext, docname: str, namespace: str, path: str, data_obj: 'SymsManifestContent') -> 'CdmManifestDefinition': database = data_obj.database if database == None or database.type != SASEntityType.database: logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_INVALID_DB_OBJECT) return None database_properties = DatabaseProperties.deserialize(database.properties) source = DataSource(None).deserialize(database_properties.source) if database_properties == None or source == None: logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_INVALID_DB_PROP_OBJECT) return None properties = database_properties.properties manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF) manifest._folder_path = path manifest._namespace = namespace manifest.manifest_name = data_obj.database.name manifest.name = docname manifest.explanation = database_properties.description if properties is not None: if "cdm:schema" in properties: manifest.schema = properties["cdm:schema"] if "cdm:jsonSchemaSemanticVersion" in properties: manifest.json_schema_semantic_version = properties["cdm:jsonSchemaSemanticVersion"] if "cdm:documentVersion" in properties: manifest.DocumentVersion = properties["cdm:documentVersion"] if "cdm:traits" in properties: utils.add_list_to_cdm_collection(manifest.exhibits_traits, utils.create_trait_reference_array(ctx, properties["cdm:traits"])) if "cdm:imports" in properties: for import_obj in properties["cdm:imports"]: manifest.imports.append(ImportPersistence.from_data(ctx, json.loads(import_obj))) if "cdm:lastFileStatusCheckTime" in properties: manifest.last_file_status_check_time = dateutil.parser.parse(properties["cdm:lastFileStatusCheckTime"]) if "cdm:lastFileModifiedTime" in properties: manifest.last_file_modified_time = dateutil.parser.parse(properties["cdm:lastFileModifiedTime"]) if "cdm:lastChildFileModifiedTime" in properties: manifest.last_child_file_modified_time = dateutil.parser.parse(properties["cdm:lastChildFileModifiedTime"]) t2pm = TraitToPropertyMap(manifest) source_trait = t2pm._fetch_trait_reference(ManifestPersistence.db_location_trait) if source_trait == None: source_trait = utils.create_source_trait(ctx, ManifestPersistence.db_location_trait, ManifestPersistence.db_location_trait_arg_name) manifest.exhibits_traits.append(source_trait) adls_path = utils.syms_path_to_adls_adapter_path(source.location) adls_corpus_path = ctx.corpus.storage.adapter_path_to_corpus_path(adls_path) if not adls_corpus_path: path_tuple = StorageUtils.split_namespace_path(source_trait.arguments[0].value) obj = utils.create_and_mount_adls_adapter_from_adls_path(ctx.corpus.storage, adls_path, path_tuple[0]) if obj == None: logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_ADLS_ADAPTER_NOT_MOUNTED, adls_path) return None if data_obj.entities is not None: for item in data_obj.entities.items: entity_obj = TableEntity(None, None).deserialize(item) if entity_obj.type == SASEntityType.table: entity = LocalEntityDeclarationPersistence.from_data(ctx, entity_obj, manifest, database_properties.source.location) if entity is not None: manifest.entities.append(entity) else: logger.warning(ctx, _TAG, 'from_object', None, CdmLogCode.WARN_PERSIST_SYMS_ENTITY_SKIPPED, entity_obj.name) if not (x for x in manifest.imports if x.corpus_path == Constants._FOUNDATIONS_CORPUS_PATH) or len(manifest.imports) == 0: manifest.imports.append(Constants._FOUNDATIONS_CORPUS_PATH) if data_obj.relationships is not None: for item in data_obj.relationships.items: relationship_entity = RelationshipEntity(None, None).deserialize(item) manifest.relationships.extend(E2ERelationshipPersistence.from_data(ctx, relationship_entity)) # TODO: Submanifest return manifest