async def _add_elevated_traits_and_relationships( self, rel: 'CdmE2ERelationship') -> None: """ Adds imports for elevated purpose traits for relationships, then adds the relationships to the manifest. The last import has the highest priority, so we insert the imports for traits to the beginning of the list. """ res_opt = ResolveOptions(self) for trait_ref in rel.exhibits_traits: trait_def = self.ctx.corpus._resolve_symbol_reference( res_opt, self, trait_ref.fetch_object_definition_name(), CdmObjectType.TRAIT_DEF, True) if trait_def is None: abs_path = rel._elevated_trait_corpus_path[trait_ref] relative_path = self.ctx.corpus.storage.create_relative_corpus_path( abs_path, self) # Adds the import to this manifest file self.imports.insert(0, CdmImport(self.ctx, relative_path, None)) # Fetches the actual file of the import and indexes it import_document = await self.ctx.corpus.fetch_object_async( abs_path) # type: 'CdmDocumentDefinition' if not isinstance(import_document, CdmDocumentDefinition): logger.error( self.ctx, self._TAG, self._add_elevated_traits_and_relationships.__name__, self.at_corpus_path, CdmLogCode.ERR_INVALID_CAST, abspath, 'CdmDocumentDefinition') continue await import_document._index_if_needed(res_opt) # Resolves the imports in the manifests await self.ctx.corpus._resolve_imports_async( self, set(self.at_corpus_path), res_opt) # Calls `GetImportPriorities` to prioritize all imports properly after a new import added (which requires `ImportPriorities` set to null) self._import_priorities = None self._get_import_priorities() # As adding a new import above set the manifest needsIndexing to true, we want to avoid over indexing for each import insertion # so we handle the indexing for the new import above seperately in this case, no indexing needed at this point self._needs_indexing = False self.relationships.append(self._localize_rel_to_manifest(rel))
def validate(self) -> bool: missing_fields = [] if not bool(self.name): missing_fields.append('name') if bool(self.cardinality): if not bool(self.cardinality.minimum): missing_fields.append('cardinality.minimum') if not bool(self.cardinality.maximum): missing_fields.append('cardinality.maximum') if missing_fields: logger.error( self.ctx, self._TAG, 'validate', self.at_corpus_path, CdmLogCode.ERR_VALDN_INTEGRITY_CHECK_FAILURE, self.at_corpus_path, ', '.join(map(lambda s: '\'' + s + '\'', missing_fields))) return False if bool(self.cardinality): if not CardinalitySettings._is_minimum_valid( self.cardinality.minimum): logger.error(self.ctx, self._TAG, 'validate', self.at_corpus_path, CdmLogCode.ERR_VALDN_INVALID_MIN_CARDINALITY, self.cardinality.minimum) return False if not CardinalitySettings._is_maximum_valid( self.cardinality.maximum): logger.error(self.ctx, self._TAG, 'validate', self.at_corpus_path, CdmLogCode.ERR_VALDN_INVALID_MAX_CARDINALITY, self.cardinality.maximum) return False return True
def corpus_path_to_adapter_path(self, corpus_path: str) -> Optional[str]: """Takes a corpus path, figures out the right adapter to use and then return an adapter domain path""" with logger._enter_scope(self._TAG, self._ctx, self.corpus_path_to_adapter_path.__name__): if not corpus_path: logger.error( self._ctx, self._TAG, StorageManager.corpus_path_to_adapter_path.__name__, None, CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH) return None result = None # Break the corpus path into namespace and ... path path_tuple = StorageUtils.split_namespace_path(corpus_path) if not path_tuple: logger.error(self._ctx, self._TAG, self.corpus_path_to_adapter_path.__name__, None, CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH) return None namespace = path_tuple[0] or self.default_namespace # Get the adapter registered for this namespace namespace_adapter = self.fetch_adapter(namespace) if not namespace_adapter: logger.error( self._ctx, self._TAG, StorageManager.corpus_path_to_adapter_path.__name__, None, CdmLogCode.ERR_STORAGE_NAMESPACE_NOT_REGISTERED, namespace) else: # Ask the storage adapter to 'adapt' this path result = namespace_adapter.create_adapter_path(path_tuple[1]) return result
def validate(self) -> bool: missing_fields = [] if not bool(self.name): missing_fields.append('name') if bool(self.cardinality): if not bool(self.cardinality.minimum): missing_fields.append('cardinality.minimum') if not bool(self.cardinality.maximum): missing_fields.append('cardinality.maximum') if missing_fields: logger.error( self._TAG, self.ctx, Errors.validate_error_string(self.at_corpus_path, missing_fields)) return False if bool(self.cardinality): if not CardinalitySettings._is_minimum_valid( self.cardinality.minimum): logger.error( self._TAG, self.ctx, 'Invalid minimum cardinality {}.'.format( self.cardinality.minimum)) return False if not CardinalitySettings._is_maximum_valid( self.cardinality.maximum): logger.error( self._TAG, self.ctx, 'Invalid maximum cardinality {}.'.format( self.cardinality.maximum)) return False return True
def corpus_path_to_adapter_path(self, corpus_path: str) -> Optional[str]: """Takes a corpus path, figures out the right adapter to use and then return an adapter domain path""" if not corpus_path: logger.error(self._TAG, self._ctx, 'The corpus path is null or empty.', StorageManager.corpus_path_to_adapter_path.__name__) return None result = None # Break the corpus path into namespace and ... path path_tuple = StorageUtils.split_namespace_path(corpus_path) if not path_tuple: logger.error(self._TAG, self._ctx, 'The corpus path cannot be null or empty.', self.corpus_path_to_adapter_path.__name__) return None namespace = path_tuple[0] or self.default_namespace # Get the adapter registered for this namespace namespace_adapter = self.fetch_adapter(namespace) if not namespace_adapter: logger.error(self._TAG, self._ctx, 'The namespace cannot be null or empty.'.format( namespace), StorageManager.corpus_path_to_adapter_path.__name__) else: # Ask the storage adapter to 'adapt' this path result = namespace_adapter.create_adapter_path(path_tuple[1]) return result
async def _save_dirty_link(self, relative: str, options: 'CopyOptions') -> None: """helper that fixes a path from local to absolute, gets the object from that path then looks at the document where the object is found. if dirty, the document is saved with the original name""" # get the document object from the import doc_path = self.ctx.corpus.storage.create_absolute_corpus_path( relative, self) if doc_path is None: logger.error(self._TAG, self.ctx, 'Invalid corpus path {}'.format(relative), self._save_dirty_link.__name__) return False obj_at = await self.ctx.corpus.fetch_object_async(doc_path) if obj_at is None: logger.error(self._TAG, self.ctx, 'Couldn\'t get object from path {}'.format(doc_path), self._save_dirty_link.__name__) return False doc_imp = cast('CdmDocumentDefinition', obj_at.in_document) if doc_imp: if doc_imp._is_dirty: # save it with the same name if not await doc_imp.save_as_async(doc_imp.name, True, options): logger.error( self._TAG, self.ctx, 'failed saving document {}'.format(doc_imp.name), self._save_dirty_link.__name__) return False return True
async def _save_dirty_link(self, relative: str, options: 'CopyOptions') -> bool: """Helper that fixes a path from local to absolute, gets the object from that path then looks at the document where the object is found. if dirty, the document is saved with the original name""" # get the document object from the import doc_path = self.ctx.corpus.storage.create_absolute_corpus_path( relative, self) if doc_path is None: logger.error(self.ctx, self._TAG, self._save_dirty_link.__name__, self.at_corpus_path, CdmLogCode.ERR_VALDN_INVALID_CORPUS_PATH, relative) return False obj_at = await self.ctx.corpus.fetch_object_async(doc_path) if obj_at is None: logger.error(self.ctx, self._TAG, self._save_dirty_link.__name__, self.at_corpus_path, CdmLogCode.ERR_PERSIST_OBJECT_NOT_FOUND, doc_path) return False doc_imp = cast('CdmDocumentDefinition', obj_at.in_document) if doc_imp: if doc_imp._is_dirty: # save it with the same name if not await doc_imp.save_as_async(doc_imp.name, True, options): logger.error(self.ctx, self._TAG, self._save_dirty_link.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_ENTITY_DOC_SAVING_FAILURE, doc_imp.name) return False return True
def unmount(self, namespace: str) -> None: """unregisters a storage adapter and its root folder""" if not namespace: logger.error(self._TAG, self._ctx, 'The namespace cannot be null or empty.', StorageManager.unmount.__name__) return None if namespace in self.namespace_adapters: self.namespace_adapters.pop(namespace, None) self._namespace_folders.pop(namespace, None) if namespace in self._system_defined_namespaces: self._system_defined_namespaces.remove(namespace) # The special case, use Resource adapter. if (namespace == 'cdm'): self.mount(namespace, ResourceAdapter()) else: logger.warning( self._TAG, self._ctx, 'Cannot remove the adapter from non-existing namespace.', StorageManager.mount.__name__)
def adapter_path_to_corpus_path(self, adapter_path: str) -> Optional[str]: """Takes a storage adapter domain path, figures out the right adapter to use and then return a corpus path""" result = None # Keep trying adapters until one of them likes what it sees if self.namespace_adapters: for key, value in self.namespace_adapters.items(): result = value.create_corpus_path(adapter_path) if result: # Got one, add the prefix result = '{}:{}'.format(key, result) break if not result: logger.error( self._TAG, self._ctx, 'No registered storage adapter understood the path "{}"'. format(adapter_path), StorageManager.adapter_path_to_corpus_path.__name__) return result
def _check_and_add_item_modifications(self, document: 'CdmDocumentDefinition') -> bool: if self.item(document.name) is not None: logger.error(self.ctx, _TAG, '_check_and_add_item_modifications', document.at_corpus_path, CdmLogCode.ERR_DOC_ALREADY_EXIST, document.name, lambda x:self.owner.at_corpus_path if self.owner.at_corpus_path is not None else self.owner.name) return False; if document.owner and document.owner is not self.owner: # this is fun! the document is moving from one folder to another # it must be removed from the old folder for sure, but also now # there will be a problem with any corpus paths that are relative to that old folder location. # so, whip through the document and change any corpus paths to be relative to this folder document._localize_corpus_paths(self.owner) # returns false if it fails, but ... who cares? we tried document.owner.documents.remove(document.name) document._folder_path = self.owner._folder_path document.owner = self.owner document._namespace = self.owner._namespace super()._make_document_dirty() # set the document to dirty so it will get saved in the new folder location if saved self.owner._corpus._add_document_objects(self.owner, document) return True
def adapter_path_to_corpus_path(self, adapter_path: str) -> Optional[str]: """Takes a storage adapter domain path, figures out the right adapter to use and then return a corpus path""" with logger._enter_scope(self._TAG, self._ctx, self.adapter_path_to_corpus_path.__name__): result = None # Keep trying adapters until one of them likes what it sees if self.namespace_adapters: for key, value in self.namespace_adapters.items(): result = value.create_corpus_path(adapter_path) if result: # Got one, add the prefix result = '{}:{}'.format(key, result) break if not result: logger.error( self._ctx, self._TAG, StorageManager.adapter_path_to_corpus_path.__name__, None, CdmLogCode.ERR_STORAGE_INVALID_ADAPTER_PATH, adapter_path) return result
async def _save_odi_documents(self, doc: Any, adapter: 'StorageAdapter', new_name: str) -> None: if doc is None: raise Exception('Failed to persist document because doc is null.') # ask the adapter to make it happen. try: old_document_path = doc.documentPath new_document_path = old_document_path[ 0:len(old_document_path) - len(self.ODI_EXTENSION)] + new_name content = doc.encode() await adapter.write_async(new_document_path, content) except Exception as e: logger.error( self._TAG, self._ctx, 'Failed to write to the file \'{}\' for reason {}.'.format( doc.documentPath, e), self._save_odi_documents.__name__) # Save linked documents. if doc.get('linkedDocuments') is not None: for linked_doc in doc.linkedDocuments: await self._save_odi_documents(linked_doc, adapter, new_name)
def unmount(self, namespace: str) -> None: """unregisters a storage adapter and its root folder""" with logger._enter_scope(self._TAG, self._ctx, self.unmount.__name__): if not namespace: logger.error(self._ctx, self._TAG, StorageManager.unmount.__name__, None, CdmLogCode.ERR_STORAGE_NULL_NAMESPACE) return None if namespace in self.namespace_adapters: self.namespace_adapters.pop(namespace, None) self._namespace_folders.pop(namespace, None) if namespace in self._system_defined_namespaces: self._system_defined_namespaces.remove(namespace) # The special case, use Resource adapter. if (namespace == 'cdm'): self.mount(namespace, ResourceAdapter()) else: logger.warning(self._ctx, self._TAG, StorageManager.unmount.__name__, None, CdmLogCode.WARN_STORAGE_REMOVE_ADAPTER_FAILED)
def mount(self, namespace: str, adapter: 'StorageAdapterBase') -> None: """registers a namespace and assigns creates a storage adapter for that namespace""" if not namespace: logger.error(self._TAG, self._ctx, 'The namespace cannot be null or empty.', StorageManager.mount.__name__) return None from cdm.objectmodel import CdmFolderDefinition if adapter: self.namespace_adapters[namespace] = adapter fd = CdmFolderDefinition(self._ctx, '') fd._corpus = self._corpus fd.namespace = namespace fd.folder_path = '/' self._namespace_folders[namespace] = fd if namespace in self._system_defined_namespaces: self._system_defined_namespaces.remove(namespace) else: logger.error(self._TAG, self._ctx, 'The adapter cannot be null.', StorageManager.mount.__name__)
def from_data(ctx: 'CdmCorpusContext', obj: StorageDescriptor, syms_root_path: str, format_type: str) -> CdmDataPartitionDefinition: new_partition = ctx.corpus.make_object( CdmObjectType.DATA_PARTITION_DEF ) # type: CdmDataPartitionDefinition syms_path = utils.create_syms_absolute_path(syms_root_path, obj.source.location) new_partition.location = utils.syms_path_to_corpus_path( syms_path, ctx.corpus.storage) trait = utils.create_partition_trait(obj.format.properties, ctx, format_type) if trait is not None: new_partition.exhibits_traits.append(trait) else: logger.error(ctx, _TAG, DataPartitionPersistence.from_data.__name__, None, CdmLogCode.ERR_PERSIST_SYMS_UNSUPPORTED_TABLE_FORMAT) return None properties = obj.properties if properties is not None: if 'cdm:name' in properties: new_partition.name = properties['cdm:name'] if 'cdm:lastFileStatusCheckTime' in properties: new_partition.last_file_status_check_time = dateutil.parser.parse( properties['cdm:lastFileStatusCheckTime']) if 'cdm:lastFileModifiedTime' in properties: new_partition.last_file_modified_time = dateutil.parser.parse( properties['cdm:lastFileModifiedTime']) if 'cdm:traits' in properties: utils.add_list_to_cdm_collection( new_partition.exhibits_traits, utils.create_trait_reference_array( ctx, properties['cdm:traits'])) return new_partition
async def to_data(instance: 'CdmReferencedEntityDeclarationDefinition', res_opt: 'ResolveOptions', options: 'CopyOptions') -> Optional['ReferenceEntity']: source_index = instance.entity_path.rfind('/') if source_index == -1: logger.error( instance.ctx, _TAG, 'to_data', instance.at_corpus_path, CdmLogCode. ERR_PERSIST_MODELJSON_ENTITY_PARTITION_CONVERSION_ERROR, instance.at_corpus_path) return None reference_entity = ReferenceEntity() t2pm = TraitToPropertyMap(instance) reference_entity.type = 'ReferenceEntity' reference_entity.name = instance.entity_name reference_entity.source = instance.entity_path[source_index + 1:] reference_entity.description = instance.explanation reference_entity.lastFileStatusCheckTime = utils.get_formatted_date_string( instance.last_file_status_check_time) reference_entity.lastFileModifiedTime = utils.get_formatted_date_string( instance.last_file_modified_time) reference_entity.isHidden = bool( t2pm._fetch_trait_reference('is.hidden')) or None properties_trait = t2pm._fetch_trait_reference( 'is.propertyContent.multiTrait') if properties_trait: reference_entity.modelId = properties_trait.arguments[0].value utils.process_traits_and_annotations_to_data(instance.ctx, reference_entity, instance.exhibits_traits) return reference_entity
async def to_data(instance: 'CdmEntityDefinition', res_opt: 'ResolveOptions', options: 'CopyOptions', ctx: 'CdmCorpusContext') -> Optional['LocalEntity']: data = LocalEntity() data.type = 'LocalEntity' data.name = instance.entity_name data.description = instance._get_property("description") utils.process_traits_and_annotations_to_data(instance.ctx, data, instance.exhibits_traits) if instance.attributes: data.attributes = [] for element in instance.attributes: if element.object_type != CdmObjectType.TYPE_ATTRIBUTE_DEF: logger.error(ctx, EntityPersistence.__name__, EntityPersistence.to_data.__name__, element.at_corpus_path, CdmLogCode.ERR_PERSIST_MANIFEST_SAVING_FAILURE) return None attribute = await TypeAttributePersistence.to_data(element, res_opt, options) if attribute: data.attributes.append(attribute) else: logger.error(ctx, EntityPersistence.__name__, EntityPersistence.to_data.__name__, CdmLogCode.ERR_PERSIST_MODELJSON_ATTR_CONVERSION_FAILURE) return None return data
async def _save_odi_documents(self, doc: Any, adapter: 'StorageAdapter', new_name: str) -> None: if doc is None: raise Exception('Failed to persist document because doc is null.') # ask the adapter to make it happen. try: old_document_path = doc.documentPath new_document_path = old_document_path[0: len(old_document_path) - len(self.ODI_EXTENSION)] + new_name # Remove namespace from path path_tuple = StorageUtils.split_namespace_path(new_document_path) if not path_tuple: logger.error(self._TAG, self._ctx, 'The object path cannot be null or empty.', self._save_odi_documents.__name__) return content = doc.encode() await adapter.write_async(path_tuple[1], content) except Exception as e: logger.error(self._TAG, self._ctx, 'Failed to write to the file \'{}\' for reason {}.'.format( doc.documentPath, e), self._save_odi_documents.__name__) # Save linked documents. if doc.get('linkedDocuments') is not None: for linked_doc in doc.linkedDocuments: await self._save_odi_documents(linked_doc, adapter, new_name)
async def _index_if_needed(self, res_opt: 'ResolveOptions', load_imports: bool = False) -> bool: if not self._needs_indexing or self._currently_indexing: return True if not self.folder: logger.error(self.ctx, self._TAG, self._index_if_needed.__name__, self.at_corpus_path, CdmLogCode.ERR_VALDN_MISSING_DOC, self.name) return False corpus = self.folder._corpus # if the imports load strategy is "LAZY_LOAD", loadImports value will be the one sent by the called function. if res_opt.imports_load_strategy == ImportsLoadStrategy.DO_NOT_LOAD: load_imports = False elif res_opt.imports_load_strategy == ImportsLoadStrategy.LOAD: load_imports = True if load_imports: await corpus._resolve_imports_async(self, res_opt) # make the corpus internal machinery pay attention to this document for this call corpus._document_library._mark_document_for_indexing(self) return corpus._index_documents(res_opt, load_imports)
def from_data( ctx: CdmCorpusContext, prefix_path: str, data: ReferencedEntityDeclaration ) -> CdmReferencedEntityDeclarationDefinition: referenced_entity = ctx.corpus.make_object( CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF, data.entityName) entity_path = data.get('entityPath') or data.get('entityDeclaration') if not entity_path: logger.error( ctx, _TAG, ReferencedEntityDeclarationPersistence.from_data.__name__, None, CdmLogCode.ERR_PERSIST_ENTITY_PATH_NOT_FOUND) # The entity path has to be absolute. # If the namespace is not present then add the "prefixPath" which has the absolute folder path. if entity_path and entity_path.find(':/') == -1: entity_path = '{}{}'.format(prefix_path, entity_path) referenced_entity.entity_path = entity_path referenced_entity.explanation = data.get('explanation') if data.get('lastFileStatusCheckTime'): referenced_entity.last_file_status_check_time = dateutil.parser.parse( data.lastFileStatusCheckTime) if data.get('lastFileModifiedTime'): referenced_entity.last_file_modified_time = dateutil.parser.parse( data.lastFileModifiedTime) if data.get('exhibitsTraits'): exhibits_traits = utils.create_trait_reference_array( ctx, data.exhibitsTraits) referenced_entity.exhibits_traits.extend(exhibits_traits) return referenced_entity
def _create_new_projection_attribute_state_set( proj_ctx: 'ProjectionContext', proj_output_set: 'ProjectionAttributeStateSet', new_res_attr_FK: 'ResolvedAttribute', ref_attr_name: str) -> 'ProjectionAttributeStateSet': pas_list = ProjectionResolutionCommonUtil._get_leaf_list( proj_ctx, ref_attr_name) if pas_list is not None: # update the new foreign key resolved attribute with trait param with reference details reqd_trait = new_res_attr_FK.resolved_traits.find( proj_ctx._projection_directive._res_opt, 'is.linkedEntity.identifier') if reqd_trait: trait_param_ent_ref = ProjectionResolutionCommonUtil._create_foreign_key_linked_entity_identifier_trait_parameter( proj_ctx._projection_directive, proj_output_set._ctx.corpus, pas_list) reqd_trait.parameter_values.update_parameter_value( proj_ctx._projection_directive._res_opt, 'entityReferences', trait_param_ent_ref) # Create new output projection attribute state set for FK and add prevPas as previous state set new_proj_attr_state_FK = ProjectionAttributeState( proj_output_set._ctx) new_proj_attr_state_FK._current_resolved_attribute = new_res_attr_FK new_proj_attr_state_FK._previous_state_list = pas_list proj_output_set._add(new_proj_attr_state_FK) else: # Log error & return proj_output_set without any change logger.error( CdmOperationReplaceAsForeignKey.__name__, proj_output_set._ctx, 'Unable to locate state for reference attribute \"{}\".'. format(ref_attr_name), CdmOperationReplaceAsForeignKey. _create_new_projection_attribute_state_set.__name__) return proj_output_set
def mount_from_config(self, adapter_config: str, does_return_error_list: bool = False) -> List['StorageAdapter']: if not adapter_config: logger.error(self._TAG, self._ctx, 'Adapter config cannot be null or empty.', StorageManager.mount_from_config.__name__) return None adapter_config_json = json.loads(adapter_config) adapers_module = importlib.import_module('cdm.storage') if adapter_config_json.get('appId'): self._corpus.app_id = adapter_config_json['appId'] if adapter_config_json.get('defaultNamespace'): self.default_namespace = adapter_config_json['defaultNamespace'] unrecognized_adapters = [] for item in adapter_config_json['adapters']: namespace = None # Check whether the namespace exists. if item.get('namespace'): namespace = item['namespace'] else: logger.error(self._TAG, self._ctx, 'The namespace is missing for one of the adapters in the JSON config.') continue configs = None # Check whether the config exists. if item.get('config'): configs = item['config'] else: logger.error(self._TAG, self._ctx, 'Missing JSON config for the namespace {}.'.format(namespace)) continue if not item.get('type'): logger.error(self._TAG, self._ctx, 'Missing type in the JSON config for the namespace {}.'.format(namespace)) continue adapter_type = self._registered_adapter_types.get(item['type'], None) if adapter_type is None: unrecognized_adapters.append(json.dumps(item)) else: adapter = getattr(adapers_module, adapter_type)() adapter.update_config(json.dumps(configs)) self.mount(namespace, adapter) return unrecognized_adapters if does_return_error_list else None
def to_data(instance: CdmTypeAttributeDefinition, ctx: CdmCorpusContext, res_opt: ResolveOptions, options: CopyOptions) -> TypeAttribute: properties = TypeAttributePersistence.create_properties( instance, res_opt, options) origin_data_type_name = TypeInfo( type_name='', properties=properties, is_complex_type=False, is_nullable=instance._get_property('isNullable'), type_family='cdm') t2pm = TraitToPropertyMap(instance) numeric_traits = t2pm._fetch_trait_reference( 'is.data_format.numeric.shaped') if numeric_traits is not None: for numeric_traits_arg in numeric_traits.argument: if numeric_traits_arg.name == 'precision': origin_data_type_name.precision = numeric_traits_arg.value if numeric_traits_arg.Name == 'scale': origin_data_type_name.scale = numeric_traits_arg.value data_format = instance._get_property('dataFormat') origin_data_type_name = utils.cdm_data_format_to_syms_data_type( data_format, origin_data_type_name) if origin_data_type_name == None: return None if origin_data_type_name.type_name == None: logger.error(ctx, _TAG, 'toData', instance.at_corpus_path, CdmLogCode.ERR_PERSIST_SYMS_UNKNOWN_DATA_FORMAT, instance.display_name) return None data_col = DataColumn(origin_data_type_name=origin_data_type_name, name=instance.name) return data_col
async def to_data_async(document_object_or_path, manifest: CdmManifestDefinition, ctx: CdmCorpusContext, res_opt: ResolveOptions, options: CopyOptions) -> TableEntity: if isinstance(document_object_or_path, str): obje = await ctx.corpus.fetch_object_async(document_object_or_path, manifest) if isinstance(obje, CdmEntityDefinition): cdm_entity = obje table_entity = EntityPersistence.to_data( cdm_entity, ctx, res_opt, options) te_properties = table_entity.properties te_properties.namespace = TableNamespace( manifest.manifest_name) if cdm_entity.owner is not None and isinstance( cdm_entity.owner, CdmDocumentDefinition): document = cdm_entity.owner if len(document.imports) > 0: imports = copy_data_utils._array_copy_data( res_opt, document.imports, options) te_properties.properties["cdm:imports"] = imports else: logger.warning(ctx, _TAG, 'to_data_async', manifest.at_corpus_path, CdmLogCode.WARN_PERSIST_SYMS_ENTITY_MISIING, cdm_entity.name) return table_entity else: logger.error(ctx, _TAG, 'to_data_async', manifest.at_corpus_path, CdmLogCode.ERR_PERSIST_SYMS_ENTITY_FETCH_ERROR, document_object_or_path) return None return None
async def from_data(ctx: 'CdmCorpusContext', data_obj: 'LocalEntity', extension_trait_def_list: List['CdmTraitDefinition'], local_extension_trait_def_list: List['CdmTraitDefinition']) -> Optional['CdmDocumentDefinition']: doc = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, '{}.cdm.json'.format(data_obj.name)) # import at least foundations doc.imports.append('cdm:/foundations.cdm.json') entity_dec = await EntityPersistence.from_data(ctx, data_obj, extension_trait_def_list, local_extension_trait_def_list) if not entity_dec: logger.error(ctx, DocumentPersistence.__name__, DocumentPersistence.from_data.__name__, None, CdmLogCode.ERR_PERSIST_MODELJSON_ENTITY_CONVERSION_ERROR, data_obj.name) return None if data_obj.get('imports'): for element in data_obj.imports: if element.corpusPath == 'cdm:/foundations.cdm.json': # don't add foundations twice continue doc.imports.append(CdmImportPersistence.from_data(ctx, element)) doc.definitions.append(entity_dec) return doc
async def from_data(ctx: 'CdmCorpusContext', data_obj: 'LocalEntity', extension_trait_def_list: List['CdmTraitDefinition'], local_extension_trait_def_list: List['CdmTraitDefinition']) -> Optional['CdmDocumentDefinition']: doc = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, '{}.cdm.json'.format(data_obj.name)) # import at least foundations doc.imports.append('cdm:/foundations.cdm.json') entity_dec = await EntityPersistence.from_data(ctx, data_obj, extension_trait_def_list, local_extension_trait_def_list) if not entity_dec: logger.error(DocumentPersistence.__name__, ctx, 'There was an error while trying to convert a model.json entity to the CDM entity.') return None if data_obj.get('imports'): for element in data_obj.imports: if element.corpusPath == 'cdm:/foundations.cdm.json': # don't add foundations twice continue doc.imports.append(CdmImportPersistence.from_data(ctx, element)) doc.definitions.append(entity_dec) return doc
async def to_data_async(instance: CdmLocalEntityDeclarationDefinition, manifest: CdmManifestDefinition, syms_root_path: str, res_opt: 'ResolveOptions', options: 'CopyOptions') -> TableEntity: table_entity = await DocumentPersistence.to_data_async(instance.entity_path, manifest, instance.ctx, res_opt, options) if table_entity is not None: te_properties = table_entity.properties properties = LocalEntityDeclarationPersistence.create_table_propertybags(instance, res_opt, options, te_properties.properties) if instance.data_partitions is not None and len(instance.data_partitions) > 0: paths = [] for element in instance.data_partitions: if element.location is not None: adls_path = instance.ctx.corpus.storage.corpus_path_to_adapter_path(element.location) location = element.location if adls_path == None: logger.error(instance.ctx, _TAG, 'to_data_async', instance.at_corpus_path, CdmLogCode.ERR_PERSIST_SYMS_ADLS_ADAPTER_MISSING, element.location) return None syms_path = utils.adls_adapter_path_to_syms_path(adls_path) if syms_path is not None: location = syms_path else: path_tuple = StorageUtils.split_namespace_path(element.location) location = utils.create_syms_absolute_path(syms_root_path, path_tuple[1]) paths.append(location) te_properties.storage_descriptor = DataPartitionPersistence.to_data(element, te_properties.storage_descriptor, res_opt, options) # Logic to find common root folder. source = DataSource(''.join(c[0] for c in takewhile(lambda x:all(x[0] == y for y in x), zip(*paths)))) te_properties.storage_descriptor.source = source else: # location and format is mandatory for syms. source = DataSource(utils.create_syms_absolute_path(syms_root_path, instance.entity_name)) te_properties.storage_descriptor.source = source te_properties.properties = properties return table_entity
def create_absolute_corpus_path(self, object_path: str, obj: 'CdmObject' = None) -> Optional[str]: """Takes a corpus path (relative or absolute) and creates a valid absolute path with namespace""" if not object_path: logger.error(self._TAG, self._ctx, 'The namespace cannot be null or empty.', StorageManager.create_absolute_corpus_path.__name__) return None if self._contains_unsupported_path_format(object_path): # Already called status_rpt when checking for unsupported path format. return None path_tuple = StorageUtils.split_namespace_path(object_path) if not path_tuple: logger.error(self._TAG, self._ctx, 'The object path cannot be null or empty.', self.create_absolute_corpus_path.__name__) return None final_namespace = '' prefix = None namespace_from_obj = None if obj and hasattr(obj, 'namespace') and hasattr(obj, 'folder_path'): prefix = obj.folder_path namespace_from_obj = obj.namespace elif obj: prefix = obj.in_document.folder_path namespace_from_obj = obj.in_document.namespace if prefix and self._contains_unsupported_path_format(prefix): # Already called status_rpt when checking for unsupported path format. return None if prefix and prefix[-1] != '/': logger.warning( self._TAG, self._ctx, 'Expected path prefix to end in /, but it didn\'t. Appended the /', prefix) prefix += '/' # check if this is a relative path if path_tuple[1][0] != '/': if not obj: # relative path and no other info given, assume default and root prefix = '/' if path_tuple[0] and path_tuple[0] != namespace_from_obj: logger.warning( self._TAG, self._ctx, 'The namespace "{}" found on the path does not match the namespace found on the object' .format(path_tuple[0])) return None path_tuple = (path_tuple[0], prefix + path_tuple[1]) final_namespace = namespace_from_obj or path_tuple[ 0] or self.default_namespace else: final_namespace = path_tuple[ 0] or namespace_from_obj or self.default_namespace return '{}:{}'.format( final_namespace, path_tuple[1]) if final_namespace else path_tuple[1]
async def _save_linked_documents_async(self, options: 'CopyOptions') -> bool: if self.imports: for imp in self.imports: if not await self._save_dirty_link(imp.corpus_path, options): logger.error(self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_IMPORT_SAVING_FAILURE, imp.at_corpus_path) return False # only the local entity declarations please for entity_def in self.entities: if entity_def.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: if not await self._save_dirty_link(entity_def.entity_path, options): logger.error(self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_ENTITY_DOC_SAVING_FAILURE, entity_def.entity_path) return False # also, partitions can have their own schemas if entity_def.data_partitions: for partition in entity_def.data_partitions: if partition.specialized_schema: if not await self._save_dirty_link( entity_def.entity_path, options): logger.error( self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode. ERR_DOC_ENTITY_DOC_SAVING_FAILURE, entity_def.entity_path) return False # so can patterns if entity_def.data_partition_patterns: for pattern in entity_def.data_partition_patterns: if pattern.specialized_schema: if not await self._save_dirty_link( pattern.specialized_schema, options): logger.error( self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode. ERR_DOC_PARTITION_SCHEMA_SAVING_FAILURE, pattern.specialized_schema) return False if self.sub_manifests: for sub in self.sub_manifests: if not await self._save_dirty_link(sub.definition, options): logger.error( self.ctx, self._TAG, self._save_linked_documents_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_SUB_MANIFEST_SAVING_FAILURE, sub.definition) return False return True
async def create_resolved_manifest_async( self, new_manifest_name: str, new_entity_document_name_format: Optional[str], directives: Optional[AttributeResolutionDirectiveSet] = None ) -> Optional['CdmManifestDefinition']: """Creates a resolved copy of the manifest. new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities. The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any sub-folders described by the pattern should exist in the corpus prior to calling this function. """ with logger._enter_scope(self._TAG, self.ctx, self.create_resolved_manifest_async.__name__): if self.entities is None: return None if not self.folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_MANIFEST_FAILED, self.manifest_name) return None if new_entity_document_name_format is None: new_entity_document_name_format = '{f}resolved/{n}.cdm.json' elif new_entity_document_name_format == '': # for back compat new_entity_document_name_format = '{n}.cdm.json' elif '{n}' not in new_entity_document_name_format: # for back compat new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json' source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.at_corpus_path, self) source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.folder.at_corpus_path, self) resolved_manifest_path_split = new_manifest_name.rfind('/') + 1 resolved_manifest_folder = None if resolved_manifest_path_split > 0: resolved_manifest_path = new_manifest_name[ 0:resolved_manifest_path_split] new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( resolved_manifest_path, self) resolved_manifest_folder = await self.ctx.corpus.fetch_object_async( new_folder_path) # type: CdmFolderDefinition if resolved_manifest_folder is None: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND, new_folder_path) return None new_manifest_name = new_manifest_name[ resolved_manifest_path_split:] else: resolved_manifest_folder = self.owner logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'resolving manifest {}'.format(source_manifest_path)) # using the references present in the resolved entities, get an entity # create an imports doc with all the necessary resolved entity references and then resolve it # sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name if new_manifest_name.lower().endswith('.manifest.cdm.json'): new_manifest_name = new_manifest_name[0:( len(new_manifest_name) - len('.manifest.cdm.json'))] resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name) # bring over any imports in this document or other bobbles resolved_manifest.schema = self.schema resolved_manifest.explanation = self.explanation resolved_manifest.document_version = self.document_version for imp in self.imports: resolved_manifest.imports.append(imp.copy()) # add the new document to the folder if resolved_manifest_folder.documents.append( resolved_manifest) is None: # when would this happen? return None for entity in self.entities: ent_def = await self._get_entity_from_reference(entity, self) if not ent_def: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_ENTITY_REF_ERROR) return None if not ent_def.in_document.folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_IS_NOT_FOLDERformat, ent_def.entity_name) return None # get the path from this manifest to the source entity. this will be the {f} replacement value source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.in_document.folder.at_corpus_path, self) f = '' if source_entity_full_path.startswith( source_manifest_folder_path): f = source_entity_full_path[len(source_manifest_folder_path ):] new_document_full_path = new_entity_document_name_format.replace( '{n}', ent_def.entity_name).replace('{f}', f) new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( new_document_full_path, self) new_document_path_split = new_document_full_path.rfind('/') + 1 new_document_path = new_document_full_path[ 0:new_document_path_split] new_document_name = new_document_full_path[ new_document_path_split:] # make sure the new folder exists folder = await self.ctx.corpus.fetch_object_async( new_document_path) # type: CdmFolderDefinition if not folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND, new_document_path) return None # next create the resolved entity. with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives res_opt = ResolveOptions(ent_def.in_document, with_directives.copy()) logger.debug( self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'resolving entity {} to document {}'.format( source_entity_full_path, new_document_full_path)) resolved_entity = await ent_def.create_resolved_entity_async( ent_def.entity_name, res_opt, folder, new_document_name) if not resolved_entity: # fail all resolution, if any one entity resolution fails return None result = entity.copy(res_opt) if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path( resolved_entity.at_corpus_path, resolved_manifest) result.entity_path = relative_entity_path or result.at_corpus_path resolved_manifest.entities.append(result) logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'calculating relationships') # Calculate the entity graph for just this manifest. await self.ctx.corpus.calculate_entity_graph_async( resolved_manifest) # Stick results into the relationships list for the manifest. await resolved_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) # needed until Matt's changes with collections where I can propigate resolved_manifest._is_dirty = True return resolved_manifest