async def _add_elevated_traits_and_relationships(
self, rel: 'CdmE2ERelationship') -> None:
"""
Adds imports for elevated purpose traits for relationships, then adds the relationships to the manifest.
The last import has the highest priority, so we insert the imports for traits to the beginning of the list.
"""
res_opt = ResolveOptions(self)
for trait_ref in rel.exhibits_traits:
trait_def = self.ctx.corpus._resolve_symbol_reference(
res_opt, self, trait_ref.fetch_object_definition_name(),
CdmObjectType.TRAIT_DEF, True)
if trait_def is None:
abs_path = rel._elevated_trait_corpus_path[trait_ref]
relative_path = self.ctx.corpus.storage.create_relative_corpus_path(
abs_path, self)
# Adds the import to this manifest file
self.imports.insert(0, CdmImport(self.ctx, relative_path,
None))
# Fetches the actual file of the import and indexes it
import_document = await self.ctx.corpus.fetch_object_async(
abs_path) # type: 'CdmDocumentDefinition'
if not isinstance(import_document, CdmDocumentDefinition):
logger.error(
self.ctx, self._TAG,
self._add_elevated_traits_and_relationships.__name__,
self.at_corpus_path, CdmLogCode.ERR_INVALID_CAST,
abspath, 'CdmDocumentDefinition')
continue
await import_document._index_if_needed(res_opt)
# Resolves the imports in the manifests
await self.ctx.corpus._resolve_imports_async(
self, set(self.at_corpus_path), res_opt)
# Calls `GetImportPriorities` to prioritize all imports properly after a new import added (which requires `ImportPriorities` set to null)
self._import_priorities = None
self._get_import_priorities()
# As adding a new import above set the manifest needsIndexing to true, we want to avoid over indexing for each import insertion
# so we handle the indexing for the new import above seperately in this case, no indexing needed at this point
self._needs_indexing = False
self.relationships.append(self._localize_rel_to_manifest(rel))
def validate(self) -> bool:
missing_fields = []
if not bool(self.name):
missing_fields.append('name')
if bool(self.cardinality):
if not bool(self.cardinality.minimum):
missing_fields.append('cardinality.minimum')
if not bool(self.cardinality.maximum):
missing_fields.append('cardinality.maximum')
if missing_fields:
logger.error(
self.ctx, self._TAG, 'validate', self.at_corpus_path,
CdmLogCode.ERR_VALDN_INTEGRITY_CHECK_FAILURE,
self.at_corpus_path,
', '.join(map(lambda s: '\'' + s + '\'', missing_fields)))
return False
if bool(self.cardinality):
if not CardinalitySettings._is_minimum_valid(
self.cardinality.minimum):
logger.error(self.ctx, self._TAG, 'validate',
self.at_corpus_path,
CdmLogCode.ERR_VALDN_INVALID_MIN_CARDINALITY,
self.cardinality.minimum)
return False
if not CardinalitySettings._is_maximum_valid(
self.cardinality.maximum):
logger.error(self.ctx, self._TAG, 'validate',
self.at_corpus_path,
CdmLogCode.ERR_VALDN_INVALID_MAX_CARDINALITY,
self.cardinality.maximum)
return False
return True
def corpus_path_to_adapter_path(self, corpus_path: str) -> Optional[str]:
"""Takes a corpus path, figures out the right adapter to use and then return
an adapter domain path"""
with logger._enter_scope(self._TAG, self._ctx,
self.corpus_path_to_adapter_path.__name__):
if not corpus_path:
logger.error(
self._ctx, self._TAG,
StorageManager.corpus_path_to_adapter_path.__name__, None,
CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH)
return None
result = None
# Break the corpus path into namespace and ... path
path_tuple = StorageUtils.split_namespace_path(corpus_path)
if not path_tuple:
logger.error(self._ctx, self._TAG,
self.corpus_path_to_adapter_path.__name__, None,
CdmLogCode.ERR_STORAGE_NULL_CORPUS_PATH)
return None
namespace = path_tuple[0] or self.default_namespace
# Get the adapter registered for this namespace
namespace_adapter = self.fetch_adapter(namespace)
if not namespace_adapter:
logger.error(
self._ctx, self._TAG,
StorageManager.corpus_path_to_adapter_path.__name__, None,
CdmLogCode.ERR_STORAGE_NAMESPACE_NOT_REGISTERED, namespace)
else:
# Ask the storage adapter to 'adapt' this path
result = namespace_adapter.create_adapter_path(path_tuple[1])
return result
def validate(self) -> bool:
missing_fields = []
if not bool(self.name):
missing_fields.append('name')
if bool(self.cardinality):
if not bool(self.cardinality.minimum):
missing_fields.append('cardinality.minimum')
if not bool(self.cardinality.maximum):
missing_fields.append('cardinality.maximum')
if missing_fields:
logger.error(
self._TAG, self.ctx,
Errors.validate_error_string(self.at_corpus_path,
missing_fields))
return False
if bool(self.cardinality):
if not CardinalitySettings._is_minimum_valid(
self.cardinality.minimum):
logger.error(
self._TAG, self.ctx,
'Invalid minimum cardinality {}.'.format(
self.cardinality.minimum))
return False
if not CardinalitySettings._is_maximum_valid(
self.cardinality.maximum):
logger.error(
self._TAG, self.ctx,
'Invalid maximum cardinality {}.'.format(
self.cardinality.maximum))
return False
return True
def corpus_path_to_adapter_path(self, corpus_path: str) -> Optional[str]:
"""Takes a corpus path, figures out the right adapter to use and then return
an adapter domain path"""
if not corpus_path:
logger.error(self._TAG, self._ctx, 'The corpus path is null or empty.', StorageManager.corpus_path_to_adapter_path.__name__)
return None
result = None
# Break the corpus path into namespace and ... path
path_tuple = StorageUtils.split_namespace_path(corpus_path)
if not path_tuple:
logger.error(self._TAG, self._ctx, 'The corpus path cannot be null or empty.', self.corpus_path_to_adapter_path.__name__)
return None
namespace = path_tuple[0] or self.default_namespace
# Get the adapter registered for this namespace
namespace_adapter = self.fetch_adapter(namespace)
if not namespace_adapter:
logger.error(self._TAG, self._ctx, 'The namespace cannot be null or empty.'.format(
namespace), StorageManager.corpus_path_to_adapter_path.__name__)
else:
# Ask the storage adapter to 'adapt' this path
result = namespace_adapter.create_adapter_path(path_tuple[1])
return result
async def _save_dirty_link(self, relative: str,
options: 'CopyOptions') -> None:
"""helper that fixes a path from local to absolute, gets the object from that path
then looks at the document where the object is found.
if dirty, the document is saved with the original name"""
# get the document object from the import
doc_path = self.ctx.corpus.storage.create_absolute_corpus_path(
relative, self)
if doc_path is None:
logger.error(self._TAG, self.ctx,
'Invalid corpus path {}'.format(relative),
self._save_dirty_link.__name__)
return False
obj_at = await self.ctx.corpus.fetch_object_async(doc_path)
if obj_at is None:
logger.error(self._TAG, self.ctx,
'Couldn\'t get object from path {}'.format(doc_path),
self._save_dirty_link.__name__)
return False
doc_imp = cast('CdmDocumentDefinition', obj_at.in_document)
if doc_imp:
if doc_imp._is_dirty:
# save it with the same name
if not await doc_imp.save_as_async(doc_imp.name, True,
options):
logger.error(
self._TAG, self.ctx,
'failed saving document {}'.format(doc_imp.name),
self._save_dirty_link.__name__)
return False
return True
async def _save_dirty_link(self, relative: str,
options: 'CopyOptions') -> bool:
"""Helper that fixes a path from local to absolute, gets the object from that path
then looks at the document where the object is found.
if dirty, the document is saved with the original name"""
# get the document object from the import
doc_path = self.ctx.corpus.storage.create_absolute_corpus_path(
relative, self)
if doc_path is None:
logger.error(self.ctx, self._TAG, self._save_dirty_link.__name__,
self.at_corpus_path,
CdmLogCode.ERR_VALDN_INVALID_CORPUS_PATH, relative)
return False
obj_at = await self.ctx.corpus.fetch_object_async(doc_path)
if obj_at is None:
logger.error(self.ctx, self._TAG, self._save_dirty_link.__name__,
self.at_corpus_path,
CdmLogCode.ERR_PERSIST_OBJECT_NOT_FOUND, doc_path)
return False
doc_imp = cast('CdmDocumentDefinition', obj_at.in_document)
if doc_imp:
if doc_imp._is_dirty:
# save it with the same name
if not await doc_imp.save_as_async(doc_imp.name, True,
options):
logger.error(self.ctx, self._TAG,
self._save_dirty_link.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_ENTITY_DOC_SAVING_FAILURE,
doc_imp.name)
return False
return True
def unmount(self, namespace: str) -> None:
"""unregisters a storage adapter and its root folder"""
if not namespace:
logger.error(self._TAG, self._ctx,
'The namespace cannot be null or empty.',
StorageManager.unmount.__name__)
return None
if namespace in self.namespace_adapters:
self.namespace_adapters.pop(namespace, None)
self._namespace_folders.pop(namespace, None)
if namespace in self._system_defined_namespaces:
self._system_defined_namespaces.remove(namespace)
# The special case, use Resource adapter.
if (namespace == 'cdm'):
self.mount(namespace, ResourceAdapter())
else:
logger.warning(
self._TAG, self._ctx,
'Cannot remove the adapter from non-existing namespace.',
StorageManager.mount.__name__)
def adapter_path_to_corpus_path(self, adapter_path: str) -> Optional[str]:
"""Takes a storage adapter domain path, figures out the right adapter to use
and then return a corpus path"""
result = None
# Keep trying adapters until one of them likes what it sees
if self.namespace_adapters:
for key, value in self.namespace_adapters.items():
result = value.create_corpus_path(adapter_path)
if result:
# Got one, add the prefix
result = '{}:{}'.format(key, result)
break
if not result:
logger.error(
self._TAG, self._ctx,
'No registered storage adapter understood the path "{}"'.
format(adapter_path),
StorageManager.adapter_path_to_corpus_path.__name__)
return result
def _check_and_add_item_modifications(self, document: 'CdmDocumentDefinition') -> bool:
if self.item(document.name) is not None:
logger.error(self.ctx, _TAG, '_check_and_add_item_modifications', document.at_corpus_path,
CdmLogCode.ERR_DOC_ALREADY_EXIST, document.name,
lambda x:self.owner.at_corpus_path if self.owner.at_corpus_path is not None else self.owner.name)
return False;
if document.owner and document.owner is not self.owner:
# this is fun! the document is moving from one folder to another
# it must be removed from the old folder for sure, but also now
# there will be a problem with any corpus paths that are relative to that old folder location.
# so, whip through the document and change any corpus paths to be relative to this folder
document._localize_corpus_paths(self.owner) # returns false if it fails, but ... who cares? we tried
document.owner.documents.remove(document.name)
document._folder_path = self.owner._folder_path
document.owner = self.owner
document._namespace = self.owner._namespace
super()._make_document_dirty() # set the document to dirty so it will get saved in the new folder location if saved
self.owner._corpus._add_document_objects(self.owner, document)
return True
def adapter_path_to_corpus_path(self, adapter_path: str) -> Optional[str]:
"""Takes a storage adapter domain path, figures out the right adapter to use
and then return a corpus path"""
with logger._enter_scope(self._TAG, self._ctx,
self.adapter_path_to_corpus_path.__name__):
result = None
# Keep trying adapters until one of them likes what it sees
if self.namespace_adapters:
for key, value in self.namespace_adapters.items():
result = value.create_corpus_path(adapter_path)
if result:
# Got one, add the prefix
result = '{}:{}'.format(key, result)
break
if not result:
logger.error(
self._ctx, self._TAG,
StorageManager.adapter_path_to_corpus_path.__name__, None,
CdmLogCode.ERR_STORAGE_INVALID_ADAPTER_PATH, adapter_path)
return result
async def _save_odi_documents(self, doc: Any, adapter: 'StorageAdapter',
new_name: str) -> None:
if doc is None:
raise Exception('Failed to persist document because doc is null.')
# ask the adapter to make it happen.
try:
old_document_path = doc.documentPath
new_document_path = old_document_path[
0:len(old_document_path) - len(self.ODI_EXTENSION)] + new_name
content = doc.encode()
await adapter.write_async(new_document_path, content)
except Exception as e:
logger.error(
self._TAG, self._ctx,
'Failed to write to the file \'{}\' for reason {}.'.format(
doc.documentPath, e), self._save_odi_documents.__name__)
# Save linked documents.
if doc.get('linkedDocuments') is not None:
for linked_doc in doc.linkedDocuments:
await self._save_odi_documents(linked_doc, adapter, new_name)
def unmount(self, namespace: str) -> None:
"""unregisters a storage adapter and its root folder"""
with logger._enter_scope(self._TAG, self._ctx, self.unmount.__name__):
if not namespace:
logger.error(self._ctx, self._TAG,
StorageManager.unmount.__name__, None,
CdmLogCode.ERR_STORAGE_NULL_NAMESPACE)
return None
if namespace in self.namespace_adapters:
self.namespace_adapters.pop(namespace, None)
self._namespace_folders.pop(namespace, None)
if namespace in self._system_defined_namespaces:
self._system_defined_namespaces.remove(namespace)
# The special case, use Resource adapter.
if (namespace == 'cdm'):
self.mount(namespace, ResourceAdapter())
else:
logger.warning(self._ctx, self._TAG,
StorageManager.unmount.__name__, None,
CdmLogCode.WARN_STORAGE_REMOVE_ADAPTER_FAILED)
def mount(self, namespace: str, adapter: 'StorageAdapterBase') -> None:
"""registers a namespace and assigns creates a storage adapter for that namespace"""
if not namespace:
logger.error(self._TAG, self._ctx,
'The namespace cannot be null or empty.',
StorageManager.mount.__name__)
return None
from cdm.objectmodel import CdmFolderDefinition
if adapter:
self.namespace_adapters[namespace] = adapter
fd = CdmFolderDefinition(self._ctx, '')
fd._corpus = self._corpus
fd.namespace = namespace
fd.folder_path = '/'
self._namespace_folders[namespace] = fd
if namespace in self._system_defined_namespaces:
self._system_defined_namespaces.remove(namespace)
else:
logger.error(self._TAG, self._ctx, 'The adapter cannot be null.',
StorageManager.mount.__name__)
def from_data(ctx: 'CdmCorpusContext', obj: StorageDescriptor,
syms_root_path: str,
format_type: str) -> CdmDataPartitionDefinition:
new_partition = ctx.corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF
) # type: CdmDataPartitionDefinition
syms_path = utils.create_syms_absolute_path(syms_root_path,
obj.source.location)
new_partition.location = utils.syms_path_to_corpus_path(
syms_path, ctx.corpus.storage)
trait = utils.create_partition_trait(obj.format.properties, ctx,
format_type)
if trait is not None:
new_partition.exhibits_traits.append(trait)
else:
logger.error(ctx, _TAG,
DataPartitionPersistence.from_data.__name__, None,
CdmLogCode.ERR_PERSIST_SYMS_UNSUPPORTED_TABLE_FORMAT)
return None
properties = obj.properties
if properties is not None:
if 'cdm:name' in properties:
new_partition.name = properties['cdm:name']
if 'cdm:lastFileStatusCheckTime' in properties:
new_partition.last_file_status_check_time = dateutil.parser.parse(
properties['cdm:lastFileStatusCheckTime'])
if 'cdm:lastFileModifiedTime' in properties:
new_partition.last_file_modified_time = dateutil.parser.parse(
properties['cdm:lastFileModifiedTime'])
if 'cdm:traits' in properties:
utils.add_list_to_cdm_collection(
new_partition.exhibits_traits,
utils.create_trait_reference_array(
ctx, properties['cdm:traits']))
return new_partition
async def to_data(instance: 'CdmReferencedEntityDeclarationDefinition',
res_opt: 'ResolveOptions',
options: 'CopyOptions') -> Optional['ReferenceEntity']:
source_index = instance.entity_path.rfind('/')
if source_index == -1:
logger.error(
instance.ctx, _TAG, 'to_data', instance.at_corpus_path,
CdmLogCode.
ERR_PERSIST_MODELJSON_ENTITY_PARTITION_CONVERSION_ERROR,
instance.at_corpus_path)
return None
reference_entity = ReferenceEntity()
t2pm = TraitToPropertyMap(instance)
reference_entity.type = 'ReferenceEntity'
reference_entity.name = instance.entity_name
reference_entity.source = instance.entity_path[source_index + 1:]
reference_entity.description = instance.explanation
reference_entity.lastFileStatusCheckTime = utils.get_formatted_date_string(
instance.last_file_status_check_time)
reference_entity.lastFileModifiedTime = utils.get_formatted_date_string(
instance.last_file_modified_time)
reference_entity.isHidden = bool(
t2pm._fetch_trait_reference('is.hidden')) or None
properties_trait = t2pm._fetch_trait_reference(
'is.propertyContent.multiTrait')
if properties_trait:
reference_entity.modelId = properties_trait.arguments[0].value
utils.process_traits_and_annotations_to_data(instance.ctx,
reference_entity,
instance.exhibits_traits)
return reference_entity
async def to_data(instance: 'CdmEntityDefinition', res_opt: 'ResolveOptions', options: 'CopyOptions', ctx: 'CdmCorpusContext') -> Optional['LocalEntity']:
data = LocalEntity()
data.type = 'LocalEntity'
data.name = instance.entity_name
data.description = instance._get_property("description")
utils.process_traits_and_annotations_to_data(instance.ctx, data, instance.exhibits_traits)
if instance.attributes:
data.attributes = []
for element in instance.attributes:
if element.object_type != CdmObjectType.TYPE_ATTRIBUTE_DEF:
logger.error(ctx, EntityPersistence.__name__, EntityPersistence.to_data.__name__, element.at_corpus_path, CdmLogCode.ERR_PERSIST_MANIFEST_SAVING_FAILURE)
return None
attribute = await TypeAttributePersistence.to_data(element, res_opt, options)
if attribute:
data.attributes.append(attribute)
else:
logger.error(ctx, EntityPersistence.__name__, EntityPersistence.to_data.__name__, CdmLogCode.ERR_PERSIST_MODELJSON_ATTR_CONVERSION_FAILURE)
return None
return data
async def _save_odi_documents(self, doc: Any, adapter: 'StorageAdapter', new_name: str) -> None:
if doc is None:
raise Exception('Failed to persist document because doc is null.')
# ask the adapter to make it happen.
try:
old_document_path = doc.documentPath
new_document_path = old_document_path[0: len(old_document_path) - len(self.ODI_EXTENSION)] + new_name
# Remove namespace from path
path_tuple = StorageUtils.split_namespace_path(new_document_path)
if not path_tuple:
logger.error(self._TAG, self._ctx, 'The object path cannot be null or empty.', self._save_odi_documents.__name__)
return
content = doc.encode()
await adapter.write_async(path_tuple[1], content)
except Exception as e:
logger.error(self._TAG, self._ctx, 'Failed to write to the file \'{}\' for reason {}.'.format(
doc.documentPath, e), self._save_odi_documents.__name__)
# Save linked documents.
if doc.get('linkedDocuments') is not None:
for linked_doc in doc.linkedDocuments:
await self._save_odi_documents(linked_doc, adapter, new_name)
async def _index_if_needed(self, res_opt: 'ResolveOptions', load_imports: bool = False) -> bool:
if not self._needs_indexing or self._currently_indexing:
return True
if not self.folder:
logger.error(self.ctx, self._TAG, self._index_if_needed.__name__, self.at_corpus_path, CdmLogCode.ERR_VALDN_MISSING_DOC, self.name)
return False
corpus = self.folder._corpus
# if the imports load strategy is "LAZY_LOAD", loadImports value will be the one sent by the called function.
if res_opt.imports_load_strategy == ImportsLoadStrategy.DO_NOT_LOAD:
load_imports = False
elif res_opt.imports_load_strategy == ImportsLoadStrategy.LOAD:
load_imports = True
if load_imports:
await corpus._resolve_imports_async(self, res_opt)
# make the corpus internal machinery pay attention to this document for this call
corpus._document_library._mark_document_for_indexing(self)
return corpus._index_documents(res_opt, load_imports)
def from_data(
ctx: CdmCorpusContext, prefix_path: str,
data: ReferencedEntityDeclaration
) -> CdmReferencedEntityDeclarationDefinition:
referenced_entity = ctx.corpus.make_object(
CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF, data.entityName)
entity_path = data.get('entityPath') or data.get('entityDeclaration')
if not entity_path:
logger.error(
ctx, _TAG,
ReferencedEntityDeclarationPersistence.from_data.__name__,
None, CdmLogCode.ERR_PERSIST_ENTITY_PATH_NOT_FOUND)
# The entity path has to be absolute.
# If the namespace is not present then add the "prefixPath" which has the absolute folder path.
if entity_path and entity_path.find(':/') == -1:
entity_path = '{}{}'.format(prefix_path, entity_path)
referenced_entity.entity_path = entity_path
referenced_entity.explanation = data.get('explanation')
if data.get('lastFileStatusCheckTime'):
referenced_entity.last_file_status_check_time = dateutil.parser.parse(
data.lastFileStatusCheckTime)
if data.get('lastFileModifiedTime'):
referenced_entity.last_file_modified_time = dateutil.parser.parse(
data.lastFileModifiedTime)
if data.get('exhibitsTraits'):
exhibits_traits = utils.create_trait_reference_array(
ctx, data.exhibitsTraits)
referenced_entity.exhibits_traits.extend(exhibits_traits)
return referenced_entity
def _create_new_projection_attribute_state_set(
proj_ctx: 'ProjectionContext',
proj_output_set: 'ProjectionAttributeStateSet',
new_res_attr_FK: 'ResolvedAttribute',
ref_attr_name: str) -> 'ProjectionAttributeStateSet':
pas_list = ProjectionResolutionCommonUtil._get_leaf_list(
proj_ctx, ref_attr_name)
if pas_list is not None:
# update the new foreign key resolved attribute with trait param with reference details
reqd_trait = new_res_attr_FK.resolved_traits.find(
proj_ctx._projection_directive._res_opt,
'is.linkedEntity.identifier')
if reqd_trait:
trait_param_ent_ref = ProjectionResolutionCommonUtil._create_foreign_key_linked_entity_identifier_trait_parameter(
proj_ctx._projection_directive,
proj_output_set._ctx.corpus, pas_list)
reqd_trait.parameter_values.update_parameter_value(
proj_ctx._projection_directive._res_opt,
'entityReferences', trait_param_ent_ref)
# Create new output projection attribute state set for FK and add prevPas as previous state set
new_proj_attr_state_FK = ProjectionAttributeState(
proj_output_set._ctx)
new_proj_attr_state_FK._current_resolved_attribute = new_res_attr_FK
new_proj_attr_state_FK._previous_state_list = pas_list
proj_output_set._add(new_proj_attr_state_FK)
else:
# Log error & return proj_output_set without any change
logger.error(
CdmOperationReplaceAsForeignKey.__name__, proj_output_set._ctx,
'Unable to locate state for reference attribute \"{}\".'.
format(ref_attr_name), CdmOperationReplaceAsForeignKey.
_create_new_projection_attribute_state_set.__name__)
return proj_output_set
def mount_from_config(self, adapter_config: str, does_return_error_list: bool = False) -> List['StorageAdapter']:
if not adapter_config:
logger.error(self._TAG, self._ctx, 'Adapter config cannot be null or empty.', StorageManager.mount_from_config.__name__)
return None
adapter_config_json = json.loads(adapter_config)
adapers_module = importlib.import_module('cdm.storage')
if adapter_config_json.get('appId'):
self._corpus.app_id = adapter_config_json['appId']
if adapter_config_json.get('defaultNamespace'):
self.default_namespace = adapter_config_json['defaultNamespace']
unrecognized_adapters = []
for item in adapter_config_json['adapters']:
namespace = None
# Check whether the namespace exists.
if item.get('namespace'):
namespace = item['namespace']
else:
logger.error(self._TAG, self._ctx, 'The namespace is missing for one of the adapters in the JSON config.')
continue
configs = None
# Check whether the config exists.
if item.get('config'):
configs = item['config']
else:
logger.error(self._TAG, self._ctx, 'Missing JSON config for the namespace {}.'.format(namespace))
continue
if not item.get('type'):
logger.error(self._TAG, self._ctx, 'Missing type in the JSON config for the namespace {}.'.format(namespace))
continue
adapter_type = self._registered_adapter_types.get(item['type'], None)
if adapter_type is None:
unrecognized_adapters.append(json.dumps(item))
else:
adapter = getattr(adapers_module, adapter_type)()
adapter.update_config(json.dumps(configs))
self.mount(namespace, adapter)
return unrecognized_adapters if does_return_error_list else None
def to_data(instance: CdmTypeAttributeDefinition, ctx: CdmCorpusContext,
res_opt: ResolveOptions,
options: CopyOptions) -> TypeAttribute:
properties = TypeAttributePersistence.create_properties(
instance, res_opt, options)
origin_data_type_name = TypeInfo(
type_name='',
properties=properties,
is_complex_type=False,
is_nullable=instance._get_property('isNullable'),
type_family='cdm')
t2pm = TraitToPropertyMap(instance)
numeric_traits = t2pm._fetch_trait_reference(
'is.data_format.numeric.shaped')
if numeric_traits is not None:
for numeric_traits_arg in numeric_traits.argument:
if numeric_traits_arg.name == 'precision':
origin_data_type_name.precision = numeric_traits_arg.value
if numeric_traits_arg.Name == 'scale':
origin_data_type_name.scale = numeric_traits_arg.value
data_format = instance._get_property('dataFormat')
origin_data_type_name = utils.cdm_data_format_to_syms_data_type(
data_format, origin_data_type_name)
if origin_data_type_name == None:
return None
if origin_data_type_name.type_name == None:
logger.error(ctx, _TAG, 'toData', instance.at_corpus_path,
CdmLogCode.ERR_PERSIST_SYMS_UNKNOWN_DATA_FORMAT,
instance.display_name)
return None
data_col = DataColumn(origin_data_type_name=origin_data_type_name,
name=instance.name)
return data_col
async def to_data_async(document_object_or_path,
manifest: CdmManifestDefinition,
ctx: CdmCorpusContext, res_opt: ResolveOptions,
options: CopyOptions) -> TableEntity:
if isinstance(document_object_or_path, str):
obje = await ctx.corpus.fetch_object_async(document_object_or_path,
manifest)
if isinstance(obje, CdmEntityDefinition):
cdm_entity = obje
table_entity = EntityPersistence.to_data(
cdm_entity, ctx, res_opt, options)
te_properties = table_entity.properties
te_properties.namespace = TableNamespace(
manifest.manifest_name)
if cdm_entity.owner is not None and isinstance(
cdm_entity.owner, CdmDocumentDefinition):
document = cdm_entity.owner
if len(document.imports) > 0:
imports = copy_data_utils._array_copy_data(
res_opt, document.imports, options)
te_properties.properties["cdm:imports"] = imports
else:
logger.warning(ctx, _TAG, 'to_data_async',
manifest.at_corpus_path,
CdmLogCode.WARN_PERSIST_SYMS_ENTITY_MISIING,
cdm_entity.name)
return table_entity
else:
logger.error(ctx, _TAG, 'to_data_async',
manifest.at_corpus_path,
CdmLogCode.ERR_PERSIST_SYMS_ENTITY_FETCH_ERROR,
document_object_or_path)
return None
return None
async def from_data(ctx: 'CdmCorpusContext', data_obj: 'LocalEntity', extension_trait_def_list: List['CdmTraitDefinition'],
local_extension_trait_def_list: List['CdmTraitDefinition']) -> Optional['CdmDocumentDefinition']:
doc = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, '{}.cdm.json'.format(data_obj.name))
# import at least foundations
doc.imports.append('cdm:/foundations.cdm.json')
entity_dec = await EntityPersistence.from_data(ctx, data_obj, extension_trait_def_list, local_extension_trait_def_list)
if not entity_dec:
logger.error(ctx, DocumentPersistence.__name__, DocumentPersistence.from_data.__name__, None, CdmLogCode.ERR_PERSIST_MODELJSON_ENTITY_CONVERSION_ERROR, data_obj.name)
return None
if data_obj.get('imports'):
for element in data_obj.imports:
if element.corpusPath == 'cdm:/foundations.cdm.json':
# don't add foundations twice
continue
doc.imports.append(CdmImportPersistence.from_data(ctx, element))
doc.definitions.append(entity_dec)
return doc
async def from_data(ctx: 'CdmCorpusContext', data_obj: 'LocalEntity', extension_trait_def_list: List['CdmTraitDefinition'],
local_extension_trait_def_list: List['CdmTraitDefinition']) -> Optional['CdmDocumentDefinition']:
doc = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, '{}.cdm.json'.format(data_obj.name))
# import at least foundations
doc.imports.append('cdm:/foundations.cdm.json')
entity_dec = await EntityPersistence.from_data(ctx, data_obj, extension_trait_def_list, local_extension_trait_def_list)
if not entity_dec:
logger.error(DocumentPersistence.__name__, ctx, 'There was an error while trying to convert a model.json entity to the CDM entity.')
return None
if data_obj.get('imports'):
for element in data_obj.imports:
if element.corpusPath == 'cdm:/foundations.cdm.json':
# don't add foundations twice
continue
doc.imports.append(CdmImportPersistence.from_data(ctx, element))
doc.definitions.append(entity_dec)
return doc
async def to_data_async(instance: CdmLocalEntityDeclarationDefinition, manifest: CdmManifestDefinition, syms_root_path: str, res_opt: 'ResolveOptions', options: 'CopyOptions') -> TableEntity:
table_entity = await DocumentPersistence.to_data_async(instance.entity_path, manifest, instance.ctx, res_opt,
options)
if table_entity is not None:
te_properties = table_entity.properties
properties = LocalEntityDeclarationPersistence.create_table_propertybags(instance, res_opt, options, te_properties.properties)
if instance.data_partitions is not None and len(instance.data_partitions) > 0:
paths = []
for element in instance.data_partitions:
if element.location is not None:
adls_path = instance.ctx.corpus.storage.corpus_path_to_adapter_path(element.location)
location = element.location
if adls_path == None:
logger.error(instance.ctx, _TAG, 'to_data_async', instance.at_corpus_path, CdmLogCode.ERR_PERSIST_SYMS_ADLS_ADAPTER_MISSING, element.location)
return None
syms_path = utils.adls_adapter_path_to_syms_path(adls_path)
if syms_path is not None:
location = syms_path
else:
path_tuple = StorageUtils.split_namespace_path(element.location)
location = utils.create_syms_absolute_path(syms_root_path, path_tuple[1])
paths.append(location)
te_properties.storage_descriptor = DataPartitionPersistence.to_data(element, te_properties.storage_descriptor,
res_opt, options)
# Logic to find common root folder.
source = DataSource(''.join(c[0] for c in takewhile(lambda x:all(x[0] == y for y in x), zip(*paths))))
te_properties.storage_descriptor.source = source
else:
# location and format is mandatory for syms.
source = DataSource(utils.create_syms_absolute_path(syms_root_path, instance.entity_name))
te_properties.storage_descriptor.source = source
te_properties.properties = properties
return table_entity
def create_absolute_corpus_path(self,
object_path: str,
obj: 'CdmObject' = None) -> Optional[str]:
"""Takes a corpus path (relative or absolute) and creates a valid absolute
path with namespace"""
if not object_path:
logger.error(self._TAG, self._ctx,
'The namespace cannot be null or empty.',
StorageManager.create_absolute_corpus_path.__name__)
return None
if self._contains_unsupported_path_format(object_path):
# Already called status_rpt when checking for unsupported path format.
return None
path_tuple = StorageUtils.split_namespace_path(object_path)
if not path_tuple:
logger.error(self._TAG, self._ctx,
'The object path cannot be null or empty.',
self.create_absolute_corpus_path.__name__)
return None
final_namespace = ''
prefix = None
namespace_from_obj = None
if obj and hasattr(obj, 'namespace') and hasattr(obj, 'folder_path'):
prefix = obj.folder_path
namespace_from_obj = obj.namespace
elif obj:
prefix = obj.in_document.folder_path
namespace_from_obj = obj.in_document.namespace
if prefix and self._contains_unsupported_path_format(prefix):
# Already called status_rpt when checking for unsupported path format.
return None
if prefix and prefix[-1] != '/':
logger.warning(
self._TAG, self._ctx,
'Expected path prefix to end in /, but it didn\'t. Appended the /',
prefix)
prefix += '/'
# check if this is a relative path
if path_tuple[1][0] != '/':
if not obj:
# relative path and no other info given, assume default and root
prefix = '/'
if path_tuple[0] and path_tuple[0] != namespace_from_obj:
logger.warning(
self._TAG, self._ctx,
'The namespace "{}" found on the path does not match the namespace found on the object'
.format(path_tuple[0]))
return None
path_tuple = (path_tuple[0], prefix + path_tuple[1])
final_namespace = namespace_from_obj or path_tuple[
0] or self.default_namespace
else:
final_namespace = path_tuple[
0] or namespace_from_obj or self.default_namespace
return '{}:{}'.format(
final_namespace,
path_tuple[1]) if final_namespace else path_tuple[1]
async def _save_linked_documents_async(self,
options: 'CopyOptions') -> bool:
if self.imports:
for imp in self.imports:
if not await self._save_dirty_link(imp.corpus_path, options):
logger.error(self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_IMPORT_SAVING_FAILURE,
imp.at_corpus_path)
return False
# only the local entity declarations please
for entity_def in self.entities:
if entity_def.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
if not await self._save_dirty_link(entity_def.entity_path,
options):
logger.error(self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_ENTITY_DOC_SAVING_FAILURE,
entity_def.entity_path)
return False
# also, partitions can have their own schemas
if entity_def.data_partitions:
for partition in entity_def.data_partitions:
if partition.specialized_schema:
if not await self._save_dirty_link(
entity_def.entity_path, options):
logger.error(
self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path, CdmLogCode.
ERR_DOC_ENTITY_DOC_SAVING_FAILURE,
entity_def.entity_path)
return False
# so can patterns
if entity_def.data_partition_patterns:
for pattern in entity_def.data_partition_patterns:
if pattern.specialized_schema:
if not await self._save_dirty_link(
pattern.specialized_schema, options):
logger.error(
self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path, CdmLogCode.
ERR_DOC_PARTITION_SCHEMA_SAVING_FAILURE,
pattern.specialized_schema)
return False
if self.sub_manifests:
for sub in self.sub_manifests:
if not await self._save_dirty_link(sub.definition, options):
logger.error(
self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_SUB_MANIFEST_SAVING_FAILURE,
sub.definition)
return False
return True
async def create_resolved_manifest_async(
self,
new_manifest_name: str,
new_entity_document_name_format: Optional[str],
directives: Optional[AttributeResolutionDirectiveSet] = None
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
with logger._enter_scope(self._TAG, self.ctx,
self.create_resolved_manifest_async.__name__):
if self.entities is None:
return None
if not self.folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_MANIFEST_FAILED,
self.manifest_name)
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND,
new_folder_path)
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
logger.debug(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
'resolving manifest {}'.format(source_manifest_path))
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
# sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name
if new_manifest_name.lower().endswith('.manifest.cdm.json'):
new_manifest_name = new_manifest_name[0:(
len(new_manifest_name) - len('.manifest.cdm.json'))]
resolved_manifest = CdmManifestDefinition(self.ctx,
new_manifest_name)
# bring over any imports in this document or other bobbles
resolved_manifest.schema = self.schema
resolved_manifest.explanation = self.explanation
resolved_manifest.document_version = self.document_version
for imp in self.imports:
resolved_manifest.imports.append(imp.copy())
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_ENTITY_REF_ERROR)
return None
if not ent_def.in_document.folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_IS_NOT_FOLDERformat,
ent_def.entity_name)
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(
source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path
):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND,
new_document_path)
return None
# next create the resolved entity.
with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives
res_opt = ResolveOptions(ent_def.in_document,
with_directives.copy())
logger.debug(
self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
'resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
logger.debug(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path, 'calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus.calculate_entity_graph_async(
resolved_manifest)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
