def to_data(instance: CdmImport, res_opt: ResolveOptions,
options: CopyOptions) -> Import:
result = Import()
if not StringUtils.is_blank_by_cdm_standard(instance.corpus_path):
result.corpusPath = instance.corpus_path
if not StringUtils.is_blank_by_cdm_standard(instance.moniker):
result.moniker = instance.moniker
return result
async def from_data(
ctx: 'CdmCorpusContext', data: 'LocalEntity',
extension_trait_def_list: List['CdmTraitDefinition'],
local_extension_trait_def_list: List['CdmTraitDefinition']
) -> Optional['CdmEntityDefinition']:
entity = ctx.corpus.make_object(CdmObjectType.ENTITY_DEF, data.name)
if not StringUtils.is_blank_by_cdm_standard(data.get('description')):
entity.description = data.description
await utils.process_annotations_from_data(ctx, data,
entity.exhibits_traits)
for element in (data.get('attributes') or []):
type_attribute = await TypeAttributePersistence.from_data(
ctx, element, extension_trait_def_list,
local_extension_trait_def_list)
if type_attribute is not None:
entity.attributes.append(type_attribute)
else:
logger.error(
ctx, EntityPersistence.__name__,
EntityPersistence.from_data.__name__,
element.at_corpus_path, CdmLogCode.
ERR_PERSIST_MODELJSON_TO_ATTR_CONVERSION_FAILURE)
return
extension_helper.process_extension_from_json(
ctx, data, entity.exhibits_traits, extension_trait_def_list,
local_extension_trait_def_list)
return entity
async def to_data(document_object_or_path: Union[CdmDocumentDefinition,
str],
manifest: 'CdmManifestDefinition',
res_opt: 'ResolveOptions', options: 'CopyOptions',
ctx: 'CdmCorpusContext') -> Optional['LocalEntity']:
if isinstance(document_object_or_path, str):
# Fetch the document from entity schema.
cdm_entity = await ctx.corpus.fetch_object_async(
document_object_or_path, manifest)
if not isinstance(cdm_entity, CdmEntityDefinition):
logger.error(ctx, DocumentPersistence.__name__,
DocumentPersistence.to_data.__name__,
manifest.at_corpus_path,
CdmLogCode.ERR_INVALID_CAST,
document_object_or_path, 'CdmEntityDefinition')
return None
if not cdm_entity:
logger.error(ctx, DocumentPersistence.__name__,
DocumentPersistence.to_data.__name__,
manifest.at_corpus_path,
CdmLogCode.ERR_PERSIST_CDM_ENTITY_FETCH_ERROR)
return None
entity = await EntityPersistence.to_data(cdm_entity, res_opt,
options, ctx)
if cdm_entity.owner and cdm_entity.owner.object_type == CdmObjectType.DOCUMENT_DEF:
document = cdm_entity.owner # type: CdmDocumentDefinition
entity.imports = []
for element in document.imports:
imp = CdmImportPersistence.to_data(element, res_opt,
options)
# the corpus path in the imports are relative to the document where it was defined.
# when saving in model.json the documents are flattened to the manifest level
# so it is necessary to recalculate the path to be relative to the manifest.
absolute_path = ctx.corpus.storage.create_absolute_corpus_path(
imp.corpusPath, document)
if not StringUtils.is_blank_by_cdm_standard(
document._namespace) and absolute_path.startswith(
document._namespace + ':'):
absolute_path = absolute_path[len(document._namespace
) + 1:]
imp.corpusPath = ctx.corpus.storage.create_relative_corpus_path(
absolute_path, manifest)
entity.imports.append(imp)
else:
logger.warning(ctx, _TAG, DocumentPersistence.to_data.__name__,
manifest.at_corpus_path,
CdmLogCode.WARN_PERSIST_ENTITY_MISSING,
cdm_entity.get_name())
return entity
else:
# TODO: Do something else when document_object_or_path is an object.
pass
def from_data(ctx: CdmCorpusContext, obj: Import) -> CdmImport:
imp = ctx.corpus.make_object(CdmObjectType.IMPORT)
corpus_path = obj.get('corpusPath')
if StringUtils.is_blank_by_cdm_standard(corpus_path):
corpus_path = obj.uri
imp.corpus_path = corpus_path
imp.moniker = obj.get('moniker')
return imp
async def create_or_update_syms_entities(syms_manifest_content: 'SymsManifestContent', adapter: 'StorageAdapter'):
failed_updated_tables = {}
failed_updated_relationships = {}
failed_removed_tables = {}
failed_removed_relationships = {}
error_mesg = ''
if syms_manifest_content.intial_sync:
await create_or_update_database(syms_manifest_content.database, adapter)
if syms_manifest_content.removed_entities is not None:
for remove_table in syms_manifest_content.removed_entities:
try:
await remove_table_entity(remove_table, syms_manifest_content.database.name, adapter)
except Exception as e:
failed_removed_tables[remove_table] = str(e)
if len(failed_removed_tables) > 0:
error_mesg += 'Failed removed tables : ' + str(failed_removed_tables)
if syms_manifest_content.removed_relationships is not None:
for remove_relationship in syms_manifest_content.removed_relationships:
try:
await remove_relationship_entity(remove_relationship, syms_manifest_content.database.name, adapter)
except Exception as e:
failed_removed_relationships[remove_relationship] = str(e)
if len(failed_removed_relationships) > 0:
error_mesg += 'Failed removed relationships :' + str(failed_removed_relationships)
if syms_manifest_content.entities is not None:
for table in syms_manifest_content.entities:
try:
await create_or_update_table_entity(table, adapter)
except Exception as e:
failed_updated_tables[table.name] = str(e)
if len(failed_updated_tables) > 0:
error_mesg += 'Failed updated tables : ' + str(failed_updated_tables)
if syms_manifest_content.relationships is not None:
for relationship in syms_manifest_content.relationships:
try:
await create_or_update_relationship_entity(relationship, adapter)
except Exception as e:
failed_updated_relationships[relationship.name] = str(e)
if len(failed_updated_relationships) > 0:
error_mesg += 'Failed updated relationships : ' + str(failed_updated_relationships)
if not StringUtils.is_blank_by_cdm_standard(error_mesg):
raise Exception (error_mesg)
def to_data(instance: 'CdmE2ERelationship',
dbname: str,
res_opt: ResolveOptions,
options: CopyOptions,
relationship=None) -> RelationshipEntity:
properties = {}
if instance.exhibits_traits is not None and len(
instance.exhibits_traits) > 0:
properties[
"cdm:exhibitsTraits"] = copy_data_utils._array_copy_data(
res_opt, instance.exhibits_traits, options)
ns = Namespace(dbname)
column_relationship_informations = []
col_info = ColumnRelationshipInformation(
from_column_name=instance.from_entity_attribute,
to_column_name=instance.to_entity_attribute)
column_relationship_informations.append(col_info)
relationship_properties = RelationshipProperties(
namespace=ns,
from_table_name=utils.extract_table_name_from_entity_path(
instance.from_entity),
to_table_name=utils.extract_table_name_from_entity_path(
instance.to_entity),
properties=properties,
publish_status=PublishStatus.published,
relationship_type=RelationshipType.manytoone,
column_relationship_informations=column_relationship_informations)
relationship_name = instance.name
if StringUtils.is_blank_by_cdm_standard(instance.name):
relationship_name = "{}_{}_relationship".format(
relationship_properties.from_table_name,
relationship_properties.to_table_name)
rel = RelationshipEntity(name=relationship_name,
properties=relationship_properties,
type=SASEntityType.relationship)
return rel
def to_data(instance: 'CdmArgumentDefinition', res_opt: 'ResolveOptions',
options: 'CopyOptions') -> CdmJsonType:
from cdm.objectmodel import CdmObject
value = None
if instance.value is not None:
if isinstance(instance.value, CdmObject):
value = PersistenceLayer.to_data(instance.value, res_opt,
options,
PersistenceLayer.CDM_FOLDER)
else:
value = instance.value
# Skip the argument if just a value
if StringUtils.is_blank_by_cdm_standard(instance.name):
return value
arg = Argument()
arg.explanation = instance.explanation
arg.name = instance.name
arg.value = value
return arg
def to_data(instance: CdmObjectReference, res_opt: ResolveOptions,
options: CopyOptions) -> CdmJsonType:
# We don't know what object we are creating to initialize to any
copy = None
replace = None
if not StringUtils.is_blank_by_cdm_standard(instance.named_reference):
identifier = utils.copy_identifier_ref(instance, res_opt, options)
if instance.simple_named_reference:
return identifier
replace = CdmObjectRefPersistence._copy_ref_data(
instance, res_opt, copy, identifier, options)
if replace:
copy = replace
elif instance.explicit_reference:
er_copy = PersistenceLayer.to_data(instance.explicit_reference,
res_opt, options,
PersistenceLayer.CDM_FOLDER)
replace = CdmObjectRefPersistence._copy_ref_data(
instance, res_opt, copy, er_copy, options)
if replace:
copy = replace
if instance.optional is not None:
copy.optional = instance.optional
if instance.applied_traits:
# We don't know if the object we are copying has applied traits or not and hence use any
copy.appliedTraits = copy_data_utils._array_copy_data(
res_opt, instance.applied_traits, options)
return copy
def from_object(ctx: CdmCorpusContext, name: str, namespace: str,
path: str,
data: 'DocumentContent') -> 'CdmDocumentDefinition':
document = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, name)
document._folder_path = path
document._namespace = namespace
if data:
if not StringUtils.is_blank_by_cdm_standard(data.schema):
document.schema = data.schema
# support old model syntax
if not StringUtils.is_blank_by_cdm_standard(data.schemaVersion):
document.json_schema_semantic_version = data.schemaVersion
if not StringUtils.is_blank_by_cdm_standard(data.documentVersion):
document.document_version = data.documentVersion
if data.get('imports'):
for import_obj in data.imports:
document.imports.append(
ImportPersistence.from_data(ctx, import_obj))
if data.get('definitions') and isinstance(data.definitions, List):
for definition in data.definitions:
if definition.get('dataTypeName'):
document.definitions.append(
DataTypePersistence.from_data(ctx, definition))
elif definition.get('purposeName'):
document.definitions.append(
PurposePersistence.from_data(ctx, definition))
elif definition.get('attributeGroupName'):
document.definitions.append(
AttributeGroupPersistence.from_data(
ctx, definition))
elif definition.get('traitName'):
document.definitions.append(
TraitPersistence.from_data(ctx, definition))
elif definition.get('traitGroupName'):
document.definitions.append(
TraitGroupPersistence.from_data(ctx, definition))
elif definition.get('entityShape'):
document.definitions.append(
ConstantEntityPersistence.from_data(
ctx, definition))
elif definition.get('entityName'):
document.definitions.append(
EntityPersistence.from_data(ctx, definition))
is_resolved_doc = False
if len(document.definitions) == 1 and isinstance(
document.definitions[0], CdmEntityDefinition):
entity = document.definitions[0] # type: CdmEntityDefinition
resolved_trait = entity.exhibits_traits.item(
'has.entitySchemaAbstractionLevel')
# Tries to figure out if the document is in resolved form by looking for the schema abstraction trait
# or the presence of the attribute context.
is_resolved_doc = resolved_trait and resolved_trait.arguments[
0].value == 'resolved'
is_resolved_doc = is_resolved_doc or entity.attribute_context
if data.jsonSchemaSemanticVersion:
document.json_schema_semantic_version = data.jsonSchemaSemanticVersion
if DocumentPersistence._compare_json_semantic_version(
ctx, document.json_schema_semantic_version) > 0:
if is_resolved_doc:
logger.warning(
ctx, _TAG, DocumentPersistence.from_data.__name__,
None,
CdmLogCode.WARN_PERSIST_UNSUPPORTED_JSON_SEM_VER,
DocumentPersistence.json_semantic_version,
document.json_schema_semantic_version)
else:
logger.error(
ctx, _TAG, DocumentPersistence.from_data.__name__,
None,
CdmLogCode.ERR_PERSIST_UNSUPPORTED_JSON_SEM_VER,
DocumentPersistence.json_semantic_version,
document.json_schema_semantic_version)
else:
logger.warning(ctx, _TAG,
DocumentPersistence.from_data.__name__,
document.at_corpus_path,
CdmLogCode.WARN_PERSIST_JSON_SEM_VER_MANDATORY)
return document
async def from_object(
ctx: 'CdmCorpusContext', obj: 'Model',
folder: 'CdmFolderDefinition'
) -> Optional['CdmManifestDefinition']:
extension_trait_def_list = []
manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF, obj.name)
# we need to set up folder path and namespace of a folio to be able to retrieve that object.
folder.documents.append(manifest, manifest.name)
imports = obj.get('imports')
if imports:
for an_import in imports:
import_obj = ImportPersistence.from_data(ctx, an_import)
manifest.imports.append(import_obj)
if not any(
(import_present.corpus_path == Constants._FOUNDATIONS_CORPUS_PATH
for import_present in manifest.imports)):
manifest.imports.append(Constants._FOUNDATIONS_CORPUS_PATH)
if obj.get('modifiedTime'):
manifest.last_file_modified_time = dateutil.parser.parse(
obj.get('modifiedTime'))
if obj.get('lastChildFileModifiedTime'):
manifest.last_child_file_modified_time = dateutil.parser.parse(
obj.get('lastChildFileModifiedTime'))
if obj.get('lastFileStatusCheckTime'):
manifest.last_file_status_check_time = dateutil.parser.parse(
obj.get('lastFileStatusCheckTime'))
if not StringUtils.is_blank_by_cdm_standard(
obj.get('documentVersion')):
manifest.document_version = obj.get('documentVersion')
if obj.get('application'):
application_trait = ctx.corpus.make_object(CdmObjectType.TRAIT_REF,
'is.managedBy', False)
arg = ctx.corpus.make_object(CdmObjectType.ARGUMENT_DEF,
'application')
arg.value = obj.application
application_trait.arguments.append(arg)
manifest.exhibits_traits.append(application_trait)
if obj.get('version'):
version_trait = ctx.corpus.make_object(
CdmObjectType.TRAIT_REF, 'is.modelConversion.modelVersion',
False)
arg = ctx.corpus.make_object(CdmObjectType.ARGUMENT_DEF, 'version')
arg.value = obj.version
version_trait.arguments.append(arg)
manifest.exhibits_traits.append(version_trait)
if obj.get('culture'):
culture_trait = ctx.corpus.make_object(CdmObjectType.TRAIT_REF,
'is.partition.culture',
False)
arg = ctx.corpus.make_object(CdmObjectType.ARGUMENT_DEF, 'culture')
arg.value = obj.culture
culture_trait.arguments.append(arg)
manifest.exhibits_traits.append(culture_trait)
isHidden = obj.get('isHidden')
if isHidden:
is_hidden_trait = ctx.corpus.make_object(CdmObjectType.TRAIT_REF,
'is.hidden', True)
is_hidden_trait.is_from_property = True
manifest.exhibits_traits.append(is_hidden_trait)
reference_models = {}
if obj.get('referenceModels'):
# Create a trait and put reference models inside an argument that goes inside the trait.
reference_models_trait = ctx.corpus.make_object(
CdmObjectType.TRAIT_REF,
'is.modelConversion.referenceModelMap', False)
reference_model_arg = ctx.corpus.make_object(
CdmObjectType.ARGUMENT_DEF, 'referenceModelMap')
reference_model_arg.value = obj.referenceModels
reference_models_trait.arguments.append(reference_model_arg)
manifest.exhibits_traits.append(reference_models_trait)
for element in obj.referenceModels:
reference_models[element.id] = element.location
entity_schema_by_name = {}
for element in (obj.entities or []):
entity = None
if element.type == 'LocalEntity':
entity = await LocalEntityDeclarationPersistence.from_data(
ctx, folder, element, extension_trait_def_list, manifest)
elif element.type == 'ReferenceEntity':
reference_entity = element
location = reference_models.get(reference_entity.modelId)
if not location:
logger.error(
ctx, _TAG, "from_object", None,
CdmLogCode.ERR_PERSIST_MODELJSON_MODEL_ID_NOT_FOUND,
reference_entity.modelId, reference_entity.name)
return None
entity = await ReferencedEntityDeclarationPersistence.from_data(
ctx, reference_entity, location)
else:
logger.error(
ctx, _TAG, 'from_object', None,
CdmLogCode.ERR_PERSIST_MODELJSON_ENTITY_PARSING_ERROR)
if entity:
manifest.entities.append(entity)
entity_schema_by_name[entity.entity_name] = entity.entity_path
else:
logger.error(
ctx, _TAG, 'from_object', None,
CdmLogCode.ERR_PERSIST_MODELJSON_ENTITY_PARSING_ERROR)
if obj.get('relationships'):
for relationship in obj.get('relationships'):
relationship = await RelationshipPersistence.from_data(
ctx, relationship, entity_schema_by_name)
if relationship:
manifest.relationships.append(relationship)
else:
logger.warning(
ctx, _TAG, ManifestPersistence.from_object.__name__,
None,
CdmLogCode.WARN_PERSIST_MODELJSON_REL_READ_FAILED)
await utils.process_annotations_from_data(ctx, obj,
manifest.exhibits_traits)
local_extension_trait_def_list = [] # type: List['CdmTraitDefinition']
extension_helper.process_extension_from_json(
ctx, obj, manifest.exhibits_traits, extension_trait_def_list,
local_extension_trait_def_list)
import_docs = await extension_helper.standard_import_detection(
ctx, extension_trait_def_list,
local_extension_trait_def_list) # type: List[CdmImport]
extension_helper.add_import_docs_to_manifest(ctx, import_docs,
manifest)
ManifestPersistence.create_extension_doc_and_add_to_folder_and_imports(
ctx, extension_trait_def_list, folder)
return manifest
async def to_data(instance: 'CdmManifestDefinition',
res_opt: 'ResolveOptions',
options: 'CopyOptions') -> Optional['Model']:
result = Model()
# process_traits_and_annotations_to_data also processes extensions.
await utils.process_traits_and_annotations_to_data(
instance.ctx, result, instance.exhibits_traits)
result.name = instance.manifest_name
result.description = instance.explanation
result.modifiedTime = utils.get_formatted_date_string(
instance.last_file_modified_time)
result.lastChildFileModifiedTime = utils.get_formatted_date_string(
instance.last_child_file_modified_time)
result.lastFileStatusCheckTime = utils.get_formatted_date_string(
instance.last_file_status_check_time)
result.documentVersion = instance.document_version
t2pm = TraitToPropertyMap(instance)
result.isHidden = bool(
t2pm._fetch_trait_reference('is.hidden')) or None
application_trait = t2pm._fetch_trait_reference('is.managedBy')
if application_trait:
result.application = application_trait.arguments[0].value
version_trait = t2pm._fetch_trait_reference(
'is.modelConversion.modelVersion')
if version_trait:
result.version = version_trait.arguments[0].value
else:
result.version = '1.0'
culture_trait = t2pm._fetch_trait_reference('is.partition.culture')
if culture_trait:
result.culture = culture_trait.arguments[0].value
reference_entity_locations = {}
reference_models = OrderedDict()
reference_models_trait = t2pm._fetch_trait_reference(
'is.modelConversion.referenceModelMap')
if reference_models_trait:
ref_models = reference_models_trait.arguments[0].value
for element in ref_models:
reference_models[element.id] = element.location
reference_entity_locations[element.location] = element.id
if instance.entities:
result.entities = []
# Schedule processing of each entity to be added to the manifest
for entity in instance.entities:
element = None
if entity.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
element = await LocalEntityDeclarationPersistence.to_data(
entity, instance, res_opt, options)
elif entity.object_type == CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF:
element = await ReferencedEntityDeclarationPersistence.to_data(
entity, res_opt, options)
location = instance.ctx.corpus.storage.corpus_path_to_adapter_path(
entity.entity_path)
if StringUtils.is_blank_by_cdm_standard(location):
logger.error(
instance.ctx, _TAG, 'to_data',
instance.at_corpus_path, CdmLogCode.
ERR_PERSIST_MODELJSON_INVALID_ENTITY_PATH,
entity.entity_name)
element = None
reference_entity = element # type: ReferenceEntity
if reference_entity:
# path separator can differ depending on the adapter, cover the case where path uses '/' or '\'
last_slash_location = location.rfind(
'/') if location.rfind('/') > location.rfind(
'\\') else location.rfind('\\')
if last_slash_location > 0:
location = location[:last_slash_location]
if reference_entity.modelId:
saved_location = reference_models.get(
reference_entity.modelId)
if saved_location is not None and saved_location != location:
logger.error(
instance.ctx, 'to_data',
instance.at_corpus_path, _TAG, CdmLogCode.
ERR_PERSIST_MODELJSON_MODEL_ID_DUPLICATION)
element = None
elif saved_location is None:
reference_models[
reference_entity.modelId] = location
reference_entity_locations[
location] = reference_entity.modelId
elif not reference_entity.modelId and location in reference_entity_locations:
reference_entity.modelId = reference_entity_locations[
location]
else:
reference_entity.modelId = str(uuid.uuid4())
reference_models[
reference_entity.modelId] = location
reference_entity_locations[
location] = reference_entity.modelId
if element:
result.entities.append(element)
else:
logger.error(
instance.ctx, _TAG, 'to_data', instance.at_corpus_path,
CdmLogCode.
ERR_PERSIST_MODELJSON_ENTITY_DECLARATION_CONVERSION_ERROR,
entity.entity_name)
if reference_models:
result.referenceModels = []
for value, key in reference_models.items():
model = ReferenceModel()
model.id = value
model.location = key
result.referenceModels.append(model)
if instance.relationships is not None and len(
instance.relationships) > 0:
result.relationships = [] # type: List[SingleKeyRelationship]
for cdm_relationship in instance.relationships:
relationship = await RelationshipPersistence.to_data(
cdm_relationship, res_opt, options, instance.ctx)
if relationship is not None:
result.relationships.append(relationship)
result.imports = []
if instance.imports:
for element in instance.imports:
import_obj = ImportPersistence.to_data(element, res_opt,
options)
if import_obj:
result.imports.append(import_obj)
# Importing foundations.cdm.json to resolve trait properly on manifest
if instance.imports is None or instance.imports.item(
Constants._FOUNDATIONS_CORPUS_PATH,
check_moniker=False) is None:
foundations_import = Import()
foundations_import.corpusPath = Constants._FOUNDATIONS_CORPUS_PATH
result.imports.append(foundations_import)
return result
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder._folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder._namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
if StringUtils.is_blank_by_cdm_standard(json_data):
error_msg = "Json Data is null or empty."
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, error_msg)
return None
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, folder._namespace, e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if StringUtils.is_blank_by_cdm_standard(doc_name):
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
from cdm.persistence.syms import utils
if utils.check_if_syms_adapter(adapter):
from cdm.persistence.syms import ManifestDatabasesPersistence
from cdm.persistence.syms.types import SymsDatabasesResponse
if doc_name_lower == self.SYMS_DATABASES:
from cdm.persistence.syms.models.query_artifacts_response import QueryArtifactsResponse
databases = QueryArtifactsResponse()
databases = databases.deserialize(json.loads(json_data))
doc_content = ManifestDatabasesPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, databases)
elif self.MANIFEST_EXTENSION in doc_name_lower:
from cdm.persistence.syms import ManifestPersistence
manifest_content = await utils.get_syms_model(
adapter, json_data, doc_path)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, manifest_content)
elif self.CDM_EXTENSION in doc_name_lower:
from cdm.persistence.syms.models import TableEntity
from cdm.persistence.syms import DocumentPersistence
table = TableEntity(None, None).deserialize(
json.loads(json_data))
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, table)
elif doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder._namespace,
folder._folder_path, document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
error_msg = str(e)
if e.__context__ is not None:
error_msg += ' Implicitly chained exception: '.format(
str(e.__context__))
if e.__cause__ is not None:
error_msg += ' Explicitly chained exception: '.format(
str(e.__cause__))
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, error_msg)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
exist_id_list = [
x for x in folder.documents if x.id == doc_content.id
]
if not exist_id_list:
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
def from_object(ctx: CdmCorpusContext, name: str, namespace: str,
path: str,
data: 'ManifestContent') -> 'CdmManifestDefinition':
if data is None:
return None
if not StringUtils.is_blank_by_cdm_standard(data.manifestName):
manifest_name = data.manifestName
elif not StringUtils.is_blank_by_cdm_standard(data.get('folioName')):
manifest_name = data.get('folioName')
elif name:
manifest_name = name.replace(PersistenceLayer.MANIFEST_EXTENSION,
'').replace(
PersistenceLayer.FOLIO_EXTENSION,
'')
else:
manifest_name = ''
manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF,
manifest_name)
manifest.name = name # this is the document name which is assumed by constructor to be related to the the manifest name, but may not be
manifest._folder_path = path
manifest._namespace = namespace
manifest.explanation = data.get('explanation')
if not StringUtils.is_blank_by_cdm_standard(data.schema):
manifest.schema = data.schema
# support old model syntax
if not StringUtils.is_blank_by_cdm_standard(data.schemaVersion):
manifest.json_schema_semantic_version = data.schemaVersion
if not StringUtils.is_blank_by_cdm_standard(
data.jsonSchemaSemanticVersion):
manifest.json_schema_semantic_version = data.jsonSchemaSemanticVersion
if manifest.json_schema_semantic_version != '0.9.0' and manifest.json_schema_semantic_version != '1.0.0':
# TODO: validate that this is a version we can understand with the OM
pass
if not StringUtils.is_blank_by_cdm_standard(data.documentVersion):
manifest.document_version = data.documentVersion
utils.add_list_to_cdm_collection(
manifest.exhibits_traits,
utils.create_trait_reference_array(ctx, data.exhibitsTraits))
if data.get('imports'):
for import_obj in data.imports:
manifest.imports.append(
ImportPersistence.from_data(ctx, import_obj))
if data.get('definitions'):
for definition in data.definitions:
if 'dataTypeName' in definition:
manifest.definitions.append(
DataTypePersistence.from_data(ctx, definition))
elif 'purposeName' in definition:
manifest.definitions.append(
PurposePersistence.from_data(ctx, definition))
elif 'attributeGroupName' in definition:
manifest.definitions.append(
AttributeGroupPersistence.from_data(ctx, definition))
elif 'traitName' in definition:
manifest.definitions.append(
TraitPersistence.from_data(ctx, definition))
elif 'entityShape' in definition:
manifest.definitions.append(
ConstantEntityPersistence.from_data(ctx, definition))
elif 'entityName' in definition:
manifest.definitions.append(
EntityPersistence.from_data(ctx, definition))
if data.get('lastFileStatusCheckTime'):
manifest.last_file_status_check_time = dateutil.parser.parse(
data.lastFileStatusCheckTime)
if data.get('lastFileModifiedTime'):
manifest.last_file_modified_time = dateutil.parser.parse(
data.lastFileModifiedTime)
if data.get('lastChildFileModifiedTime'):
manifest.last_child_file_modified_time = dateutil.parser.parse(
data.lastChildFileModifiedTime)
if data.get('entities'):
full_path = '{}:{}'.format(
namespace, path
) if not StringUtils.is_blank_by_cdm_standard(namespace) else path
for entity_obj in data.entities:
if entity_obj.get(
'type'
) == 'LocalEntity' or 'entitySchema' in entity_obj:
manifest.entities.append(
LocalEntityDeclarationPersistence.from_data(
ctx, full_path, entity_obj))
elif entity_obj.get(
'type'
) == 'ReferencedEntity' or 'entityDeclaration' in entity_obj:
manifest.entities.append(
ReferencedEntityDeclarationPersistence.from_data(
ctx, full_path, entity_obj))
else:
logger.error(
ctx, _TAG, ManifestPersistence.from_object.__name__,
None,
CdmLogCode.ERR_PERSIST_ENTITY_DECLARATION_MISSING,
entity_obj.get('entityName'))
return None
# Checks if incremental trait is needed from foundations.cdm.json
ManifestPersistence._import_foundations_if_incremental_partition_trait_exist(
manifest)
if data.get('relationships'):
for relationship in data.relationships:
manifest.relationships.append(
E2ERelationshipPersistence.from_data(ctx, relationship))
if data.get('subManifests'):
sub_manifests = data.subManifests
elif data.get('subFolios'):
sub_manifests = data.subFolios
else:
sub_manifests = []
for sub_manifest in sub_manifests:
manifest.sub_manifests.append(
ManifestDeclarationPersistence.from_data(ctx, sub_manifest))
return manifest
def to_data(instance: 'CdmProjection', res_opt: 'ResolveOptions',
options: 'CopyOptions') -> 'Projection':
from cdm.persistence.cdmfolder.entity_reference_persistence import EntityReferencePersistence
if not instance:
return None
if instance.source and isinstance(instance.source, str):
source = instance.source
elif instance.source and not StringUtils.is_blank_by_cdm_standard(
instance.source.named_reference
) and instance.source.explicit_reference is None:
source = instance.source.named_reference
elif instance.source and instance.source.object_type == CdmObjectType.ENTITY_REF:
source = EntityReferencePersistence.to_data(
instance.source, res_opt, options)
operations = None
if instance.operations and len(instance.operations) > 0:
operations = []
for operation in instance.operations:
if operation.object_type == CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF:
add_count_attribute_op = OperationAddCountAttributePersistence.to_data(
operation, res_opt, options)
operations.append(add_count_attribute_op)
elif operation.object_type == CdmObjectType.OPERATION_ADD_SUPPORTING_ATTRIBUTE_DEF:
add_supporting_attribute_op = OperationAddSupportingAttributePersistence.to_data(
operation, res_opt, options)
operations.append(add_supporting_attribute_op)
elif operation.object_type == CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF:
add_type_attribute_op = OperationAddTypeAttributePersistence.to_data(
operation, res_opt, options)
operations.append(add_type_attribute_op)
elif operation.object_type == CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF:
exclude_attributes_op = OperationExcludeAttributesPersistence.to_data(
operation, res_opt, options)
operations.append(exclude_attributes_op)
elif operation.object_type == CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF:
array_expansion_op = OperationArrayExpansionPersistence.to_data(
operation, res_opt, options)
operations.append(array_expansion_op)
elif operation.object_type == CdmObjectType.OPERATION_COMBINE_ATTRIBUTES_DEF:
combine_attributes_op = OperationCombineAttributesPersistence.to_data(
operation, res_opt, options)
operations.append(combine_attributes_op)
elif operation.object_type == CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF:
rename_attributes_op = OperationRenameAttributesPersistence.to_data(
operation, res_opt, options)
operations.append(rename_attributes_op)
elif operation.object_type == CdmObjectType.OPERATION_REPLACE_AS_FOREIGN_KEY_DEF:
replace_as_foreign_key_op = OperationReplaceAsForeignKeyPersistence.to_data(
operation, res_opt, options)
operations.append(replace_as_foreign_key_op)
elif operation.object_type == CdmObjectType.OPERATION_INCLUDE_ATTRIBUTES_DEF:
include_attributes_op = OperationIncludeAttributesPersistence.to_data(
operation, res_opt, options)
operations.append(include_attributes_op)
elif operation.object_type == CdmObjectType.OPERATION_ADD_ATTRIBUTE_GROUP_DEF:
add_attribute_group_op = OperationAddAttributeGroupPersistence.to_data(
operation, res_opt, options)
operations.append(add_attribute_group_op)
elif operation.object_type == CdmObjectType.OPERATION_ALTER_TRAITS_DEF:
alter_traits_op = OperationAlterTraitsPersistence.to_data(
operation, res_opt, options)
operations.append(alter_traits_op)
elif operation.object_type == CdmObjectType.OPERATION_ADD_ARTIFACT_ATTRIBUTE_DEF:
add_artifact_attribute_op = OperationAddArtifactAttributePersistence.to_data(
operation, res_opt, options)
operations.append(add_artifact_attribute_op)
else:
base_op = OperationBase()
base_op.type = OperationTypeConvertor._operation_type_to_string(
CdmOperationType.ERROR)
operations.append(base_op)
obj = Projection()
obj.explanation = instance.explanation
obj.source = source
obj.operations = operations
obj.condition = instance.condition
obj.runSequentially = instance.run_sequentially
return obj
async def standard_import_detection(
ctx: 'CdmCorpusContext',
extension_trait_def_list: List['CdmTraitDefinition'],
local_extension_trait_def_list: List['CdmTraitDefinition']
) -> Optional[List['CdmImport']]:
imports_list = []
trait_index = len(local_extension_trait_def_list) - 1
has_custom_extension_import = False
while trait_index >= 0:
extension_trait_def = local_extension_trait_def_list[trait_index]
if StringUtils.is_blank_by_cdm_standard(
extension_trait_def.trait_name
) or not extension_trait_def.trait_name.startswith(
EXTENSION_TRAIT_NAME_PREFIX):
logger.error(
ctx, _TAG, standard_import_detection.__name__,
extension_trait_def.at_corpus_path,
CdmLogCode.ERR_PERSIST_MODELJSON_INVALID_EXTENSION_TRAIT,
extension_trait_def.trait_name, EXTENSION_TRAIT_NAME_PREFIX)
return None
extension_breakdown = extension_trait_def.trait_name[
len(EXTENSION_TRAIT_NAME_PREFIX):].split(':')
if len(extension_breakdown) > 1:
extension_name = extension_breakdown[0]
if extension_name not in SUPPORTED_EXTENSIONS:
if not has_custom_extension_import:
import_object = ctx.corpus.make_object(
CdmObjectType.IMPORT)
import_object.corpus_path = EXTENSION_DOC_NAME
imports_list.append(import_object)
has_custom_extension_import = True
trait_index -= 1
continue
file_name = '{}.extension.cdm.json'.format(extension_name)
file_corpus_path = 'cdm:/extensions/{}'.format(file_name)
extension_doc = await fetch_def_doc(ctx, file_name)
# if no document was found for that extensionName, the trait does not have a document with it's definition.
# trait will be kept in extensionTraitDefList (a document with its definition will be created locally)
if not extension_doc:
trait_index -= 1
continue
# there is a document with extensionName, now we search for the trait in the document.
# if we find it, we remove the trait from extensionTraitDefList and add the document to imports.
matching_traits = [
definition for definition in extension_doc.definitions
if definition.object_type == CdmObjectType.TRAIT_DEF
and definition.get_name() == extension_trait_def.trait_name
]
if matching_traits:
parameter_list = matching_traits[0].parameters
if all(
any(def_parameter.name == extension_parameter.name
for def_parameter in parameter_list) for
extension_parameter in extension_trait_def.parameters):
extension_trait_def_list.remove(extension_trait_def)
if not any(import_doc.corpus_path == file_corpus_path
for import_doc in imports_list):
import_object = ctx.corpus.make_object(
CdmObjectType.IMPORT)
import_object.corpus_path = file_corpus_path
imports_list.append(import_object)
trait_index -= 1
return imports_list
def from_data(ctx: CdmCorpusContext,
data: AttributeContext) -> Optional[CdmAttributeContext]:
if data is None:
return None
attribute_context = ctx.corpus.make_object(
CdmObjectType.ATTRIBUTE_CONTEXT_DEF, data.name)
# TEMPORARY CODE
temp = map_type_name_to_enum[data.type]
attribute_context.type = temp
if data.get('parent'):
attribute_context.parent = AttributeContextReferencePersistence.from_data(
ctx, data.get('parent'))
if not StringUtils.is_blank_by_cdm_standard(data.get('explanation')):
attribute_context.explanation = data.get('explanation')
if data.get('definition'):
if attribute_context.type == CdmAttributeContextType.ENTITY or attribute_context.type == CdmAttributeContextType.ENTITY_REFERENCE_EXTENDS:
attribute_context.definition = EntityReferencePersistence.from_data(
ctx, data.definition)
elif attribute_context.type == CdmAttributeContextType.ATTRIBUTE_GROUP:
attribute_context.definition = AttributeGroupReferencePersistence.from_data(
ctx, data.definition)
elif attribute_context.type == CdmAttributeContextType.ADDED_ATTRIBUTE_NEW_ARTIFACT \
or attribute_context.type == CdmAttributeContextType.ADDED_ATTRIBUTE_SUPPORTING \
or attribute_context.type == CdmAttributeContextType.ADDED_ATTRIBUTE_IDENTITY \
or attribute_context.type == CdmAttributeContextType.ADDED_ATTRIBUTE_NEW_ARTIFACT \
or attribute_context.type == CdmAttributeContextType.ADDED_ATTRIBUTE_EXPANSION_TOTAL \
or attribute_context.type == CdmAttributeContextType.ADDED_ATTRIBUTE_SELECTED_TYPE \
or attribute_context.type == CdmAttributeContextType.ATTRIBUTE_DEFINITION \
or attribute_context.type == CdmAttributeContextType.ATTRIBUTE_EXCLUDED:
attribute_context.definition = AttributeReferencePersistence.from_data(
ctx, data.definition)
# I know the trait collection names look wrong. but I wanted to use the def baseclass
utils.add_list_to_cdm_collection(
attribute_context.exhibits_traits,
utils.create_trait_reference_array(ctx, data.get('appliedTraits')))
if data.get('contents'):
if attribute_context.contents is None:
attribute_context.contents = []
for elem in data.contents:
if isinstance(elem, str):
attribute_context.contents.append(
AttributeReferencePersistence.from_data(ctx, elem))
else:
attribute_context.contents.append(
AttributeContextPersistence.from_data(ctx, elem))
if data.get('lineage'):
if attribute_context.lineage is None:
attribute_context.lineage = CdmCollection(
ctx, attribute_context,
CdmObjectType.ATTRIBUTE_CONTEXT_REF)
for elem in data.lineage:
attribute_context.lineage.append(
AttributeContextReferencePersistence.from_data(ctx, elem))
return attribute_context
async def _save_document_as_async(self, doc: 'CdmDocumentDefinition',
options: 'CopyOptions', new_name: str,
save_referenced: bool) -> bool:
"""a manifest or document can be saved with a new or exisitng name. This function on the corpus does all the actual work
because the corpus knows about persistence types and about the storage adapters
if saved with the same name, then consider this document 'clean' from changes. if saved with a back compat model or
to a different name, then the source object is still 'dirty'
an option will cause us to also save any linked documents."""
# find out if the storage adapter is able to write.
namespace = StorageUtils.split_namespace_path(new_name)[0]
if StringUtils.is_blank_by_cdm_standard(namespace):
namespace = doc._namespace
if not namespace:
namespace = self._corpus.storage.default_namespace
adapter = self._corpus.storage.fetch_adapter(namespace)
if adapter is None:
logger.error(
self._ctx, self._TAG, self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_ADAPTER_NOT_FOUND_FOR_NAMESPACE,
namespace)
return False
if not adapter.can_write():
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_ADAPTER_WRITE_FAILURE,
namespace)
return False
if StringUtils.is_blank_by_cdm_standard(new_name):
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
# what kind of document is requested?
persistence_type = ''
from cdm.persistence.syms import utils
if utils.check_if_syms_adapter(adapter):
if new_name == self.SYMS_DATABASES:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_SYMS_UNSUPPORTED_MANIFEST,
new_name)
return False
elif not new_name.lower().endswith(
self.MANIFEST_EXTENSION) and new_name.lower().endswith(
self.CDM_EXTENSION):
logger.error(
self._ctx, self._TAG,
self._save_document_as_async.__name__, doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_SYMS_UNSUPPORTED_CDM_CONVERSION,
new_name)
return False
persistence_type = self.SYMS
else:
if new_name.lower().endswith(self.MODEL_JSON_EXTENSION):
persistence_type = self.MODEL_JSON
else:
persistence_type = self.CDM_FOLDER
if persistence_type == self.MODEL_JSON and new_name.lower(
) != self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path, CdmLogCode.ERR_PERSIST_FAILURE,
new_name, self.MODEL_JSON_EXTENSION)
return False
# save the object into a json blob
res_opt = {
'wrt_doc': doc,
'directives': AttributeResolutionDirectiveSet()
}
persisted_doc = None
try:
if new_name.lower().endswith(
PersistenceLayer.MODEL_JSON_EXTENSION) or new_name.lower(
).endswith(
PersistenceLayer.MANIFEST_EXTENSION) or new_name.lower(
).endswith(PersistenceLayer.FOLIO_EXTENSION):
if persistence_type == self.CDM_FOLDER:
from cdm.persistence.cdmfolder import ManifestPersistence
persisted_doc = ManifestPersistence.to_data(
doc, res_opt, options)
elif persistence_type == self.SYMS:
from cdm.persistence.syms.manifest_persistence import ManifestPersistence
persisted_doc = await ManifestPersistence.convert_manifest_to_syms(
doc, adapter, new_name, res_opt, options)
else:
if new_name != self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FAILURE, new_name)
return False
from cdm.persistence.modeljson import ManifestPersistence
persisted_doc = await ManifestPersistence.to_data(
doc, res_opt, options)
elif new_name.lower().endswith(PersistenceLayer.CDM_EXTENSION):
if persistence_type == self.CDM_FOLDER:
from cdm.persistence.cdmfolder import DocumentPersistence
persisted_doc = DocumentPersistence.to_data(
doc, res_opt, options)
elif persistence_type == self.SYMS:
from cdm.persistence.syms.document_persistence import DocumentPersistence
persisted_doc = await DocumentPersistence.convert_doc_to_syms_table(
self._ctx, doc, adapter, new_name, res_opt, options)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_CLASS_MISSING, new_name)
return False
except Exception as e:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_PERSIST_ERROR, new_name,
e)
return False
if not persisted_doc:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_PERSIST_FAILED, new_name)
return False
# turn the name into a path
new_path = '{}{}'.format(doc._folder_path, new_name)
new_path = self._ctx.corpus.storage.create_absolute_corpus_path(
new_path, doc)
if new_path.startswith(namespace + ':'):
new_path = new_path[len(namespace) + 1:]
# ask the adapter to make it happen
try:
if persistence_type == self.SYMS:
from cdm.persistence.syms import utils
if new_name.lower().endswith(self.MANIFEST_EXTENSION):
await utils.create_or_update_syms_entities(
persisted_doc, adapter)
elif new_name.lower().endswith(self.CDM_EXTENSION):
await utils.create_or_update_table_entity(
persisted_doc, adapter)
else:
content = persisted_doc.encode()
await adapter.write_async(new_path, content)
doc._file_system_modified_time = await adapter.compute_last_modified_time_async(
new_path)
# Write the adapter's config.
if options.save_config_file is not False and options._is_top_level_document and persistence_type != self.SYMS:
await self._corpus.storage.save_adapters_config_async(
'/config.json', adapter)
# The next document won't be top level, so reset the flag.
options._is_top_level_document = False
except Exception as e:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_FILE_WRITE_FAILURE, new_name,
e)
return False
# if we also want to save referenced docs, then it depends on what kind of thing just got saved
# if a model.json there are none. If a manifest or definition doc then ask the docs to do the right things
# definition will save imports, manifests will save imports, schemas, sub manifests
if save_referenced and persistence_type == self.CDM_FOLDER:
saved_linked_docs = await doc._save_linked_documents_async(options)
if not saved_linked_docs:
logger.error(self._ctx, self._TAG,
self._save_document_as_async.__name__,
doc.at_corpus_path,
CdmLogCode.ERR_PERSIST_SAVE_LINK_DOCS, new_name)
return False
return True