async def to_data_async(instance: CdmLocalEntityDeclarationDefinition,
manifest: CdmManifestDefinition,
syms_root_path: str, res_opt: 'ResolveOptions',
options: 'CopyOptions') -> TableEntity:
table_entity = await DocumentPersistence.to_data_async(
instance.entity_path, manifest, instance.ctx, res_opt, options)
if table_entity is not None:
te_properties = table_entity.properties
properties = LocalEntityDeclarationPersistence.create_table_propertybags(
instance, res_opt, options, te_properties.properties)
if instance.data_partitions is not None and len(
instance.data_partitions) > 0:
paths = []
for element in instance.data_partitions:
if element.location is not None:
adls_path = instance.ctx.corpus.storage.corpus_path_to_adapter_path(
element.location)
location = element.location
if adls_path == None:
logger.error(
instance.ctx, _TAG, 'to_data_async',
instance.at_corpus_path, CdmLogCode.
ERR_PERSIST_SYMS_ADLS_ADAPTER_MISSING,
element.location)
return None
syms_path = utils.adls_adapter_path_to_syms_path(
adls_path)
if syms_path is not None:
location = syms_path
else:
path_tuple = StorageUtils.split_namespace_path(
element.location)
location = utils.create_syms_absolute_path(
syms_root_path, path_tuple[1])
paths.append(location)
te_properties.storage_descriptor = DataPartitionPersistence.to_data(
element, te_properties.storage_descriptor, res_opt,
options)
# Logic to find common root folder.
source = DataSource(''.join(c[0] for c in takewhile(
lambda x: all(x[0] == y for y in x), zip(*paths))))
te_properties.storage_descriptor.source = source
else:
# location and format is mandatory for syms.
source = DataSource(
utils.create_syms_absolute_path(syms_root_path,
instance.entity_name))
te_properties.storage_descriptor.source = source
te_properties.properties = properties
return table_entity
def create_data_source(instance: CdmManifestDefinition) -> DataSource:
source = DataSource(None)
t2pm = TraitToPropertyMap(instance)
source_traits = t2pm._fetch_trait_reference(ManifestPersistence.db_location_trait)
if source_traits is not None and source_traits.arguments is not None and len(source_traits.arguments) == 1 and \
source_traits.arguments[0].name == ManifestPersistence.db_location_trait_arg_name:
source.location = utils.corpus_path_to_syms_path(source_traits.arguments[0].value, instance.ctx.corpus.storage)
if source.location == None:
logger.error(instance.ctx, _TAG, 'create_data_source', instance.at_corpus_path, CdmLogCode.ERR_PERSIST_SYMS_STORAGE_SOURCE_TRAIT_ERROR, ManifestPersistence.db_location_trait , ManifestPersistence.db_location_trait_arg_name)
return None
return source
def from_object(ctx: CdmCorpusContext, docname: str, namespace: str, path: str,
data_obj: 'SymsManifestContent') -> 'CdmManifestDefinition':
database = data_obj.database
if database == None or database.type != SASEntityType.database:
logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_INVALID_DB_OBJECT)
return None
database_properties = DatabaseProperties.deserialize(database.properties)
source = DataSource(None).deserialize(database_properties.source)
if database_properties == None or source == None:
logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_INVALID_DB_PROP_OBJECT)
return None
properties = database_properties.properties
manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF)
manifest._folder_path = path
manifest._namespace = namespace
manifest.manifest_name = data_obj.database.name
manifest.name = docname
manifest.explanation = database_properties.description
if properties is not None:
if "cdm:schema" in properties:
manifest.schema = properties["cdm:schema"]
if "cdm:jsonSchemaSemanticVersion" in properties:
manifest.json_schema_semantic_version = properties["cdm:jsonSchemaSemanticVersion"]
if "cdm:documentVersion" in properties:
manifest.DocumentVersion = properties["cdm:documentVersion"]
if "cdm:traits" in properties:
utils.add_list_to_cdm_collection(manifest.exhibits_traits,
utils.create_trait_reference_array(ctx, properties["cdm:traits"]))
if "cdm:imports" in properties:
for import_obj in properties["cdm:imports"]:
manifest.imports.append(ImportPersistence.from_data(ctx, json.loads(import_obj)))
if "cdm:lastFileStatusCheckTime" in properties:
manifest.last_file_status_check_time = dateutil.parser.parse(properties["cdm:lastFileStatusCheckTime"])
if "cdm:lastFileModifiedTime" in properties:
manifest.last_file_modified_time = dateutil.parser.parse(properties["cdm:lastFileModifiedTime"])
if "cdm:lastChildFileModifiedTime" in properties:
manifest.last_child_file_modified_time = dateutil.parser.parse(properties["cdm:lastChildFileModifiedTime"])
t2pm = TraitToPropertyMap(manifest)
source_trait = t2pm._fetch_trait_reference(ManifestPersistence.db_location_trait)
if source_trait == None:
source_trait = utils.create_source_trait(ctx, ManifestPersistence.db_location_trait, ManifestPersistence.db_location_trait_arg_name)
manifest.exhibits_traits.append(source_trait)
adls_path = utils.syms_path_to_adls_adapter_path(source.location)
adls_corpus_path = ctx.corpus.storage.adapter_path_to_corpus_path(adls_path)
if not adls_corpus_path:
path_tuple = StorageUtils.split_namespace_path(source_trait.arguments[0].value)
obj = utils.create_and_mount_adls_adapter_from_adls_path(ctx.corpus.storage, adls_path, path_tuple[0])
if obj == None:
logger.error(ctx, _TAG, 'from_object', None, CdmLogCode.ERR_PERSIST_SYMS_ADLS_ADAPTER_NOT_MOUNTED, adls_path)
return None
if data_obj.entities is not None:
for item in data_obj.entities.items:
entity_obj = TableEntity(None, None).deserialize(item)
if entity_obj.type == SASEntityType.table:
entity = LocalEntityDeclarationPersistence.from_data(ctx, entity_obj, manifest, database_properties.source.location)
if entity is not None:
manifest.entities.append(entity)
else:
logger.warning(ctx, _TAG, 'from_object', None, CdmLogCode.WARN_PERSIST_SYMS_ENTITY_SKIPPED, entity_obj.name)
if not (x for x in manifest.imports if x.corpus_path == Constants._FOUNDATIONS_CORPUS_PATH) or len(manifest.imports) == 0:
manifest.imports.append(Constants._FOUNDATIONS_CORPUS_PATH)
if data_obj.relationships is not None:
for item in data_obj.relationships.items:
relationship_entity = RelationshipEntity(None, None).deserialize(item)
manifest.relationships.extend(E2ERelationshipPersistence.from_data(ctx, relationship_entity))
# TODO: Submanifest
return manifest