def copy(
self,
res_opt: Optional['ResolveOptions'] = None,
host: Optional['CdmDocumentDefinition'] = None
) -> 'CdmDocumentDefinition':
res_opt = res_opt if res_opt is not None else ResolveOptions(
wrt_doc=self,
directives=self.ctx.corpus.default_resolution_directives)
if host is None:
copy = CdmDocumentDefinition(self.ctx, self.name)
else:
copy = host
copy.ctx = self.ctx
copy.name = self.name
copy.definitions.clear()
copy._declarations_indexed = False
copy.internal_declarations = {}
copy._needs_indexing = True
copy.imports.clear()
copy._imports_indexed = False
copy.in_document = copy
copy._is_dirty = True
copy._folder_path = self._folder_path
copy.schema = self.schema
copy.json_schema_semantic_version = self.json_schema_semantic_version
copy.document_version = self.document_version
for definition in self.definitions:
copy.definitions.append(definition)
for imp in self.imports:
copy.imports.append(imp)
return copy
async def _save_resolved(
self, corpus: 'CdmCorpusDefinition',
manifest: 'CdmManifestDefinition', test_name: str,
input_entity: 'CdmEntityDefinition',
resolution_options: List[str]) -> 'CdmEntityDefinition':
ro_hash_set = set()
for i in range(len(resolution_options)):
ro_hash_set.add(resolution_options[i])
file_name_suffix = self._get_resolution_option_name_suffix(
resolution_options)
resolved_entity_name = 'Resolved_{}{}.cdm.json'.format(
input_entity.entity_name, file_name_suffix)
ro = ResolveOptions(
input_entity.in_document,
directives=AttributeResolutionDirectiveSet(ro_hash_set))
resolved_folder = corpus.storage.fetch_root_folder('output')
resolved_entity = await input_entity.create_resolved_entity_async(
resolved_entity_name, ro, resolved_folder)
return resolved_entity
def copy(
self,
res_opt: Optional['ResolveOptions'] = None,
host: Optional['CdmManifestDeclarationDefinition'] = None
) -> 'CdmManifestDeclarationDefinition':
if not res_opt:
res_opt = ResolveOptions(
wrt_doc=self,
directives=self.ctx.corpus.default_resolution_directives)
if not host:
copy = CdmManifestDeclarationDefinition(self.ctx,
self.manifest_name)
else:
copy = host
copy.ctx = self.ctx
copy.manifest_name = self.get_name()
copy.definition = self.definition
copy.last_file_status_check_time = self.last_file_status_check_time
copy.last_file_modified_time = self.last_file_modified_time
self._copy_def(res_opt, copy)
return copy
def copy(
self,
res_opt: Optional['ResolveOptions'] = None,
host: Optional['CdmE2ERelationship'] = None
) -> 'CdmE2ERelationship':
if not res_opt:
res_opt = ResolveOptions(
wrt_doc=self,
directives=self.ctx.corpus.default_resolution_directives)
if not host:
copy = CdmE2ERelationship(self.ctx, self.get_name())
else:
copy = host
copy.ctx = self.ctx
copy.relationship_name = self.get_name()
copy.from_entity = self.from_entity
copy.from_entity_attribute = self.from_entity_attribute
copy.to_entity = self.to_entity
copy.to_entity_attribute = self.to_entity_attribute
self._copy_def(res_opt, copy)
return copy
async def test_cardinality_persistence(self):
'''
Testing that cardinality settings are loaded and saved correctly
'''
corpus = TestHelper.get_local_corpus(self.tests_subpath,
'test_cardinality_persistence')
# test from_data
entity = await corpus.fetch_object_async(
'local:/someEntity.cdm.json/someEntity'
) # type: CdmEntityDefinition
attribute = entity.attributes[0] # type: CdmTypeAttributeDefinition
self.assertIsNotNone(attribute.cardinality)
self.assertEqual(attribute.cardinality.minimum, '0')
self.assertEqual(attribute.cardinality.maximum, '1')
# test to_data
attribute_data = TypeAttributePersistence.to_data(
attribute, ResolveOptions(entity.in_document), CopyOptions())
self.assertIsNotNone(attribute_data.cardinality)
self.assertEqual(attribute_data.cardinality.minimum, '0')
self.assertEqual(attribute_data.cardinality.maximum, '1')
async def test_circular_reference(self):
cdm_corpus = TestHelper.get_local_corpus(self.tests_subpath, 'TestCircularReference')
customer = await cdm_corpus.fetch_object_async('local:/Customer.cdm.json/Customer')
res_customer_structured = await customer.create_resolved_entity_async('resCustomer', ResolveOptions(customer.in_document, AttributeResolutionDirectiveSet({'normalized', 'structured', 'noMaxDepth'})))
# check that the circular reference attribute has a single id attribute
store_group_att = res_customer_structured.attributes[1].explicit_reference
customer_group_att = store_group_att.members[1].explicit_reference
self.assertEqual(len(customer_group_att.members), 1)
self.assertEqual(customer_group_att.members[0].name, 'customerId')
async def run_test(self, test_name: str, source_entity_name: str) -> None:
test_input_path = TestHelper.get_input_folder_path(self.tests_subpath, test_name)
test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name)
test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name)
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('localInput', LocalAdapter(test_input_path))
corpus.storage.mount('localExpectedOutput', LocalAdapter(test_expected_output_path))
corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path))
corpus.storage.mount('cdm', LocalAdapter(TestHelper.get_schema_docs_root()))
corpus.storage.default_namespace = 'localInput'
src_entity_def = await corpus.fetch_object_async('localInput:/{0}.cdm.json/{0}'.format(source_entity_name)) # type: CdmEntityDefinition
self.assertIsNotNone(src_entity_def)
res_opt = ResolveOptions(src_entity_def.in_document, directives=AttributeResolutionDirectiveSet(set()))
actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition
resolved_entity_def = None
output_entity_file_name = ''
entity_file_name = ''
entity_file_name = 'default'
res_opt.directives = AttributeResolutionDirectiveSet(set())
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'normalized'
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'structured'
res_opt.directives = AttributeResolutionDirectiveSet({'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_normalized'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'normalized', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
entity_file_name = 'referenceOnly_normalized_structured'
res_opt.directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized', 'structured'})
output_entity_file_name = '{}_Resolved_{}.cdm.json'.format(source_entity_name, entity_file_name)
resolved_entity_def = await src_entity_def.create_resolved_entity_async(output_entity_file_name, res_opt, actual_output_folder)
if await resolved_entity_def.in_document.save_as_async(output_entity_file_name, True):
self.validate_output(output_entity_file_name, test_expected_output_path, test_actual_output_path)
async def create_resolved_manifest_async(
self,
new_manifest_name: str,
new_entity_document_name_format: Optional[str],
directives: Optional[AttributeResolutionDirectiveSet] = None
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
with logger._enter_scope(self._TAG, self.ctx,
self.create_resolved_manifest_async.__name__):
if self.entities is None:
return None
if not self.folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_MANIFEST_FAILED,
self.manifest_name)
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND,
new_folder_path)
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
logger.debug(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
'resolving manifest {}'.format(source_manifest_path))
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
# sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name
if new_manifest_name.lower().endswith('.manifest.cdm.json'):
new_manifest_name = new_manifest_name[0:(
len(new_manifest_name) - len('.manifest.cdm.json'))]
resolved_manifest = CdmManifestDefinition(self.ctx,
new_manifest_name)
# bring over any imports in this document or other bobbles
resolved_manifest.schema = self.schema
resolved_manifest.explanation = self.explanation
resolved_manifest.document_version = self.document_version
for imp in self.imports:
resolved_manifest.imports.append(imp.copy())
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_ENTITY_REF_ERROR)
return None
if not ent_def.in_document.folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_IS_NOT_FOLDERformat,
ent_def.entity_name)
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(
source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path
):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND,
new_document_path)
return None
# next create the resolved entity.
with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives
res_opt = ResolveOptions(ent_def.in_document,
with_directives.copy())
logger.debug(
self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
'resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
logger.debug(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path, 'calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus.calculate_entity_graph_async(
resolved_manifest)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an ArrayExpansion operation and a condition using the object model"""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an ArrayExpansion operation
array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF)
array_expansion_op.start_ordinal = 1
array_expansion_op.end_ordinal = 2
projection.operations.append(array_expansion_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity
projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF)
projection2.source = projection_entity_ref
# Create a RenameAttributes operation
rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF)
rename_attrs_op.rename_format = '{m}{o}'
projection2.operations.append(rename_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref2.explicit_reference = projection2
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref2
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(8, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id1', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name1', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value1', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date1', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual('id2', resolved_entity_with_reference_only.attributes[4].name)
self.assertEqual('name2', resolved_entity_with_reference_only.attributes[5].name)
self.assertEqual('value2', resolved_entity_with_reference_only.attributes[6].name)
self.assertEqual('date2', resolved_entity_with_reference_only.attributes[7].name)
# Now resolve the entity with the default directives
res_opt.directives = AttributeResolutionDirectiveSet(set([]))
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id', resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name', resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value', resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder.Namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, str(folder.namespace), e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if not doc_name:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, e)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
async def create_resolved_entity_async(self, new_ent_name: str, res_opt: Optional['ResolveOptions'] = None, folder: 'CdmFolderDefinition' = None,
new_doc_name: str = None) -> 'CdmEntityDefinition':
"""Create a resolved copy of the entity."""
if not res_opt:
res_opt = ResolveOptions(self, self.ctx.corpus.default_resolution_directives)
if not res_opt.wrt_doc:
logger.error(self._TAG, self.ctx, 'No WRT document was supplied.', self.create_resolved_entity_async.__name__)
return None
if not new_ent_name:
logger.error(self._TAG, self.ctx, 'No Entity Name provided.', self.create_resolved_entity_async.__name__)
return None
folder = folder or self.in_document.folder
file_name = new_doc_name or new_ent_name + PersistenceLayer.CDM_EXTENSION
orig_doc = self.in_document.at_corpus_path
# Don't overwite the source document.
target_at_corpus_path = self.ctx.corpus.storage.create_absolute_corpus_path(folder.at_corpus_path, folder) + file_name
if StringUtils.equals_with_ignore_case(target_at_corpus_path, orig_doc):
logger.error(self._TAG, self.ctx, 'Attempting to replace source entity\'s document \'{}\''.format(
target_at_corpus_path), self.create_resolved_entity_async.__name__)
return None
if not await res_opt.wrt_doc._index_if_needed(res_opt, True):
logger.error(self._TAG, self.ctx, 'Couldn\'t index source document.', self.create_resolved_entity_async.__name__)
return None
# Make the top level attribute context for this entity.
# for this whole section where we generate the attribute context tree and get resolved attributes
# set the flag that keeps all of the parent changes and document dirty from from happening
was_resolving = self.ctx.corpus._is_currently_resolving
self.ctx.corpus._is_currently_resolving = True
ent_name = new_ent_name
ctx = self.ctx
att_ctx_ent = ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_CONTEXT_DEF, ent_name, True) # type: CdmAttributeContext
att_ctx_ent.ctx = ctx
att_ctx_ent.in_document = self.in_document
# Cheating a bit to put the paths in the right place.
acp = AttributeContextParameters()
acp._under = att_ctx_ent
acp._type = CdmAttributeContextType.ATTRIBUTE_GROUP
acp._name = 'attributeContext'
att_ctx_ac = CdmAttributeContext._create_child_under(res_opt, acp)
acp_ent = AttributeContextParameters()
acp_ent._under = att_ctx_ac
acp_ent._type = CdmAttributeContextType.ENTITY
acp_ent._name = ent_name
acp_ent._regarding = ctx.corpus.make_ref(CdmObjectType.ENTITY_REF, self.get_name(), True)
# Use this whenever we need to keep references pointing at things that were already found.
# Used when 'fixing' references by localizing to a new document.
res_opt_copy = CdmObject._copy_resolve_options(res_opt)
res_opt_copy._save_resolutions_on_copy = True
# Resolve attributes with this context. the end result is that each resolved attribute points to the level of
# the context where it was created.
ras = self._fetch_resolved_attributes(res_opt_copy, acp_ent)
if ras is None:
self._resolving_attributes = False
return None
# Create a new copy of the attribute context for this entity.
# TODO: all_att_ctx type ideally should be ordered set
all_att_ctx = [] # type: List[CdmAttributeContext]
new_node = cast('CdmAttributeContext', att_ctx_ent.copy_node(res_opt))
att_ctx_ent = att_ctx_ent._copy_attribute_context_tree(res_opt, new_node, ras, all_att_ctx, 'resolvedFrom')
att_ctx = cast('CdmAttributeContext', cast('CdmAttributeContext', att_ctx_ent.contents[0]).contents[0])
self.ctx.corpus._is_currently_resolving = was_resolving
# make a new document in given folder if provided or the same folder as the source entity
folder.documents.remove(file_name)
doc_res = folder.documents.append(file_name)
# add a import of the source document
orig_doc = self.ctx.corpus.storage.create_relative_corpus_path(orig_doc, doc_res) # just in case we missed the prefix
doc_res.imports.append(orig_doc, "resolvedFrom")
# make the empty entity
ent_resolved = doc_res.definitions.append(ent_name)
# set the context to the copy of the tree. fix the docs on the context nodes
ent_resolved.attribute_context = att_ctx
def visit_callback(obj: 'CdmObject', path: str) -> bool:
obj.in_document = doc_res
return False
att_ctx.visit('{}/attributeContext/'.format(ent_name), visit_callback, None)
# add the traits of the entity
rts_ent = self._fetch_resolved_traits(res_opt)
for rt in rts_ent.rt_set:
trait_ref = CdmObject._resolved_trait_to_trait_ref(res_opt_copy, rt)
ent_resolved.exhibits_traits.append(trait_ref)
# The attributes have been named, shaped, etc. for this entity so now it is safe to go and make each attribute
# context level point at these final versions of attributes.
att_path_to_order = {} # type: Dict[str, int]
def point_context_at_resolved_atts(ras_sub: 'ResolvedAttributeSet', path: str) -> None:
for ra in ras_sub._set:
ra_ctx_in_ent = [] # type: List[CdmAttributeContext]
ra_ctx_set = ras_sub.rattr_to_attctxset.get(ra)
# find the correct attctx for this copy, intersect these two sets
# (interate over the shortest list)
if len(all_att_ctx) < len(ra_ctx_set):
for curr_att_ctx in all_att_ctx:
if curr_att_ctx in ra_ctx_set:
ra_ctx_in_ent.append(curr_att_ctx)
else:
for curr_att_ctx in ra_ctx_set:
if curr_att_ctx in all_att_ctx:
ra_ctx_in_ent.append(curr_att_ctx)
for ra_ctx in ra_ctx_in_ent:
refs = ra_ctx.contents
# there might be more than one explanation for where and attribute came from when things get merges as they do
# This won't work when I add the structured attributes to avoid name collisions.
att_ref_path = path + ra.resolved_name
if isinstance(ra.target, CdmObject):
if not att_ref_path in att_path_to_order:
att_ref = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_REF, att_ref_path, True) # type: CdmObjectReference
# only need one explanation for this path to the insert order
att_path_to_order[att_ref.named_reference] = ra.insert_order
refs.append(att_ref)
else:
att_ref_path += '/members/'
point_context_at_resolved_atts(cast('ResolvedAttributeSet', ra.target), att_ref_path)
point_context_at_resolved_atts(ras, ent_name + '/hasAttributes/')
# generated attribute structures may end up with 0 attributes after that. prune them
def clean_sub_group(sub_item, under_generated) -> bool:
if sub_item.object_type == CdmObjectType.ATTRIBUTE_REF:
return True # not empty
ac = sub_item # type: CdmAttributeContext
if ac.type == CdmAttributeContextType.GENERATED_SET:
under_generated = True
if not ac.contents:
return False # empty
# look at all children, make a set to remove
to_remove = [] # type: List[CdmAttributeContext]
for sub_sub in ac.contents:
if not clean_sub_group(sub_sub, under_generated):
potential_target = under_generated
if not potential_target:
# cast is safe because we returned false meaning empty and not a attribute ref
# so is this the set holder itself?
potential_target = sub_sub.type == CdmAttributeContextType.GENERATED_SET
if potential_target:
to_remove.append(sub_sub)
for to_die in to_remove:
ac.contents.remove(to_die)
return bool(ac.contents)
clean_sub_group(att_ctx, False)
# Create an all-up ordering of attributes at the leaves of this tree based on insert order. Sort the attributes
# in each context by their creation order and mix that with the other sub-contexts that have been sorted.
def get_order_num(item: 'CdmObject') -> int:
if item.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF:
return order_contents(item)
if item.object_type == CdmObjectType.ATTRIBUTE_REF:
return att_path_to_order[item.named_reference]
# put the mystery item on top.
return -1
def order_contents(under: 'CdmAttributeContext') -> int:
if under._lowest_order is None:
under._lowest_order = -1
if len(under.contents) == 1:
under._lowest_order = get_order_num(under.contents[0])
else:
def get_and_update_order(item1: 'CdmObject', item2: 'CdmObject'):
left_order = get_order_num(item1)
right_order = get_order_num(item2)
if left_order != -1 and (under._lowest_order == -1 or left_order < under._lowest_order):
under._lowest_order = left_order
if right_order != -1 and (under._lowest_order == -1 or right_order < under._lowest_order):
under._lowest_order = right_order
return left_order - right_order
under.contents.sort(key=functools.cmp_to_key(get_and_update_order))
return under._lowest_order
order_contents(att_ctx)
# Resolved attributes can gain traits that are applied to an entity when referenced since these traits are
# described in the context, it is redundant and messy to list them in the attribute. So, remove them. Create and
# cache a set of names to look for per context. There is actually a hierarchy to this. All attributes from the
# base entity should have all traits applied independently of the sub-context they come from. Same is true of
# attribute entities. So do this recursively top down.
ctx_to_trait_names = {} # type: Dict[CdmAttributeContext, Set[str]]
def collect_context_traits(sub_att_ctx: 'CdmAttributeContext', inherited_trait_names: Set[str]) -> None:
trait_names_here = set(inherited_trait_names)
traits_here = sub_att_ctx.exhibits_traits
if traits_here:
for tat in traits_here:
trait_names_here.add(tat.named_reference)
ctx_to_trait_names[sub_att_ctx] = trait_names_here
for cr in sub_att_ctx.contents:
if cr.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF:
# do this for all types?
collect_context_traits(cast('CdmAttributeContext', cr), trait_names_here)
collect_context_traits(att_ctx, set())
# add the attributes, put them in attribute groups if structure needed.
res_att_to_ref_path = {} # type: Dict[ResolvedAttribute, str]
def add_attributes(ras_sub: 'ResolvedAttributeSet', container: 'Union[CdmEntityDefinition, CdmAttributeGroupDefinition]', path: str) -> None:
for ra in ras_sub._set:
att_path = path + ra.resolved_name
# use the path of the context associated with this attribute to find the new context that matches on path.
ra_ctx_set = ras_sub.rattr_to_attctxset[ra]
# find the correct att_ctx for this copy.
# ra_ctx = next((ac for ac in all_att_ctx if ac in ra_ctx_set), None) # type: CdmAttributeContext
if len(all_att_ctx) < len(ra_ctx_set):
for curr_att_ctx in all_att_ctx:
if curr_att_ctx in ra_ctx_set:
ra_ctx = curr_att_ctx
break
else:
for curr_att_ctx in ra_ctx_set:
if curr_att_ctx in all_att_ctx:
ra_ctx = curr_att_ctx
break
if isinstance(ra.target, ResolvedAttributeSet) and cast('ResolvedAttributeSet', ra.target)._set:
# this is a set of attributes. Make an attribute group to hold them.
att_grp = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_GROUP_DEF, ra.resolved_name) # type: CdmAttributeGroupDefinition
att_grp.attribute_context = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_CONTEXT_REF, ra_ctx.at_corpus_path, True)
# take any traits from the set and make them look like traits exhibited by the group.
avoid_set = ctx_to_trait_names[ra_ctx]
rts_att = ra.resolved_traits
for rt in rts_att.rt_set:
if not rt.trait.ugly: # Don't mention your ugly traits.
if not avoid_set or rt.trait_name not in avoid_set: # Avoid the ones from the context.
trait_ref = CdmObject._resolved_trait_to_trait_ref(res_opt_copy, rt)
cast('CdmObjectDefinition', att_grp).exhibits_traits.append(trait_ref, isinstance(trait_ref, str))
# wrap it in a reference and then recurse with this as the new container.
att_grp_ref = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_GROUP_REF, None) # type: CdmAttributeGroupReference
att_grp_ref.explicit_reference = att_grp
container._add_attribute_def(att_grp_ref)
# isn't this where ...
add_attributes(cast('ResolvedAttributeSet', ra.target), att_grp, att_path + '/(object)/members/')
else:
att = self.ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, ra.resolved_name) # type: CdmTypeAttributeDefinition
att.attribute_context = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_CONTEXT_REF, ra_ctx.at_corpus_path, True)
avoid_set = ctx_to_trait_names[ra_ctx]
rts_att = ra.resolved_traits
for rt in rts_att.rt_set:
if not rt.trait.ugly: # Don't mention your ugly traits.
if not avoid_set or rt.trait_name not in avoid_set: # Avoid the ones from the context.
trait_ref = CdmObject._resolved_trait_to_trait_ref(res_opt_copy, rt)
cast('CdmTypeAttributeDefinition', att).applied_traits.append(trait_ref, isinstance(trait_ref, str))
# none of the dataformat traits have the bit set that will make them turn into a property
# this is intentional so that the format traits make it into the resolved object
# but, we still want a guess as the data format, so get it and set it.
implied_data_format = att.data_format
if implied_data_format:
att.data_format = implied_data_format
container._add_attribute_def(att)
res_att_to_ref_path[ra] = att_path
add_attributes(ras, ent_resolved, ent_name + '/hasAttributes/')
# Any resolved traits that hold arguments with attribute refs should get 'fixed' here.
def replace_trait_att_ref(tr: 'CdmTraitReference', entity_hint: str) -> None:
if tr.arguments:
for arg in tr.arguments:
v = arg._unresolved_value or arg.value
# Is this an attribute reference?
if v and isinstance(v, CdmObject) and cast('CdmObject', v).object_type == CdmObjectType.ATTRIBUTE_REF:
# Only try this if the reference has no path to it (only happens with intra-entity att refs).
att_ref = cast('CdmAttributeReference', v)
if att_ref.named_reference and att_ref.named_reference.find('/') == -1:
if not arg._unresolved_value:
arg._unresolved_value = arg.value
# Give a promise that can be worked out later.
# Assumption is that the attribute must come from this entity.
new_att_ref = self.ctx.corpus.make_object(CdmObjectType.ATTRIBUTE_REF, entity_hint +
'/(resolvedAttributes)/' + att_ref.named_reference, True)
# inDocument is not propagated during resolution, so set it here
new_att_ref.in_document = arg.in_document
arg.set_value(new_att_ref)
# Fix entity traits.
if ent_resolved.exhibits_traits:
for et in ent_resolved.exhibits_traits:
replace_trait_att_ref(et, new_ent_name)
# Fix context traits.
def fix_context_traits(sub_att_ctx: 'CdmAttributeContext', entity_hint: str) -> None:
traits_here = sub_att_ctx.exhibits_traits
if traits_here:
for tr in traits_here:
replace_trait_att_ref(tr, entity_hint)
for cr in sub_att_ctx.contents:
if cr.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF:
# If this is a new entity context, get the name to pass along.
sub_sub_att_ctx = cast('CdmAttributeContext', cr)
sub_entity_hint = entity_hint
if sub_sub_att_ctx.type == CdmAttributeContextType.ENTITY:
sub_entity_hint = sub_sub_att_ctx.definition.named_reference
# Do this for all types.
fix_context_traits(sub_sub_att_ctx, sub_entity_hint)
fix_context_traits(att_ctx, new_ent_name)
# And the attribute traits.
ent_atts = ent_resolved.attributes
if ent_atts:
for attribute in ent_atts:
att_traits = attribute.applied_traits
if att_traits:
for tr in att_traits:
replace_trait_att_ref(tr, new_ent_name)
# We are about to put this content created in the context of various documents (like references to attributes
# from base entities, etc.) into one specific document. All of the borrowed refs need to work. so, re-write all
# string references to work from this new document. The catch-22 is that the new document needs these fixes done
# before it can be used to make these fixes. The fix needs to happen in the middle of the refresh trigger the
# document to refresh current content into the resolved OM.
cast('CdmAttributeContext', att_ctx).parent = None # Remove the fake parent that made the paths work.
res_opt_new = CdmObject._copy_resolve_options(res_opt)
res_opt_new.wrt_doc = doc_res
res_opt_new._localize_references_for = doc_res
if not await doc_res.refresh_async(res_opt_new):
logger.error(self._TAG, self.ctx, 'Failed to index the resolved document.', self.create_resolved_entity_async.__name__)
return None
# Get a fresh ref.
ent_resolved = cast('CdmEntityDefinition', doc_res._fetch_object_from_document_path(ent_name, res_opt_new))
self.ctx.corpus._res_ent_map[self.at_corpus_path] = ent_resolved.at_corpus_path
return ent_resolved
def _fetch_resolved_attributes(
self,
res_opt: 'ResolveOptions',
acp_in_context: Optional['AttributeContextParameters'] = None
) -> 'ResolvedAttributeSet':
from cdm.resolvedmodel import ResolvedAttributeSet
from cdm.utilities import SymbolSet
from .cdm_attribute_context import CdmAttributeContext
from .cdm_corpus_def import CdmCorpusDefinition
was_previously_resolving = self.ctx.corpus.is_currently_resolving
self.ctx.corpus.is_currently_resolving = True
if not res_opt:
res_opt = ResolveOptions(self)
kind = 'rasb'
ctx = self.ctx
cache_tag = ctx.corpus._fetch_definition_cache_tag(
res_opt, self, kind, 'ctx' if acp_in_context else '')
rasb_cache = ctx.cache.get(cache_tag) if cache_tag else None
under_ctx = None
# store the previous symbol set, we will need to add it with
# children found from the constructResolvedTraits call
curr_sym_ref_set = res_opt.symbol_ref_set or SymbolSet()
res_opt.symbol_ref_set = SymbolSet()
# get the moniker that was found and needs to be appended to all
# refs in the children attribute context nodes
from_moniker = res_opt._from_moniker
res_opt._from_moniker = None
if not rasb_cache:
if self._resolving_attributes:
# re-entered self attribute through some kind of self or looping reference.
self.ctx.corpus.is_currently_resolving = was_previously_resolving
return ResolvedAttributeSet()
self._resolving_attributes = True
# if a new context node is needed for these attributes, make it now
if acp_in_context:
under_ctx = CdmAttributeContext._create_child_under(
res_opt, acp_in_context)
rasb_cache = self._construct_resolved_attributes(
res_opt, under_ctx)
self._resolving_attributes = False
# register set of possible docs
odef = self.fetch_object_definition(res_opt)
if odef is not None:
ctx.corpus._register_definition_reference_symbols(
odef, kind, res_opt.symbol_ref_set)
# get the new cache tag now that we have the list of symbols
cache_tag = ctx.corpus._fetch_definition_cache_tag(
res_opt, self, kind, 'ctx' if acp_in_context else '')
# save this as the cached version
if cache_tag:
ctx.cache[cache_tag] = rasb_cache
if from_moniker and acp_in_context and cast(
'CdmObjectReference', self).named_reference:
# create a fresh context
old_context = acp_in_context.under.contents[-1]
acp_in_context.under.contents.pop()
under_ctx = CdmAttributeContext._create_child_under(
res_opt, acp_in_context)
old_context._copy_attribute_context_tree(
res_opt, under_ctx, rasb_cache.ras, None, from_moniker)
else:
# get the SymbolSet for this cached object and pass that back
key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind)
res_opt.symbol_ref_set = ctx.corpus._definition_reference_symbols.get(
key)
# cache found. if we are building a context, then fix what we got instead of making a new one
if acp_in_context:
# make the new context
under_ctx = CdmAttributeContext._create_child_under(
res_opt, acp_in_context)
rasb_cache.ras.attribute_context._copy_attribute_context_tree(
res_opt, under_ctx, rasb_cache.ras, None, from_moniker)
# merge child reference symbols set with current
curr_sym_ref_set.merge(res_opt.symbol_ref_set)
res_opt.symbol_ref_set = curr_sym_ref_set
self.ctx.corpus.is_currently_resolving = was_previously_resolving
return rasb_cache.ras
async def create_resolved_manifest_async(
self, new_manifest_name: str, new_entity_document_name_format: str
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
if self.entities is None:
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
self.ctx.logger.error(
'New folder for manifest not found {}'.format(
new_folder_path), 'create_resolved_manifest_async')
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
self.ctx.logger.debug(
'resolving manifest {}'.format(source_manifest_path),
'create_resolved_manifest_async')
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name)
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
# mapping from entity path to resolved entity path for translating relationhsip paths
res_ent_map = {} # type: Dict[str, str]
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
self.ctx.logger.error('Unable to get entity from reference')
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
self.ctx.logger.error(
'New folder not found {}'.format(new_document_path))
return None
# next create the resolved entity.
res_opt = ResolveOptions()
res_opt.wrt_doc = ent_def.in_document
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
self.ctx.logger.debug(
' resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
# absolute path is needed for generating relationships
absolute_ent_path = self.ctx.corpus.storage.create_absolute_corpus_path(
result.entity_path, resolved_manifest)
res_ent_map[self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.at_corpus_path,
ent_def.in_document)] = absolute_ent_path
self.ctx.logger.debug(' calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus._calculate_entity_graph_async(
resolved_manifest, res_ent_map)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
def fetch_object_definition(self, res_opt: 'ResolveOptions') -> 'CdmObjectDefinition':
if res_opt is None:
res_opt = ResolveOptions(self)
return self._fetch_resolved_reference(res_opt)
def create_simple_reference(self, res_opt: Optional['ResolveOptions'] = None) -> 'CdmObjectReference':
if not res_opt:
res_opt = ResolveOptions(self)
if self.named_reference:
return self._copy_ref_object(res_opt, self.named_reference, True)
return self._copy_ref_object(res_opt, self._declared_path + self.explicit_reference.get_name(), True)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an ExcludeAttributes operation and a condition using the object model"""
corpus = TestHelper.get_local_corpus(
self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount(
'local',
LocalAdapter(
TestHelper.get_actual_output_folder_path(
self.tests_subpath,
'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an ExcludeAttributes operation
exclude_attrs_op = corpus.make_object(
CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF)
exclude_attrs_op.exclude_attributes.append('id')
exclude_attrs_op.exclude_attributes.append('date')
projection.operations.append(exclude_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(
set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: ['id', 'date']
self.assertEqual(2,
len(resolved_entity_with_reference_only.attributes))
self.assertEqual(
'name', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual(
'value', resolved_entity_with_reference_only.attributes[1].name)
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(
set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operation
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: none, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id',
resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name',
resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value',
resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date',
resolved_entity_with_structured.attributes[3].name)
async def test_resolved_attribute_limit(self):
corpus = TestHelper.get_local_corpus(
self.tests_subpath,
'test_resolved_attribute_limit') # type: CdmCorpusDefinition
main_entity = await corpus.fetch_object_async(
'local:/mainEntity.cdm.json/mainEntity'
) # type: CdmEntityDefinition
res_opt = ResolveOptions(wrt_doc=main_entity.in_document)
# if attribute limit is reached, entity should be None
res_opt._resolved_attribute_limit = 4
resEnt = await main_entity.create_resolved_entity_async(
'{}_zeroAtts'.format(main_entity.entity_name),
res_opt) # type: CdmEntityDefinition
self.assertIsNone(resEnt)
# when the attribute limit is set to null, there should not be a limit on the possible number of attributes
res_opt._resolved_attribute_limit = None
ras = main_entity._fetch_resolved_attributes(
res_opt) # type: ResolvedAttributeSet
resEnt = await main_entity.create_resolved_entity_async(
'{}_normalized_referenceOnly'.format(main_entity.entity_name),
res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
self.assertEqual(5, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(
2, len(main_entity.attributes[2].explicit_reference.members))
# using the default limit number
res_opt = ResolveOptions(wrt_doc=main_entity.in_document)
ras = main_entity._fetch_resolved_attributes(res_opt)
resEnt = await main_entity.create_resolved_entity_async(
'{}_normalized_referenceOnly'.format(main_entity.entity_name),
res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
self.assertEqual(5, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(
2, len(main_entity.attributes[2].explicit_reference.members))
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
ras = main_entity._fetch_resolved_attributes(res_opt)
resEnt = await main_entity.create_resolved_entity_async(
'{}_normalized_structured'.format(main_entity.entity_name),
res_opt)
# there are 5 total attributes
self.assertEqual(5, ras._resolved_attribute_count)
# the attribute count is different because one attribute is a group that contains two different attributes
self.assertEqual(4, len(ras._set))
self.assertEqual(3, len(main_entity.attributes))
# again there are 2 attributes grouped in an entity attribute
# and 2 attributes grouped in an attribute group
self.assertEqual(
2, len(main_entity.attributes[2].explicit_reference.members))
def _fetch_resolved_traits(
self, res_opt: 'ResolveOptions') -> 'ResolvedTraitSet':
from cdm.resolvedmodel import ResolvedTraitSet, ResolvedTraitSetBuilder
from cdm.utilities import SymbolSet
was_previously_resolving = self.ctx.corpus.is_currently_resolving
self.ctx.corpus.is_currently_resolving = True
if not res_opt:
res_opt = ResolveOptions(self)
kind = 'rtsb'
ctx = self.ctx
cache_tag_a = ctx.corpus._fetch_definition_cache_tag(
res_opt, self, kind)
rtsb_all = None # type: ResolvedTraitSetBuilder
if self._trait_cache is None:
self._trait_cache = {}
elif cache_tag_a:
rtsb_all = self._trait_cache.get(cache_tag_a)
# store the previous document set, we will need to add it with
# children found from the constructResolvedTraits call
curr_doc_ref_set = res_opt.symbol_ref_set
if curr_doc_ref_set is None:
curr_doc_ref_set = SymbolSet()
res_opt.symbol_ref_set = SymbolSet()
if rtsb_all is None:
rtsb_all = ResolvedTraitSetBuilder()
if not self._resolving_traits:
self._resolving_traits = True
self._construct_resolved_traits(rtsb_all, res_opt)
self._resolving_traits = False
obj_def = self.fetch_object_definition(res_opt)
if obj_def:
# register set of possible docs
ctx.corpus._register_definition_reference_symbols(
obj_def, kind, res_opt.symbol_ref_set)
if rtsb_all.resolved_trait_set is None:
# nothing came back, but others will assume there is a set in this builder
rtsb_all.resolved_trait_set = ResolvedTraitSet(res_opt)
# get the new cache tag now that we have the list of docs
cache_tag_a = ctx.corpus._fetch_definition_cache_tag(
res_opt, self, kind)
if cache_tag_a:
self._trait_cache[cache_tag_a] = rtsb_all
else:
# cache was found
# get the SymbolSet for this cached object
from .cdm_corpus_def import CdmCorpusDefinition
key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind)
temp_doc_ref_set = ctx.corpus._definition_reference_symbols.get(
key)
res_opt.symbol_ref_set = temp_doc_ref_set
# merge child document set with current
curr_doc_ref_set.merge(res_opt.symbol_ref_set)
res_opt.symbol_ref_set = curr_doc_ref_set
self.ctx.corpus.is_currently_resolving = was_previously_resolving
return rtsb_all.resolved_trait_set
def is_derived_from(self, base: str, res_opt: Optional['ResolveOptions'] = None) -> bool:
res_opt = res_opt if res_opt is not None else ResolveOptions(wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives)
return self._is_derived_from_def(res_opt, self.extends_entity, self.get_name(), base)
def _fetch_resolved_traits(
self, res_opt: 'ResolveOptions') -> 'ResolvedTraitSet':
from cdm.resolvedmodel import ResolvedTrait, ResolvedTraitSet
from cdm.utilities import SymbolSet
from .cdm_corpus_def import CdmCorpusDefinition
res_opt = res_opt if res_opt is not None else ResolveOptions(
self, self.ctx.corpus.default_resolution_directives)
kind = 'rtsb'
ctx = self.ctx
# this may happen 0, 1 or 2 times. so make it fast
base_trait = None # type: CdmTraitDefinition
base_values = None # type: List[CdmArgumentValue]
def get_base_info(base_trait, base_values):
if self.extends_trait:
base_trait = self.extends_trait.fetch_object_definition(
res_opt)
if base_trait:
base_rts = self.extends_trait._fetch_resolved_traits(
res_opt)
if base_rts and base_rts.size == 1:
base_pv = base_rts.get(
base_trait).parameter_values if base_rts.get(
base_trait) else None
if base_pv:
base_values = base_pv.values
return base_trait, base_values
# see if one is already cached
# if this trait has parameters, then the base trait found through the reference might be a different reference
# because trait references are unique per argument value set. so use the base as a part of the cache tag
# since it is expensive to figure out the extra tag, cache that too!
if self._base_is_known_to_have_parameters is None:
base_trait, base_values = get_base_info(base_trait, base_values)
# is a cache tag needed? then make one
self._base_is_known_to_have_parameters = False
if base_values:
self._base_is_known_to_have_parameters = True
cache_tag_extra = ''
if self._base_is_known_to_have_parameters:
cache_tag_extra = str(self.extends_trait.id)
cache_tag = ctx.corpus._create_definition_cache_tag(
res_opt, self, kind, cache_tag_extra)
rts_result = ctx._cache.get(cache_tag) if cache_tag else None
# store the previous reference symbol set, we will need to add it with
# children found from the _construct_resolved_traits call
curr_sym_ref_set = res_opt._symbol_ref_set or SymbolSet()
res_opt._symbol_ref_set = SymbolSet()
# if not, then make one and save it
if not rts_result:
base_trait, base_values = get_base_info(base_trait, base_values)
if base_trait:
# get the resolution of the base class and use the values as a starting point for this trait's values
if not self._has_set_flags:
# inherit these flags
if self.elevated is None:
self.elevated = base_trait.elevated
if self.ugly is None:
self.ugly = base_trait.ugly
if self.associated_properties is None:
self.associated_properties = base_trait.associated_properties
self._has_set_flags = True
pc = self._fetch_all_parameters(res_opt)
av = [] # type: List[CdmArgumentValue]
was_set = [] # type: List[bool]
self._this_is_known_to_have_parameters = bool(pc.sequence)
for i in range(len(pc.sequence)):
# either use the default value or (higher precidence) the value taken from the base reference
value = pc.sequence[i].default_value
if base_values and i < len(base_values):
base_value = base_values[i]
if base_value:
value = base_value
av.append(value)
was_set.append(False)
# save it
res_trait = ResolvedTrait(self, pc, av, was_set)
rts_result = ResolvedTraitSet(res_opt)
rts_result.merge(res_trait, False)
# register set of possible symbols
ctx.corpus._register_definition_reference_symbols(
self.fetch_object_definition(res_opt), kind,
res_opt._symbol_ref_set)
# get the new cache tag now that we have the list of docs
cache_tag = ctx.corpus._create_definition_cache_tag(
res_opt, self, kind, cache_tag_extra)
if cache_tag:
ctx._cache[cache_tag] = rts_result
else:
# cache found
# get the SymbolSet for this cached object
key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind)
res_opt._symbol_ref_set = ctx.corpus._definition_reference_symbols.get(
key)
# merge child document set with current
curr_sym_ref_set._merge(res_opt._symbol_ref_set)
res_opt._symbol_ref_set = curr_sym_ref_set
return rts_result
def _fetch_resolved_traits(
self, res_opt: 'ResolveOptions') -> 'ResolvedTraitSet':
from cdm.utilities import SymbolSet
from .cdm_corpus_def import CdmCorpusDefinition
res_opt = res_opt if res_opt is not None else ResolveOptions(
self, self.ctx.corpus.default_resolution_directives)
kind = 'rtsb'
ctx = self.ctx
# get referenced trait
trait = self.fetch_object_definition(res_opt)
rts_trait = None
if not trait:
return ctx.corpus._fetch_empty_resolved_trait_set(res_opt)
# see if one is already cached
# cache by name unless there are parameter
if trait._this_is_known_to_have_parameters is None:
# never been resolved, it will happen soon, so why not now?
rts_trait = trait._fetch_resolved_traits(res_opt)
cache_by_path = True
if trait._this_is_known_to_have_parameters is not None:
cache_by_path = not trait._this_is_known_to_have_parameters
cache_tag = ctx.corpus._create_definition_cache_tag(
res_opt, self, kind, '', cache_by_path, trait.at_corpus_path)
rts_result = ctx._cache.get(cache_tag) if cache_tag else None
# store the previous reference symbol set, we will need to add it with
# children found from the _construct_resolved_traits call
curr_sym_ref_set = res_opt._symbol_ref_set or SymbolSet()
res_opt._symbol_ref_set = SymbolSet()
# if not, then make one and save it
if not rts_result:
# get the set of resolutions, should just be this one trait
if not rts_trait:
# store current doc ref set
new_doc_ref_set = res_opt._symbol_ref_set
res_opt._symbol_ref_set = SymbolSet()
rts_trait = trait._fetch_resolved_traits(res_opt)
# bubble up symbol reference set from children
if new_doc_ref_set:
new_doc_ref_set._merge(res_opt._symbol_ref_set)
res_opt._symbol_ref_set = new_doc_ref_set
if rts_trait:
rts_result = rts_trait.deep_copy()
# now if there are argument for this application, set the values in the array
if self.arguments and rts_result:
# if never tried to line up arguments with parameters, do that
if not self._resolved_arguments:
self._resolved_arguments = True
params = trait._fetch_all_parameters(res_opt)
param_found = None
a_value = None
for index, argument in enumerate(self.arguments):
param_found = params.resolve_parameter(
index, argument.get_name())
argument._resolved_parameter = param_found
a_value = argument.value
a_value = ctx.corpus._const_type_check(
res_opt, self.in_document, param_found, a_value)
argument.value = a_value
for argument in self.arguments:
rts_result.set_parameter_value_from_argument(
trait, argument)
# register set of possible symbols
ctx.corpus._register_definition_reference_symbols(
self.fetch_object_definition(res_opt), kind,
res_opt._symbol_ref_set)
# get the new cache tag now that we have the list of docs
cache_tag = ctx.corpus._create_definition_cache_tag(
res_opt, self, kind, '', cache_by_path, trait.at_corpus_path)
if cache_tag:
ctx._cache[cache_tag] = rts_result
else:
# cache was found
# get the SymbolSet for this cached object
key = CdmCorpusDefinition._fetch_cache_key_from_object(self, kind)
res_opt._symbol_ref_set = ctx.corpus._definition_reference_symbols.get(
key)
# merge child document set with current
curr_sym_ref_set._merge(res_opt._symbol_ref_set)
res_opt._symbol_ref_set = curr_sym_ref_set
return rts_result
async def logical_manipulation_using_projections(
self, corpus: CdmCorpusDefinition):
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async(
'output:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('local:/Address.cdm.json')
entity = logical_doc.definitions.append(
'Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append(
'name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append(
'age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append(
'email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print(
'Get \'resolved\' folder where the resolved entities will be saved.'
)
resolved_folder = await corpus.fetch_object_async(
'output:/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
res_normalized_entity = await entity.create_resolved_entity_async(
f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(
f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
res_reference_only_entity = await entity.create_resolved_entity_async(
f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(
f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
res_structured_entity = await entity.create_resolved_entity_async(
f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(
f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({})
res_array_entity = await entity.create_resolved_entity_async(
f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(
f'{res_array_entity.entity_name}.cdm.json')
def fetch_resolved_entity_references(
self,
res_opt: Optional['ResolveOptions'] = None
) -> 'ResolvedEntityReferenceSet':
res_opt = res_opt if res_opt is not None else ResolveOptions(
wrt_doc=self)
from cdm.resolvedmodel import AttributeResolutionContext, ResolvedEntityReference, ResolvedEntityReferenceSide, ResolvedEntityReferenceSet
from cdm.utilities import ResolveOptions
from .cdm_object import CdmObject
rts_this_att = self._fetch_resolved_traits(res_opt)
res_guide = self.resolution_guidance
# this context object holds all of the info about what needs to happen to resolve these attributes
arc = AttributeResolutionContext(res_opt, res_guide, rts_this_att)
rel_info = self._get_relationship_info(res_opt, arc)
if not rel_info.is_by_ref or rel_info.is_array:
return None
# only place this is used, so logic here instead of encapsulated.
# make a set and the one ref it will hold
rers = ResolvedEntityReferenceSet(res_opt)
rer = ResolvedEntityReference()
# referencing attribute(s) come from this attribute
rer.referencing._rasb.merge_attributes(
self._fetch_resolved_attributes(res_opt))
def resolve_side(
ent_ref: 'CdmEntityReference') -> ResolvedEntityReferenceSide:
side_other = ResolvedEntityReferenceSide()
if ent_ref:
# reference to the other entity, hard part is the attribue name.
# by convention, this is held in a trait that identifies the key
side_other.entity = ent_ref.fetch_object_definition(res_opt)
if side_other.entity:
other_attribute = None
other_opts = ResolveOptions(wrt_doc=res_opt.wrt_doc,
directives=res_opt.directives)
trait = ent_ref._fetch_resolved_traits(other_opts).find(
other_opts, 'is.identifiedBy')
if trait and trait.parameter_values and trait.parameter_values.length:
other_ref = trait.parameter_values.fetch_parameter_value(
'attribute').value
if isinstance(other_ref, CdmObject):
other_attribute = other_ref.fetch_object_definition(
other_opts)
if other_attribute:
side_other._rasb.own_one(
side_other.entity.
_fetch_resolved_attributes(other_opts).get(
other_attribute.get_name()).copy())
return side_other
# either several or one entity
# for now, a sub for the 'select one' idea
if self.entity.explicit_reference:
ent_pick_from = self.entity.fetch_object_definition(res_opt)
atts_pick = ent_pick_from.attributes
if ent_pick_from and atts_pick:
for attribute in atts_pick:
if attribute.object_type == CdmObjectType.ENTITY_ATTRIBUTE_DEF:
entity = attribute.entity
rer.referenced.append(resolve_side(entity))
else:
rer.referenced.append(resolve_side(self.entity))
rers.rer_set.append(rer)
return rers
def fetch_replacement_value(cls, res_opt: 'ResolveOptions',
old_value: 'CdmArgumentValue',
new_value: 'CdmArgumentValue',
was_set: bool) -> 'CdmArgumentValue':
from cdm.objectmodel import CdmObject
if old_value is None:
return new_value
if not was_set:
# Must explicitly set a value to override if a new value is not set, then new_value holds nothing or the
# default. In this case, if there was already a value in this argument then just keep using it.
return old_value
if not isinstance(old_value, CdmObject):
return new_value
ov = old_value # type: CdmObject
nv = new_value # type: CdmObject
# Replace an old table with a new table? Actually just mash them together.
if (ov is not None and ov.object_type == CdmObjectType.ENTITY_REF
and nv is not None and not isinstance(nv, str)
and nv.object_type == CdmObjectType.ENTITY_REF):
old_ent = ov.fetch_object_definition(
res_opt) # type: CdmConstantEntityDefinition
new_ent = nv.fetch_object_definition(
res_opt) # type: CdmConstantEntityDefinition
# Check that the entities are the same shape.
if new_ent is None:
return ov
# BUG
ent_def_shape = old_ent.entity_shape.fetch_object_definition(
res_opt)
if (old_ent is None or
(ent_def_shape !=
new_ent.entity_shape.fetch_object_definition(res_opt))):
return nv
old_cv = old_ent.constant_values
new_cv = new_ent.constant_values
# Rows in old?
if not old_cv:
return nv
# Rows in new?
if not new_cv:
return ov
# Make a set of rows in the old one and add the new ones. This will union the two find rows in the new
# one that are not in the old one. Slow, but these are small usually.
unioned_rows = OrderedDict()
# see if any of the entity atts are the primary key, meaning, the only thing that causes us to merge dups unique.
# i know this makes you think about a snake eating its own tail, but fetch the resolved attributes of the constant shape
pk_att = -1
if ent_def_shape:
from cdm.utilities import ResolveOptions
res_opt_shape = ResolveOptions(ent_def_shape.in_document)
res_atts_shape = ent_def_shape._fetch_resolved_attributes(
res_opt_shape) # type: ResolvedAttributeSet
if res_atts_shape:
tmp_item = next(
filter(
lambda ra: (ra.resolved_traits.find(
res_opt_shape, 'is.identifiedBy') is not None),
res_atts_shape._set), None)
pk_att = res_atts_shape._set.index(
tmp_item) if tmp_item else -1
for i in range(len(old_cv)):
row = old_cv[i]
# the entity might have a PK, if so, only look at that values as the key
if pk_att != -1:
key = row[pk_att]
else:
key = '::'.join(row)
unioned_rows[key] = row
for i in range(len(new_cv)):
row = new_cv[i]
# the entity might have a PK, if so, only look at that values as the key
if pk_att != -1:
key = row[pk_att]
else:
key = '::'.join(row)
unioned_rows[key] = row
if len(unioned_rows) == len(old_cv):
return ov
replacement_ent = old_ent.copy(
res_opt) # type: CdmConstantEntityDefinition
replacement_ent.constant_values = list(unioned_rows.values())
return res_opt.wrt_doc.ctx.corpus.make_ref(
CdmObjectType.ENTITY_REF, replacement_ent, False)
return new_value
async def populate_manifest_relationships_async(
self,
option: CdmRelationshipDiscoveryStyle = CdmRelationshipDiscoveryStyle.
ALL
) -> None:
"""Populate the relationships that the entities in the current manifest are involved in."""
with logger._enter_scope(
self._TAG, self.ctx,
self.populate_manifest_relationships_async.__name__):
self.relationships.clear()
rel_cache = set() # Set[str]
for ent_dec in self.entities:
ent_path = await self._get_entity_path_from_declaration(
ent_dec, self)
curr_entity = await self.ctx.corpus.fetch_object_async(
ent_path) # type: Optional[CdmEntityDefinition]
if curr_entity is None:
continue
# handle the outgoing relationships
outgoing_rels = self.ctx.corpus.fetch_outgoing_relationships(
curr_entity) # List[CdmE2ERelationship]
if outgoing_rels:
for rel in outgoing_rels:
cache_key = rel2_cache_key(rel)
if cache_key not in rel_cache and self._is_rel_allowed(
rel, option):
self.relationships.append(
self._localize_rel_to_manifest(rel))
rel_cache.add(cache_key)
incoming_rels = self.ctx.corpus.fetch_incoming_relationships(
curr_entity) # type: List[CdmE2ERelationship]
if incoming_rels:
for in_rel in incoming_rels:
# get entity object for current toEntity
current_in_base = await self.ctx.corpus.fetch_object_async(
in_rel.to_entity,
self) # type: Optional[CdmEntityDefinition]
if not current_in_base:
continue
# create graph of inheritance for to current_in_base
# graph represented by an array where entity at i extends entity at i+1
to_inheritance_graph = [
] # type: List[CdmEntityDefinition]
while current_in_base:
res_opt = ResolveOptions(
wrt_doc=current_in_base.in_document)
current_in_base = current_in_base.extends_entity.fetch_object_definition(
res_opt
) if current_in_base.extends_entity else None
if current_in_base:
to_inheritance_graph.append(current_in_base)
# add current incoming relationship
cache_key = rel2_cache_key(in_rel)
if cache_key not in rel_cache and self._is_rel_allowed(
in_rel, option):
self.relationships.append(
self._localize_rel_to_manifest(in_rel))
rel_cache.add(cache_key)
# if A points at B, A's base classes must point at B as well
for base_entity in to_inheritance_graph:
incoming_rels_for_base = self.ctx.corpus.fetch_incoming_relationships(
base_entity) # type: List[CdmE2ERelationship]
if incoming_rels_for_base:
for in_rel_base in incoming_rels_for_base:
new_rel = self.ctx.corpus.make_object(
CdmObjectType.E2E_RELATIONSHIP_DEF, '')
new_rel.from_entity = in_rel_base.from_entity
new_rel.from_entity_attribute = in_rel_base.from_entity_attribute
new_rel.to_entity = in_rel.to_entity
new_rel.to_entity_attribute = in_rel.to_entity_attribute
base_rel_cache_key = rel2_cache_key(
new_rel)
if base_rel_cache_key not in rel_cache and self._is_rel_allowed(
new_rel, option):
self.relationships.append(
self._localize_rel_to_manifest(
new_rel))
rel_cache.add(base_rel_cache_key)
if self.sub_manifests:
for sub_manifest_def in self.sub_manifests:
corpus_path = self.ctx.corpus.storage.create_absolute_corpus_path(
sub_manifest_def.definition, self)
sub_manifest = await self.ctx.corpus.fetch_object_async(
corpus_path) # type: Optional[CdmManifestDefinition]
await sub_manifest.populate_manifest_relationships_async(
option)
async def refresh_async(self, res_opt: Optional['ResolveOptions'] = None) -> bool:
"""updates indexes for document content, call this after modifying objects in the document"""
res_opt = res_opt if res_opt is not None else ResolveOptions(wrt_doc=self)
self._needs_indexing = True
return await self._index_if_needed(res_opt)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddTypeAttribute operation and a condition using the object model"""
corpus = ProjectionTestUtils.get_local_corpus(
self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount(
'local',
LocalAdapter(
TestHelper.get_actual_output_folder_path(
self.tests_subpath,
'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an AddTypeAttribute operation
add_type_attr_op = corpus.make_object(
CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF)
add_type_attr_op.type_attribute = corpus.make_object(
CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testType')
add_type_attr_op.type_attribute.data_type = corpus.make_ref(
CdmObjectType.DATA_TYPE_REF, 'entityName', True)
projection.operations.append(add_type_attr_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(
set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the AddTypeAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# Type attribute: "testType"
self.assertEqual(5,
len(resolved_entity_with_reference_only.attributes))
self.assertEqual(
'id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual(
'name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual(
'value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual(
'date', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual(
'testType', resolved_entity_with_reference_only.attributes[4].name)
self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4].
applied_traits.item('is.linkedEntity.name'))
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(
set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), res_opt,
local_root)
# Verify correctness of the resolved attributes after running the AddTypeAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# No Type attribute added, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id',
resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name',
resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value',
resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date',
resolved_entity_with_structured.attributes[3].name)
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._TAG, self._ctx, 'request file: {}'.format(doc_path), self._load_document_from_path_async.__name__)
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._TAG, self._ctx, 'received file: {}'.format(doc_path), self._load_document_from_path_async.__name__)
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._TAG, self._ctx, 'fail file: {}'.format(doc_path), self._load_document_from_path_async.__name__)
message = 'Could not read {} from the \'{}\' namespace.\n Reason: {}'.format(doc_path, folder.namespace, e)
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__)
else:
logger.error(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(doc_path)
except Exception as e:
logger.warning(self._TAG, self._ctx, 'Failed to compute file last modified time. Reason {}'.format(e), self._load_document_from_path_async.__name__)
if not doc_name:
logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._load_document_from_path_async.__name__)
return None
# If loading an odi.json/model.json file, check that it is named correctly.
if doc_name.lower().endswith(self.ODI_EXTENSION) and not doc_name.lower() == self.ODI_EXTENSION:
logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format(
doc_name, self.ODI_EXTENSION), self._load_document_from_path_async.__name__)
return None
if doc_name.lower().endswith(self.MODEL_JSON_EXTENSION) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format(
doc_name, self.MODEL_JSON_EXTENSION), self._load_document_from_path_async.__name__)
return None
# Fetch the correct persistence class to use.
persistence_class = self._fetch_registered_persistence_format(doc_name)
if persistence_class:
try:
method = persistence_class.from_data
parameters = [self._ctx, doc_name, json_data, folder]
# check if from_data() is asynchronous for this persistence class.
if persistence_class not in self._is_registered_persistence_async:
# Cache whether this persistence class has async methods.
self._is_registered_persistence_async[persistence_class] = persistence_class.is_persistence_async
if self._is_registered_persistence_async[persistence_class]:
doc_content = await method(*parameters)
else:
doc_content = method(*parameters)
except Exception as e:
logger.error(self._TAG, self._ctx, 'Could not convert \'{}\'. Reason \'{}\''.format(doc_name, e), self._load_document_from_path_async.__name__)
return None
else:
# could not find a registered persistence class to handle this document type.
logger.error(self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\''.format(
doc_name), self._load_document_from_path_async.__name__)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus.default_resolution_directives), doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
def _finalize_attribute_context(self, res_opt: ResolveOptions,
path_start: str,
doc_home: CdmDocumentDefinition,
doc_from: CdmDocumentDefinition,
moniker_for_doc_from: str, finished: bool):
# run over the attCtx tree again and 'fix it' fix means replace the parent and lineage reference path strings with
# final values from new home and set the inDocument and fix any references to definitions
# keep track of the paths to documents for fixing symbol refs. expensive to compute
found_doc_paths = OrderedDict()
if moniker_for_doc_from and not moniker_for_doc_from.isspace():
moniker_for_doc_from = '{}/'.format(moniker_for_doc_from)
# first step makes sure every node in the tree has a good path for itself and a good document
# second pass uses the paths from nodes to fix references to other nodes
def _fix_att_ctx_node_paths(sub_item: CdmObject,
path_from: str) -> None:
ac = cast(CdmAttributeContext, sub_item)
if ac is None:
return
ac.in_document = doc_home
# fix up the reference to definition. need to get path from this document to the
# add moniker if this is a reference
if ac.definition:
ac.definition.in_document = doc_home
if ac.definition and ac.definition.named_reference:
# need the real path to this thing from the explicitRef held in the portable reference
# the real path is {monikerFrom/}{path from 'from' document to document holding the explicit ref/{declaredPath of explicitRef}}
# if we have never looked up the path between docs, do that now
doc_from_def = ac.definition.explicit_reference.in_document # if all parts not set, this is a broken portal ref!
path_between_docs = found_doc_paths[
doc_from_def] if doc_from_def in found_doc_paths else None
if path_between_docs is None:
path_between_docs = doc_from._import_path_to_doc(
doc_from_def)
if path_between_docs is None:
# hmm. hmm.
path_between_docs = ''
found_doc_paths[doc_from] = path_between_docs
cast('CdmObjectReference',
ac.definition)._localize_portable_reference(
res_opt, '{}{}'.format(moniker_for_doc_from,
path_between_docs))
# doc of parent ref
if ac.parent:
ac.parent.in_document = doc_home
# doc of lineage refs
if ac.lineage:
for lin in ac.lineage:
lin.in_document = doc_home
divider = '/' if not ac.at_corpus_path or not path_from.endswith(
'/') else ''
ac.at_corpus_path = '{}{}{}'.format(path_from, divider, ac.name)
if not ac.contents:
return
# look at all children
for sub_sub in ac.contents:
if sub_sub.object_type == CdmObjectType.ATTRIBUTE_CONTEXT_DEF:
_fix_att_ctx_node_paths(sub_sub, ac.at_corpus_path)
_fix_att_ctx_node_paths(self, path_start)
# now fix any lineage and parent references
def _fix_att_ctx_node_lineage(sub_item) -> None:
ac = cast(CdmAttributeContext,
sub_item) # type: CdmAttributeContext
if not ac:
return
# for debugLindeage, write id
# ac.name = '{}({})'.format(ac.name, ac.id)
# parent ref
if ac.parent and ac.parent.explicit_reference:
ac.parent.named_reference = cast(
CdmAttributeContext,
ac.parent.explicit_reference).at_corpus_path
# for debugLindeage, write id
# ac.parent.named_reference = cast(CdmAttributeContext, ac.parent.explicit_reference).at_corpus_path + '(' + ac.parent.explicit_reference.id + ')'
# fix lineage
if ac.lineage:
for lin in ac.lineage:
if lin.explicit_reference:
# use the new path as the ref
lin.named_reference = cast(
CdmAttributeContext,
lin.explicit_reference).at_corpus_path
# for debugLindeage, write id
# lin.named_reference = cast(CdmAttributeContext, lin.explicit_reference).at_corpus_path + '(' + lin.explicit_reference.id + ')'
if not ac.contents:
return
# look at all children
for sub_sub in ac.contents:
_fix_att_ctx_node_lineage(sub_sub)
_fix_att_ctx_node_lineage(self)
if finished:
res_opt._save_resolutions_on_copy = False
res_opt._map_old_ctx_to_new_ctx = None
return True
async def logical_manipulation_using_projections():
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
A single logical definition can be resolved into multiple physical layouts. The directives are used to instruct the ObjectModel about how it should to
resolve the logical definition provided. To achieve this, we define projections that run conditionally, depending on the directives provided when
calling create_resolved_entity_async.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
# Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
corpus = CdmCorpusDefinition()
print('Configure storage adapters')
# Configure storage adapters to point at the target local manifest location and at the fake public standards
path_from_exe_to_example_root = '../../'
corpus.storage.mount(
'local',
LocalAdapter(path_from_exe_to_example_root +
'8-logical-manipulation-using-projections/sample-data'))
corpus.storage.default_namespace = 'local' # local is our default. so any paths that start out navigating without a device tag will assume local
# Fake cdm, normaly use the CDM Standards adapter
# Mount it as the 'cdm' device, not the default so must use 'cdm:/folder' to get there
corpus.storage.mount(
'cdm',
LocalAdapter(path_from_exe_to_example_root +
'example-public-standards'))
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async(
'local:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('Address.cdm.json')
entity = logical_doc.definitions.append(
'Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append(
'name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append(
'age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append(
'email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print('Get \'resolved\' folder where the resolved entities will be saved.')
resolved_folder = await corpus.fetch_object_async(
'local:/resolved/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({'normalized'})
res_normalized_entity = await entity.create_resolved_entity_async(
f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(
f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
res_reference_only_entity = await entity.create_resolved_entity_async(
f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(
f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'structured'})
res_structured_entity = await entity.create_resolved_entity_async(
f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(
f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({})
res_array_entity = await entity.create_resolved_entity_async(
f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(
f'{res_array_entity.entity_name}.cdm.json')
