async def test_not_saving_config_file(self):
"""Test setting SaveConfigFile to false and checking if the file is not saved."""
test_name = 'test_not_saving_config_file'
corpus = TestHelper.get_local_corpus(self.test_subpath, test_name)
# Load manifest from input folder.
manifest = await corpus.fetch_object_async('default.manifest.cdm.json')
# Move manifest to output folder.
output_folder = corpus.storage.fetch_root_folder('output')
for entity_dec in manifest.entities:
entity = await corpus.fetch_object_async(entity_dec.entity_path, manifest)
output_folder.documents.append(entity.in_document)
output_folder.documents.append(manifest)
# Make sure the output folder is empty.
TestHelper.delete_files_from_actual_output(TestHelper.get_actual_output_folder_path(self.test_subpath, test_name))
# Save manifest to output folder.
copy_options = CopyOptions()
copy_options.save_config_file = False
await manifest.save_as_async("default.manifest.cdm.json", False, copy_options)
# Compare the result.
TestHelper.compare_folder_files_equality(
TestHelper.get_expected_output_folder_path(self.test_subpath, test_name),
TestHelper.get_actual_output_folder_path(self.test_subpath, test_name))
async def test_ref_entity_with_slash_path(self):
test_name = 'test_ref_entity_with_slash_path'
slash_corpus = TestHelper.get_local_corpus(self.tests_subpath,
test_name)
slash_local_path = (
slash_corpus.storage.namespace_adapters.get('local')).root
slash_adapter = LocalAdapterWithSlashPath(slash_local_path, '/')
slash_corpus.storage.mount('slash', slash_adapter)
slash_corpus.storage.defaultNamespace = 'slash'
# load model.json files with paths generated using both '/' and '\'
slash_manifest = await slash_corpus.fetch_object_async(
'slash:/model.json')
# manually add the reference model location, path will vary on each machine
ref_model_trait = slash_manifest.exhibits_traits.item(
'is.modelConversion.referenceModelMap')
entity_path = slash_manifest.entities[0].entity_path
ref_model_trait.arguments[0].value[
0].location = slash_adapter.create_adapter_path(
entity_path[0:entity_path.rindex('/')])
slash_model = await ManifestPersistence.to_data(
slash_manifest, ResolveOptions(), CopyOptions())
self.assertIsNotNone(slash_model)
self.assertEqual(1, len(slash_model.entities))
back_slash_corpus = TestHelper.get_local_corpus(
self.tests_subpath, test_name)
back_slash_local_path = (
back_slash_corpus.storage.namespace_adapters.get('local')).root
back_slash_adapter = LocalAdapterWithSlashPath(back_slash_local_path,
'\\')
back_slash_corpus.storage.mount('backslash', back_slash_adapter)
back_slash_corpus.storage.default_namespace = 'backslash'
back_slash_manifest = await back_slash_corpus.fetch_object_async(
'backslash:/model.json')
# manually add the reference model location, path will vary on each machine
back_slash_ref_model_trait = back_slash_manifest.exhibits_traits.item(
'is.modelConversion.referenceModelMap')
back_slash_entity_path = back_slash_manifest.entities[0].entity_path
back_slash_ref_model_trait.arguments[0].value[
0].location = back_slash_adapter.create_adapter_path(
back_slash_entity_path[0:back_slash_entity_path.
rindex('/')]).replace('/', '\\\\')
back_slash_model = await ManifestPersistence.to_data(
back_slash_manifest, ResolveOptions(), CopyOptions())
self.assertIsNotNone(back_slash_model)
self.assertEqual(1, len(back_slash_model.entities))
async def save_as_async(self,
new_name: str,
save_referenced: bool = False,
options: Optional['CopyOptions'] = None) -> bool:
"""saves the document back through the adapter in the requested format
format is specified via document name/extension based on conventions:
'model.json' for back compat model, '*.manifest.json' for manifest, '*.json' for cdm defs
save_referenced (default False) when true will also save any schema defintion documents that are
linked from the source doc and that have been modified. existing document names are used for those."""
options = options if options is not None else CopyOptions()
index_if_needed = await self._index_if_needed(
ResolveOptions(
wrt_doc=self,
directives=self.ctx.corpus.default_resolution_directives))
if not index_if_needed:
logger.error(
self._TAG, self.ctx,
'Failed to index document prior to save {}.'.format(self.name),
self.save_as_async.__name__)
return False
if new_name == self.name:
self._is_dirty = False
return await self.ctx.corpus.persistence._save_document_as_async(
self, options, new_name, save_referenced)
async def _save_linked_documents_async(self,
options: 'CopyOptions') -> bool:
docs = []
if not options:
options = CopyOptions()
# the only linked documents would be the imports
if self.imports:
for imp in self.imports:
# get the document object from the import
doc_path = self.ctx.corpus.storage.create_absolute_corpus_path(
imp.corpus_path, self)
if not doc_path:
logger.error(self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_VALDN_INVALID_CORPUS_PATH,
imp.corpus_path)
return False
try:
obj_at = await self.ctx.corpus.fetch_object_async(doc_path)
if not isinstance(obj_at, CdmDocumentDefinition):
logger.error(
self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path, CdmLogCode.ERR_INVALID_CAST,
doc_path, 'CdmDocumentDefinition')
return False
elif not obj_at:
logger.error(
self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_PERSIST_OBJECT_NOT_FOUND,
imp.corpus_path)
return False
doc_imp = obj_at.in_document
if doc_imp is not None and doc_imp._is_dirty:
docs.append(doc_imp)
except Exception as e:
logger.error(self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_PERSIST_OBJECT_NOT_FOUND,
imp.corpus_path + ' ' + str(e))
return False
for doc_imp in docs:
# save it with the same name
if not await doc_imp.save_as_async(doc_imp.name, True,
options):
logger.error(self.ctx, self._TAG,
self._save_linked_documents_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_IMPORT_SAVING_FAILURE,
doc_imp.name)
return False
return True
async def save_as_async(self,
new_name: str,
save_referenced: bool = False,
options: Optional['CopyOptions'] = None) -> bool:
"""saves the document back through the adapter in the requested format
format is specified via document name/extension based on conventions:
'model.json' for back compat model, '*.manifest.json' for manifest, '*.json' for cdm defs
save_referenced (default False) when true will also save any schema defintion documents that are
linked from the source doc and that have been modified. existing document names are used for those."""
with logger._enter_scope(self._TAG, self.ctx,
self.save_as_async.__name__):
options = options if options is not None else CopyOptions()
index_if_needed = await self._index_if_needed(
ResolveOptions(
wrt_doc=self,
directives=self.ctx.corpus.default_resolution_directives))
if not index_if_needed:
logger.error(self.ctx, self._TAG, self.save_as_async.__name__,
self.at_corpus_path, CdmLogCode.ERR_INDEX_FAILED,
self.name)
return False
if new_name == self.name:
self._is_dirty = False
# Import here to avoid circular import
from .cdm_entity_def import CdmEntityDefinition
from .cdm_manifest_def import CdmManifestDefinition
if not await self.ctx.corpus.persistence._save_document_as_async(
self, options, new_name, save_referenced):
return False
# Log the telemetry if the document is a manifest
if isinstance(self, CdmManifestDefinition):
for entity in self.entities:
if isinstance(entity, CdmLocalEntityDeclarationDefinition):
entity.reset_last_file_modified_old_time()
for relationship in self.relationships:
relationship.reset_last_file_modified_old_time()
logger._ingest_manifest_telemetry(
self, self.ctx, CdmDocumentDefinition.__name__,
self.save_as_async.__name__, self.at_corpus_path)
# Log the telemetry of all entities contained in the document
else:
for obj in self.definitions:
if isinstance(obj, CdmEntityDefinition):
logger._ingest_entity_telemetry(
obj, self.ctx, CdmDocumentDefinition.__name__,
self.save_as_async.__name__, obj.at_corpus_path)
return True
def fetch_value_string(self, res_opt: 'ResolveOptions') -> str:
from cdm.objectmodel import CdmObject
if self.value is None:
return ''
if isinstance(self.value, str):
return self.value
elif isinstance(self.value, CdmObject):
# If this is a constant table, then expand into an HTML table.
object_def = self.value.fetch_object_definition(
res_opt) # type: CdmConstantEntityDefinition
if self.value.object_type == CdmObjectType.ENTITY_REF and object_def is not None and object_def.object_type == CdmObjectType.CONSTANT_ENTITY_DEF:
ent_shape = object_def.entity_shape
ent_values = object_def.constant_values
if not ent_values:
return ''
rows = []
shape_atts = ent_shape._fetch_resolved_attributes(res_opt)
if shape_atts is not None and shape_atts._set is not None:
for row_data in ent_values:
if not row_data:
continue
row = OrderedDict()
for (c, tvalue) in enumerate(row_data):
col_att = shape_atts._set[c]
if col_att is not None and tvalue is not None:
row[col_att.resolved_name] = tvalue
rows.append(row)
rows_string = [self._spew_dict(row) for row in rows]
return '[' + ','.join(rows_string) + ']'
# Should be a reference to an object.
from cdm.persistence import PersistenceLayer
from cdm.utilities import CopyOptions
data = PersistenceLayer.to_data(self.value, res_opt,
CopyOptions(string_refs=False),
PersistenceLayer.CDM_FOLDER)
if isinstance(data, str):
return data
# TODO: the line bellow won't work, the result is going to be the address of the object.
return str(data)
else:
return str(self.value)
return ''
async def save_as_async(self, new_name: str, save_referenced: bool = False, options: Optional['CopyOptions'] = None) -> bool:
"""saves the document back through the adapter in the requested format
format is specified via document name/extension based on conventions:
'model.json' for back compat model, '*.manifest.json' for manifest, '*.json' for cdm defs
save_referenced (default false) when true will also save any schema defintion documents that are
linked from the source doc and that have been modified. existing document names are used for those."""
options = options if options is not None else CopyOptions()
if new_name == self.name:
self._is_dirty = False
return await self.ctx.corpus._save_document_as(self, options, new_name, save_referenced)
async def test_imports_for_rel_elevated_purpose_traits(self):
"""
Testing that import for elevated purpose traits for relationships are added.
"""
test_name = 'test_imports_for_rel_elevated_purpose_traits'
corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
root_manifest = await corpus.fetch_object_async(
'local:/default.manifest.cdm.json'
) # type: 'CdmManifestDefinition'
sub_manifest = await corpus.fetch_object_async(
root_manifest.sub_manifests[0].definition)
await corpus.calculate_entity_graph_async(root_manifest)
await root_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
self.assertEqual('specialized/Gold.cdm.json',
root_manifest.imports[0].corpus_path)
self.assertEqual('/Lead.cdm.json', sub_manifest.imports[0].corpus_path)
corpus.storage.fetch_root_folder('output').documents.append(
root_manifest)
corpus.storage.fetch_root_folder('output').documents.append(
sub_manifest)
copy_options = CopyOptions()
copy_options.save_config_file = False
await root_manifest.save_as_async('output:/default.manifest.cdm.json',
False, copy_options)
# "acct.trait" in Acct.cdm.json. relationships in the manifests contain these 2 traits,
# so the manifest should import these two entity documents, but Lead.cdm.json imports Acct.cdm.json.
# Thus, the manifest can only import Lead.cdm.json
await sub_manifest.save_as_async(
'output:/default-submanifest.manifest.cdm.json', False,
copy_options)
TestHelper.compare_folder_files_equality(
TestHelper.get_expected_output_folder_path(self.tests_subpath,
test_name),
TestHelper.get_actual_output_folder_path(self.tests_subpath,
test_name))
async def test_loading_and_saving_csv_partition_traits(self):
"""Tests that the trait is.partition.format.CSV is saved when contains arguments not supported by fileFormatSettings."""
cdm_corpus = TestHelper.get_local_corpus(
self.tests_subpath, 'test_loading_and_saving_csv_partition_traits')
manifest = await cdm_corpus.fetch_object_async(
'model.json') # type: CdmManifestDefinition
# If the data partition trait is.partition.format.CSV being saved has arguments that are not supported by fileFormatSettings
# the trait should also be persisted.
manifestData = await ManifestPersistence.to_data(
manifest, ResolveOptions(manifest.in_document), CopyOptions())
localEntity = manifestData.entities[0]
self.assertEqual(len(localEntity.partitions[0].traits), 1)
# Remove the argument that is not supported by fileFormatSettings and check if the trait is removed after that.
csv_trait = manifest.entities[0].data_partitions[0].exhibits_traits[0]
csv_trait.arguments.remove(csv_trait.arguments.item('newline'))
manifestData = await ManifestPersistence.to_data(
manifest, ResolveOptions(manifest.in_document), CopyOptions())
localEntity = manifestData.entities[0]
self.assertIsNone(localEntity.partitions[0].traits)
async def save_actual_entity_and_validate_with_expected(
self,
expected_path: str,
actual_resolved_entity_def: CdmEntityDefinition,
update_expected_output: bool = False) -> None:
"""Runs validation to test actual output vs expected output for attributes collection vs attribute context."""
co = CopyOptions()
co._is_top_level_document = False
await actual_resolved_entity_def.in_document.save_as_async(
actual_resolved_entity_def.in_document.name, options=co)
actual_path = actual_resolved_entity_def.ctx.corpus.storage.corpus_path_to_adapter_path(
actual_resolved_entity_def.in_document.at_corpus_path)
with open(actual_path, 'r', encoding='utf-8') as actual_file:
actual_ctx = actual_file.read()
if update_expected_output:
with open(expected_path, 'w', encoding='utf-8') as expected_file:
expected_file.write(actual_ctx)
with open(expected_path, 'r', encoding='utf-8') as expected_file:
expected_ctx = expected_file.read()
self.assertEqual(expected_ctx, actual_ctx)
async def test_from_and_to_data_with_elevated_traits(self):
corpus = TestHelper.get_local_corpus(
self.tests_subpath, 'test_from_and_to_data_with_elevated_traits')
# need to set schema docs to the cdm namespace instead of using resources
corpus.storage.mount("cdm",
LocalAdapter(TestHelper.get_schema_docs_root()))
def callback(level, message):
self.assertTrue('unable to resolve an entity' not in message)
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
entity = await corpus.fetch_object_async(
'local:/Account.cdm.json/Account') # type: CdmEntityDefinition
res_entity = await entity.create_resolved_entity_async(
'{}_'.format(entity.entity_name)) # type: CdmEntityDefinition
PersistenceLayer.to_data(
res_entity, ResolveOptions(wrt_doc=res_entity.in_document),
CopyOptions(string_refs=True), PersistenceLayer.CDM_FOLDER)
async def test_cardinality_persistence(self):
'''
Testing that cardinality settings are loaded and saved correctly
'''
corpus = TestHelper.get_local_corpus(self.tests_subpath,
'test_cardinality_persistence')
# test from_data
entity = await corpus.fetch_object_async(
'local:/someEntity.cdm.json/someEntity'
) # type: CdmEntityDefinition
attribute = entity.attributes[0] # type: CdmTypeAttributeDefinition
self.assertIsNotNone(attribute.cardinality)
self.assertEqual(attribute.cardinality.minimum, '0')
self.assertEqual(attribute.cardinality.maximum, '1')
# test to_data
attribute_data = TypeAttributePersistence.to_data(
attribute, ResolveOptions(entity.in_document), CopyOptions())
self.assertIsNotNone(attribute_data.cardinality)
self.assertEqual(attribute_data.cardinality.minimum, '0')
self.assertEqual(attribute_data.cardinality.maximum, '1')
async def test_update_relationships(self):
test_name = 'test_update_relationships'
expected_rels = TestHelper.get_expected_output_data(
self.tests_subpath, test_name, 'expectedRels.json')
temp_from_file_path = 'fromEntTemp.cdm.json'
temp_from_entity_path = 'local:/fromEntTemp.cdm.json/fromEnt'
temp_to_entity_path = 'local:/toEnt.cdm.json/toEnt'
corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name)
manifest = await corpus.fetch_object_async(
'local:/main.manifest.cdm.json') # type: CdmManifestDefinition
manifest_no_to_ent = await corpus.fetch_object_async(
'local:/mainNoToEnt.manifest.cdm.json'
) # type: CdmManifestDefinition
from_ent = await corpus.fetch_object_async(
'local:/fromEnt.cdm.json/fromEnt') # type: CdmEntityDefinition
co = CopyOptions()
co._is_top_level_document = False
await from_ent.in_document.save_as_async(temp_from_file_path,
options=co)
async def reload_from_entity():
await from_ent.in_document.save_as_async(temp_from_file_path,
options=co)
# fetch again to reset the cache
await corpus.fetch_object_async(temp_from_entity_path,
None,
shallow_validation=False,
force_reload=True)
try:
# 1. test when entity attribute is removed
await corpus.calculate_entity_graph_async(manifest)
await manifest.populate_manifest_relationships_async()
# check that the relationship has been created correctly
self.verify_relationships(manifest, expected_rels)
# now remove the entity attribute, which removes the relationship
removed_attribute = from_ent.attributes[
0] # type: CdmAttributeItem
from_ent.attributes.pop(0)
await reload_from_entity()
await corpus.calculate_entity_graph_async(manifest)
await manifest.populate_manifest_relationships_async()
# check that the relationship has been removed
self.verify_relationships(manifest, [])
# 2. test when the to entity is removed
# restore the entity to the original state
from_ent.attributes.append(removed_attribute)
await reload_from_entity()
await corpus.calculate_entity_graph_async(manifest)
await manifest.populate_manifest_relationships_async()
# check that the relationship has been created correctly
self.verify_relationships(manifest, expected_rels)
# remove the to entity
from_ent.attributes.pop(0)
await reload_from_entity()
# fetch again to reset the cache
await corpus.fetch_object_async(temp_to_entity_path, None, False,
True)
await corpus.calculate_entity_graph_async(manifest_no_to_ent)
await manifest_no_to_ent.populate_manifest_relationships_async()
# check that the relationship has been removed
self.verify_relationships(manifest_no_to_ent, [])
finally:
# clean up the file created
from_path = corpus.storage.corpus_path_to_adapter_path(
'local:/' + temp_from_file_path)
if os.path.exists(from_path):
os.remove(from_path)
def fetch_value_string(self, res_opt: 'ResolveOptions') -> str:
from cdm.objectmodel import CdmObject
if self.value is None:
return ''
if isinstance(self.value, str):
return self.value
elif isinstance(self.value, CdmObject):
# If this is a constant table, then expand into an HTML table.
object_def = self.value.fetch_object_definition(
res_opt) # type: CdmConstantEntityDefinition
if self.value.object_type == CdmObjectType.ENTITY_REF and object_def is not None and object_def.object_type == CdmObjectType.CONSTANT_ENTITY_DEF:
ent_shape = object_def.entity_shape
ent_values = object_def.constant_values
if not ent_values:
return ''
rows = []
shape_atts = ent_shape._fetch_resolved_attributes(res_opt)
for row_data in ent_values:
if not row_data:
continue
row = {}
for (c, tvalue) in enumerate(row_data):
col_att = shape_atts.set[c]
if col_att is not None and tvalue is not None:
row[col_att.resolved_name] = tvalue
rows.append(row)
if rows:
keys = list(rows[0].keys())
keys.sort()
first_key = keys[0]
second_key = keys[1] if len(keys) > 1 else keys[0]
rows.sort(key=lambda row: (row[first_key].lower(), row[
second_key].lower()))
rows_string = [self._spew_dict(row) for row in rows]
return '[' + ','.join(rows_string) + ']'
# Should be a reference to an object.
from cdm.persistence import persistence_layer
from cdm.utilities import CopyOptions
data = persistence_layer.to_data(self.value, res_opt, 'CdmFolder',
CopyOptions(string_refs=False))
if isinstance(data, str):
return data
# TODO: the line bellow won't work, the result is going to be the address of the object.
return str(data)
else:
return str(self.value)
return ''
def test_persist_attribute_group_definition(self):
corpus = CdmCorpusDefinition()
att_group = CdmAttributeGroupDefinition(corpus.ctx, 'attGroup')
persisted_group = AttributeGroupPersistence.to_data(
att_group, ResolveOptions(att_group.in_document), CopyOptions())
self.assertIsNotNone(persisted_group.members)
