async def test_loads_and_sets_times_correctly(self):
"""Test modified times for manifest and files beneath it"""
input_path = TestHelper.get_input_folder_path(self.tests_subpath, 'test_loads_and_sets_times_correctly')
time_before_load = datetime.now(timezone.utc)
cdm_corpus = self.get_corpus()
cdm_corpus.storage.mount('someNamespace', LocalAdapter(input_path))
cdm_corpus.storage.mount('local', LocalAdapter(input_path))
cdm_corpus.storage.unmount('cdm')
cdm_corpus.storage.default_namespace = 'local'
cdm_manifest = await cdm_corpus.fetch_object_async('someNamespace:/default.manifest.cdm.json')
status_time_at_load = cdm_manifest.last_file_status_check_time
# hard coded because the time comes from inside the file
self.assertEqual(time_utils._get_formatted_date_string(status_time_at_load), '2019-02-01T15:36:19.410Z')
self.assertIsNotNone(cdm_manifest._file_system_modified_time)
self.assertGreater(time_before_load, cdm_manifest._file_system_modified_time)
time.sleep(1)
await cdm_manifest.file_status_check_async()
self.assertGreater(cdm_manifest.last_file_status_check_time, time_before_load)
self.assertGreater(cdm_manifest.last_file_status_check_time, status_time_at_load)
self.assertEqual(1, len(cdm_manifest.sub_manifests))
self.assertGreater(cdm_manifest.sub_manifests[0].last_file_status_check_time, time_before_load)
self.assertEqual(1, len(cdm_manifest.entities))
self.assertEqual(1, len(cdm_manifest.entities[0].data_partitions))
entity = cdm_manifest.entities[0]
sub_manifest = cdm_manifest.sub_manifests[0]
max_time = time_utils._max_time(entity.last_file_modified_time, sub_manifest.last_file_modified_time)
self.assertEqual(time_utils._get_formatted_date_string(cdm_manifest.last_child_file_modified_time), time_utils._get_formatted_date_string(max_time))
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None,
is_language_specific: Optional[bool] = False, expected_codes: Optional[set] = None,
no_input_and_output_folder: Optional[bool] = False):
"""
Creates a corpus to be used by the tests, which mounts inputFolder, outputFolder, cdm, and remoteAdapter. Will fail on any unexpected warning/error.
@param testSubpath The root of the corpus files.
@param testName The test name.
@param testInputDir The test input directory.
@param isLanguageSpecific Indicate whether there is subfolder called Java, it's used when input is different compared with other languages
@param expectedCodes The error codes that are expected, and they should not block the test.
@param noInputAndOutputFolder No input and output folder needed.
"""
if no_input_and_output_folder:
test_input_dir = 'C:\\dummpyPath'
test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name,
is_language_specific)
test_output_dir = test_input_dir if no_input_and_output_folder else TestHelper.get_actual_output_folder_path(test_subpath, test_name)
cdm_corpus = CdmCorpusDefinition()
def callback(level: CdmStatusLevel, message: str):
TestHelper._fail_on_unexpected_failure(cdm_corpus, message, expected_codes)
cdm_corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir))
cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir))
cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.schema_documents_path))
cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'}))
return cdm_corpus
async def test_from_and_to_data(self):
with open(os.path.join(ROOT_PATH,
'default.manifest.cdm.json')) as manifestFile:
manifestContent = ManifestContent()
manifestContent.decode(manifestFile.read())
corpus = CdmCorpusDefinition()
corpus.storage.default_namespace = 'local'
corpus.ctx.update_logging_options(level='WARNING')
corpus.storage.mount('cdm',
LocalAdapter(root="../CDM.SchemaDocuments"))
corpus.storage.mount('local', LocalAdapter(root=ROOT_PATH))
corpus.storage.mount(
'remote', RemoteAdapter(hosts={"contoso": "http://contoso.com"}))
folder = corpus.storage.fetch_root_folder('local')
manifest = await corpus.fetch_object_async('default.manifest.cdm.json',
folder)
data = await ManifestPersistence.to_data(manifest, None, None)
for entity in manifest.entities:
entity_def = await corpus.fetch_object_async(
entity.entity_path, manifest)
await manifest.save_as_async('test_output/new.manifest.cdm.json',
save_referenced=True)
self.assertDictEqual(json.loads(manifestContent.encode()),
json.loads(data.encode()))
def get_local_corpus(self, test_files_input_root: str, test_files_output_root: Optional[str] = None):
"""Gets local corpus."""
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter(test_files_input_root))
if test_files_output_root:
cdm_corpus.storage.mount('target', LocalAdapter(test_files_output_root))
return cdm_corpus
async def test_syms_saving_and_fetching_document(self):
syms_adapter = SymsTestHelper.create_adapter_with_clientid()
await SymsTestHelper.clean_database(syms_adapter,
SymsTestHelper.DATABASE_NAME)
test_input_path = TestHelper.get_input_folder_path(
self.test_subpath, 'TestSymsSavingAndFetchingDocument')
test_act_output_path = TestHelper.get_actual_output_folder_path(
self.test_subpath, 'TestSymsSavingAndFetchingDocument')
test_exp_output_path = TestHelper.get_expected_output_folder_path(
self.test_subpath, 'TestSymsSavingAndFetchingDocument')
corpus = CdmCorpusDefinition()
adls_adapter1 = SymsTestHelper.create_adapter_clientid_with_shared_key(
1)
adls_adapter2 = SymsTestHelper.create_adapter_clientid_with_shared_key(
2)
local_input_adapter = LocalAdapter(test_input_path)
local_act_output_adapter = LocalAdapter(test_act_output_path)
local_exp_output_adapter = LocalAdapter(test_exp_output_path)
corpus.storage.mount('adls1', adls_adapter1)
corpus.storage.mount('adls2', adls_adapter2)
corpus.storage.mount('syms', syms_adapter)
corpus.storage.mount('localInput', local_input_adapter)
corpus.storage.mount('localActOutput', local_act_output_adapter)
corpus.storage.mount('localExpOutput', local_exp_output_adapter)
corpus.storage.unmount('cdm')
corpus.storage.default_namespace = 'localInput'
manifest = await corpus.fetch_object_async('default.manifest.cdm.json')
manifest.manifest_name = SymsTestHelper.DATABASE_NAME
await self.run_syms_save_manifest(manifest)
await self.run_syms_fetch_manifest(corpus, manifest,
'default.manifest.cdm.json')
await self.run_syms_fetch_document(corpus, manifest)
manifest_modified = await corpus.fetch_object_async(
'defaultmodified.manifest.cdm.json')
manifest_modified.manifest_name = SymsTestHelper.DATABASE_NAME
manifest_modified.entities[0].set_last_file_modified_time(
datetime.datetime.now(datetime.timezone.utc))
await self.run_syms_save_manifest(manifest_modified)
await self.run_syms_fetch_manifest(
corpus, manifest_modified, 'defaultmodified.manifest.cdm.json')
await self.run_syms_fetch_document(corpus, manifest_modified)
await self.run_syms_smart_adls_adapter_mount_logic()
await SymsTestHelper.clean_database(syms_adapter,
SymsTestHelper.DATABASE_NAME)
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None):
test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name)
test_output_dir = TestHelper.get_actual_output_folder_path(test_subpath, test_name)
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.ctx.report_at_level = CdmStatusLevel.WARNING
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir))
cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir))
cdm_corpus.storage.mount('cdm', LocalAdapter('../../schemaDocuments'))
cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'}))
return cdm_corpus
def setup_cdm_corpus(self):
# Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating
# objects and paths.
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.ctx.report_at_level = CdmStatusLevel.ERROR
print('Configure storage adapters')
cdm_corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, self.test_name)))
# Local is our default. So any paths that start out navigating without a device tag will assume local.
cdm_corpus.storage.default_namespace = 'local'
# Fake cdm, normally use the CDM Standards adapter.
cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.sample_schema_folder_path))
return cdm_corpus
async def test_alter_traits_on_enti_attr_proj(self):
"""Test AlterTraits on an entity attribute"""
test_name = 'test_alter_traits_on_enti_attr_proj'
entity_name = 'NewPerson'
corpus = ProjectionTestUtils.get_local_corpus(
self.tests_subpath, test_name) # type: CdmCorpusDefinition
corpus.storage.mount('traitGroup',
LocalAdapter(self.trait_group_file_path))
for res_opt in self.res_opts_combinations:
await ProjectionTestUtils.load_entity_for_resolution_option_and_save(
self, corpus, test_name, self.tests_subpath, entity_name,
res_opt)
entity = await corpus.fetch_object_async(
'local:/{0}.cdm.json/{0}'.format(entity_name)
) # type: CdmEntityDefinition
resolved_entity = await ProjectionTestUtils.get_resolved_entity(
corpus, entity, ['structured'])
self.assertEqual(5, len(resolved_entity.attributes))
self.validate_trait(resolved_entity.attributes[0], 'name', True)
self.validate_trait(resolved_entity.attributes[1], 'age')
self.validate_trait(resolved_entity.attributes[2], 'address')
self.validate_trait(resolved_entity.attributes[3], 'phoneNumber')
self.validate_trait(resolved_entity.attributes[4], 'email')
async def test_combine_ops_nested_proj(self):
"""Test AlterTraits operation nested with IncludeAttributes and RenameAttribute"""
test_name = 'test_combine_ops_nested_proj'
entity_name = 'NewPerson'
corpus = ProjectionTestUtils.get_local_corpus(
self.tests_subpath, test_name) # type: CdmCorpusDefinition
corpus.storage.mount('traitGroup',
LocalAdapter(self.trait_group_file_path))
for res_opt in self.res_opts_combinations:
await ProjectionTestUtils.load_entity_for_resolution_option_and_save(
self, corpus, test_name, self.tests_subpath, entity_name,
res_opt)
entity = await corpus.fetch_object_async(
'local:/{0}.cdm.json/{0}'.format(entity_name)
) # type: CdmEntityDefinition
resolved_entity = await ProjectionTestUtils.get_resolved_entity(
corpus, entity, [])
# Original set of attributes: ["name", "age", "address", "phoneNumber", "email"]
# Include attributes: ["age", "phoneNumber", "name"]
# Add attribute: ["newName"]
# alter traits on ["newName", "name", + { "means.TraitG100" , "JobTitleBase" } - { "means.TraitG300" } ]
# Rename attribute ["newName" -> "renaming-{m}" ]
# alter traits on ["renaming-newName", + { "means.TraitG4(precision:5, scale:15)" } ]
self.assertEqual(4, len(resolved_entity.attributes))
self.validate_trait(resolved_entity.attributes[0], 'age', False, True)
self.validate_trait(resolved_entity.attributes[1], 'phoneNumber',
False, True)
self.validate_trait(resolved_entity.attributes[2], 'name')
self.validate_trait(resolved_entity.attributes[3], 'renaming-newName',
True)
def __init__(self, class_name: str, test_name: str):
self._foundation_json_path = 'cdm:/foundations.cdm.json' # type: str
self._local_output_storage_ns = 'output' # type: str
self._manifest_name = 'default' # type: str
self._manifest_doc_name = '{}.manifest.cdm.json'.format(
self._manifest_name) # type: str
self._all_imports_name = '_allImports' # type: str
self._all_imports_doc_name = '{}.cdm.json'.format(
self._all_imports_name) # type: str
self._class_name = class_name # type: str
self._test_name = test_name # type: str
self._tests_sub_path = os.path.join('Cdm', 'Projection',
class_name) # type: str
self._input_path = TestHelper.get_input_folder_path(
self._tests_sub_path, test_name) # type: str
self._expected_output_path = TestHelper.get_expected_output_folder_path(
self._tests_sub_path, test_name) # type: str
self._actual_output_path = TestHelper.get_actual_output_folder_path(
self._tests_sub_path, test_name) # type: str
self._corpus = ProjectionTestUtils.get_local_corpus(
self._tests_sub_path, test_name) # type: CdmCorpusDefinition
self._corpus.storage.mount(self._local_output_storage_ns,
LocalAdapter(self._actual_output_path))
self._corpus.storage.defaultNamespace = self._local_output_storage_ns
self._local_storage_root = self._corpus.storage.fetch_root_folder(
self._local_output_storage_ns) # type CdmFolderDefinition
self._default_manifest = self.create_default_manifest(
) # type: CdmManifestDefinition
self._all_imports = self.create_and_initialize_all_imports_file(
) # type: CdmDocumentDefinition
def test_manifest_add_list_of_entity_declarations(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = "local"
cdm_corpus.storage.mount("local", LocalAdapter("CdmCorpus/LocalPath"))
ctx = cdm_corpus.ctx
cdmDocument = CdmDocumentDefinition(ctx, "NameOfDocument")
collection = CdmEntityCollection(ctx, cdmDocument)
entity_list = []
for i in range(0, 2):
entity = CdmEntityDefinition(cdm_corpus.ctx,
'entityName_{}'.format(i), None)
create_document_for_entity(cdm_corpus, entity)
entity_list.append(entity)
self.assertEqual(0, len(collection))
collection.extend(entity_list)
self.assertEqual(2, len(collection))
for i in range(0, 2):
self.assertEqual('entityName_{}'.format(i),
collection[i].entity_name)
def setup_cdm_corpus(self) -> CdmCorpusDefinition:
corpus = CdmCorpusDefinition()
corpus.default_namespace = 'local'
corpus.storage.mount(
'cdm', LocalAdapter(TestHelper.sample_schema_folder_path))
corpus.storage.mount(
'local',
LocalAdapter(
AdlsModelJsonTestHelper.get_actual_sub_folder_path(
self.tests_subpath, self.test_name,
AdlsModelJsonTestHelper.input_folder_name)))
corpus.storage.mount(
'adls',
AdlsTestHelper.create_adapter_with_client_id(
self.root_relative_path))
return corpus
def test_manifest_cannot_add_entity_definition_without_creating_document(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = ''
def callback(status_level: 'CdmStatusLevel', message: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message
cdm_corpus.set_event_callback(callback)
cdm_corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path'))
manifest = CdmManifestDefinition(cdm_corpus.ctx, 'manifest')
manifest._folder_path = '/'
manifest._namespace = 'local'
entity = CdmEntityDefinition(manifest.ctx, 'entityName', None)
manifest.entities.append(entity)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue('Expected entity to have an \'Owner\' document set. Cannot create entity declaration to add to manifest.' in function_parameter2)
async def test_entity_properties(self):
test_input_path = TestHelper.get_input_folder_path(
self.tests_subpath, 'test_entity_properties')
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A')
att = obj.attributes[0] # type: CdmTypeAttributeDefinition
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNotNone(result)
self.assertEqual(att.maximum_length, 30)
self.assertIsNone(att.maximum_value)
self.assertIsNone(att.minimum_value)
# removing the only argument should remove the trait
att.maximum_length = None
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNone(att.maximum_length)
self.assertIsNone(result)
async def test_entity_properties(self):
test_input_path = os.path.join(ROOT_PATH, 'entity')
corpus = CdmCorpusDefinition()
corpus.ctx.update_logging_options(level='WARNING')
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A')
att = obj.attributes[0] # type: CdmTypeAttributeDefinition
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNotNone(result)
self.assertEqual(att.maximum_length, 30)
self.assertIsNone(att.maximum_value)
self.assertIsNone(att.minimum_value)
# removing the only argument should remove the trait
att.maximum_length = None
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNone(att.maximum_length)
self.assertIsNone(result)
def __init__(self, corpus: 'CdmCorpusDefinition'):
# the default namespace to be used when not specified.
self.default_namespace = None # type: Optional[str]
# Internal
self._corpus = corpus
self._namespace_adapters = OrderedDict() # type: Dict[str, StorageAdapterBase]
self._namespace_folders = OrderedDict() # type: Dict[str, CdmFolderDefinition]
self._registered_adapter_types = {
'local': 'LocalAdapter',
'adls': 'ADLSAdapter',
'remote': 'RemoteAdapter',
'github': 'GithubAdapter'
}
# The namespaces that have default adapters defined by the program and not by a user.
self._system_defined_namespaces = set() # type: Set
# set up default adapters.
self.mount('local', LocalAdapter(root=os.getcwd()))
self.mount('cdm', GithubAdapter())
self._system_defined_namespaces.add('local')
self._system_defined_namespaces.add('cdm')
self._TAG = StorageManager.__name__
async def test_nested_proj_using_object_model(self):
"""Test for creating nested projections with ExcludeAttributes operations using the object model"""
corpus = ProjectionTestUtils.get_local_corpus(
self.tests_subpath, 'test_nested_proj_using_object_model')
corpus.storage.mount(
'local',
LocalAdapter(
TestHelper.get_actual_output_folder_path(
self.tests_subpath,
'test_nested_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection
projection = ProjectionTestUtils.create_projection(corpus, local_root)
# Create an ExcludeAttributes operation
exclude_attrs_op = corpus.make_object(
CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF)
exclude_attrs_op.exclude_attributes = ['id', 'date']
projection.operations.append(exclude_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref.explicit_reference = projection
# Create another projection that uses the previous projection as its source
projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF)
projection2.source = projection_entity_ref
# Create an ExcludeAttributes operation
exclude_attrs_op2 = corpus.make_object(
CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF)
exclude_attrs_op2.exclude_attributes = ['value']
projection2.operations.append(exclude_attrs_op2)
# Create an entity reference to hold this projection
projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref2.explicit_reference = projection2
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref2
entity.attributes.append(entity_attribute)
# Resolve the entity
resolved_entity = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), None,
local_root)
# Verify correctness of the resolved attributes after running the ExcludeAttributes operations
# Original set of attributes: ['id', 'name', 'value', 'date']
# Excluded attributes: ['id', 'date'], ['value']
self.assertEqual(1, len(resolved_entity.attributes))
self.assertEqual('name', resolved_entity.attributes[0].name)
def test_cdm_collection_appending_list(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath'))
ctx = cdm_corpus.ctx
cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument')
collection = CdmCollection(ctx, cdm_document,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity_list = []
for i in range(0, 2):
entity = CdmEntityDefinition(cdm_corpus.ctx,
'entityName_{}'.format(i), None)
create_document_for_entity(cdm_corpus, entity)
entity_declaration = cdm_corpus.make_object(
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, entity.entity_name,
False) # type: CdmLocalEntityDeclarationDefinition
entity_declaration.entity_path = '{}/{}'.format(
entity.owner.at_corpus_path, entity.entity_name)
entity_declaration.owner = entity.owner
entity_list.append(entity_declaration)
self.assertEqual(0, len(collection))
collection.extend(entity_list)
self.assertEqual(2, len(collection))
for i in range(0, 2):
self.assertEqual('entityName_{}'.format(i),
collection[i].entity_name)
async def test_cdm_folder_to_data_type_attribute(self):
"""
Testing that "is.localized.describedAs" trait with a table of three entries (en, rs and cn) is fully preserved when running CdmFolder TypeAttributePersistence ToData.
"""
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path'))
corpus.storage.default_namespace = 'local'
cdm_type_attribute_definition = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'TestSavingTraitAttribute', False)
english_constants_list = ['en', 'Some description in English language']
serbian_constants_list = ['sr', 'Opis na srpskom jeziku']
chinese_constants_list = ['cn', '一些中文描述']
list_of_const_lists = [english_constants_list, serbian_constants_list, chinese_constants_list]
const_ent_def = corpus.make_object(CdmObjectType.CONSTANT_ENTITY_DEF, 'localizedDescriptions', False)
const_ent_def.constant_values = list_of_const_lists
const_ent_def.entity_shape = corpus.make_ref(CdmObjectType.ENTITY_REF, 'localizedTable', True)
trait_reference2 = corpus.make_object(CdmObjectType.TRAIT_REF, 'is.localized.describedAs', False)
trait_reference2.arguments.append('localizedDisplayText', corpus.make_ref(CdmObjectType.ENTITY_REF, const_ent_def, True))
cdm_type_attribute_definition.applied_traits.append(trait_reference2)
result = TypeAttribute().decode(PersistenceLayer.to_data(cdm_type_attribute_definition, None, None, PersistenceLayer.CDM_FOLDER))
self.assertIsNotNone(result.appliedTraits)
argument = result.appliedTraits[0].arguments[0]
constant_values = argument.value.entityReference.constantValues
self.assertEqual('en', constant_values[0][0])
self.assertEqual('Some description in English language', constant_values[0][1])
self.assertEqual('sr', constant_values[1][0])
self.assertEqual('Opis na srpskom jeziku', constant_values[1][1])
self.assertEqual('cn', constant_values[2][0])
self.assertEqual('一些中文描述', constant_values[2][1])
async def test_imposed_directives(self):
"""
Resolution Guidance Test - With structured/normal imposed directives
This test directly read imposed directives from json file instead of setting resOpt in code as RunTest()
"""
test_name = 'test_imposed_directives'
test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name)
test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name)
corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) # type: CdmCorpusDefinition
corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path))
actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition
# Test "structured" imposed directive
entity = await corpus.fetch_object_async('local:/Person_Structured.cdm.json/Person') # type: CdmEntityDefinition
resolved_entity = await entity.create_resolved_entity_async('Person_Resolved', None, actual_output_folder)
await resolved_entity.in_document.save_as_async('Person_Structured_Resolved.cdm.json')
self.validate_output('Person_Structured_Resolved.cdm.json', test_expected_output_path, test_actual_output_path)
# Test default imposed directive
entity = await corpus.fetch_object_async('local:/Person_Default.cdm.json/Person') # type: CdmEntityDefinition
resolved_entity = await entity.create_resolved_entity_async('Person_Resolved', None, actual_output_folder)
await resolved_entity.in_document.save_as_async('Person_Default_Resolved.cdm.json')
self.validate_output('Person_Default_Resolved.cdm.json', test_expected_output_path, test_actual_output_path)
async def test_resolving_manifest_not_in_folder(self):
"""Test that resolving a manifest that hasn't been added to a folder doesn't throw any exceptions."""
try:
corpus = CdmCorpusDefinition()
corpus.storage.mount('local', LocalAdapter('C:\\path'))
corpus.storage.default_namespace = 'local'
def callback(level, message):
# We should see the following error message be logged. If not, fail.
if 'Cannot resolve the manifest \'test\' because it has not been added to a folder' not in message:
self.fail()
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test')
entity = corpus.make_object(CdmObjectType.ENTITY_DEF, 'entity')
document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION))
document.definitions.append(entity)
# Don't add the document containing the entity to a folder either.
manifest.entities.append(entity)
await manifest.create_resolved_manifest_async('resolved', None)
except Exception:
self.fail('Exception should not be thrown when resolving a manifest that is not in a folder.')
def test_cdm_collection_append_method(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath'))
ctx = cdm_corpus.ctx
cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument')
collection = CdmCollection(ctx, cdm_document,
CdmObjectType.ATTRIBUTE_CONTEXT_DEF)
appended_attribute_context = collection.append('nameOfNewAttribute')
self.assertEqual(1, len(collection))
self.assertEqual('nameOfNewAttribute', collection[0].name)
self.assertEqual(cdm_document, collection[0].owner)
self.assertEqual(ctx, collection[0].ctx)
self.assertEqual(collection[0], appended_attribute_context)
attribute_context = CdmAttributeContext(ctx, 'NameOfAttributeContext')
appended_attribute = collection.append(attribute_context)
self.assertEqual(2, len(collection))
self.assertEqual(attribute_context, appended_attribute)
self.assertEqual(attribute_context, collection[1])
self.assertEqual(cdm_document, attribute_context.owner)
async def test_programmatically_create_partitions(self):
corpus = CdmCorpusDefinition()
corpus.storage.mount('local', LocalAdapter())
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest')
entity = manifest.entities.append('entity')
relative_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'relative partition')
relative_partition.location = 'relative/path'
absolute_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'absolute partition')
absolute_partition.location = 'local:/absolute/path'
entity.data_partitions.append(relative_partition)
entity.data_partitions.append(absolute_partition)
manifest_data = await ManifestPersistence.to_data(manifest, None, None)
self.assertEqual(len(manifest_data.entities), 1)
entityData = manifest_data.entities[0]
partitions_list = entityData.dataPartitions
self.assertEqual(len(partitions_list), 2)
relative_partition_data = partitions_list[0]
absolute_partition_data = partitions_list[-1]
self.assertEqual(relative_partition_data.location,
relative_partition.location)
self.assertEqual(absolute_partition_data.location,
absolute_partition.location)
async def test_referenced_entity_declaration_resolution(self):
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('cdm', LocalAdapter(root='../../schemaDocuments'))
corpus.storage.default_namespace = 'cdm'
manifest = CdmManifestDefinition(corpus.ctx, 'manifest')
manifest.entities.append(
'Account', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account')
referenced_entity_path = \
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/electronicMedicalRecords.manifest.cdm.json/Address'
referenced_entity = CdmReferencedEntityDeclarationDefinition(corpus.ctx, 'Address')
referenced_entity.entity_path = referenced_entity_path
manifest.entities.append(referenced_entity)
corpus.storage.fetch_root_folder('cdm').documents.append(manifest)
resolved_manifest = await manifest.create_resolved_manifest_async('resolved_manifest', None)
self.assertEqual(2, len(resolved_manifest.entities))
self.assertEqual('core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/resolved/Account.cdm.json/Account',
resolved_manifest.entities[0].entity_path)
self.assertEqual(referenced_entity_path, resolved_manifest.entities[1].entity_path)
async def test_attributes_that_are_replaced(self):
corpus = TestHelper.get_local_corpus(
self.tests_subpath, 'test_attributes_that_are_replaced')
corpus.storage.mount('cdm',
LocalAdapter(TestHelper.get_schema_docs_root()))
extended_entity = await corpus.fetch_object_async(
'local:/extended.cdm.json/extended') # type: CdmEntityDefinition
res_extended_ent = await extended_entity.create_resolved_entity_async(
'resExtended')
# the attribute from the base class should be merged with the attribute
# from the extended class into a single attribute
self.assertEqual(1, len(res_extended_ent.attributes))
# check that traits from the base class merged with the traits from the extended class
attribute = res_extended_ent.attributes[0]
# base trait
self.assertNotEqual(
-1, attribute.applied_traits.index('means.identity.brand'))
# extended trait
self.assertNotEqual(
-1, attribute.applied_traits.index('means.identity.company.name'))
# make sure the attribute context and entity foreign key were maintained correctly
foreign_key_for_base_attribute = res_extended_ent.attribute_context.contents[
1].contents[1]
self.assertEqual('_generatedAttributeSet',
foreign_key_for_base_attribute.name)
fk_reference = foreign_key_for_base_attribute.contents[0].contents[
0].contents[0] # type: CdmAttributeReference
self.assertEqual('resExtended/hasAttributes/regardingObjectId',
fk_reference.named_reference)
async def test_conditional_proj_using_object_model(self):
"""Test for creating a projection with an AddCountAttribute operation and a condition using the object model"""
corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly'
projection = ProjectionTestUtils.create_projection(corpus, local_root)
projection.condition = 'referenceOnly==True'
# Create an AddCountAttribute operation
add_count_attr_op = corpus.make_object(CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF)
add_count_attr_op.count_attribute = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testCount')
add_count_attr_op.count_attribute.data_type = corpus.make_ref(CdmObjectType.DATA_TYPE_REF, 'integer', True)
projection.operations.append(add_count_attr_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Create resolution options with the 'referenceOnly' directive
res_opt = ResolveOptions(entity.in_document)
res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly']))
# Resolve the entity with 'referenceOnly'
resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddCountAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# Count attribute: "testCount"
self.assertEqual(5, len(resolved_entity_with_reference_only.attributes))
self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name)
self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name)
self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name)
self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name)
self.assertEqual('testCount', resolved_entity_with_reference_only.attributes[4].name)
self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4].applied_traits.item('is.linkedEntity.array.count'))
# Now resolve the entity with the 'structured' directive
res_opt.directives = AttributeResolutionDirectiveSet(set(['structured']))
resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root)
# Verify correctness of the resolved attributes after running the AddCountAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# No Count attribute added, condition was false
self.assertEqual(4, len(resolved_entity_with_structured.attributes))
self.assertEqual('id', resolved_entity_with_structured.attributes[0].name)
self.assertEqual('name', resolved_entity_with_structured.attributes[1].name)
self.assertEqual('value', resolved_entity_with_structured.attributes[2].name)
self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_entity_attribute_proj_using_object_model(self):
"""Test for creating a projection with an AddTypeAttribute operation on an entity attribute using the object model"""
corpus = ProjectionTestUtils.get_local_corpus(
self.tests_subpath,
'test_entity_attribute_proj_using_object_model')
corpus.storage.mount(
'local',
LocalAdapter(
TestHelper.get_actual_output_folder_path(
self.tests_subpath,
'test_entity_attribute_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection
projection = ProjectionTestUtils.create_projection(corpus, local_root)
# Create an AddTypeAttribute operation
add_type_attr_op = corpus.make_object(
CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF)
add_type_attr_op.type_attribute = corpus.make_object(
CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testType')
add_type_attr_op.type_attribute.data_type = corpus.make_ref(
CdmObjectType.DATA_TYPE_REF, 'entityName', True)
projection.operations.append(add_type_attr_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF,
None)
projection_entity_ref.explicit_reference = projection
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(
CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref
entity.attributes.append(entity_attribute)
# Resolve the entity
resolved_entity = await entity.create_resolved_entity_async(
'Resolved_{}.cdm.json'.format(entity.entity_name), None,
local_root)
# Verify correctness of the resolved attributes after running the AddTypeAttribute operation
# Original set of attributes: ["id", "name", "value", "date"]
# Type attribute: "testType"
self.assertEqual(5, len(resolved_entity.attributes))
self.assertEqual('id', resolved_entity.attributes[0].name)
self.assertEqual('name', resolved_entity.attributes[1].name)
self.assertEqual('value', resolved_entity.attributes[2].name)
self.assertEqual('date', resolved_entity.attributes[3].name)
self.assertEqual('testType', resolved_entity.attributes[4].name)
self.assertEqual(
'is.linkedEntity.name',
resolved_entity.attributes[4].applied_traits[4].named_reference)
async def test_loading_invalid_model_json_name(self):
test_input_path = TestHelper.get_input_folder_path(self.test_subpath, 'test_loading_invalid_model_json_name')
corpus = CdmCorpusDefinition()
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
# We are trying to load a file with an invalid name, so fetch_object_async should just return None.
invalid_model_json = await corpus.fetch_object_async('test.model.json')
self.assertIsNone(invalid_model_json)
async def test_resolve_test_corpus(self):
self.assertTrue(os.path.exists(TestHelper.get_schema_docs_root()))
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(TestHelper.get_schema_docs_root()))
manifest = await corpus.fetch_object_async(TestHelper.cdm_standards_schema_path) # type: CdmManifestDefinition
directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'})
all_resolved = await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher())
self.assertNotEqual(all_resolved, '')
async def test_entity_attribute_proj_using_object_model(self):
"""Test for creating a projection with an ArrayExpansion operation on an entity attribute using the object model"""
corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_entity_attribute_proj_using_object_model')
corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_entity_attribute_proj_using_object_model')))
local_root = corpus.storage.fetch_root_folder('local')
# Create an entity
entity = ProjectionTestUtils.create_entity(corpus, local_root)
# Create a projection
projection = ProjectionTestUtils.create_projection(corpus, local_root)
# Create an ArrayExpansion operation
array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF)
array_expansion_op.start_ordinal = 1
array_expansion_op.end_ordinal = 2
projection.operations.append(array_expansion_op)
# Create an entity reference to hold this projection
projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref.explicit_reference = projection
# Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity
projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF)
projection2.source = projection_entity_ref
# Create a RenameAttributes operation
rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF)
rename_attrs_op.rename_format = '{m}{o}'
projection2.operations.append(rename_attrs_op)
# Create an entity reference to hold this projection
projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None)
projection_entity_ref2.explicit_reference = projection2
# Create an entity attribute that contains this projection and add this to the entity
entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute')
entity_attribute.entity = projection_entity_ref2
entity.attributes.append(entity_attribute)
# Resolve the entity
resolved_entity = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), None, local_root)
# Verify correctness of the resolved attributes after running the projections
# Original set of attributes: ["id", "name", "value", "date"]
# Expand 1...2, renameFormat = {m}{o}
self.assertEqual(8, len(resolved_entity.attributes))
self.assertEqual('id1', resolved_entity.attributes[0].name)
self.assertEqual('name1', resolved_entity.attributes[1].name)
self.assertEqual('value1', resolved_entity.attributes[2].name)
self.assertEqual('date1', resolved_entity.attributes[3].name)
self.assertEqual('id2', resolved_entity.attributes[4].name)
self.assertEqual('name2', resolved_entity.attributes[5].name)
self.assertEqual('value2', resolved_entity.attributes[6].name)
self.assertEqual('date2', resolved_entity.attributes[7].name)
