async def test_loads_and_sets_times_correctly(self): """Test modified times for manifest and files beneath it""" input_path = TestHelper.get_input_folder_path(self.tests_subpath, 'test_loads_and_sets_times_correctly') time_before_load = datetime.now(timezone.utc) cdm_corpus = self.get_corpus() cdm_corpus.storage.mount('someNamespace', LocalAdapter(input_path)) cdm_corpus.storage.mount('local', LocalAdapter(input_path)) cdm_corpus.storage.unmount('cdm') cdm_corpus.storage.default_namespace = 'local' cdm_manifest = await cdm_corpus.fetch_object_async('someNamespace:/default.manifest.cdm.json') status_time_at_load = cdm_manifest.last_file_status_check_time # hard coded because the time comes from inside the file self.assertEqual(time_utils._get_formatted_date_string(status_time_at_load), '2019-02-01T15:36:19.410Z') self.assertIsNotNone(cdm_manifest._file_system_modified_time) self.assertGreater(time_before_load, cdm_manifest._file_system_modified_time) time.sleep(1) await cdm_manifest.file_status_check_async() self.assertGreater(cdm_manifest.last_file_status_check_time, time_before_load) self.assertGreater(cdm_manifest.last_file_status_check_time, status_time_at_load) self.assertEqual(1, len(cdm_manifest.sub_manifests)) self.assertGreater(cdm_manifest.sub_manifests[0].last_file_status_check_time, time_before_load) self.assertEqual(1, len(cdm_manifest.entities)) self.assertEqual(1, len(cdm_manifest.entities[0].data_partitions)) entity = cdm_manifest.entities[0] sub_manifest = cdm_manifest.sub_manifests[0] max_time = time_utils._max_time(entity.last_file_modified_time, sub_manifest.last_file_modified_time) self.assertEqual(time_utils._get_formatted_date_string(cdm_manifest.last_child_file_modified_time), time_utils._get_formatted_date_string(max_time))
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None, is_language_specific: Optional[bool] = False, expected_codes: Optional[set] = None, no_input_and_output_folder: Optional[bool] = False): """ Creates a corpus to be used by the tests, which mounts inputFolder, outputFolder, cdm, and remoteAdapter. Will fail on any unexpected warning/error. @param testSubpath The root of the corpus files. @param testName The test name. @param testInputDir The test input directory. @param isLanguageSpecific Indicate whether there is subfolder called Java, it's used when input is different compared with other languages @param expectedCodes The error codes that are expected, and they should not block the test. @param noInputAndOutputFolder No input and output folder needed. """ if no_input_and_output_folder: test_input_dir = 'C:\\dummpyPath' test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name, is_language_specific) test_output_dir = test_input_dir if no_input_and_output_folder else TestHelper.get_actual_output_folder_path(test_subpath, test_name) cdm_corpus = CdmCorpusDefinition() def callback(level: CdmStatusLevel, message: str): TestHelper._fail_on_unexpected_failure(cdm_corpus, message, expected_codes) cdm_corpus.set_event_callback(callback, CdmStatusLevel.WARNING) cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir)) cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir)) cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.schema_documents_path)) cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'})) return cdm_corpus
async def test_from_and_to_data(self): with open(os.path.join(ROOT_PATH, 'default.manifest.cdm.json')) as manifestFile: manifestContent = ManifestContent() manifestContent.decode(manifestFile.read()) corpus = CdmCorpusDefinition() corpus.storage.default_namespace = 'local' corpus.ctx.update_logging_options(level='WARNING') corpus.storage.mount('cdm', LocalAdapter(root="../CDM.SchemaDocuments")) corpus.storage.mount('local', LocalAdapter(root=ROOT_PATH)) corpus.storage.mount( 'remote', RemoteAdapter(hosts={"contoso": "http://contoso.com"})) folder = corpus.storage.fetch_root_folder('local') manifest = await corpus.fetch_object_async('default.manifest.cdm.json', folder) data = await ManifestPersistence.to_data(manifest, None, None) for entity in manifest.entities: entity_def = await corpus.fetch_object_async( entity.entity_path, manifest) await manifest.save_as_async('test_output/new.manifest.cdm.json', save_referenced=True) self.assertDictEqual(json.loads(manifestContent.encode()), json.loads(data.encode()))
def get_local_corpus(self, test_files_input_root: str, test_files_output_root: Optional[str] = None): """Gets local corpus.""" cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter(test_files_input_root)) if test_files_output_root: cdm_corpus.storage.mount('target', LocalAdapter(test_files_output_root)) return cdm_corpus
async def test_syms_saving_and_fetching_document(self): syms_adapter = SymsTestHelper.create_adapter_with_clientid() await SymsTestHelper.clean_database(syms_adapter, SymsTestHelper.DATABASE_NAME) test_input_path = TestHelper.get_input_folder_path( self.test_subpath, 'TestSymsSavingAndFetchingDocument') test_act_output_path = TestHelper.get_actual_output_folder_path( self.test_subpath, 'TestSymsSavingAndFetchingDocument') test_exp_output_path = TestHelper.get_expected_output_folder_path( self.test_subpath, 'TestSymsSavingAndFetchingDocument') corpus = CdmCorpusDefinition() adls_adapter1 = SymsTestHelper.create_adapter_clientid_with_shared_key( 1) adls_adapter2 = SymsTestHelper.create_adapter_clientid_with_shared_key( 2) local_input_adapter = LocalAdapter(test_input_path) local_act_output_adapter = LocalAdapter(test_act_output_path) local_exp_output_adapter = LocalAdapter(test_exp_output_path) corpus.storage.mount('adls1', adls_adapter1) corpus.storage.mount('adls2', adls_adapter2) corpus.storage.mount('syms', syms_adapter) corpus.storage.mount('localInput', local_input_adapter) corpus.storage.mount('localActOutput', local_act_output_adapter) corpus.storage.mount('localExpOutput', local_exp_output_adapter) corpus.storage.unmount('cdm') corpus.storage.default_namespace = 'localInput' manifest = await corpus.fetch_object_async('default.manifest.cdm.json') manifest.manifest_name = SymsTestHelper.DATABASE_NAME await self.run_syms_save_manifest(manifest) await self.run_syms_fetch_manifest(corpus, manifest, 'default.manifest.cdm.json') await self.run_syms_fetch_document(corpus, manifest) manifest_modified = await corpus.fetch_object_async( 'defaultmodified.manifest.cdm.json') manifest_modified.manifest_name = SymsTestHelper.DATABASE_NAME manifest_modified.entities[0].set_last_file_modified_time( datetime.datetime.now(datetime.timezone.utc)) await self.run_syms_save_manifest(manifest_modified) await self.run_syms_fetch_manifest( corpus, manifest_modified, 'defaultmodified.manifest.cdm.json') await self.run_syms_fetch_document(corpus, manifest_modified) await self.run_syms_smart_adls_adapter_mount_logic() await SymsTestHelper.clean_database(syms_adapter, SymsTestHelper.DATABASE_NAME)
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None): test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name) test_output_dir = TestHelper.get_actual_output_folder_path(test_subpath, test_name) cdm_corpus = CdmCorpusDefinition() cdm_corpus.ctx.report_at_level = CdmStatusLevel.WARNING cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir)) cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir)) cdm_corpus.storage.mount('cdm', LocalAdapter('../../schemaDocuments')) cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'})) return cdm_corpus
def setup_cdm_corpus(self): # Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating # objects and paths. cdm_corpus = CdmCorpusDefinition() cdm_corpus.ctx.report_at_level = CdmStatusLevel.ERROR print('Configure storage adapters') cdm_corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, self.test_name))) # Local is our default. So any paths that start out navigating without a device tag will assume local. cdm_corpus.storage.default_namespace = 'local' # Fake cdm, normally use the CDM Standards adapter. cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.sample_schema_folder_path)) return cdm_corpus
async def test_alter_traits_on_enti_attr_proj(self): """Test AlterTraits on an entity attribute""" test_name = 'test_alter_traits_on_enti_attr_proj' entity_name = 'NewPerson' corpus = ProjectionTestUtils.get_local_corpus( self.tests_subpath, test_name) # type: CdmCorpusDefinition corpus.storage.mount('traitGroup', LocalAdapter(self.trait_group_file_path)) for res_opt in self.res_opts_combinations: await ProjectionTestUtils.load_entity_for_resolution_option_and_save( self, corpus, test_name, self.tests_subpath, entity_name, res_opt) entity = await corpus.fetch_object_async( 'local:/{0}.cdm.json/{0}'.format(entity_name) ) # type: CdmEntityDefinition resolved_entity = await ProjectionTestUtils.get_resolved_entity( corpus, entity, ['structured']) self.assertEqual(5, len(resolved_entity.attributes)) self.validate_trait(resolved_entity.attributes[0], 'name', True) self.validate_trait(resolved_entity.attributes[1], 'age') self.validate_trait(resolved_entity.attributes[2], 'address') self.validate_trait(resolved_entity.attributes[3], 'phoneNumber') self.validate_trait(resolved_entity.attributes[4], 'email')
async def test_combine_ops_nested_proj(self): """Test AlterTraits operation nested with IncludeAttributes and RenameAttribute""" test_name = 'test_combine_ops_nested_proj' entity_name = 'NewPerson' corpus = ProjectionTestUtils.get_local_corpus( self.tests_subpath, test_name) # type: CdmCorpusDefinition corpus.storage.mount('traitGroup', LocalAdapter(self.trait_group_file_path)) for res_opt in self.res_opts_combinations: await ProjectionTestUtils.load_entity_for_resolution_option_and_save( self, corpus, test_name, self.tests_subpath, entity_name, res_opt) entity = await corpus.fetch_object_async( 'local:/{0}.cdm.json/{0}'.format(entity_name) ) # type: CdmEntityDefinition resolved_entity = await ProjectionTestUtils.get_resolved_entity( corpus, entity, []) # Original set of attributes: ["name", "age", "address", "phoneNumber", "email"] # Include attributes: ["age", "phoneNumber", "name"] # Add attribute: ["newName"] # alter traits on ["newName", "name", + { "means.TraitG100" , "JobTitleBase" } - { "means.TraitG300" } ] # Rename attribute ["newName" -> "renaming-{m}" ] # alter traits on ["renaming-newName", + { "means.TraitG4(precision:5, scale:15)" } ] self.assertEqual(4, len(resolved_entity.attributes)) self.validate_trait(resolved_entity.attributes[0], 'age', False, True) self.validate_trait(resolved_entity.attributes[1], 'phoneNumber', False, True) self.validate_trait(resolved_entity.attributes[2], 'name') self.validate_trait(resolved_entity.attributes[3], 'renaming-newName', True)
def __init__(self, class_name: str, test_name: str): self._foundation_json_path = 'cdm:/foundations.cdm.json' # type: str self._local_output_storage_ns = 'output' # type: str self._manifest_name = 'default' # type: str self._manifest_doc_name = '{}.manifest.cdm.json'.format( self._manifest_name) # type: str self._all_imports_name = '_allImports' # type: str self._all_imports_doc_name = '{}.cdm.json'.format( self._all_imports_name) # type: str self._class_name = class_name # type: str self._test_name = test_name # type: str self._tests_sub_path = os.path.join('Cdm', 'Projection', class_name) # type: str self._input_path = TestHelper.get_input_folder_path( self._tests_sub_path, test_name) # type: str self._expected_output_path = TestHelper.get_expected_output_folder_path( self._tests_sub_path, test_name) # type: str self._actual_output_path = TestHelper.get_actual_output_folder_path( self._tests_sub_path, test_name) # type: str self._corpus = ProjectionTestUtils.get_local_corpus( self._tests_sub_path, test_name) # type: CdmCorpusDefinition self._corpus.storage.mount(self._local_output_storage_ns, LocalAdapter(self._actual_output_path)) self._corpus.storage.defaultNamespace = self._local_output_storage_ns self._local_storage_root = self._corpus.storage.fetch_root_folder( self._local_output_storage_ns) # type CdmFolderDefinition self._default_manifest = self.create_default_manifest( ) # type: CdmManifestDefinition self._all_imports = self.create_and_initialize_all_imports_file( ) # type: CdmDocumentDefinition
def test_manifest_add_list_of_entity_declarations(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = "local" cdm_corpus.storage.mount("local", LocalAdapter("CdmCorpus/LocalPath")) ctx = cdm_corpus.ctx cdmDocument = CdmDocumentDefinition(ctx, "NameOfDocument") collection = CdmEntityCollection(ctx, cdmDocument) entity_list = [] for i in range(0, 2): entity = CdmEntityDefinition(cdm_corpus.ctx, 'entityName_{}'.format(i), None) create_document_for_entity(cdm_corpus, entity) entity_list.append(entity) self.assertEqual(0, len(collection)) collection.extend(entity_list) self.assertEqual(2, len(collection)) for i in range(0, 2): self.assertEqual('entityName_{}'.format(i), collection[i].entity_name)
def setup_cdm_corpus(self) -> CdmCorpusDefinition: corpus = CdmCorpusDefinition() corpus.default_namespace = 'local' corpus.storage.mount( 'cdm', LocalAdapter(TestHelper.sample_schema_folder_path)) corpus.storage.mount( 'local', LocalAdapter( AdlsModelJsonTestHelper.get_actual_sub_folder_path( self.tests_subpath, self.test_name, AdlsModelJsonTestHelper.input_folder_name))) corpus.storage.mount( 'adls', AdlsTestHelper.create_adapter_with_client_id( self.root_relative_path)) return corpus
def test_manifest_cannot_add_entity_definition_without_creating_document(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' function_was_called = False function_parameter1 = CdmStatusLevel.INFO function_parameter2 = '' def callback(status_level: 'CdmStatusLevel', message: str): nonlocal function_was_called, function_parameter1, function_parameter2 function_was_called = True function_parameter1 = status_level function_parameter2 = message cdm_corpus.set_event_callback(callback) cdm_corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path')) manifest = CdmManifestDefinition(cdm_corpus.ctx, 'manifest') manifest._folder_path = '/' manifest._namespace = 'local' entity = CdmEntityDefinition(manifest.ctx, 'entityName', None) manifest.entities.append(entity) self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue('Expected entity to have an \'Owner\' document set. Cannot create entity declaration to add to manifest.' in function_parameter2)
async def test_entity_properties(self): test_input_path = TestHelper.get_input_folder_path( self.tests_subpath, 'test_entity_properties') corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A') att = obj.attributes[0] # type: CdmTypeAttributeDefinition result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNotNone(result) self.assertEqual(att.maximum_length, 30) self.assertIsNone(att.maximum_value) self.assertIsNone(att.minimum_value) # removing the only argument should remove the trait att.maximum_length = None result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNone(att.maximum_length) self.assertIsNone(result)
async def test_entity_properties(self): test_input_path = os.path.join(ROOT_PATH, 'entity') corpus = CdmCorpusDefinition() corpus.ctx.update_logging_options(level='WARNING') corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A') att = obj.attributes[0] # type: CdmTypeAttributeDefinition result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNotNone(result) self.assertEqual(att.maximum_length, 30) self.assertIsNone(att.maximum_value) self.assertIsNone(att.minimum_value) # removing the only argument should remove the trait att.maximum_length = None result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNone(att.maximum_length) self.assertIsNone(result)
def __init__(self, corpus: 'CdmCorpusDefinition'): # the default namespace to be used when not specified. self.default_namespace = None # type: Optional[str] # Internal self._corpus = corpus self._namespace_adapters = OrderedDict() # type: Dict[str, StorageAdapterBase] self._namespace_folders = OrderedDict() # type: Dict[str, CdmFolderDefinition] self._registered_adapter_types = { 'local': 'LocalAdapter', 'adls': 'ADLSAdapter', 'remote': 'RemoteAdapter', 'github': 'GithubAdapter' } # The namespaces that have default adapters defined by the program and not by a user. self._system_defined_namespaces = set() # type: Set # set up default adapters. self.mount('local', LocalAdapter(root=os.getcwd())) self.mount('cdm', GithubAdapter()) self._system_defined_namespaces.add('local') self._system_defined_namespaces.add('cdm') self._TAG = StorageManager.__name__
async def test_nested_proj_using_object_model(self): """Test for creating nested projections with ExcludeAttributes operations using the object model""" corpus = ProjectionTestUtils.get_local_corpus( self.tests_subpath, 'test_nested_proj_using_object_model') corpus.storage.mount( 'local', LocalAdapter( TestHelper.get_actual_output_folder_path( self.tests_subpath, 'test_nested_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection projection = ProjectionTestUtils.create_projection(corpus, local_root) # Create an ExcludeAttributes operation exclude_attrs_op = corpus.make_object( CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF) exclude_attrs_op.exclude_attributes = ['id', 'date'] projection.operations.append(exclude_attrs_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create another projection that uses the previous projection as its source projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF) projection2.source = projection_entity_ref # Create an ExcludeAttributes operation exclude_attrs_op2 = corpus.make_object( CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF) exclude_attrs_op2.exclude_attributes = ['value'] projection2.operations.append(exclude_attrs_op2) # Create an entity reference to hold this projection projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref2.explicit_reference = projection2 # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object( CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref2 entity.attributes.append(entity_attribute) # Resolve the entity resolved_entity = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), None, local_root) # Verify correctness of the resolved attributes after running the ExcludeAttributes operations # Original set of attributes: ['id', 'name', 'value', 'date'] # Excluded attributes: ['id', 'date'], ['value'] self.assertEqual(1, len(resolved_entity.attributes)) self.assertEqual('name', resolved_entity.attributes[0].name)
def test_cdm_collection_appending_list(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath')) ctx = cdm_corpus.ctx cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument') collection = CdmCollection(ctx, cdm_document, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity_list = [] for i in range(0, 2): entity = CdmEntityDefinition(cdm_corpus.ctx, 'entityName_{}'.format(i), None) create_document_for_entity(cdm_corpus, entity) entity_declaration = cdm_corpus.make_object( CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, entity.entity_name, False) # type: CdmLocalEntityDeclarationDefinition entity_declaration.entity_path = '{}/{}'.format( entity.owner.at_corpus_path, entity.entity_name) entity_declaration.owner = entity.owner entity_list.append(entity_declaration) self.assertEqual(0, len(collection)) collection.extend(entity_list) self.assertEqual(2, len(collection)) for i in range(0, 2): self.assertEqual('entityName_{}'.format(i), collection[i].entity_name)
async def test_cdm_folder_to_data_type_attribute(self): """ Testing that "is.localized.describedAs" trait with a table of three entries (en, rs and cn) is fully preserved when running CdmFolder TypeAttributePersistence ToData. """ corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path')) corpus.storage.default_namespace = 'local' cdm_type_attribute_definition = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'TestSavingTraitAttribute', False) english_constants_list = ['en', 'Some description in English language'] serbian_constants_list = ['sr', 'Opis na srpskom jeziku'] chinese_constants_list = ['cn', '一些中文描述'] list_of_const_lists = [english_constants_list, serbian_constants_list, chinese_constants_list] const_ent_def = corpus.make_object(CdmObjectType.CONSTANT_ENTITY_DEF, 'localizedDescriptions', False) const_ent_def.constant_values = list_of_const_lists const_ent_def.entity_shape = corpus.make_ref(CdmObjectType.ENTITY_REF, 'localizedTable', True) trait_reference2 = corpus.make_object(CdmObjectType.TRAIT_REF, 'is.localized.describedAs', False) trait_reference2.arguments.append('localizedDisplayText', corpus.make_ref(CdmObjectType.ENTITY_REF, const_ent_def, True)) cdm_type_attribute_definition.applied_traits.append(trait_reference2) result = TypeAttribute().decode(PersistenceLayer.to_data(cdm_type_attribute_definition, None, None, PersistenceLayer.CDM_FOLDER)) self.assertIsNotNone(result.appliedTraits) argument = result.appliedTraits[0].arguments[0] constant_values = argument.value.entityReference.constantValues self.assertEqual('en', constant_values[0][0]) self.assertEqual('Some description in English language', constant_values[0][1]) self.assertEqual('sr', constant_values[1][0]) self.assertEqual('Opis na srpskom jeziku', constant_values[1][1]) self.assertEqual('cn', constant_values[2][0]) self.assertEqual('一些中文描述', constant_values[2][1])
async def test_imposed_directives(self): """ Resolution Guidance Test - With structured/normal imposed directives This test directly read imposed directives from json file instead of setting resOpt in code as RunTest() """ test_name = 'test_imposed_directives' test_expected_output_path = TestHelper.get_expected_output_folder_path(self.tests_subpath, test_name) test_actual_output_path = TestHelper.get_actual_output_folder_path(self.tests_subpath, test_name) corpus = TestHelper.get_local_corpus(self.tests_subpath, test_name) # type: CdmCorpusDefinition corpus.storage.mount('localActualOutput', LocalAdapter(test_actual_output_path)) actual_output_folder = await corpus.fetch_object_async('localActualOutput:/') # type: CdmFolderDefinition # Test "structured" imposed directive entity = await corpus.fetch_object_async('local:/Person_Structured.cdm.json/Person') # type: CdmEntityDefinition resolved_entity = await entity.create_resolved_entity_async('Person_Resolved', None, actual_output_folder) await resolved_entity.in_document.save_as_async('Person_Structured_Resolved.cdm.json') self.validate_output('Person_Structured_Resolved.cdm.json', test_expected_output_path, test_actual_output_path) # Test default imposed directive entity = await corpus.fetch_object_async('local:/Person_Default.cdm.json/Person') # type: CdmEntityDefinition resolved_entity = await entity.create_resolved_entity_async('Person_Resolved', None, actual_output_folder) await resolved_entity.in_document.save_as_async('Person_Default_Resolved.cdm.json') self.validate_output('Person_Default_Resolved.cdm.json', test_expected_output_path, test_actual_output_path)
async def test_resolving_manifest_not_in_folder(self): """Test that resolving a manifest that hasn't been added to a folder doesn't throw any exceptions.""" try: corpus = CdmCorpusDefinition() corpus.storage.mount('local', LocalAdapter('C:\\path')) corpus.storage.default_namespace = 'local' def callback(level, message): # We should see the following error message be logged. If not, fail. if 'Cannot resolve the manifest \'test\' because it has not been added to a folder' not in message: self.fail() corpus.set_event_callback(callback, CdmStatusLevel.WARNING) manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test') entity = corpus.make_object(CdmObjectType.ENTITY_DEF, 'entity') document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION)) document.definitions.append(entity) # Don't add the document containing the entity to a folder either. manifest.entities.append(entity) await manifest.create_resolved_manifest_async('resolved', None) except Exception: self.fail('Exception should not be thrown when resolving a manifest that is not in a folder.')
def test_cdm_collection_append_method(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath')) ctx = cdm_corpus.ctx cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument') collection = CdmCollection(ctx, cdm_document, CdmObjectType.ATTRIBUTE_CONTEXT_DEF) appended_attribute_context = collection.append('nameOfNewAttribute') self.assertEqual(1, len(collection)) self.assertEqual('nameOfNewAttribute', collection[0].name) self.assertEqual(cdm_document, collection[0].owner) self.assertEqual(ctx, collection[0].ctx) self.assertEqual(collection[0], appended_attribute_context) attribute_context = CdmAttributeContext(ctx, 'NameOfAttributeContext') appended_attribute = collection.append(attribute_context) self.assertEqual(2, len(collection)) self.assertEqual(attribute_context, appended_attribute) self.assertEqual(attribute_context, collection[1]) self.assertEqual(cdm_document, attribute_context.owner)
async def test_programmatically_create_partitions(self): corpus = CdmCorpusDefinition() corpus.storage.mount('local', LocalAdapter()) manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest') entity = manifest.entities.append('entity') relative_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'relative partition') relative_partition.location = 'relative/path' absolute_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'absolute partition') absolute_partition.location = 'local:/absolute/path' entity.data_partitions.append(relative_partition) entity.data_partitions.append(absolute_partition) manifest_data = await ManifestPersistence.to_data(manifest, None, None) self.assertEqual(len(manifest_data.entities), 1) entityData = manifest_data.entities[0] partitions_list = entityData.dataPartitions self.assertEqual(len(partitions_list), 2) relative_partition_data = partitions_list[0] absolute_partition_data = partitions_list[-1] self.assertEqual(relative_partition_data.location, relative_partition.location) self.assertEqual(absolute_partition_data.location, absolute_partition.location)
async def test_referenced_entity_declaration_resolution(self): corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('cdm', LocalAdapter(root='../../schemaDocuments')) corpus.storage.default_namespace = 'cdm' manifest = CdmManifestDefinition(corpus.ctx, 'manifest') manifest.entities.append( 'Account', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account') referenced_entity_path = \ 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/electronicMedicalRecords.manifest.cdm.json/Address' referenced_entity = CdmReferencedEntityDeclarationDefinition(corpus.ctx, 'Address') referenced_entity.entity_path = referenced_entity_path manifest.entities.append(referenced_entity) corpus.storage.fetch_root_folder('cdm').documents.append(manifest) resolved_manifest = await manifest.create_resolved_manifest_async('resolved_manifest', None) self.assertEqual(2, len(resolved_manifest.entities)) self.assertEqual('core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/resolved/Account.cdm.json/Account', resolved_manifest.entities[0].entity_path) self.assertEqual(referenced_entity_path, resolved_manifest.entities[1].entity_path)
async def test_attributes_that_are_replaced(self): corpus = TestHelper.get_local_corpus( self.tests_subpath, 'test_attributes_that_are_replaced') corpus.storage.mount('cdm', LocalAdapter(TestHelper.get_schema_docs_root())) extended_entity = await corpus.fetch_object_async( 'local:/extended.cdm.json/extended') # type: CdmEntityDefinition res_extended_ent = await extended_entity.create_resolved_entity_async( 'resExtended') # the attribute from the base class should be merged with the attribute # from the extended class into a single attribute self.assertEqual(1, len(res_extended_ent.attributes)) # check that traits from the base class merged with the traits from the extended class attribute = res_extended_ent.attributes[0] # base trait self.assertNotEqual( -1, attribute.applied_traits.index('means.identity.brand')) # extended trait self.assertNotEqual( -1, attribute.applied_traits.index('means.identity.company.name')) # make sure the attribute context and entity foreign key were maintained correctly foreign_key_for_base_attribute = res_extended_ent.attribute_context.contents[ 1].contents[1] self.assertEqual('_generatedAttributeSet', foreign_key_for_base_attribute.name) fk_reference = foreign_key_for_base_attribute.contents[0].contents[ 0].contents[0] # type: CdmAttributeReference self.assertEqual('resExtended/hasAttributes/regardingObjectId', fk_reference.named_reference)
async def test_conditional_proj_using_object_model(self): """Test for creating a projection with an AddCountAttribute operation and a condition using the object model""" corpus = ProjectionTestUtils.get_local_corpus(self.tests_subpath, 'test_conditional_proj_using_object_model') corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_conditional_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection with a condition that states the operation should only execute when the resolution directive is 'referenceOnly' projection = ProjectionTestUtils.create_projection(corpus, local_root) projection.condition = 'referenceOnly==True' # Create an AddCountAttribute operation add_count_attr_op = corpus.make_object(CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF) add_count_attr_op.count_attribute = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testCount') add_count_attr_op.count_attribute.data_type = corpus.make_ref(CdmObjectType.DATA_TYPE_REF, 'integer', True) projection.operations.append(add_count_attr_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Create resolution options with the 'referenceOnly' directive res_opt = ResolveOptions(entity.in_document) res_opt.directives = AttributeResolutionDirectiveSet(set(['referenceOnly'])) # Resolve the entity with 'referenceOnly' resolved_entity_with_reference_only = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddCountAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # Count attribute: "testCount" self.assertEqual(5, len(resolved_entity_with_reference_only.attributes)) self.assertEqual('id', resolved_entity_with_reference_only.attributes[0].name) self.assertEqual('name', resolved_entity_with_reference_only.attributes[1].name) self.assertEqual('value', resolved_entity_with_reference_only.attributes[2].name) self.assertEqual('date', resolved_entity_with_reference_only.attributes[3].name) self.assertEqual('testCount', resolved_entity_with_reference_only.attributes[4].name) self.assertIsNotNone(resolved_entity_with_reference_only.attributes[4].applied_traits.item('is.linkedEntity.array.count')) # Now resolve the entity with the 'structured' directive res_opt.directives = AttributeResolutionDirectiveSet(set(['structured'])) resolved_entity_with_structured = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), res_opt, local_root) # Verify correctness of the resolved attributes after running the AddCountAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # No Count attribute added, condition was false self.assertEqual(4, len(resolved_entity_with_structured.attributes)) self.assertEqual('id', resolved_entity_with_structured.attributes[0].name) self.assertEqual('name', resolved_entity_with_structured.attributes[1].name) self.assertEqual('value', resolved_entity_with_structured.attributes[2].name) self.assertEqual('date', resolved_entity_with_structured.attributes[3].name)
async def test_entity_attribute_proj_using_object_model(self): """Test for creating a projection with an AddTypeAttribute operation on an entity attribute using the object model""" corpus = ProjectionTestUtils.get_local_corpus( self.tests_subpath, 'test_entity_attribute_proj_using_object_model') corpus.storage.mount( 'local', LocalAdapter( TestHelper.get_actual_output_folder_path( self.tests_subpath, 'test_entity_attribute_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection projection = ProjectionTestUtils.create_projection(corpus, local_root) # Create an AddTypeAttribute operation add_type_attr_op = corpus.make_object( CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF) add_type_attr_op.type_attribute = corpus.make_object( CdmObjectType.TYPE_ATTRIBUTE_DEF, 'testType') add_type_attr_op.type_attribute.data_type = corpus.make_ref( CdmObjectType.DATA_TYPE_REF, 'entityName', True) projection.operations.append(add_type_attr_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object( CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref entity.attributes.append(entity_attribute) # Resolve the entity resolved_entity = await entity.create_resolved_entity_async( 'Resolved_{}.cdm.json'.format(entity.entity_name), None, local_root) # Verify correctness of the resolved attributes after running the AddTypeAttribute operation # Original set of attributes: ["id", "name", "value", "date"] # Type attribute: "testType" self.assertEqual(5, len(resolved_entity.attributes)) self.assertEqual('id', resolved_entity.attributes[0].name) self.assertEqual('name', resolved_entity.attributes[1].name) self.assertEqual('value', resolved_entity.attributes[2].name) self.assertEqual('date', resolved_entity.attributes[3].name) self.assertEqual('testType', resolved_entity.attributes[4].name) self.assertEqual( 'is.linkedEntity.name', resolved_entity.attributes[4].applied_traits[4].named_reference)
async def test_loading_invalid_model_json_name(self): test_input_path = TestHelper.get_input_folder_path(self.test_subpath, 'test_loading_invalid_model_json_name') corpus = CdmCorpusDefinition() corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' # We are trying to load a file with an invalid name, so fetch_object_async should just return None. invalid_model_json = await corpus.fetch_object_async('test.model.json') self.assertIsNone(invalid_model_json)
async def test_resolve_test_corpus(self): self.assertTrue(os.path.exists(TestHelper.get_schema_docs_root())) corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(TestHelper.get_schema_docs_root())) manifest = await corpus.fetch_object_async(TestHelper.cdm_standards_schema_path) # type: CdmManifestDefinition directives = AttributeResolutionDirectiveSet({'referenceOnly', 'normalized'}) all_resolved = await self.list_all_resolved(corpus, directives, manifest, StringSpewCatcher()) self.assertNotEqual(all_resolved, '')
async def test_entity_attribute_proj_using_object_model(self): """Test for creating a projection with an ArrayExpansion operation on an entity attribute using the object model""" corpus = TestHelper.get_local_corpus(self.tests_subpath, 'test_entity_attribute_proj_using_object_model') corpus.storage.mount('local', LocalAdapter(TestHelper.get_actual_output_folder_path(self.tests_subpath, 'test_entity_attribute_proj_using_object_model'))) local_root = corpus.storage.fetch_root_folder('local') # Create an entity entity = ProjectionTestUtils.create_entity(corpus, local_root) # Create a projection projection = ProjectionTestUtils.create_projection(corpus, local_root) # Create an ArrayExpansion operation array_expansion_op = corpus.make_object(CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF) array_expansion_op.start_ordinal = 1 array_expansion_op.end_ordinal = 2 projection.operations.append(array_expansion_op) # Create an entity reference to hold this projection projection_entity_ref = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref.explicit_reference = projection # Create another projection that does a rename so that we can see the expanded attributes in the final resolved entity projection2 = corpus.make_object(CdmObjectType.PROJECTION_DEF) projection2.source = projection_entity_ref # Create a RenameAttributes operation rename_attrs_op = corpus.make_object(CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF) rename_attrs_op.rename_format = '{m}{o}' projection2.operations.append(rename_attrs_op) # Create an entity reference to hold this projection projection_entity_ref2 = corpus.make_object(CdmObjectType.ENTITY_REF, None) projection_entity_ref2.explicit_reference = projection2 # Create an entity attribute that contains this projection and add this to the entity entity_attribute = corpus.make_object(CdmObjectType.ENTITY_ATTRIBUTE_DEF, 'TestEntityAttribute') entity_attribute.entity = projection_entity_ref2 entity.attributes.append(entity_attribute) # Resolve the entity resolved_entity = await entity.create_resolved_entity_async('Resolved_{}.cdm.json'.format(entity.entity_name), None, local_root) # Verify correctness of the resolved attributes after running the projections # Original set of attributes: ["id", "name", "value", "date"] # Expand 1...2, renameFormat = {m}{o} self.assertEqual(8, len(resolved_entity.attributes)) self.assertEqual('id1', resolved_entity.attributes[0].name) self.assertEqual('name1', resolved_entity.attributes[1].name) self.assertEqual('value1', resolved_entity.attributes[2].name) self.assertEqual('date1', resolved_entity.attributes[3].name) self.assertEqual('id2', resolved_entity.attributes[4].name) self.assertEqual('name2', resolved_entity.attributes[5].name) self.assertEqual('value2', resolved_entity.attributes[6].name) self.assertEqual('date2', resolved_entity.attributes[7].name)