async def test_aad_app_authorization_exception(self): """ Test with invalid AAD App credentials, which should fail the authorization. """ corpus = CdmCorpusDefinition() # Initialize with some dummy credentials tenant_id = os.environ.get('KUSTO_TENANTID') kusto_config = TelemetryConfig(tenant_id=tenant_id, client_id='client_id', secret='secret', cluster_name='cluster_name', database_name='database_name', ingest_at_level=EnvironmentType.DEV, remove_user_content=False) corpus.telemetry_client = TelemetryKustoClient(corpus.ctx, kusto_config) try: await cast( 'TelemetryKustoClient', corpus.telemetry_client).post_kusto_query("some random query") except Exception as ex: self.assertTrue( str(ex).startswith( 'There was an error while acquiring Kusto authorization Token with client ID/secret authentication.' ))
async def test_programmatically_create_partitions(self): corpus = CdmCorpusDefinition() corpus.storage.mount('local', LocalAdapter()) manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest') entity = manifest.entities.append('entity') relative_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'relative partition') relative_partition.location = 'relative/path' absolute_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'absolute partition') absolute_partition.location = 'local:/absolute/path' entity.data_partitions.append(relative_partition) entity.data_partitions.append(absolute_partition) manifest_data = await ManifestPersistence.to_data(manifest, None, None) self.assertEqual(len(manifest_data.entities), 1) entityData = manifest_data.entities[0] partitions_list = entityData.dataPartitions self.assertEqual(len(partitions_list), 2) relative_partition_data = partitions_list[0] absolute_partition_data = partitions_list[-1] self.assertEqual(relative_partition_data.location, relative_partition.location) self.assertEqual(absolute_partition_data.location, absolute_partition.location)
def test_cdm_collection_appending_list(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath')) ctx = cdm_corpus.ctx cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument') collection = CdmCollection(ctx, cdm_document, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity_list = [] for i in range(0, 2): entity = CdmEntityDefinition(cdm_corpus.ctx, 'entityName_{}'.format(i), None) create_document_for_entity(cdm_corpus, entity) entity_declaration = cdm_corpus.make_object( CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, entity.entity_name, False) # type: CdmLocalEntityDeclarationDefinition entity_declaration.entity_path = '{}/{}'.format( entity.owner.at_corpus_path, entity.entity_name) entity_declaration.owner = entity.owner entity_list.append(entity_declaration) self.assertEqual(0, len(collection)) collection.extend(entity_list) self.assertEqual(2, len(collection)) for i in range(0, 2): self.assertEqual('entityName_{}'.format(i), collection[i].entity_name)
def test_data_format(self): corpus = CdmCorpusDefinition() att = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'att') for format in CdmDataFormat: att.dataFormat = format self.assertEqual(att.dataFormat, format)
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None, is_language_specific: Optional[bool] = False, expected_codes: Optional[set] = None, no_input_and_output_folder: Optional[bool] = False): """ Creates a corpus to be used by the tests, which mounts inputFolder, outputFolder, cdm, and remoteAdapter. Will fail on any unexpected warning/error. @param testSubpath The root of the corpus files. @param testName The test name. @param testInputDir The test input directory. @param isLanguageSpecific Indicate whether there is subfolder called Java, it's used when input is different compared with other languages @param expectedCodes The error codes that are expected, and they should not block the test. @param noInputAndOutputFolder No input and output folder needed. """ if no_input_and_output_folder: test_input_dir = 'C:\\dummpyPath' test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name, is_language_specific) test_output_dir = test_input_dir if no_input_and_output_folder else TestHelper.get_actual_output_folder_path(test_subpath, test_name) cdm_corpus = CdmCorpusDefinition() def callback(level: CdmStatusLevel, message: str): TestHelper._fail_on_unexpected_failure(cdm_corpus, message, expected_codes) cdm_corpus.set_event_callback(callback, CdmStatusLevel.WARNING) cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir)) cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir)) cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.schema_documents_path)) cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'})) return cdm_corpus
def test_path_that_does_not_end_in_slash(self): """FolderPath should always end with a / This checks the behavior if FolderPath does not end with a / ('/' should be appended and a warning be sent through callback function)""" corpus = CdmCorpusDefinition() function_was_called = False function_parameter1 = CdmStatusLevel.INFO function_parameter2 = None def callback(status_level: CdmStatusLevel, message1: str): nonlocal function_was_called, function_parameter1, function_parameter2 function_was_called = True function_parameter1 = status_level function_parameter2 = message1 corpus.set_event_callback(callback) manifest = CdmManifestDefinition(None, None) manifest.namespace = 'cdm' manifest.folder_path = 'Mnp' absolute_path = corpus.storage.create_absolute_corpus_path('Abc', manifest) self.assertEqual('cdm:Mnp/Abc', absolute_path) self.assertEqual(function_was_called, True) self.assertEqual(function_parameter1, CdmStatusLevel.WARNING) self.assertTrue(function_parameter2.find('Expected path prefix to end in /, but it didn\'t. Appended the /') != -1)
def test_manifest_cannot_add_entity_definition_without_creating_document(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' function_was_called = False function_parameter1 = CdmStatusLevel.INFO function_parameter2 = '' def callback(status_level: 'CdmStatusLevel', message: str): nonlocal function_was_called, function_parameter1, function_parameter2 function_was_called = True function_parameter1 = status_level function_parameter2 = message cdm_corpus.set_event_callback(callback) cdm_corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path')) manifest = CdmManifestDefinition(cdm_corpus.ctx, 'manifest') manifest._folder_path = '/' manifest._namespace = 'local' entity = CdmEntityDefinition(manifest.ctx, 'entityName', None) manifest.entities.append(entity) self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue('Expected entity to have an \'Owner\' document set. Cannot create entity declaration to add to manifest.' in function_parameter2)
def test_path_root_invalid_folder_path(self): """"Tests absolute paths cannot be created with wrong parameters. Checks behavior if FolderPath is invalid.""" corpus = CdmCorpusDefinition() function_was_called = False function_parameter1 = CdmStatusLevel.INFO function_parameter2 = None def callback(status_level: CdmStatusLevel, message1: str): nonlocal function_was_called, function_parameter1, function_parameter2 function_was_called = True function_parameter1 = status_level function_parameter2 = message1 corpus.set_event_callback(callback) manifest = CdmManifestDefinition(None, None) manifest.namespace = 'cdm' manifest.folder_path = './Mnp' corpus.storage.create_absolute_corpus_path('Abc', manifest) self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue(function_parameter2.find('The path should not start with ./') != -1) function_was_called = False manifest.namespace = 'cdm' manifest.folder_path = '/./Mnp' corpus.storage.create_absolute_corpus_path('Abc', manifest) self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue(function_parameter2.find('The path should not contain /./') != -1) function_was_called = False manifest.namespace = 'cdm' manifest.folder_path = '../Mnp' corpus.storage.create_absolute_corpus_path('Abc', manifest) function_parameter2 = function_parameter2.split('|')[1].strip() self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue(function_parameter2.find('The path should not contain ../') != -1) function_was_called = False manifest.namespace = 'cdm' manifest.folder_path = 'Mnp/./Qrs' corpus.storage.create_absolute_corpus_path('Abc', manifest) function_parameter2 = function_parameter2.split('|')[1].strip() self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue(function_parameter2.find('The path should not contain /./') != -1) function_was_called = False manifest.namespace = 'cdm' manifest.folder_path = 'Mnp/../Qrs' corpus.storage.create_absolute_corpus_path('Abc', manifest) function_parameter2 = function_parameter2.split('|')[1].strip() self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue(function_parameter2.find('The path should not contain ../') != -1)
async def create_manifest(self, cdm_corpus: CdmCorpusDefinition): print('Make placeholder manifest') # Make the temp manifest and add it to the root of the local documents in the corpus. manifest_abstract = cdm_corpus.make_object(CdmObjectType.MANIFEST_DEF, 'temp_abstract') # type: CdmManifestDefinition # Add each declaration, this example is about medical appointments and care plans manifest_abstract.entities.append('Account', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account') manifest_abstract.entities.append('Address', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address') manifest_abstract.entities.append('CarePlan', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan') manifest_abstract.entities.append('CodeableConcept', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept') manifest_abstract.entities.append('Contact', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact') manifest_abstract.entities.append('Device', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device') manifest_abstract.entities.append('EmrAppointment', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment') manifest_abstract.entities.append('Encounter', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter') manifest_abstract.entities.append('EpisodeOfCare', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare') manifest_abstract.entities.append('Location', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location') # Add the temp manifest to the root of the local documents in the corpus. local_root = cdm_corpus.storage.fetch_root_folder('local') local_root.documents.append(manifest_abstract) # Create the resolved version of everything in the root folder too. print('Resolve the placeholder') manifest_resolved = await manifest_abstract.create_resolved_manifest_async('default', '') # Add an import to the foundations doc so the traits about partitons will resolve nicely. manifest_resolved.imports.append('cdm:/foundations.cdm.json', '') print('Save the documents') for e_def in manifest_resolved.entities: # Get the entity being pointed at. local_e_def = cast(CdmLocalEntityDeclarationDefinition, e_def) # Turns a relative path from manifest_resolved into an absolute path. ent_def = cast(CdmEntityDefinition, await cdm_corpus.fetch_object_async(local_e_def.entity_path, manifest_resolved)) # Make a fake partition, just to demo that. part = cdm_corpus.make_object(CdmObjectType.DATA_PARTITION_DEF, '{}-data-description'.format(ent_def.entity_name)) # type: CdmDataPartitionDefinition local_e_def.data_partitions.append(part) part.explanation = 'not real data, just for demo' # Define the location of the partition, relative to the manifest local_location = 'local:/{}/partition-data.csv'.format(ent_def.entity_name) part.location = cdm_corpus.storage.create_relative_corpus_path(local_location, manifest_resolved) # Add trait to partition for csv params. csv_trait = part.exhibits_traits.append('is.partition.format.CSV', False) csv_trait.arguments.append('columnHeaders', 'true') csv_trait.arguments.append('delimiter', ',') # Get the actual location of the partition file from the corpus. part_path = cdm_corpus.storage.corpus_path_to_adapter_path(local_location) # Make a fake file with nothing but header for columns. header = ','.join([att.name for att in ent_def.attributes]) os.makedirs(cdm_corpus.storage.corpus_path_to_adapter_path('local:/{}'.format(ent_def.entity_name)), exist_ok=True) with open(part_path, 'w') as file: file.write(header) await manifest_resolved.save_as_async('{}.manifest.cdm.json'.format(manifest_resolved.manifest_name), True)
async def read_local_save_adls(self, corpus: CdmCorpusDefinition): # --------------------------------------------------------------------------------------------- # Create manifest from a local model.json file manifest = await corpus.fetch_object_async('local:/model.json') # ------------------------------------------------------------------------------------------------------------ # Explore entities and partitions defined in the model print('Listing entity declarations:') for decl in manifest.entities: print(' ' + decl.entity_name) if decl.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: for data_part in decl.data_partitions: print(' ' + data_part.location) # --------------------------------------------------------------------------------------------- # Make a few changes to the model # Create a new document where the new entity's definition will be stored new_entity_doc = corpus.make_object( CdmObjectType.DOCUMENT_DEF, 'NewEntity.cdm.json') # type: CdmDocumentDefinition new_entity_doc.imports.append('cdm:/foundations.cdm.json') corpus.storage.fetch_root_folder('local').documents.append( new_entity_doc) new_entity = new_entity_doc.definitions.append( 'NewEntity', CdmObjectType.ENTITY_DEF, False) # type: CdmEntityDefinition # Define new string attribute and add it to the entity definition new_attribute = corpus.make_object( CdmObjectType.TYPE_ATTRIBUTE_DEF, 'NewAttribute') # type: CdmTypeAttributeDefinition new_attribute.data_format = CdmDataFormat.STRING new_entity.attributes.append(new_attribute) # Call will create entity_declaration_definition based on entity definition and add it to manifest.entities new_entity_decl = manifest.entities.append( new_entity) # type: CdmLocalEntityDeclaration # Define a partition and add it to the local declaration new_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'NewPartition', False) # type: CdmDataPartitionDefinition new_partition.location = 'adls:/NewPartition.csv' new_entity_decl.data_partitions.append(new_partition) # --------------------------------------------------------------------------------------------- # Save the file to ADLSg2 - we achieve that by adding the manifest to the root folder of # the ADLS file-system and performing a save on the manifest adls_folder = corpus.storage.fetch_root_folder( 'adls') # type: CdmFolderDefinition adls_folder.documents.append(manifest) await manifest.save_as_async('model.json', True)
async def test_programmatically_create_partitions(self): corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest') entity = manifest.entities.append('entity') relative_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'relative partition') relative_partition.location = 'relative/path' relative_partition.arguments['test1'] = ['argument1'] relative_partition.arguments['test2'] = ['argument2', 'argument3'] absolute_partition = corpus.make_object( CdmObjectType.DATA_PARTITION_DEF, 'absolute partition') absolute_partition.location = 'local:/absolute/path' # add an empty arguments list to test empty list should not be displayed in ToData json. absolute_partition.arguments['test'] = [] entity.data_partitions.append(relative_partition) entity.data_partitions.append(absolute_partition) manifest_data = ManifestPersistence.to_data(manifest, None, None) self.assertEqual(len(manifest_data.entities), 1) entityData = manifest_data.entities[0] partitions_list = entityData.dataPartitions self.assertEqual(len(partitions_list), 2) relative_partition_data = partitions_list[0] absolute_partition_data = partitions_list[-1] self.assertEqual(relative_partition_data.location, relative_partition.location) arguments_list = relative_partition_data.arguments self.assertEqual(3, len(arguments_list)) checked_arguments = [] for argument in arguments_list: self.assertEqual(3, len(argument)) checked_arguments.append(argument.value) if argument.value == 'argument1': self.assertEqual('test1', argument.name) elif argument.value == 'argument2': self.assertEqual('test2', argument.name) elif argument.value == 'argument3': self.assertEqual('test2', argument.name) else: raise Exception('unexpected argument in data partitions') self.assertTrue('argument1' in checked_arguments) self.assertTrue('argument2' in checked_arguments) self.assertTrue('argument3' in checked_arguments) self.assertEqual(absolute_partition_data.location, absolute_partition.location) # test if empty argument list is set to null self.assertEqual(absolute_partition_data.arguments, None)
def test_path_root_invalid_object_path(self): """Tests absolute paths cannot be created with wrong parameters. Checks behavior if objectPath is invalid.""" corpus = CdmCorpusDefinition() function_was_called = False function_parameter1 = CdmStatusLevel.INFO function_parameter2 = None def callback(status_level: CdmStatusLevel, message1: str): nonlocal function_was_called, function_parameter1, function_parameter2 function_was_called = True function_parameter1 = status_level function_parameter2 = message1 corpus.set_event_callback(callback) corpus.storage.create_absolute_corpus_path('./Abc') self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue( function_parameter2.find('The path should not start with ./') != -1 ) function_was_called = False corpus.storage.create_absolute_corpus_path('/./Abc') self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue( function_parameter2.find('The path should not contain /./') != -1) function_was_called = False corpus.storage.create_absolute_corpus_path('../Abc') self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue( function_parameter2.find('The path should not contain ../') != -1) function_was_called = False corpus.storage.create_absolute_corpus_path('Abc/./Def') self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue( function_parameter2.find('The path should not contain /./') != -1) function_was_called = False corpus.storage.create_absolute_corpus_path('Abc/../Def') self.assertTrue(function_was_called) self.assertEqual(CdmStatusLevel.ERROR, function_parameter1) self.assertTrue( function_parameter2.find('The path should not contain ../') != -1)
async def test_entity_properties(self): test_input_path = TestHelper.get_input_folder_path( self.tests_subpath, 'test_entity_properties') corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A') att = obj.attributes[0] # type: CdmTypeAttributeDefinition result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNotNone(result) self.assertEqual(att.maximum_length, 30) self.assertIsNone(att.maximum_value) self.assertIsNone(att.minimum_value) # removing the only argument should remove the trait att.maximum_length = None result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNone(att.maximum_length) self.assertIsNone(result)
async def test_from_and_to_data(self): with open(os.path.join(ROOT_PATH, 'default.manifest.cdm.json')) as manifestFile: manifestContent = ManifestContent() manifestContent.decode(manifestFile.read()) corpus = CdmCorpusDefinition() corpus.storage.default_namespace = 'local' corpus.ctx.update_logging_options(level='WARNING') corpus.storage.mount('cdm', LocalAdapter(root="../CDM.SchemaDocuments")) corpus.storage.mount('local', LocalAdapter(root=ROOT_PATH)) corpus.storage.mount( 'remote', RemoteAdapter(hosts={"contoso": "http://contoso.com"})) folder = corpus.storage.fetch_root_folder('local') manifest = await corpus.fetch_object_async('default.manifest.cdm.json', folder) data = await ManifestPersistence.to_data(manifest, None, None) for entity in manifest.entities: entity_def = await corpus.fetch_object_async( entity.entity_path, manifest) await manifest.save_as_async('test_output/new.manifest.cdm.json', save_referenced=True) self.assertDictEqual(json.loads(manifestContent.encode()), json.loads(data.encode()))
def test_manifest_copy(self): """Tests if the copy function creates copies of the sub objects""" corpus = CdmCorpusDefinition() manifest = CdmManifestDefinition(corpus.ctx, 'name') entity_name = 'entity' sub_manifest_name = 'sub_manifest' relationship_name = 'relName' trait_name = 'traitName' entity_dec = manifest.entities.append(entity_name) sub_manifest = manifest.sub_manifests.append(sub_manifest_name) relationship = manifest.relationships.append(relationship_name) trait = manifest.exhibits_traits.append(trait_name) copy = manifest.copy() # type: CdmManifestDefinition copy.entities[0].entity_name = 'newEntity' copy.sub_manifests[0].manifest_name = 'newSubManifest' copy.relationships[0].name = 'newRelName' copy.exhibits_traits[0].named_reference = 'newTraitName' self.assertEqual(entity_name, entity_dec.entity_name) self.assertEqual(sub_manifest_name, sub_manifest.manifest_name) self.assertEqual(relationship_name, relationship.name) self.assertEqual(trait_name, trait.named_reference)
def test_load_local_entity_with_data_partition_pattern(self): content = TestHelper.get_input_file_content( self.test_subpath, "test_load_local_entity_with_data_partition_pattern", "entities.manifest.cdm.json") manifest_content = ManifestContent() manifest_content.decode(content) cdmManifest = ManifestPersistence.from_object( CdmCorpusContext(CdmCorpusDefinition(), None), "entities", "testNamespace", "/", manifest_content) self.assertEqual(len(cdmManifest.entities), 1) self.assertEqual(cdmManifest.entities[0].object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity = cdmManifest.entities[0] self.assertEqual(len(entity.data_partition_patterns), 1) pattern = entity.data_partition_patterns[0] self.assertEqual(pattern.name, "testPattern") self.assertEqual(pattern.explanation, "test explanation") self.assertEqual(pattern.root_location, "test location") self.assertEqual(pattern.regular_expression, "\\s*") self.assertEqual(len(pattern.parameters), 2) self.assertEqual(pattern.parameters[0], "testParam1") self.assertEqual(pattern.parameters[1], "testParam2") self.assertEqual(pattern.specialized_schema, "test special schema") self.assertEqual(len(pattern.exhibits_traits), 1)
def test_manifest_add_list_of_entity_declarations(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = "local" cdm_corpus.storage.mount("local", LocalAdapter("CdmCorpus/LocalPath")) ctx = cdm_corpus.ctx cdmDocument = CdmDocumentDefinition(ctx, "NameOfDocument") collection = CdmEntityCollection(ctx, cdmDocument) entity_list = [] for i in range(0, 2): entity = CdmEntityDefinition(cdm_corpus.ctx, 'entityName_{}'.format(i), None) create_document_for_entity(cdm_corpus, entity) entity_list.append(entity) self.assertEqual(0, len(collection)) collection.extend(entity_list) self.assertEqual(2, len(collection)) for i in range(0, 2): self.assertEqual('entityName_{}'.format(i), collection[i].entity_name)
async def test_entity_properties(self): test_input_path = os.path.join(ROOT_PATH, 'entity') corpus = CdmCorpusDefinition() corpus.ctx.update_logging_options(level='WARNING') corpus.storage.mount('local', LocalAdapter(test_input_path)) corpus.storage.default_namespace = 'local' obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A') att = obj.attributes[0] # type: CdmTypeAttributeDefinition result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNotNone(result) self.assertEqual(att.maximum_length, 30) self.assertIsNone(att.maximum_value) self.assertIsNone(att.minimum_value) # removing the only argument should remove the trait att.maximum_length = None result = next( filter(lambda x: x.named_reference == 'is.constrained', att.applied_traits), None) self.assertIsNone(att.maximum_length) self.assertIsNone(result)
def test_create_absolute_corpus_pathWithColon(self): """Tests if create_absolute_corpus_path works correctly when provided with a path that contains a colon character.""" corpus = CdmCorpusDefinition() folder = corpus.storage.fetch_root_folder('local') absolute_namespace = 'namespace:/' file_name = 'dataPartition.csv@snapshot=2020-05-10T02:47:46.0039374Z' sub_folder_path = 'some/sub/folder:with::colon/' # Cases where the path provided is relative. self.assertEqual( 'local:/' + file_name, corpus.storage.create_absolute_corpus_path(file_name, folder)) self.assertEqual( 'local:/' + sub_folder_path + file_name, corpus.storage.create_absolute_corpus_path( sub_folder_path + file_name, folder)) # Cases where the path provided is absolute. self.assertEqual( absolute_namespace + file_name, corpus.storage.create_absolute_corpus_path( absolute_namespace + file_name, folder)) self.assertEqual( absolute_namespace + sub_folder_path + file_name, corpus.storage.create_absolute_corpus_path( absolute_namespace + sub_folder_path + file_name, folder))
def setup_cdm_corpus(self) -> CdmCorpusDefinition: corpus = CdmCorpusDefinition() corpus.default_namespace = 'local' corpus.storage.mount( 'cdm', LocalAdapter(TestHelper.sample_schema_folder_path)) corpus.storage.mount( 'local', LocalAdapter( AdlsModelJsonTestHelper.get_actual_sub_folder_path( self.tests_subpath, self.test_name, AdlsModelJsonTestHelper.input_folder_name))) corpus.storage.mount( 'adls', AdlsTestHelper.create_adapter_with_client_id( self.root_relative_path)) return corpus
async def test_referenced_entity_declaration_resolution(self): corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('cdm', LocalAdapter(root='../../schemaDocuments')) corpus.storage.default_namespace = 'cdm' manifest = CdmManifestDefinition(corpus.ctx, 'manifest') manifest.entities.append( 'Account', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account') referenced_entity_path = \ 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/electronicMedicalRecords.manifest.cdm.json/Address' referenced_entity = CdmReferencedEntityDeclarationDefinition(corpus.ctx, 'Address') referenced_entity.entity_path = referenced_entity_path manifest.entities.append(referenced_entity) corpus.storage.fetch_root_folder('cdm').documents.append(manifest) resolved_manifest = await manifest.create_resolved_manifest_async('resolved_manifest', None) self.assertEqual(2, len(resolved_manifest.entities)) self.assertEqual('core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/resolved/Account.cdm.json/Account', resolved_manifest.entities[0].entity_path) self.assertEqual(referenced_entity_path, resolved_manifest.entities[1].entity_path)
async def test_loading_and_saving_config(self): """Testing loading and saving config.""" test_name = 'test_loading_and_saving_config' test_input_path = TestHelper.get_input_folder_path( self.tests_subpath, test_name) test_output_path = TestHelper.get_expected_output_folder_path( self.tests_subpath, test_name) # Create a corpus to load the config. cdm_corpus = self.get_local_corpus(test_input_path, test_output_path) config = await cdm_corpus.storage.fetch_adapter('local').read_async( '/config.json') different_corpus = CdmCorpusDefinition() different_corpus.storage.mount_from_config(config) result_config = different_corpus.storage.fetch_config() output_config = await cdm_corpus.storage.fetch_adapter( 'target').read_async('/config.json') self.maxDiff = None self.assertDictEqual(json.loads(output_config), json.loads(result_config))
async def run_syms_smart_adls_adapter_mount_logic(self): syms_adapter = SymsTestHelper.create_adapter_with_clientid() corpus = CdmCorpusDefinition() corpus.storage.mount('syms', syms_adapter) adls_adapter1 = SymsTestHelper.create_adapter_clientid_with_shared_key( 1) adls_adapter2 = SymsTestHelper.create_adapter_clientid_with_shared_key( 2) count_adapter_count_before = len(corpus.storage.namespace_adapters) manifest_read_databases = await corpus.fetch_object_async( 'syms:/databases.manifest.cdm.json') manifest = await corpus.fetch_object_async( 'syms:/{}/{}.manifest.cdm.json'.format( manifest_read_databases.sub_manifests[0].manifest_name, manifest_read_databases.sub_manifests[0].manifest_name), manifest_read_databases, None, True) count_adapter_count_after = len(corpus.storage.namespace_adapters) self.assertEqual(count_adapter_count_before + 2, count_adapter_count_after) self.assertIsNotNone( corpus.storage.adapter_path_to_corpus_path('https://{}{}'.format( adls_adapter1.hostname, adls_adapter1.root))) self.assertIsNotNone( corpus.storage.adapter_path_to_corpus_path('https://{}{}'.format( adls_adapter2.hostname, adls_adapter2.root)))
def test_cdm_collection_append_method(self): cdm_corpus = CdmCorpusDefinition() cdm_corpus.storage.default_namespace = 'local' cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath')) ctx = cdm_corpus.ctx cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument') collection = CdmCollection(ctx, cdm_document, CdmObjectType.ATTRIBUTE_CONTEXT_DEF) appended_attribute_context = collection.append('nameOfNewAttribute') self.assertEqual(1, len(collection)) self.assertEqual('nameOfNewAttribute', collection[0].name) self.assertEqual(cdm_document, collection[0].owner) self.assertEqual(ctx, collection[0].ctx) self.assertEqual(collection[0], appended_attribute_context) attribute_context = CdmAttributeContext(ctx, 'NameOfAttributeContext') appended_attribute = collection.append(attribute_context) self.assertEqual(2, len(collection)) self.assertEqual(attribute_context, appended_attribute) self.assertEqual(attribute_context, collection[1]) self.assertEqual(cdm_document, attribute_context.owner)
def test_valid_root_path(self): """Checks Absolute corpus path can be created with valid input.""" corpus = CdmCorpusDefinition() # checks with None object absolute_path = corpus.storage.create_absolute_corpus_path('Abc/Def') self.assertEqual('/Abc/Def', absolute_path) absolute_path = corpus.storage.create_absolute_corpus_path('/Abc/Def') self.assertEqual('/Abc/Def', absolute_path) absolute_path = corpus.storage.create_absolute_corpus_path('cdm:/Abc/Def') self.assertEqual('cdm:/Abc/Def', absolute_path) manifest = CdmManifestDefinition(None, None) manifest.namespace = '' manifest.folder_path = 'Mnp/Qrs/' absolute_path = corpus.storage.create_absolute_corpus_path('Abc/Def', manifest) self.assertEqual('Mnp/Qrs/Abc/Def', absolute_path) manifest.namespace = 'cdm' manifest.folder_path = 'Mnp/Qrs/' absolute_path = corpus.storage.create_absolute_corpus_path('/Abc/Def', manifest) self.assertEqual('cdm:/Abc/Def', absolute_path) manifest.namespace = 'cdm' manifest.folder_path = 'Mnp/Qrs/' absolute_path = corpus.storage.create_absolute_corpus_path('Abc/Def', manifest) self.assertEqual('cdm:Mnp/Qrs/Abc/Def', absolute_path)
async def test_cdm_folder_to_data_type_attribute(self): """ Testing that "is.localized.describedAs" trait with a table of three entries (en, rs and cn) is fully preserved when running CdmFolder TypeAttributePersistence ToData. """ corpus = CdmCorpusDefinition() corpus.ctx.report_at_level = CdmStatusLevel.WARNING corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path')) corpus.storage.default_namespace = 'local' cdm_type_attribute_definition = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'TestSavingTraitAttribute', False) english_constants_list = ['en', 'Some description in English language'] serbian_constants_list = ['sr', 'Opis na srpskom jeziku'] chinese_constants_list = ['cn', '一些中文描述'] list_of_const_lists = [english_constants_list, serbian_constants_list, chinese_constants_list] const_ent_def = corpus.make_object(CdmObjectType.CONSTANT_ENTITY_DEF, 'localizedDescriptions', False) const_ent_def.constant_values = list_of_const_lists const_ent_def.entity_shape = corpus.make_ref(CdmObjectType.ENTITY_REF, 'localizedTable', True) trait_reference2 = corpus.make_object(CdmObjectType.TRAIT_REF, 'is.localized.describedAs', False) trait_reference2.arguments.append('localizedDisplayText', corpus.make_ref(CdmObjectType.ENTITY_REF, const_ent_def, True)) cdm_type_attribute_definition.applied_traits.append(trait_reference2) result = TypeAttribute().decode(PersistenceLayer.to_data(cdm_type_attribute_definition, None, None, PersistenceLayer.CDM_FOLDER)) self.assertIsNotNone(result.appliedTraits) argument = result.appliedTraits[0].arguments[0] constant_values = argument.value.entityReference.constantValues self.assertEqual('en', constant_values[0][0]) self.assertEqual('Some description in English language', constant_values[0][1]) self.assertEqual('sr', constant_values[1][0]) self.assertEqual('Opis na srpskom jeziku', constant_values[1][1]) self.assertEqual('cn', constant_values[2][0]) self.assertEqual('一些中文描述', constant_values[2][1])
async def test_resolving_manifest_not_in_folder(self): """Test that resolving a manifest that hasn't been added to a folder doesn't throw any exceptions.""" try: corpus = CdmCorpusDefinition() corpus.storage.mount('local', LocalAdapter('C:\\path')) corpus.storage.default_namespace = 'local' def callback(level, message): # We should see the following error message be logged. If not, fail. if 'Cannot resolve the manifest \'test\' because it has not been added to a folder' not in message: self.fail() corpus.set_event_callback(callback, CdmStatusLevel.WARNING) manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test') entity = corpus.make_object(CdmObjectType.ENTITY_DEF, 'entity') document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION)) document.definitions.append(entity) # Don't add the document containing the entity to a folder either. manifest.entities.append(entity) await manifest.create_resolved_manifest_async('resolved', None) except Exception: self.fail('Exception should not be thrown when resolving a manifest that is not in a folder.')
def test_trait_to_unknown_data_format(self): """Test trait to data format when unknown data format trait is in an attribute.""" cdm_attribute = CdmTypeAttributeDefinition(CdmCorpusContext(CdmCorpusDefinition(), None), 'SomeAttribute') cdm_attribute.applied_traits.append('is.data_format.someRandomDataFormat') trait_to_property_map = TraitToPropertyMap(cdm_attribute) data_format = trait_to_property_map._traits_to_data_format(False) self.assertEqual(CdmDataFormat.UNKNOWN, data_format)
async def test_setting_model_json_entity_description(self): """Tests that a description on a CdmFolder entity sets the description on the ModelJson entity.""" corpus = CdmCorpusDefinition() manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test') document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION)) folder = corpus.storage.fetch_root_folder('local') folder.documents.append(document) entity = document.definitions.append(CdmObjectType.ENTITY_DEF.name, 'entity') entity.description = 'test description' manifest.entities.append(entity) folder.documents.append(manifest) obtained_model_json = await ManifestPersistence.to_data(manifest, None, None) self.assertEqual('test description', obtained_model_json.entities[0].get('description'))
def test_entity_attribute_source(self): """Tests if not setting the projection "source" on an entity attribute triggers an error log""" corpus = CdmCorpusDefinition() error_count = 0 def callback(level: 'CdmStatusLevel', message: str): nonlocal error_count error_count += 1 corpus.set_event_callback(callback, CdmStatusLevel.ERROR) projection = CdmProjection(corpus.ctx) entity_attribute = CdmEntityAttributeDefinition( corpus.ctx, 'attribute') entity_attribute.entity = CdmEntityReference(corpus.ctx, projection, False) # First case, a projection without source. projection.validate() self.assertEqual(1, error_count) error_count = 0 # Second case, a projection with a nested projection. inner_projection = CdmProjection(corpus.ctx) projection.source = CdmEntityReference(corpus.ctx, inner_projection, False) projection.validate() inner_projection.validate() self.assertEqual(1, error_count) error_count = 0 # Third case, a projection with an explicit entity definition. inner_projection.source = CdmEntityReference( corpus.ctx, CdmEntityDefinition(corpus.ctx, 'Entity'), False) projection.validate() inner_projection.validate() self.assertEqual(0, error_count) # Third case, a projection with a named reference. inner_projection.source = CdmEntityReference(corpus.ctx, 'Entity', False) projection.validate() inner_projection.validate() self.assertEqual(0, error_count)