async def test_aad_app_authorization_exception(self):
"""
Test with invalid AAD App credentials, which should fail the authorization.
"""
corpus = CdmCorpusDefinition()
# Initialize with some dummy credentials
tenant_id = os.environ.get('KUSTO_TENANTID')
kusto_config = TelemetryConfig(tenant_id=tenant_id,
client_id='client_id',
secret='secret',
cluster_name='cluster_name',
database_name='database_name',
ingest_at_level=EnvironmentType.DEV,
remove_user_content=False)
corpus.telemetry_client = TelemetryKustoClient(corpus.ctx,
kusto_config)
try:
await cast(
'TelemetryKustoClient',
corpus.telemetry_client).post_kusto_query("some random query")
except Exception as ex:
self.assertTrue(
str(ex).startswith(
'There was an error while acquiring Kusto authorization Token with client ID/secret authentication.'
))
async def test_programmatically_create_partitions(self):
corpus = CdmCorpusDefinition()
corpus.storage.mount('local', LocalAdapter())
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest')
entity = manifest.entities.append('entity')
relative_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'relative partition')
relative_partition.location = 'relative/path'
absolute_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'absolute partition')
absolute_partition.location = 'local:/absolute/path'
entity.data_partitions.append(relative_partition)
entity.data_partitions.append(absolute_partition)
manifest_data = await ManifestPersistence.to_data(manifest, None, None)
self.assertEqual(len(manifest_data.entities), 1)
entityData = manifest_data.entities[0]
partitions_list = entityData.dataPartitions
self.assertEqual(len(partitions_list), 2)
relative_partition_data = partitions_list[0]
absolute_partition_data = partitions_list[-1]
self.assertEqual(relative_partition_data.location,
relative_partition.location)
self.assertEqual(absolute_partition_data.location,
absolute_partition.location)
def test_cdm_collection_appending_list(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath'))
ctx = cdm_corpus.ctx
cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument')
collection = CdmCollection(ctx, cdm_document,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity_list = []
for i in range(0, 2):
entity = CdmEntityDefinition(cdm_corpus.ctx,
'entityName_{}'.format(i), None)
create_document_for_entity(cdm_corpus, entity)
entity_declaration = cdm_corpus.make_object(
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, entity.entity_name,
False) # type: CdmLocalEntityDeclarationDefinition
entity_declaration.entity_path = '{}/{}'.format(
entity.owner.at_corpus_path, entity.entity_name)
entity_declaration.owner = entity.owner
entity_list.append(entity_declaration)
self.assertEqual(0, len(collection))
collection.extend(entity_list)
self.assertEqual(2, len(collection))
for i in range(0, 2):
self.assertEqual('entityName_{}'.format(i),
collection[i].entity_name)
def test_data_format(self):
corpus = CdmCorpusDefinition()
att = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'att')
for format in CdmDataFormat:
att.dataFormat = format
self.assertEqual(att.dataFormat, format)
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None,
is_language_specific: Optional[bool] = False, expected_codes: Optional[set] = None,
no_input_and_output_folder: Optional[bool] = False):
"""
Creates a corpus to be used by the tests, which mounts inputFolder, outputFolder, cdm, and remoteAdapter. Will fail on any unexpected warning/error.
@param testSubpath The root of the corpus files.
@param testName The test name.
@param testInputDir The test input directory.
@param isLanguageSpecific Indicate whether there is subfolder called Java, it's used when input is different compared with other languages
@param expectedCodes The error codes that are expected, and they should not block the test.
@param noInputAndOutputFolder No input and output folder needed.
"""
if no_input_and_output_folder:
test_input_dir = 'C:\\dummpyPath'
test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name,
is_language_specific)
test_output_dir = test_input_dir if no_input_and_output_folder else TestHelper.get_actual_output_folder_path(test_subpath, test_name)
cdm_corpus = CdmCorpusDefinition()
def callback(level: CdmStatusLevel, message: str):
TestHelper._fail_on_unexpected_failure(cdm_corpus, message, expected_codes)
cdm_corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir))
cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir))
cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.schema_documents_path))
cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'}))
return cdm_corpus
def test_path_that_does_not_end_in_slash(self):
"""FolderPath should always end with a /
This checks the behavior if FolderPath does not end with a /
('/' should be appended and a warning be sent through callback function)"""
corpus = CdmCorpusDefinition()
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = None
def callback(status_level: CdmStatusLevel, message1: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message1
corpus.set_event_callback(callback)
manifest = CdmManifestDefinition(None, None)
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp'
absolute_path = corpus.storage.create_absolute_corpus_path('Abc', manifest)
self.assertEqual('cdm:Mnp/Abc', absolute_path)
self.assertEqual(function_was_called, True)
self.assertEqual(function_parameter1, CdmStatusLevel.WARNING)
self.assertTrue(function_parameter2.find('Expected path prefix to end in /, but it didn\'t. Appended the /') != -1)
def test_manifest_cannot_add_entity_definition_without_creating_document(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = ''
def callback(status_level: 'CdmStatusLevel', message: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message
cdm_corpus.set_event_callback(callback)
cdm_corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path'))
manifest = CdmManifestDefinition(cdm_corpus.ctx, 'manifest')
manifest._folder_path = '/'
manifest._namespace = 'local'
entity = CdmEntityDefinition(manifest.ctx, 'entityName', None)
manifest.entities.append(entity)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue('Expected entity to have an \'Owner\' document set. Cannot create entity declaration to add to manifest.' in function_parameter2)
def test_path_root_invalid_folder_path(self):
""""Tests absolute paths cannot be created with wrong parameters.
Checks behavior if FolderPath is invalid."""
corpus = CdmCorpusDefinition()
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = None
def callback(status_level: CdmStatusLevel, message1: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message1
corpus.set_event_callback(callback)
manifest = CdmManifestDefinition(None, None)
manifest.namespace = 'cdm'
manifest.folder_path = './Mnp'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not start with ./') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = '/./Mnp'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = '../Mnp'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
function_parameter2 = function_parameter2.split('|')[1].strip()
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain ../') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp/./Qrs'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
function_parameter2 = function_parameter2.split('|')[1].strip()
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp/../Qrs'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
function_parameter2 = function_parameter2.split('|')[1].strip()
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain ../') != -1)
async def create_manifest(self, cdm_corpus: CdmCorpusDefinition):
print('Make placeholder manifest')
# Make the temp manifest and add it to the root of the local documents in the corpus.
manifest_abstract = cdm_corpus.make_object(CdmObjectType.MANIFEST_DEF, 'temp_abstract') # type: CdmManifestDefinition
# Add each declaration, this example is about medical appointments and care plans
manifest_abstract.entities.append('Account', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account')
manifest_abstract.entities.append('Address', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address')
manifest_abstract.entities.append('CarePlan', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan')
manifest_abstract.entities.append('CodeableConcept', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept')
manifest_abstract.entities.append('Contact', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact')
manifest_abstract.entities.append('Device', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device')
manifest_abstract.entities.append('EmrAppointment', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment')
manifest_abstract.entities.append('Encounter', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter')
manifest_abstract.entities.append('EpisodeOfCare', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare')
manifest_abstract.entities.append('Location', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location')
# Add the temp manifest to the root of the local documents in the corpus.
local_root = cdm_corpus.storage.fetch_root_folder('local')
local_root.documents.append(manifest_abstract)
# Create the resolved version of everything in the root folder too.
print('Resolve the placeholder')
manifest_resolved = await manifest_abstract.create_resolved_manifest_async('default', '')
# Add an import to the foundations doc so the traits about partitons will resolve nicely.
manifest_resolved.imports.append('cdm:/foundations.cdm.json', '')
print('Save the documents')
for e_def in manifest_resolved.entities:
# Get the entity being pointed at.
local_e_def = cast(CdmLocalEntityDeclarationDefinition, e_def)
# Turns a relative path from manifest_resolved into an absolute path.
ent_def = cast(CdmEntityDefinition, await cdm_corpus.fetch_object_async(local_e_def.entity_path, manifest_resolved))
# Make a fake partition, just to demo that.
part = cdm_corpus.make_object(CdmObjectType.DATA_PARTITION_DEF, '{}-data-description'.format(ent_def.entity_name)) # type: CdmDataPartitionDefinition
local_e_def.data_partitions.append(part)
part.explanation = 'not real data, just for demo'
# Define the location of the partition, relative to the manifest
local_location = 'local:/{}/partition-data.csv'.format(ent_def.entity_name)
part.location = cdm_corpus.storage.create_relative_corpus_path(local_location, manifest_resolved)
# Add trait to partition for csv params.
csv_trait = part.exhibits_traits.append('is.partition.format.CSV', False)
csv_trait.arguments.append('columnHeaders', 'true')
csv_trait.arguments.append('delimiter', ',')
# Get the actual location of the partition file from the corpus.
part_path = cdm_corpus.storage.corpus_path_to_adapter_path(local_location)
# Make a fake file with nothing but header for columns.
header = ','.join([att.name for att in ent_def.attributes])
os.makedirs(cdm_corpus.storage.corpus_path_to_adapter_path('local:/{}'.format(ent_def.entity_name)), exist_ok=True)
with open(part_path, 'w') as file:
file.write(header)
await manifest_resolved.save_as_async('{}.manifest.cdm.json'.format(manifest_resolved.manifest_name), True)
async def read_local_save_adls(self, corpus: CdmCorpusDefinition):
# ---------------------------------------------------------------------------------------------
# Create manifest from a local model.json file
manifest = await corpus.fetch_object_async('local:/model.json')
# ------------------------------------------------------------------------------------------------------------
# Explore entities and partitions defined in the model
print('Listing entity declarations:')
for decl in manifest.entities:
print(' ' + decl.entity_name)
if decl.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
for data_part in decl.data_partitions:
print(' ' + data_part.location)
# ---------------------------------------------------------------------------------------------
# Make a few changes to the model
# Create a new document where the new entity's definition will be stored
new_entity_doc = corpus.make_object(
CdmObjectType.DOCUMENT_DEF,
'NewEntity.cdm.json') # type: CdmDocumentDefinition
new_entity_doc.imports.append('cdm:/foundations.cdm.json')
corpus.storage.fetch_root_folder('local').documents.append(
new_entity_doc)
new_entity = new_entity_doc.definitions.append(
'NewEntity', CdmObjectType.ENTITY_DEF,
False) # type: CdmEntityDefinition
# Define new string attribute and add it to the entity definition
new_attribute = corpus.make_object(
CdmObjectType.TYPE_ATTRIBUTE_DEF,
'NewAttribute') # type: CdmTypeAttributeDefinition
new_attribute.data_format = CdmDataFormat.STRING
new_entity.attributes.append(new_attribute)
# Call will create entity_declaration_definition based on entity definition and add it to manifest.entities
new_entity_decl = manifest.entities.append(
new_entity) # type: CdmLocalEntityDeclaration
# Define a partition and add it to the local declaration
new_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'NewPartition',
False) # type: CdmDataPartitionDefinition
new_partition.location = 'adls:/NewPartition.csv'
new_entity_decl.data_partitions.append(new_partition)
# ---------------------------------------------------------------------------------------------
# Save the file to ADLSg2 - we achieve that by adding the manifest to the root folder of
# the ADLS file-system and performing a save on the manifest
adls_folder = corpus.storage.fetch_root_folder(
'adls') # type: CdmFolderDefinition
adls_folder.documents.append(manifest)
await manifest.save_as_async('model.json', True)
async def test_programmatically_create_partitions(self):
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'manifest')
entity = manifest.entities.append('entity')
relative_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'relative partition')
relative_partition.location = 'relative/path'
relative_partition.arguments['test1'] = ['argument1']
relative_partition.arguments['test2'] = ['argument2', 'argument3']
absolute_partition = corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, 'absolute partition')
absolute_partition.location = 'local:/absolute/path'
# add an empty arguments list to test empty list should not be displayed in ToData json.
absolute_partition.arguments['test'] = []
entity.data_partitions.append(relative_partition)
entity.data_partitions.append(absolute_partition)
manifest_data = ManifestPersistence.to_data(manifest, None, None)
self.assertEqual(len(manifest_data.entities), 1)
entityData = manifest_data.entities[0]
partitions_list = entityData.dataPartitions
self.assertEqual(len(partitions_list), 2)
relative_partition_data = partitions_list[0]
absolute_partition_data = partitions_list[-1]
self.assertEqual(relative_partition_data.location,
relative_partition.location)
arguments_list = relative_partition_data.arguments
self.assertEqual(3, len(arguments_list))
checked_arguments = []
for argument in arguments_list:
self.assertEqual(3, len(argument))
checked_arguments.append(argument.value)
if argument.value == 'argument1':
self.assertEqual('test1', argument.name)
elif argument.value == 'argument2':
self.assertEqual('test2', argument.name)
elif argument.value == 'argument3':
self.assertEqual('test2', argument.name)
else:
raise Exception('unexpected argument in data partitions')
self.assertTrue('argument1' in checked_arguments)
self.assertTrue('argument2' in checked_arguments)
self.assertTrue('argument3' in checked_arguments)
self.assertEqual(absolute_partition_data.location,
absolute_partition.location)
# test if empty argument list is set to null
self.assertEqual(absolute_partition_data.arguments, None)
def test_path_root_invalid_object_path(self):
"""Tests absolute paths cannot be created with wrong parameters.
Checks behavior if objectPath is invalid."""
corpus = CdmCorpusDefinition()
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = None
def callback(status_level: CdmStatusLevel, message1: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message1
corpus.set_event_callback(callback)
corpus.storage.create_absolute_corpus_path('./Abc')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not start with ./') != -1
)
function_was_called = False
corpus.storage.create_absolute_corpus_path('/./Abc')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
corpus.storage.create_absolute_corpus_path('../Abc')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain ../') != -1)
function_was_called = False
corpus.storage.create_absolute_corpus_path('Abc/./Def')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
corpus.storage.create_absolute_corpus_path('Abc/../Def')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain ../') != -1)
async def test_entity_properties(self):
test_input_path = TestHelper.get_input_folder_path(
self.tests_subpath, 'test_entity_properties')
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A')
att = obj.attributes[0] # type: CdmTypeAttributeDefinition
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNotNone(result)
self.assertEqual(att.maximum_length, 30)
self.assertIsNone(att.maximum_value)
self.assertIsNone(att.minimum_value)
# removing the only argument should remove the trait
att.maximum_length = None
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNone(att.maximum_length)
self.assertIsNone(result)
async def test_from_and_to_data(self):
with open(os.path.join(ROOT_PATH,
'default.manifest.cdm.json')) as manifestFile:
manifestContent = ManifestContent()
manifestContent.decode(manifestFile.read())
corpus = CdmCorpusDefinition()
corpus.storage.default_namespace = 'local'
corpus.ctx.update_logging_options(level='WARNING')
corpus.storage.mount('cdm',
LocalAdapter(root="../CDM.SchemaDocuments"))
corpus.storage.mount('local', LocalAdapter(root=ROOT_PATH))
corpus.storage.mount(
'remote', RemoteAdapter(hosts={"contoso": "http://contoso.com"}))
folder = corpus.storage.fetch_root_folder('local')
manifest = await corpus.fetch_object_async('default.manifest.cdm.json',
folder)
data = await ManifestPersistence.to_data(manifest, None, None)
for entity in manifest.entities:
entity_def = await corpus.fetch_object_async(
entity.entity_path, manifest)
await manifest.save_as_async('test_output/new.manifest.cdm.json',
save_referenced=True)
self.assertDictEqual(json.loads(manifestContent.encode()),
json.loads(data.encode()))
def test_manifest_copy(self):
"""Tests if the copy function creates copies of the sub objects"""
corpus = CdmCorpusDefinition()
manifest = CdmManifestDefinition(corpus.ctx, 'name')
entity_name = 'entity'
sub_manifest_name = 'sub_manifest'
relationship_name = 'relName'
trait_name = 'traitName'
entity_dec = manifest.entities.append(entity_name)
sub_manifest = manifest.sub_manifests.append(sub_manifest_name)
relationship = manifest.relationships.append(relationship_name)
trait = manifest.exhibits_traits.append(trait_name)
copy = manifest.copy() # type: CdmManifestDefinition
copy.entities[0].entity_name = 'newEntity'
copy.sub_manifests[0].manifest_name = 'newSubManifest'
copy.relationships[0].name = 'newRelName'
copy.exhibits_traits[0].named_reference = 'newTraitName'
self.assertEqual(entity_name, entity_dec.entity_name)
self.assertEqual(sub_manifest_name, sub_manifest.manifest_name)
self.assertEqual(relationship_name, relationship.name)
self.assertEqual(trait_name, trait.named_reference)
def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
"test_load_local_entity_with_data_partition_pattern",
"entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), "entities",
"testNamespace", "/", manifest_content)
self.assertEqual(len(cdmManifest.entities), 1)
self.assertEqual(cdmManifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdmManifest.entities[0]
self.assertEqual(len(entity.data_partition_patterns), 1)
pattern = entity.data_partition_patterns[0]
self.assertEqual(pattern.name, "testPattern")
self.assertEqual(pattern.explanation, "test explanation")
self.assertEqual(pattern.root_location, "test location")
self.assertEqual(pattern.regular_expression, "\\s*")
self.assertEqual(len(pattern.parameters), 2)
self.assertEqual(pattern.parameters[0], "testParam1")
self.assertEqual(pattern.parameters[1], "testParam2")
self.assertEqual(pattern.specialized_schema, "test special schema")
self.assertEqual(len(pattern.exhibits_traits), 1)
def test_manifest_add_list_of_entity_declarations(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = "local"
cdm_corpus.storage.mount("local", LocalAdapter("CdmCorpus/LocalPath"))
ctx = cdm_corpus.ctx
cdmDocument = CdmDocumentDefinition(ctx, "NameOfDocument")
collection = CdmEntityCollection(ctx, cdmDocument)
entity_list = []
for i in range(0, 2):
entity = CdmEntityDefinition(cdm_corpus.ctx,
'entityName_{}'.format(i), None)
create_document_for_entity(cdm_corpus, entity)
entity_list.append(entity)
self.assertEqual(0, len(collection))
collection.extend(entity_list)
self.assertEqual(2, len(collection))
for i in range(0, 2):
self.assertEqual('entityName_{}'.format(i),
collection[i].entity_name)
async def test_entity_properties(self):
test_input_path = os.path.join(ROOT_PATH, 'entity')
corpus = CdmCorpusDefinition()
corpus.ctx.update_logging_options(level='WARNING')
corpus.storage.mount('local', LocalAdapter(test_input_path))
corpus.storage.default_namespace = 'local'
obj = await corpus.fetch_object_async('local:/entA.cdm.json/Entity A')
att = obj.attributes[0] # type: CdmTypeAttributeDefinition
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNotNone(result)
self.assertEqual(att.maximum_length, 30)
self.assertIsNone(att.maximum_value)
self.assertIsNone(att.minimum_value)
# removing the only argument should remove the trait
att.maximum_length = None
result = next(
filter(lambda x: x.named_reference == 'is.constrained',
att.applied_traits), None)
self.assertIsNone(att.maximum_length)
self.assertIsNone(result)
def test_create_absolute_corpus_pathWithColon(self):
"""Tests if create_absolute_corpus_path works correctly when provided with a path that contains a colon character."""
corpus = CdmCorpusDefinition()
folder = corpus.storage.fetch_root_folder('local')
absolute_namespace = 'namespace:/'
file_name = 'dataPartition.csv@snapshot=2020-05-10T02:47:46.0039374Z'
sub_folder_path = 'some/sub/folder:with::colon/'
# Cases where the path provided is relative.
self.assertEqual(
'local:/' + file_name,
corpus.storage.create_absolute_corpus_path(file_name, folder))
self.assertEqual(
'local:/' + sub_folder_path + file_name,
corpus.storage.create_absolute_corpus_path(
sub_folder_path + file_name, folder))
# Cases where the path provided is absolute.
self.assertEqual(
absolute_namespace + file_name,
corpus.storage.create_absolute_corpus_path(
absolute_namespace + file_name, folder))
self.assertEqual(
absolute_namespace + sub_folder_path + file_name,
corpus.storage.create_absolute_corpus_path(
absolute_namespace + sub_folder_path + file_name, folder))
def setup_cdm_corpus(self) -> CdmCorpusDefinition:
corpus = CdmCorpusDefinition()
corpus.default_namespace = 'local'
corpus.storage.mount(
'cdm', LocalAdapter(TestHelper.sample_schema_folder_path))
corpus.storage.mount(
'local',
LocalAdapter(
AdlsModelJsonTestHelper.get_actual_sub_folder_path(
self.tests_subpath, self.test_name,
AdlsModelJsonTestHelper.input_folder_name)))
corpus.storage.mount(
'adls',
AdlsTestHelper.create_adapter_with_client_id(
self.root_relative_path))
return corpus
async def test_referenced_entity_declaration_resolution(self):
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('cdm', LocalAdapter(root='../../schemaDocuments'))
corpus.storage.default_namespace = 'cdm'
manifest = CdmManifestDefinition(corpus.ctx, 'manifest')
manifest.entities.append(
'Account', 'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account')
referenced_entity_path = \
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/electronicMedicalRecords.manifest.cdm.json/Address'
referenced_entity = CdmReferencedEntityDeclarationDefinition(corpus.ctx, 'Address')
referenced_entity.entity_path = referenced_entity_path
manifest.entities.append(referenced_entity)
corpus.storage.fetch_root_folder('cdm').documents.append(manifest)
resolved_manifest = await manifest.create_resolved_manifest_async('resolved_manifest', None)
self.assertEqual(2, len(resolved_manifest.entities))
self.assertEqual('core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/resolved/Account.cdm.json/Account',
resolved_manifest.entities[0].entity_path)
self.assertEqual(referenced_entity_path, resolved_manifest.entities[1].entity_path)
async def test_loading_and_saving_config(self):
"""Testing loading and saving config."""
test_name = 'test_loading_and_saving_config'
test_input_path = TestHelper.get_input_folder_path(
self.tests_subpath, test_name)
test_output_path = TestHelper.get_expected_output_folder_path(
self.tests_subpath, test_name)
# Create a corpus to load the config.
cdm_corpus = self.get_local_corpus(test_input_path, test_output_path)
config = await cdm_corpus.storage.fetch_adapter('local').read_async(
'/config.json')
different_corpus = CdmCorpusDefinition()
different_corpus.storage.mount_from_config(config)
result_config = different_corpus.storage.fetch_config()
output_config = await cdm_corpus.storage.fetch_adapter(
'target').read_async('/config.json')
self.maxDiff = None
self.assertDictEqual(json.loads(output_config),
json.loads(result_config))
async def run_syms_smart_adls_adapter_mount_logic(self):
syms_adapter = SymsTestHelper.create_adapter_with_clientid()
corpus = CdmCorpusDefinition()
corpus.storage.mount('syms', syms_adapter)
adls_adapter1 = SymsTestHelper.create_adapter_clientid_with_shared_key(
1)
adls_adapter2 = SymsTestHelper.create_adapter_clientid_with_shared_key(
2)
count_adapter_count_before = len(corpus.storage.namespace_adapters)
manifest_read_databases = await corpus.fetch_object_async(
'syms:/databases.manifest.cdm.json')
manifest = await corpus.fetch_object_async(
'syms:/{}/{}.manifest.cdm.json'.format(
manifest_read_databases.sub_manifests[0].manifest_name,
manifest_read_databases.sub_manifests[0].manifest_name),
manifest_read_databases, None, True)
count_adapter_count_after = len(corpus.storage.namespace_adapters)
self.assertEqual(count_adapter_count_before + 2,
count_adapter_count_after)
self.assertIsNotNone(
corpus.storage.adapter_path_to_corpus_path('https://{}{}'.format(
adls_adapter1.hostname, adls_adapter1.root)))
self.assertIsNotNone(
corpus.storage.adapter_path_to_corpus_path('https://{}{}'.format(
adls_adapter2.hostname, adls_adapter2.root)))
def test_cdm_collection_append_method(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter('CdmCorpus/LocalPath'))
ctx = cdm_corpus.ctx
cdm_document = CdmDocumentDefinition(ctx, 'NameOfDocument')
collection = CdmCollection(ctx, cdm_document,
CdmObjectType.ATTRIBUTE_CONTEXT_DEF)
appended_attribute_context = collection.append('nameOfNewAttribute')
self.assertEqual(1, len(collection))
self.assertEqual('nameOfNewAttribute', collection[0].name)
self.assertEqual(cdm_document, collection[0].owner)
self.assertEqual(ctx, collection[0].ctx)
self.assertEqual(collection[0], appended_attribute_context)
attribute_context = CdmAttributeContext(ctx, 'NameOfAttributeContext')
appended_attribute = collection.append(attribute_context)
self.assertEqual(2, len(collection))
self.assertEqual(attribute_context, appended_attribute)
self.assertEqual(attribute_context, collection[1])
self.assertEqual(cdm_document, attribute_context.owner)
def test_valid_root_path(self):
"""Checks Absolute corpus path can be created with valid input."""
corpus = CdmCorpusDefinition()
# checks with None object
absolute_path = corpus.storage.create_absolute_corpus_path('Abc/Def')
self.assertEqual('/Abc/Def', absolute_path)
absolute_path = corpus.storage.create_absolute_corpus_path('/Abc/Def')
self.assertEqual('/Abc/Def', absolute_path)
absolute_path = corpus.storage.create_absolute_corpus_path('cdm:/Abc/Def')
self.assertEqual('cdm:/Abc/Def', absolute_path)
manifest = CdmManifestDefinition(None, None)
manifest.namespace = ''
manifest.folder_path = 'Mnp/Qrs/'
absolute_path = corpus.storage.create_absolute_corpus_path('Abc/Def', manifest)
self.assertEqual('Mnp/Qrs/Abc/Def', absolute_path)
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp/Qrs/'
absolute_path = corpus.storage.create_absolute_corpus_path('/Abc/Def', manifest)
self.assertEqual('cdm:/Abc/Def', absolute_path)
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp/Qrs/'
absolute_path = corpus.storage.create_absolute_corpus_path('Abc/Def', manifest)
self.assertEqual('cdm:Mnp/Qrs/Abc/Def', absolute_path)
async def test_cdm_folder_to_data_type_attribute(self):
"""
Testing that "is.localized.describedAs" trait with a table of three entries (en, rs and cn) is fully preserved when running CdmFolder TypeAttributePersistence ToData.
"""
corpus = CdmCorpusDefinition()
corpus.ctx.report_at_level = CdmStatusLevel.WARNING
corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path'))
corpus.storage.default_namespace = 'local'
cdm_type_attribute_definition = corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, 'TestSavingTraitAttribute', False)
english_constants_list = ['en', 'Some description in English language']
serbian_constants_list = ['sr', 'Opis na srpskom jeziku']
chinese_constants_list = ['cn', '一些中文描述']
list_of_const_lists = [english_constants_list, serbian_constants_list, chinese_constants_list]
const_ent_def = corpus.make_object(CdmObjectType.CONSTANT_ENTITY_DEF, 'localizedDescriptions', False)
const_ent_def.constant_values = list_of_const_lists
const_ent_def.entity_shape = corpus.make_ref(CdmObjectType.ENTITY_REF, 'localizedTable', True)
trait_reference2 = corpus.make_object(CdmObjectType.TRAIT_REF, 'is.localized.describedAs', False)
trait_reference2.arguments.append('localizedDisplayText', corpus.make_ref(CdmObjectType.ENTITY_REF, const_ent_def, True))
cdm_type_attribute_definition.applied_traits.append(trait_reference2)
result = TypeAttribute().decode(PersistenceLayer.to_data(cdm_type_attribute_definition, None, None, PersistenceLayer.CDM_FOLDER))
self.assertIsNotNone(result.appliedTraits)
argument = result.appliedTraits[0].arguments[0]
constant_values = argument.value.entityReference.constantValues
self.assertEqual('en', constant_values[0][0])
self.assertEqual('Some description in English language', constant_values[0][1])
self.assertEqual('sr', constant_values[1][0])
self.assertEqual('Opis na srpskom jeziku', constant_values[1][1])
self.assertEqual('cn', constant_values[2][0])
self.assertEqual('一些中文描述', constant_values[2][1])
async def test_resolving_manifest_not_in_folder(self):
"""Test that resolving a manifest that hasn't been added to a folder doesn't throw any exceptions."""
try:
corpus = CdmCorpusDefinition()
corpus.storage.mount('local', LocalAdapter('C:\\path'))
corpus.storage.default_namespace = 'local'
def callback(level, message):
# We should see the following error message be logged. If not, fail.
if 'Cannot resolve the manifest \'test\' because it has not been added to a folder' not in message:
self.fail()
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test')
entity = corpus.make_object(CdmObjectType.ENTITY_DEF, 'entity')
document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION))
document.definitions.append(entity)
# Don't add the document containing the entity to a folder either.
manifest.entities.append(entity)
await manifest.create_resolved_manifest_async('resolved', None)
except Exception:
self.fail('Exception should not be thrown when resolving a manifest that is not in a folder.')
def test_trait_to_unknown_data_format(self):
"""Test trait to data format when unknown data format trait is in an attribute."""
cdm_attribute = CdmTypeAttributeDefinition(CdmCorpusContext(CdmCorpusDefinition(), None), 'SomeAttribute')
cdm_attribute.applied_traits.append('is.data_format.someRandomDataFormat')
trait_to_property_map = TraitToPropertyMap(cdm_attribute)
data_format = trait_to_property_map._traits_to_data_format(False)
self.assertEqual(CdmDataFormat.UNKNOWN, data_format)
async def test_setting_model_json_entity_description(self):
"""Tests that a description on a CdmFolder entity sets the description on the ModelJson entity."""
corpus = CdmCorpusDefinition()
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test')
document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION))
folder = corpus.storage.fetch_root_folder('local')
folder.documents.append(document)
entity = document.definitions.append(CdmObjectType.ENTITY_DEF.name, 'entity')
entity.description = 'test description'
manifest.entities.append(entity)
folder.documents.append(manifest)
obtained_model_json = await ManifestPersistence.to_data(manifest, None, None)
self.assertEqual('test description', obtained_model_json.entities[0].get('description'))
def test_entity_attribute_source(self):
"""Tests if not setting the projection "source" on an entity attribute triggers an error log"""
corpus = CdmCorpusDefinition()
error_count = 0
def callback(level: 'CdmStatusLevel', message: str):
nonlocal error_count
error_count += 1
corpus.set_event_callback(callback, CdmStatusLevel.ERROR)
projection = CdmProjection(corpus.ctx)
entity_attribute = CdmEntityAttributeDefinition(
corpus.ctx, 'attribute')
entity_attribute.entity = CdmEntityReference(corpus.ctx, projection,
False)
# First case, a projection without source.
projection.validate()
self.assertEqual(1, error_count)
error_count = 0
# Second case, a projection with a nested projection.
inner_projection = CdmProjection(corpus.ctx)
projection.source = CdmEntityReference(corpus.ctx, inner_projection,
False)
projection.validate()
inner_projection.validate()
self.assertEqual(1, error_count)
error_count = 0
# Third case, a projection with an explicit entity definition.
inner_projection.source = CdmEntityReference(
corpus.ctx, CdmEntityDefinition(corpus.ctx, 'Entity'), False)
projection.validate()
inner_projection.validate()
self.assertEqual(0, error_count)
# Third case, a projection with a named reference.
inner_projection.source = CdmEntityReference(corpus.ctx, 'Entity',
False)
projection.validate()
inner_projection.validate()
self.assertEqual(0, error_count)
