def test_path_that_does_not_end_in_slash(self):
"""FolderPath should always end with a /
This checks the behavior if FolderPath does not end with a /
('/' should be appended and a warning be sent through callback function)"""
corpus = CdmCorpusDefinition()
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = None
def callback(status_level: CdmStatusLevel, message1: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message1
corpus.set_event_callback(callback)
manifest = CdmManifestDefinition(None, None)
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp'
absolute_path = corpus.storage.create_absolute_corpus_path('Abc', manifest)
self.assertEqual('cdm:Mnp/Abc', absolute_path)
self.assertEqual(function_was_called, True)
self.assertEqual(function_parameter1, CdmStatusLevel.WARNING)
self.assertTrue(function_parameter2.find('Expected path prefix to end in /, but it didn\'t. Appended the /') != -1)
def test_manifest_cannot_add_entity_definition_without_creating_document(self):
cdm_corpus = CdmCorpusDefinition()
cdm_corpus.storage.default_namespace = 'local'
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = ''
def callback(status_level: 'CdmStatusLevel', message: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message
cdm_corpus.set_event_callback(callback)
cdm_corpus.storage.mount('local', LocalAdapter('C:\\Root\\Path'))
manifest = CdmManifestDefinition(cdm_corpus.ctx, 'manifest')
manifest._folder_path = '/'
manifest._namespace = 'local'
entity = CdmEntityDefinition(manifest.ctx, 'entityName', None)
manifest.entities.append(entity)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue('Expected entity to have an \'Owner\' document set. Cannot create entity declaration to add to manifest.' in function_parameter2)
async def test_resolving_manifest_not_in_folder(self):
"""Test that resolving a manifest that hasn't been added to a folder doesn't throw any exceptions."""
try:
corpus = CdmCorpusDefinition()
corpus.storage.mount('local', LocalAdapter('C:\\path'))
corpus.storage.default_namespace = 'local'
def callback(level, message):
# We should see the following error message be logged. If not, fail.
if 'Cannot resolve the manifest \'test\' because it has not been added to a folder' not in message:
self.fail()
corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
manifest = corpus.make_object(CdmObjectType.MANIFEST_DEF, 'test')
entity = corpus.make_object(CdmObjectType.ENTITY_DEF, 'entity')
document = corpus.make_object(CdmObjectType.DOCUMENT_DEF, 'entity{}'.format(PersistenceLayer.CDM_EXTENSION))
document.definitions.append(entity)
# Don't add the document containing the entity to a folder either.
manifest.entities.append(entity)
await manifest.create_resolved_manifest_async('resolved', None)
except Exception:
self.fail('Exception should not be thrown when resolving a manifest that is not in a folder.')
def get_local_corpus(test_subpath: str, test_name: str, test_input_dir: Optional[str] = None,
is_language_specific: Optional[bool] = False, expected_codes: Optional[set] = None,
no_input_and_output_folder: Optional[bool] = False):
"""
Creates a corpus to be used by the tests, which mounts inputFolder, outputFolder, cdm, and remoteAdapter. Will fail on any unexpected warning/error.
@param testSubpath The root of the corpus files.
@param testName The test name.
@param testInputDir The test input directory.
@param isLanguageSpecific Indicate whether there is subfolder called Java, it's used when input is different compared with other languages
@param expectedCodes The error codes that are expected, and they should not block the test.
@param noInputAndOutputFolder No input and output folder needed.
"""
if no_input_and_output_folder:
test_input_dir = 'C:\\dummpyPath'
test_input_dir = test_input_dir or TestHelper.get_input_folder_path(test_subpath, test_name,
is_language_specific)
test_output_dir = test_input_dir if no_input_and_output_folder else TestHelper.get_actual_output_folder_path(test_subpath, test_name)
cdm_corpus = CdmCorpusDefinition()
def callback(level: CdmStatusLevel, message: str):
TestHelper._fail_on_unexpected_failure(cdm_corpus, message, expected_codes)
cdm_corpus.set_event_callback(callback, CdmStatusLevel.WARNING)
cdm_corpus.storage.default_namespace = 'local'
cdm_corpus.storage.mount('local', LocalAdapter(root=test_input_dir))
cdm_corpus.storage.mount('output', LocalAdapter(root=test_output_dir))
cdm_corpus.storage.mount('cdm', LocalAdapter(TestHelper.schema_documents_path))
cdm_corpus.storage.mount('remote', RemoteAdapter(hosts={'contoso': 'http://contoso.com'}))
return cdm_corpus
def test_path_root_invalid_folder_path(self):
""""Tests absolute paths cannot be created with wrong parameters.
Checks behavior if FolderPath is invalid."""
corpus = CdmCorpusDefinition()
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = None
def callback(status_level: CdmStatusLevel, message1: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message1
corpus.set_event_callback(callback)
manifest = CdmManifestDefinition(None, None)
manifest.namespace = 'cdm'
manifest.folder_path = './Mnp'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not start with ./') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = '/./Mnp'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = '../Mnp'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
function_parameter2 = function_parameter2.split('|')[1].strip()
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain ../') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp/./Qrs'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
function_parameter2 = function_parameter2.split('|')[1].strip()
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
manifest.namespace = 'cdm'
manifest.folder_path = 'Mnp/../Qrs'
corpus.storage.create_absolute_corpus_path('Abc', manifest)
function_parameter2 = function_parameter2.split('|')[1].strip()
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(function_parameter2.find('The path should not contain ../') != -1)
def test_path_root_invalid_object_path(self):
"""Tests absolute paths cannot be created with wrong parameters.
Checks behavior if objectPath is invalid."""
corpus = CdmCorpusDefinition()
function_was_called = False
function_parameter1 = CdmStatusLevel.INFO
function_parameter2 = None
def callback(status_level: CdmStatusLevel, message1: str):
nonlocal function_was_called, function_parameter1, function_parameter2
function_was_called = True
function_parameter1 = status_level
function_parameter2 = message1
corpus.set_event_callback(callback)
corpus.storage.create_absolute_corpus_path('./Abc')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not start with ./') != -1
)
function_was_called = False
corpus.storage.create_absolute_corpus_path('/./Abc')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
corpus.storage.create_absolute_corpus_path('../Abc')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain ../') != -1)
function_was_called = False
corpus.storage.create_absolute_corpus_path('Abc/./Def')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain /./') != -1)
function_was_called = False
corpus.storage.create_absolute_corpus_path('Abc/../Def')
self.assertTrue(function_was_called)
self.assertEqual(CdmStatusLevel.ERROR, function_parameter1)
self.assertTrue(
function_parameter2.find('The path should not contain ../') != -1)
def test_entity_attribute_source(self):
"""Tests if not setting the projection "source" on an entity attribute triggers an error log"""
corpus = CdmCorpusDefinition()
error_count = 0
def callback(level: 'CdmStatusLevel', message: str):
nonlocal error_count
error_count += 1
corpus.set_event_callback(callback, CdmStatusLevel.ERROR)
projection = CdmProjection(corpus.ctx)
entity_attribute = CdmEntityAttributeDefinition(
corpus.ctx, 'attribute')
entity_attribute.entity = CdmEntityReference(corpus.ctx, projection,
False)
# First case, a projection without source.
projection.validate()
self.assertEqual(1, error_count)
error_count = 0
# Second case, a projection with a nested projection.
inner_projection = CdmProjection(corpus.ctx)
projection.source = CdmEntityReference(corpus.ctx, inner_projection,
False)
projection.validate()
inner_projection.validate()
self.assertEqual(1, error_count)
error_count = 0
# Third case, a projection with an explicit entity definition.
inner_projection.source = CdmEntityReference(
corpus.ctx, CdmEntityDefinition(corpus.ctx, 'Entity'), False)
projection.validate()
inner_projection.validate()
self.assertEqual(0, error_count)
# Third case, a projection with a named reference.
inner_projection.source = CdmEntityReference(corpus.ctx, 'Entity',
False)
projection.validate()
inner_projection.validate()
self.assertEqual(0, error_count)
async def test_avoid_retry_codes(self):
"""Tests if the adapter won't retry if a HttpStatusCode response with a code in AvoidRetryCodes is received."""
adls_adapter = AdlsTestHelper.create_adapter_with_shared_key()
adls_adapter.number_of_retries = 3
corpus = CdmCorpusDefinition()
corpus.storage.mount('adls', adls_adapter)
count = 0
def callback(status, message: str):
nonlocal count
if message.find('Response for request ') != -1:
count += 1
corpus.set_event_callback(callback, CdmStatusLevel.PROGRESS)
await corpus.fetch_object_async('adls:/inexistentFile.cdm.json') # type: CdmDocumentDefinition
self.assertEqual(1, count)
# 1. The OM library is available in the execution scope of this sample
# 2. The model_json_root constant points to the location of the model.json file
# 3. ADLSg2 adapter configuration is updated according to your env setup
# 4. The partition location in model.json file is specifying the same ADLSg2 account and file-system settings
# 5. Ensure the Azure user object is assigned "Storage Blob Data Contributor" role in the ADLSg2 access management page
# ---------------------------------------------------------------------------------------------
model_json_root = './4-read-local-save-adls/sample-data'
# ---------------------------------------------------------------------------------------------
# Instantiate corpus and set it to use default namespace 'adls'
corpus = CdmCorpusDefinition()
# set callback to receive error and warning logs.
corpus.set_event_callback(event_callback, CdmStatusLevel.WARNING)
corpus.default_namespace = 'local'
# ---------------------------------------------------------------------------------------------
# Set up a local, remote and adls adapters
# Fake cdm, normaly use the CDM Standards adapter
# Mount it as the 'cdm' device, not the default so must use "cdm:/folder" to get there
corpus.storage.mount(
'cdm',
LocalAdapter(
root=os.path.join(ROOT_PATH, '../../example-public-standards')))
corpus.storage.mount('local', LocalAdapter(root=model_json_root))
# Example how to mount to the ADLS - make sure the hostname and root entered here are also changed
# in the example.model.json file we load in the next section
corpus.storage.mount(
async def main():
# Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating
# objects and paths.
cdm_corpus = CdmCorpusDefinition()
# set callback to receive error and warning logs.
cdm_corpus.set_event_callback(event_callback, CdmStatusLevel.WARNING)
print('Configure storage adapters')
cdm_corpus.storage.mount(
'local', LocalAdapter(root=os.path.join(ROOT_PATH, '../sample-data')))
# Local is our default. So any paths that start out navigating without a device tag will assume local.
cdm_corpus.storage.default_namespace = 'local'
# Fake cdm, normally use the CDM Standards adapter.
cdm_corpus.storage.mount(
'cdm',
LocalAdapter(
root=os.path.join(ROOT_PATH, '../../example-public-standards')))
# This sample is going to simulate the steps a tool would follow in order to create a new manifest document in some user
# Storage folder when the shapes of the entities in that folder are all taken from some public standards with no
# extensions or additions
# The steps are:
# 1. Create a temporary 'manifest' object at the root of the corpus that contains a list of the selected entities.
# 2. Each selected entity points at an 'abstract' schema defintion in the public standards. These entity docs are too
# hard to deal with because of abstractions and inheritence, etc. So to make things concrete, we want to make a
# 'resolved' version of each entity doc in our local folder. To do this, we call create_resolved_manifest on our
# starting manifest. this will resolve everything and find all of the relationships between entities for us.
# 3. Save the new documents.
print('Make placeholder manifest')
# Make the temp manifest and add it to the root of the local documents in the corpus.
manifest_abstract = cdm_corpus.make_object(
CdmObjectType.MANIFEST_DEF,
'temp_abstract') # type: CdmManifestDefinition
# Add each declaration, this example is about medical appointments and care plans
manifest_abstract.entities.append(
'Account',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Account.cdm.json/Account'
)
manifest_abstract.entities.append(
'Address',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Address.cdm.json/Address'
)
manifest_abstract.entities.append(
'CarePlan',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CarePlan.cdm.json/CarePlan'
)
manifest_abstract.entities.append(
'CodeableConcept',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/CodeableConcept.cdm.json/CodeableConcept'
)
manifest_abstract.entities.append(
'Contact',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Contact.cdm.json/Contact'
)
manifest_abstract.entities.append(
'Device',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Device.cdm.json/Device'
)
manifest_abstract.entities.append(
'EmrAppointment',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EmrAppointment.cdm.json/EmrAppointment'
)
manifest_abstract.entities.append(
'Encounter',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Encounter.cdm.json/Encounter'
)
manifest_abstract.entities.append(
'EpisodeOfCare',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/EpisodeOfCare.cdm.json/EpisodeOfCare'
)
manifest_abstract.entities.append(
'Location',
'cdm:/core/applicationCommon/foundationCommon/crmCommon/accelerators/healthCare/electronicMedicalRecords/Location.cdm.json/Location'
)
# Add the temp manifest to the root of the local documents in the corpus.
local_root = cdm_corpus.storage.fetch_root_folder('local')
local_root.documents.append(manifest_abstract)
# Create the resolved version of everything in the root folder too.
print('Resolve the placeholder')
manifest_resolved = await manifest_abstract.create_resolved_manifest_async(
'default', '')
# Add an import to the foundations doc so the traits about partitons will resolve nicely.
manifest_resolved.imports.append('cdm:/foundations.cdm.json', '')
print('Save the documents')
for e_def in manifest_resolved.entities:
# Get the entity being pointed at.
local_e_def = cast(CdmLocalEntityDeclarationDefinition, e_def)
# Turns a relative path from manifest_resolved into an absolute path.
ent_def = cast(
CdmEntityDefinition, await
cdm_corpus.fetch_object_async(local_e_def.entity_path,
manifest_resolved))
# Make a fake partition, just to demo that.
part = cdm_corpus.make_object(
CdmObjectType.DATA_PARTITION_DEF, '{}-data-description'.format(
ent_def.entity_name)) # type: CdmDataPartitionDefinition
local_e_def.data_partitions.append(part)
part.explanation = 'not real data, just for demo'
# Define the location of the partition, relative to the manifest
local_location = 'local:/{}/partition-data.csv'.format(
ent_def.entity_name)
part.location = cdm_corpus.storage.create_relative_corpus_path(
local_location, manifest_resolved)
# Add trait to partition for csv params.
csv_trait = part.exhibits_traits.append('is.partition.format.CSV',
False)
csv_trait.arguments.append('columnHeaders', 'true')
csv_trait.arguments.append('delimiter', ',')
# Get the actual location of the partition file from the corpus.
part_path = cdm_corpus.storage.corpus_path_to_adapter_path(
local_location)
# Make a fake file with nothing but header for columns.
header = ','.join([att.name for att in ent_def.attributes])
os.makedirs(cdm_corpus.storage.corpus_path_to_adapter_path(
'local:/{}'.format(ent_def.entity_name)),
exist_ok=True)
with open(part_path, 'w') as file:
file.write(header)
await manifest_resolved.save_as_async(
'{}.manifest.cdm.json'.format(manifest_resolved.manifest_name), True)
async def logical_manipulation_using_projections():
'''This sample demonstrates how to model a set of common scenarios using projections.
The projections feature provides a way to customize the definition of a logical entity by influencing how the entity is resolved by the object model.
Here we will model three common use cases for using projections that are associated with the directives 'referenceOnly', 'structured' and 'normalized'.
A single logical definition can be resolved into multiple physical layouts. The directives are used to instruct the ObjectModel about how it should to
resolve the logical definition provided. To achieve this, we define projections that run conditionally, depending on the directives provided when
calling create_resolved_entity_async.
To get an overview of the projections feature as well as all of the supported operations refer to the link below.
https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#projection-overview
'''
# Make a corpus, the corpus is the collection of all documents and folders created or discovered while navigating objects and paths
corpus = CdmCorpusDefinition()
# set callback to receive error and warning logs.
corpus.set_event_callback(event_callback, CdmStatusLevel.WARNING)
print('Configure storage adapters')
# Configure storage adapters to point at the target local manifest location and at the fake public standards
path_from_exe_to_example_root = '../../'
corpus.storage.mount('local', LocalAdapter(path_from_exe_to_example_root + '8-logical-manipulation-using-projections/sample-data'))
corpus.storage.default_namespace = 'local' # local is our default. so any paths that start out navigating without a device tag will assume local
# Fake cdm, normaly use the CDM Standards adapter
# Mount it as the 'cdm' device, not the default so must use 'cdm:/folder' to get there
corpus.storage.mount('cdm', LocalAdapter(path_from_exe_to_example_root + 'example-public-standards'))
print('Create logical entity definition.')
logical_folder = await corpus.fetch_object_async('local:/') # type: CdmFolderDefinition
logical_doc = logical_folder.documents.append('Person.cdm.json')
logical_doc.imports.append('Address.cdm.json')
entity = logical_doc.definitions.append('Person') # type: CdmEntityDefinition
# Add 'name' data typed attribute.
name_attr = entity.attributes.append('name') # type: CdmTypeAttributeDefinition
name_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'age' data typed attribute.
age_attr = entity.attributes.append('age') # type: CdmTypeAttributeDefinition
age_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Add 'address' entity typed attribute.
entity_attr = CdmEntityAttributeDefinition(corpus.ctx, 'address')
entity_attr.entity = CdmEntityReference(corpus.ctx, 'Address', True)
apply_array_expansion(entity_attr, 1, 3, '{m}{A}{o}', 'countAttribute')
apply_default_behavior(entity_attr, 'addressFK', 'address')
entity.attributes.append(entity_attr)
# Add 'email' data typed attribute.
email_attr = entity.attributes.append('email') # type: CdmTypeAttributeDefinition
email_attr.data_type = CdmDataTypeReference(corpus.ctx, 'string', True)
# Save the logical definition of Person.
await entity.in_document.save_as_async('Person.cdm.json')
print('Get \'resolved\' folder where the resolved entities will be saved.')
resolved_folder = await corpus.fetch_object_async('local:/resolved/') # type: CdmFolderDefinition
res_opt = ResolveOptions(entity)
# To get more information about directives and their meaning refer to
# https://docs.microsoft.com/en-us/common-data-model/sdk/convert-logical-entities-resolved-entities#directives-guidance-and-the-resulting-resolved-shapes
# We will start by resolving this entity with the 'normalized' direcitve.
# This directive will be used on this and the next two examples so we can analize the resolved entity
# without the array expansion.
print('Resolving logical entity with normalized directive.')
res_opt.directives = AttributeResolutionDirectiveSet({ 'normalized' })
res_normalized_entity = await entity.create_resolved_entity_async(f'normalized_{entity.entity_name}', res_opt, resolved_folder)
await res_normalized_entity.in_document.save_as_async(f'{res_normalized_entity.entity_name}.cdm.json')
# Another common scenario is to resolve an entity using the 'referenceOnly' directive.
# This directives is used to replace the relationships with a foreign key.
print('Resolving logical entity with referenceOnly directive.')
res_opt.directives = AttributeResolutionDirectiveSet({ 'normalized', 'referenceOnly' })
res_reference_only_entity = await entity.create_resolved_entity_async(f'referenceOnly_{entity.entity_name}', res_opt, resolved_folder)
await res_reference_only_entity.in_document.save_as_async(f'{res_reference_only_entity.entity_name}.cdm.json')
# When dealing with structured data, like Json or parquet, it sometimes necessary to represent the idea that
# a property can hold a complex object. The shape of the complex object is defined by the source entity pointed by the
# entity attribute and we use the 'structured' directive to resolve the entity attribute as an attribute group.
print('Resolving logical entity with structured directive.')
res_opt.directives = AttributeResolutionDirectiveSet({ 'normalized', 'structured' })
res_structured_entity = await entity.create_resolved_entity_async(f'structured_{entity.entity_name}', res_opt, resolved_folder)
await res_structured_entity.in_document.save_as_async(f'{res_structured_entity.entity_name}.cdm.json')
# Now let us remove the 'normalized' directive so the array expansion operation can run.
print('Resolving logical entity without directives (array expansion).')
res_opt.directives = AttributeResolutionDirectiveSet({ })
res_array_entity = await entity.create_resolved_entity_async(f'array_expansion_{entity.entity_name}', res_opt, resolved_folder)
await res_array_entity.in_document.save_as_async(f'{res_array_entity.entity_name}.cdm.json')
