def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
"test_load_local_entity_with_data_partition_pattern",
"entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), "entities",
"testNamespace", "/", manifest_content)
self.assertEqual(len(cdmManifest.entities), 1)
self.assertEqual(cdmManifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdmManifest.entities[0]
self.assertEqual(len(entity.data_partition_patterns), 1)
pattern = entity.data_partition_patterns[0]
self.assertEqual(pattern.name, "testPattern")
self.assertEqual(pattern.explanation, "test explanation")
self.assertEqual(pattern.root_location, "test location")
self.assertEqual(pattern.regular_expression, "\\s*")
self.assertEqual(len(pattern.parameters), 2)
self.assertEqual(pattern.parameters[0], "testParam1")
self.assertEqual(pattern.parameters[1], "testParam2")
self.assertEqual(pattern.specialized_schema, "test special schema")
self.assertEqual(len(pattern.exhibits_traits), 1)
async def from_data(ctx: CdmCorpusContext, name: str, namespace: str, path: str, data: DocumentContent) -> CdmDocumentDefinition:
document = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, name)
document.folder_path = path
document.namespace = namespace
# Set this as the current document of the context for this operation
ctx.update_document_context(document)
if data:
if data.get('schema'):
document.schema = data.schema
# support old model syntax
if data.get('schemaVersion'):
document.json_schema_semantic_version = data.schemaVersion
if data.get('jsonSchemaSemanticVersion'):
document.json_schema_semantic_version = data.jsonSchemaSemanticVersion
if document.json_schema_semantic_version != '0.9.0':
# TODO: validate that this is a version we can understand with the OM
pass
if data.get('imports'):
for import_obj in data.imports:
document.imports.append(ImportPersistence.from_data(ctx, import_obj))
if data.get('definitions') and isinstance(data.definitions, List):
for definition in data.definitions:
if definition.get('dataTypeName'):
document.definitions.append(DataTypePersistence.from_data(ctx, definition))
elif definition.get('purposeName'):
document.definitions.append(PurposePersistence.from_data(ctx, definition))
elif definition.get('attributeGroupName'):
document.definitions.append(AttributeGroupPersistence.from_data(ctx, definition))
elif definition.get('traitName'):
document.definitions.append(TraitPersistence.from_data(ctx, definition))
elif definition.get('entityShape'):
document.definitions.append(ConstantEntityPersistence.from_data(ctx, definition))
elif definition.get('entityName'):
document.definitions.append(EntityPersistence.from_data(ctx, definition))
ctx.update_document_context(None)
return document
def test_trait_to_unknown_data_format(self):
"""Test trait to data format when unknown data format trait is in an attribute."""
cdm_attribute = CdmTypeAttributeDefinition(CdmCorpusContext(CdmCorpusDefinition(), None), 'SomeAttribute')
cdm_attribute.applied_traits.append('is.data_format.someRandomDataFormat')
trait_to_property_map = TraitToPropertyMap(cdm_attribute)
data_format = trait_to_property_map._traits_to_data_format(False)
self.assertEqual(CdmDataFormat.UNKNOWN, data_format)
def test_load_local_entity_with_data_partition_pattern(self):
content = TestHelper.get_input_file_content(
self.test_subpath,
'test_load_local_entity_with_data_partition_pattern',
'entities.manifest.cdm.json')
manifest_content = ManifestContent()
manifest_content.decode(content)
cdm_manifest = ManifestPersistence.from_object(
CdmCorpusContext(CdmCorpusDefinition(), None), 'entities',
'testNamespace', '/', manifest_content)
self.assertEqual(len(cdm_manifest.entities), 2)
entity1 = cdm_manifest.entities[0]
self.assertEqual(entity1.object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
self.assertEqual(len(entity1.data_partition_patterns), 1)
pattern1 = entity1.data_partition_patterns[0]
self.assertEqual(pattern1.name, 'testPattern')
self.assertEqual(pattern1.explanation, 'test explanation')
self.assertEqual(pattern1.root_location, 'test location')
self.assertEqual(pattern1.regular_expression, '\\s*')
self.assertEqual(len(pattern1.parameters), 2)
self.assertEqual(pattern1.parameters[0], 'testParam1')
self.assertEqual(pattern1.parameters[1], 'testParam2')
self.assertEqual(pattern1.specialized_schema, 'test special schema')
self.assertEqual(len(pattern1.exhibits_traits), 1)
entity2 = cdm_manifest.entities[1]
self.assertEqual(entity2.object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
self.assertEqual(len(entity2.data_partition_patterns), 1)
pattern2 = entity2.data_partition_patterns[0]
self.assertEqual(pattern2.name, 'testPattern2')
self.assertEqual(pattern2.root_location, 'test location2')
self.assertEqual(pattern2.glob_pattern, '/*.csv')
manifest_data = ManifestPersistence.to_data(cdm_manifest, None, None)
self.assertEqual(len(manifest_data.entities), 2)
entity_data1 = manifest_data.entities[0]
self.assertEqual(len(entity_data1.dataPartitionPatterns), 1)
pattern_data1 = entity_data1.dataPartitionPatterns[0]
self.assertEqual(pattern_data1.name, 'testPattern')
self.assertEqual(pattern_data1.explanation, 'test explanation')
self.assertEqual(pattern_data1.rootLocation, 'test location')
self.assertEqual(pattern_data1.regularExpression, '\\s*')
self.assertEqual(len(pattern_data1.parameters), 2)
self.assertEqual(pattern_data1.parameters[0], 'testParam1')
self.assertEqual(pattern_data1.parameters[1], 'testParam2')
self.assertEqual(pattern_data1.specializedSchema,
'test special schema')
self.assertEqual(len(pattern_data1.exhibitsTraits), 1)
pattern_data2 = manifest_data.entities[1].dataPartitionPatterns[0]
self.assertEqual(pattern_data2.name, 'testPattern2')
self.assertEqual(pattern_data2.rootLocation, 'test location2')
self.assertEqual(pattern_data2.globPattern, '/*.csv')
def test_trait_to_json_data_format(self):
"""Test trait to data format when calculated data format should be JSON."""
cdm_attribute = CdmTypeAttributeDefinition(CdmCorpusContext(CdmCorpusDefinition(), None), 'SomeAttribute')
cdm_attribute.applied_traits.append('is.dataFormat.array')
cdm_attribute.applied_traits.append('means.content.text.JSON')
trait_to_property_map = TraitToPropertyMap(cdm_attribute)
data_format = trait_to_property_map._traits_to_data_format(False)
self.assertEqual(CdmDataFormat.JSON, data_format)
async def test_pattern_with_non_existing_folder(self):
corpus = TestHelper.get_local_corpus(self.test_subpath, "test_pattern_with_non_existing_folder")
content = TestHelper.get_input_file_content(self.test_subpath, "test_pattern_with_non_existing_folder", "entities.manifest.cdm.json")
manifest_content = ManifestContent()
manifest_content.decode(content)
cdmManifest = ManifestPersistence.from_object(CdmCorpusContext(corpus, None), "entities", "local", "/", manifest_content)
await cdmManifest.file_status_check_async()
self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0)
# make sure the last check time is still being set
self.assertIsNotNone(cdmManifest.entities[0].data_partition_patterns[0].last_file_status_check_time)
def test_non_null_default_value_attribute(self):
the_list = [{
'languageTag': 'en',
'displayText': 'Preferred Customer',
'attributeValue': '1',
'displayOrder': '0'
},
{
'languageTag': 'en',
'displayText': 'Standard',
'attributeValue': '2',
'displayOrder': '1'
}]
input = {'defaultValue': the_list}
cdmTypeAttributeDefinition = PersistenceLayer.from_data(CdmCorpusContext(CdmCorpusDefinition(), None), JObject(input), CdmObjectType.TYPE_ATTRIBUTE_DEF, PersistenceLayer.CDM_FOLDER)
result = PersistenceLayer.to_data(cdmTypeAttributeDefinition, None, None, PersistenceLayer.CDM_FOLDER)
self.assertTrue(result)
self.assertEqual(result.get('defaultValue'), input.get('defaultValue'))
async def test_load_local_entitiy_with_data_partition(self):
content = TestHelper.get_input_file_content(
self.test_subpath, 'test_load_local_entity_with_data_partition',
'entities.manifest.cdm.json')
manifest_content = ManifestContent()
manifest_content.decode(content)
cdm_manifest = await ManifestPersistence.from_data(
CdmCorpusContext(CdmCorpusDefinition()), 'entities',
'testNamespace', '/', manifest_content)
self.assertEqual(len(cdm_manifest.entities), 1)
self.assertEqual(cdm_manifest.entities[0].object_type,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF)
entity = cdm_manifest.entities[0]
self.assertEqual(len(entity.data_partitions), 2)
relative_partition = entity.data_partitions[0]
self.assertEqual(relative_partition.name, 'Sample data partition')
self.assertEqual(relative_partition.location, 'test/location')
# self.assertEqual(TimeUtils.GetFormattedDateString(relative_partition.LastFileModifiedTime), '2008-09-15T23:53:23.000Z')
self.assertEqual(len(relative_partition.exhibits_traits), 1)
self.assertEqual(relative_partition.specialized_schema, 'teststring')
test_list = relative_partition.arguments['test']
self.assertEqual(len(test_list), 3)
self.assertEqual(test_list[0], 'something')
self.assertEqual(test_list[1], 'somethingelse')
self.assertEqual(test_list[2], 'anotherthing')
key_list = relative_partition.arguments['KEY']
self.assertEqual(len(key_list), 1)
self.assertEqual(key_list[0], 'VALUE')
self.assertFalse('wrong' in relative_partition.arguments)
absolute_partition = entity.data_partitions[1]
self.assertEqual(absolute_partition.location,
"local:/some/test/location")
for annotation in partition.get('annotations'):
kvprint(annotation.name, annotation.value, 3, 12)
kvprint('---', '', 0, 4)
if model.get('relationships'):
kvprint('relationships', '', 6, 0)
for relationship in model.get('relationships'):
kvprint('type', relationship.type, 3, 4)
kvprint('fromAttribute', relationship.fromAttribute, 3, 4)
kvprint('toAttribute', relationship.toAttribute, 3, 4)
if model.get('referenceModels'):
kvprint('reference models', '', 6, 0)
for refModel in model.get('referenceModels'):
kvprint('id', refModel.id, 3, 4)
kvprint('location', refModel.location, 3, 4)
# Now run the loaded model through PersistenceLayer
# TODO: Fails because CdmCorpusContext is abstract
# TODO: Fails because CdmObjectType is all capital SNAKE_CASE, but before it used to follow camelCase
manifest_def = from_data(CdmCorpusContext(), model, CdmObjectType.MANIFEST_DEF, 'CdmFolder') # type: CdmManifestDefinition
assert manifest_def
assert manifest_def.name == 'OrdersProductsCustomersLinked'
assert manifest_def.version == '1.0'
assert manifest_def.description == 'Description of the model OrdersProductsCustomersLinked'
assert len(manifest_def.entities) == 2
# assert manifest_def.entities[1].entityName
async def from_data(ctx: CdmCorpusContext, name: str, namespace: str,
path: str,
data: ManifestContent) -> CdmManifestDefinition:
if data is None:
return None
if data.get('manifestName'):
manifest_name = data.manifestName
elif data.get('folioName'):
manifest_name = data.folioName
elif name:
manifest_name = name.replace('.manifest.cdm.json',
'').replace('.folio.cdm.json', '')
else:
manifest_name = ''
manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF,
manifest_name)
manifest.name = name # this is the document name which is assumed by constructor to be related to the the manifest name, but may not be
manifest.folder_path = path
manifest.namespace = namespace
manifest.explanation = data.get('explanation')
# set this as the current manifest of the context for this operation
ctx.update_document_context(manifest)
if data.schema:
manifest.schema = data.schema
# support old model syntax
if data.get('schemaVersion'):
manifest.json_schema_semantic_version = data.schema_version
manifest.json_schema_semantic_version = data.get(
'jsonSchemaSemanticVersion')
if manifest.json_schema_semantic_version != '0.9.0':
# TODO: validate that this is a version we can understand with the OM
pass
if data.get('imports'):
for import_obj in data.imports:
manifest.imports.append(
ImportPersistence.from_data(ctx, import_obj))
if data.get('definitions'):
for definition in data.definitions:
if 'dataTypeName' in definition:
manifest.definitions.append(
DataTypePersistence.fromData(ctx, definition))
elif 'purposeName' in definition:
manifest.definitions.append(
PurposePersistence.fromData(ctx, definition))
elif 'attributeGroupName' in definition:
manifest.definitions.append(
AttributeGroupPersistence.fromData(ctx, definition))
elif 'traitName' in definition:
manifest.definitions.append(
TraitPersistence.fromData(ctx, definition))
elif 'entityShape' in definition:
manifest.definitions.append(
ConstantEntityPersistence.fromData(ctx, definition))
elif 'entityName' in definition:
manifest.definitions.append(
EntityPersistence.fromData(ctx, definition))
if data.get('lastFileStatusCheckTime'):
manifest.last_file_status_check_time = dateutil.parser.parse(
data.lastFileStatusCheckTime)
if data.get('lastFileModifiedTime'):
manifest.last_file_modified_time = dateutil.parser.parse(
data.lastFileModifiedTime)
if data.get('lastChildFileModifiedTime'):
manifest.last_child_file_modified_time = dateutil.parser.parse(
data.lastChildFileModifiedTime)
if data.get('exhibitsTraits'):
exhibits_traits = utils.create_trait_reference_array(
ctx, data.exhibitsTraits)
manifest.exhibits_traits.extend(exhibits_traits)
if data.get('entities'):
full_path = '{}:{}'.format(namespace, path) if namespace else path
for entity_obj in data.entities:
if entity_obj.get(
'type'
) == 'LocalEntity' or 'entitySchema' in entity_obj:
manifest.entities.append(
LocalEntityDeclarationPersistence.from_data(
ctx, full_path, entity_obj))
elif entity_obj.get(
'type'
) == 'ReferencedEntity' or 'entityDeclaration' in entity_obj:
manifest.entities.append(
ReferencedEntityDeclarationPersistence.from_data(
ctx, full_path, entity_obj))
else:
ctx.logger.error(
'Folio entity type should be either LocalEntity or Referenced entity.',
entity_obj.entityName)
return None
if data.get('relationships'):
for relationship in data.relationships:
manifest.relationships.append(
E2ERelationshipPersistence.from_data(ctx, relationship))
if data.get('subManifests'):
sub_manifests = data.subManifests
elif data.get('subFolios'):
sub_manifests = data.subFolios
else:
sub_manifests = []
for sub_manifest in sub_manifests:
manifest.sub_manifests.append(
ManifestDeclarationPersistence.from_data(ctx, sub_manifest))
return manifest
