def test_load_local_entity_with_data_partition_pattern(self): content = TestHelper.get_input_file_content( self.test_subpath, "test_load_local_entity_with_data_partition_pattern", "entities.manifest.cdm.json") manifest_content = ManifestContent() manifest_content.decode(content) cdmManifest = ManifestPersistence.from_object( CdmCorpusContext(CdmCorpusDefinition(), None), "entities", "testNamespace", "/", manifest_content) self.assertEqual(len(cdmManifest.entities), 1) self.assertEqual(cdmManifest.entities[0].object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity = cdmManifest.entities[0] self.assertEqual(len(entity.data_partition_patterns), 1) pattern = entity.data_partition_patterns[0] self.assertEqual(pattern.name, "testPattern") self.assertEqual(pattern.explanation, "test explanation") self.assertEqual(pattern.root_location, "test location") self.assertEqual(pattern.regular_expression, "\\s*") self.assertEqual(len(pattern.parameters), 2) self.assertEqual(pattern.parameters[0], "testParam1") self.assertEqual(pattern.parameters[1], "testParam2") self.assertEqual(pattern.specialized_schema, "test special schema") self.assertEqual(len(pattern.exhibits_traits), 1)
async def from_data(ctx: CdmCorpusContext, name: str, namespace: str, path: str, data: DocumentContent) -> CdmDocumentDefinition: document = ctx.corpus.make_object(CdmObjectType.DOCUMENT_DEF, name) document.folder_path = path document.namespace = namespace # Set this as the current document of the context for this operation ctx.update_document_context(document) if data: if data.get('schema'): document.schema = data.schema # support old model syntax if data.get('schemaVersion'): document.json_schema_semantic_version = data.schemaVersion if data.get('jsonSchemaSemanticVersion'): document.json_schema_semantic_version = data.jsonSchemaSemanticVersion if document.json_schema_semantic_version != '0.9.0': # TODO: validate that this is a version we can understand with the OM pass if data.get('imports'): for import_obj in data.imports: document.imports.append(ImportPersistence.from_data(ctx, import_obj)) if data.get('definitions') and isinstance(data.definitions, List): for definition in data.definitions: if definition.get('dataTypeName'): document.definitions.append(DataTypePersistence.from_data(ctx, definition)) elif definition.get('purposeName'): document.definitions.append(PurposePersistence.from_data(ctx, definition)) elif definition.get('attributeGroupName'): document.definitions.append(AttributeGroupPersistence.from_data(ctx, definition)) elif definition.get('traitName'): document.definitions.append(TraitPersistence.from_data(ctx, definition)) elif definition.get('entityShape'): document.definitions.append(ConstantEntityPersistence.from_data(ctx, definition)) elif definition.get('entityName'): document.definitions.append(EntityPersistence.from_data(ctx, definition)) ctx.update_document_context(None) return document
def test_trait_to_unknown_data_format(self): """Test trait to data format when unknown data format trait is in an attribute.""" cdm_attribute = CdmTypeAttributeDefinition(CdmCorpusContext(CdmCorpusDefinition(), None), 'SomeAttribute') cdm_attribute.applied_traits.append('is.data_format.someRandomDataFormat') trait_to_property_map = TraitToPropertyMap(cdm_attribute) data_format = trait_to_property_map._traits_to_data_format(False) self.assertEqual(CdmDataFormat.UNKNOWN, data_format)
def test_load_local_entity_with_data_partition_pattern(self): content = TestHelper.get_input_file_content( self.test_subpath, 'test_load_local_entity_with_data_partition_pattern', 'entities.manifest.cdm.json') manifest_content = ManifestContent() manifest_content.decode(content) cdm_manifest = ManifestPersistence.from_object( CdmCorpusContext(CdmCorpusDefinition(), None), 'entities', 'testNamespace', '/', manifest_content) self.assertEqual(len(cdm_manifest.entities), 2) entity1 = cdm_manifest.entities[0] self.assertEqual(entity1.object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) self.assertEqual(len(entity1.data_partition_patterns), 1) pattern1 = entity1.data_partition_patterns[0] self.assertEqual(pattern1.name, 'testPattern') self.assertEqual(pattern1.explanation, 'test explanation') self.assertEqual(pattern1.root_location, 'test location') self.assertEqual(pattern1.regular_expression, '\\s*') self.assertEqual(len(pattern1.parameters), 2) self.assertEqual(pattern1.parameters[0], 'testParam1') self.assertEqual(pattern1.parameters[1], 'testParam2') self.assertEqual(pattern1.specialized_schema, 'test special schema') self.assertEqual(len(pattern1.exhibits_traits), 1) entity2 = cdm_manifest.entities[1] self.assertEqual(entity2.object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) self.assertEqual(len(entity2.data_partition_patterns), 1) pattern2 = entity2.data_partition_patterns[0] self.assertEqual(pattern2.name, 'testPattern2') self.assertEqual(pattern2.root_location, 'test location2') self.assertEqual(pattern2.glob_pattern, '/*.csv') manifest_data = ManifestPersistence.to_data(cdm_manifest, None, None) self.assertEqual(len(manifest_data.entities), 2) entity_data1 = manifest_data.entities[0] self.assertEqual(len(entity_data1.dataPartitionPatterns), 1) pattern_data1 = entity_data1.dataPartitionPatterns[0] self.assertEqual(pattern_data1.name, 'testPattern') self.assertEqual(pattern_data1.explanation, 'test explanation') self.assertEqual(pattern_data1.rootLocation, 'test location') self.assertEqual(pattern_data1.regularExpression, '\\s*') self.assertEqual(len(pattern_data1.parameters), 2) self.assertEqual(pattern_data1.parameters[0], 'testParam1') self.assertEqual(pattern_data1.parameters[1], 'testParam2') self.assertEqual(pattern_data1.specializedSchema, 'test special schema') self.assertEqual(len(pattern_data1.exhibitsTraits), 1) pattern_data2 = manifest_data.entities[1].dataPartitionPatterns[0] self.assertEqual(pattern_data2.name, 'testPattern2') self.assertEqual(pattern_data2.rootLocation, 'test location2') self.assertEqual(pattern_data2.globPattern, '/*.csv')
def test_trait_to_json_data_format(self): """Test trait to data format when calculated data format should be JSON.""" cdm_attribute = CdmTypeAttributeDefinition(CdmCorpusContext(CdmCorpusDefinition(), None), 'SomeAttribute') cdm_attribute.applied_traits.append('is.dataFormat.array') cdm_attribute.applied_traits.append('means.content.text.JSON') trait_to_property_map = TraitToPropertyMap(cdm_attribute) data_format = trait_to_property_map._traits_to_data_format(False) self.assertEqual(CdmDataFormat.JSON, data_format)
async def test_pattern_with_non_existing_folder(self): corpus = TestHelper.get_local_corpus(self.test_subpath, "test_pattern_with_non_existing_folder") content = TestHelper.get_input_file_content(self.test_subpath, "test_pattern_with_non_existing_folder", "entities.manifest.cdm.json") manifest_content = ManifestContent() manifest_content.decode(content) cdmManifest = ManifestPersistence.from_object(CdmCorpusContext(corpus, None), "entities", "local", "/", manifest_content) await cdmManifest.file_status_check_async() self.assertEqual(len(cdmManifest.entities[0].data_partitions), 0) # make sure the last check time is still being set self.assertIsNotNone(cdmManifest.entities[0].data_partition_patterns[0].last_file_status_check_time)
def test_non_null_default_value_attribute(self): the_list = [{ 'languageTag': 'en', 'displayText': 'Preferred Customer', 'attributeValue': '1', 'displayOrder': '0' }, { 'languageTag': 'en', 'displayText': 'Standard', 'attributeValue': '2', 'displayOrder': '1' }] input = {'defaultValue': the_list} cdmTypeAttributeDefinition = PersistenceLayer.from_data(CdmCorpusContext(CdmCorpusDefinition(), None), JObject(input), CdmObjectType.TYPE_ATTRIBUTE_DEF, PersistenceLayer.CDM_FOLDER) result = PersistenceLayer.to_data(cdmTypeAttributeDefinition, None, None, PersistenceLayer.CDM_FOLDER) self.assertTrue(result) self.assertEqual(result.get('defaultValue'), input.get('defaultValue'))
async def test_load_local_entitiy_with_data_partition(self): content = TestHelper.get_input_file_content( self.test_subpath, 'test_load_local_entity_with_data_partition', 'entities.manifest.cdm.json') manifest_content = ManifestContent() manifest_content.decode(content) cdm_manifest = await ManifestPersistence.from_data( CdmCorpusContext(CdmCorpusDefinition()), 'entities', 'testNamespace', '/', manifest_content) self.assertEqual(len(cdm_manifest.entities), 1) self.assertEqual(cdm_manifest.entities[0].object_type, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF) entity = cdm_manifest.entities[0] self.assertEqual(len(entity.data_partitions), 2) relative_partition = entity.data_partitions[0] self.assertEqual(relative_partition.name, 'Sample data partition') self.assertEqual(relative_partition.location, 'test/location') # self.assertEqual(TimeUtils.GetFormattedDateString(relative_partition.LastFileModifiedTime), '2008-09-15T23:53:23.000Z') self.assertEqual(len(relative_partition.exhibits_traits), 1) self.assertEqual(relative_partition.specialized_schema, 'teststring') test_list = relative_partition.arguments['test'] self.assertEqual(len(test_list), 3) self.assertEqual(test_list[0], 'something') self.assertEqual(test_list[1], 'somethingelse') self.assertEqual(test_list[2], 'anotherthing') key_list = relative_partition.arguments['KEY'] self.assertEqual(len(key_list), 1) self.assertEqual(key_list[0], 'VALUE') self.assertFalse('wrong' in relative_partition.arguments) absolute_partition = entity.data_partitions[1] self.assertEqual(absolute_partition.location, "local:/some/test/location")
for annotation in partition.get('annotations'): kvprint(annotation.name, annotation.value, 3, 12) kvprint('---', '', 0, 4) if model.get('relationships'): kvprint('relationships', '', 6, 0) for relationship in model.get('relationships'): kvprint('type', relationship.type, 3, 4) kvprint('fromAttribute', relationship.fromAttribute, 3, 4) kvprint('toAttribute', relationship.toAttribute, 3, 4) if model.get('referenceModels'): kvprint('reference models', '', 6, 0) for refModel in model.get('referenceModels'): kvprint('id', refModel.id, 3, 4) kvprint('location', refModel.location, 3, 4) # Now run the loaded model through PersistenceLayer # TODO: Fails because CdmCorpusContext is abstract # TODO: Fails because CdmObjectType is all capital SNAKE_CASE, but before it used to follow camelCase manifest_def = from_data(CdmCorpusContext(), model, CdmObjectType.MANIFEST_DEF, 'CdmFolder') # type: CdmManifestDefinition assert manifest_def assert manifest_def.name == 'OrdersProductsCustomersLinked' assert manifest_def.version == '1.0' assert manifest_def.description == 'Description of the model OrdersProductsCustomersLinked' assert len(manifest_def.entities) == 2 # assert manifest_def.entities[1].entityName
async def from_data(ctx: CdmCorpusContext, name: str, namespace: str, path: str, data: ManifestContent) -> CdmManifestDefinition: if data is None: return None if data.get('manifestName'): manifest_name = data.manifestName elif data.get('folioName'): manifest_name = data.folioName elif name: manifest_name = name.replace('.manifest.cdm.json', '').replace('.folio.cdm.json', '') else: manifest_name = '' manifest = ctx.corpus.make_object(CdmObjectType.MANIFEST_DEF, manifest_name) manifest.name = name # this is the document name which is assumed by constructor to be related to the the manifest name, but may not be manifest.folder_path = path manifest.namespace = namespace manifest.explanation = data.get('explanation') # set this as the current manifest of the context for this operation ctx.update_document_context(manifest) if data.schema: manifest.schema = data.schema # support old model syntax if data.get('schemaVersion'): manifest.json_schema_semantic_version = data.schema_version manifest.json_schema_semantic_version = data.get( 'jsonSchemaSemanticVersion') if manifest.json_schema_semantic_version != '0.9.0': # TODO: validate that this is a version we can understand with the OM pass if data.get('imports'): for import_obj in data.imports: manifest.imports.append( ImportPersistence.from_data(ctx, import_obj)) if data.get('definitions'): for definition in data.definitions: if 'dataTypeName' in definition: manifest.definitions.append( DataTypePersistence.fromData(ctx, definition)) elif 'purposeName' in definition: manifest.definitions.append( PurposePersistence.fromData(ctx, definition)) elif 'attributeGroupName' in definition: manifest.definitions.append( AttributeGroupPersistence.fromData(ctx, definition)) elif 'traitName' in definition: manifest.definitions.append( TraitPersistence.fromData(ctx, definition)) elif 'entityShape' in definition: manifest.definitions.append( ConstantEntityPersistence.fromData(ctx, definition)) elif 'entityName' in definition: manifest.definitions.append( EntityPersistence.fromData(ctx, definition)) if data.get('lastFileStatusCheckTime'): manifest.last_file_status_check_time = dateutil.parser.parse( data.lastFileStatusCheckTime) if data.get('lastFileModifiedTime'): manifest.last_file_modified_time = dateutil.parser.parse( data.lastFileModifiedTime) if data.get('lastChildFileModifiedTime'): manifest.last_child_file_modified_time = dateutil.parser.parse( data.lastChildFileModifiedTime) if data.get('exhibitsTraits'): exhibits_traits = utils.create_trait_reference_array( ctx, data.exhibitsTraits) manifest.exhibits_traits.extend(exhibits_traits) if data.get('entities'): full_path = '{}:{}'.format(namespace, path) if namespace else path for entity_obj in data.entities: if entity_obj.get( 'type' ) == 'LocalEntity' or 'entitySchema' in entity_obj: manifest.entities.append( LocalEntityDeclarationPersistence.from_data( ctx, full_path, entity_obj)) elif entity_obj.get( 'type' ) == 'ReferencedEntity' or 'entityDeclaration' in entity_obj: manifest.entities.append( ReferencedEntityDeclarationPersistence.from_data( ctx, full_path, entity_obj)) else: ctx.logger.error( 'Folio entity type should be either LocalEntity or Referenced entity.', entity_obj.entityName) return None if data.get('relationships'): for relationship in data.relationships: manifest.relationships.append( E2ERelationshipPersistence.from_data(ctx, relationship)) if data.get('subManifests'): sub_manifests = data.subManifests elif data.get('subFolios'): sub_manifests = data.subFolios else: sub_manifests = [] for sub_manifest in sub_manifests: manifest.sub_manifests.append( ManifestDeclarationPersistence.from_data(ctx, sub_manifest)) return manifest