def BuildUniverse(): """Generates the ontology universe. Returns: Generated universe object. """ ontology_validator_exists = path.exists( path.join('..', 'ontology_validator')) ontology_exists = path.exists(path.join('..', '..', '..', 'ontology')) if not (ontology_validator_exists and ontology_exists): print('ERROR: ontology validator or ontology have changed locations') return None yaml_files = external_file_lib.RecursiveDirWalk( path.join('..', '..', '..', 'ontology', 'yaml', 'resources')) config = presubmit_validate_types_lib.SeparateConfigFiles(yaml_files) universe = presubmit_validate_types_lib.BuildUniverse(config) namespace_validation = namespace_validator.NamespaceValidator( universe.GetEntityTypeNamespaces()) if not namespace_validation.IsValid(): print('Universe is not valid') return None return universe
def testInheritTypesFromDifferentNamespace(self): type1 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F( ['/woof', '/howl'])) type2 = entity_type_lib.EntityType(filepath='path/to/INSECT/fly', typename='moth', description='flying insect.', local_field_tuples=_F(['/wings']), parents=['ANIMAL/dog']) type_namespace_1 = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace_1.InsertType(type1) type_namespace_2 = entity_type_lib.TypeNamespace(namespace='INSECT') type_namespace_2.InsertType(type2) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace_1, type_namespace_2])) self.assertFalse(namespace_validate.GetFindings()) expanded_type_namespace_1 = namespace_validate.type_namespaces_map.get( type_namespace_1.namespace) expanded_type_namespace_2 = namespace_validate.type_namespaces_map.get( type_namespace_2.namespace) type1_expanded = expanded_type_namespace_1.valid_types_map.get( type1.typename) type2_expanded = expanded_type_namespace_2.valid_types_map.get( type2.typename) self.assertEmpty(type1_expanded.inherited_field_names) self.assertLen(type2_expanded.inherited_field_names, 2)
def testDuplicateLocalFieldSets(self): type1 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F( ['/woof', '/howl'])) type2 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='wolf', description='canine animal', local_field_tuples=_F( ['/woof', '/howl'])) type2.inherited_field_names = ({'/bark': _F1('/bark', optional=True)}) type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace.InsertType(type1) type_namespace.InsertType(type2) namespace_validator.NamespaceValidator([type_namespace]) self.assertTrue( type1.HasFindingTypes( [findings_lib.DuplicateLocalFieldSetsWarning])) self.assertTrue( type2.HasFindingTypes( [findings_lib.DuplicateLocalFieldSetsWarning]))
def testInheritanceCycleAcrossNamepsaces(self): type1 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F(['/woof']), parents=['ANIMAL/wolf']) type2 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='wolf', description='canine animal', local_field_tuples=_F(['/growl']), parents=['FUZZY_ANIMAL/dingo']) type3 = entity_type_lib.EntityType( filepath='path/to/FUZZY_ANIMAL/mammal', typename='dingo', description='canine animal', local_field_tuples=_F(['/wag']), parents=['ANIMAL/dog']) namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') namespace.InsertType(type1) namespace.InsertType(type2) namespace2 = entity_type_lib.TypeNamespace(namespace='FUZZY_ANIMAL') namespace2.InsertType(type3) namespace_validate = (namespace_validator.NamespaceValidator( [namespace, namespace2])) self.assertTrue( namespace_validate.HasFindingTypes( [findings_lib.InheritanceCycleError]))
def BuildUniverse( use_simplified_universe: bool = False, modified_types_filepath: str = None, ) -> presubmit_validate_types_lib.ConfigUniverse: """Generates the ontology universe. Args: use_simplified_universe: boolean to quick load minimal universe instead of full ontology modified_types_filepath: filepath to the modified ontology types Returns: Generated universe object. """ if use_simplified_universe: yaml_files = None universe = universe_helper.create_simplified_universe() elif modified_types_filepath: modified_ontology_exists = path.exists(modified_types_filepath) if not modified_ontology_exists: print(f'Specified filepath [{modified_types_filepath}] ' 'modified ontology does not exist') return None modified_types_filepath = path.expanduser(modified_types_filepath) external_file_lib.Validate(filter_text=None, changed_directory=modified_types_filepath, original_directory=constants.ONTOLOGY_ROOT, interactive=False) yaml_files = external_file_lib.RecursiveDirWalk( modified_types_filepath) else: default_ontology_exists = path.exists(constants.ONTOLOGY_ROOT) if not default_ontology_exists: print(f'Specified filepath [{constants.ONTOLOGY_ROOT}] ' 'for default ontology does not exist') return None # use default location for ontology files yaml_files = external_file_lib.RecursiveDirWalk( constants.ONTOLOGY_ROOT) if yaml_files: config = presubmit_validate_types_lib.SeparateConfigFiles(yaml_files) universe = presubmit_validate_types_lib.BuildUniverse(config) namespace_validation = namespace_validator.NamespaceValidator( universe.GetEntityTypeNamespaces()) if not namespace_validation.IsValid(): print('Universe is not valid') return None return universe
def testGoodFieldIncrement(self): entity_type = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F(['/woof', '/woof_1'])) type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace.InsertType(entity_type) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace])) self.assertTrue(namespace_validate.IsValid())
def testGoodPassthroughType(self): passthrough_type = entity_type_lib.EntityType( filepath='path/to/ANIMAL/gateway', typename='dog', description='a gateway to dogs', allow_undefined_fields=True, local_field_tuples=_F(['/woof_1', '/woof_2', '/woof_3'])) type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace.InsertType(passthrough_type) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace])) self.assertTrue(namespace_validate.IsValid())
def setUp(self): super().setUp() self.universe = create_simplified_universe() nv.NamespaceValidator(self.universe.GetEntityTypeNamespaces()) test_entity_type = self.universe.entity_type_universe.GetEntityType( namespace_name='HVAC', typename='DMP_EDM') test_field_list = [ EntityTypeField(optwrapper.field.namespace, optwrapper.field.field[0:], optwrapper.optional, optwrapper.field.increment) for optwrapper in test_entity_type.GetAllFields().values() ] self.test_match = Match( field_list=test_field_list, entity_type=test_entity_type, match_score=1.0)
def BuildUniverse( modified_types_filepath: str = None ) -> presubmit_validate_types_lib.ConfigUniverse: """Generates the ontology universe. Args: modified_types_filepath: filepath to the modified ontology types Returns: Generated universe object. """ if modified_types_filepath: modified_ontology_exists = path.exists(modified_types_filepath) if not modified_ontology_exists: print('Specified filepath [{0}] modified ontology does not exist'. format(modified_types_filepath)) return None modified_types_filepath = path.expanduser(modified_types_filepath) external_file_lib.Validate(filter_text=None, changed_directory=modified_types_filepath, original_directory=constants.ONTOLOGY_ROOT, interactive=False) yaml_files = external_file_lib.RecursiveDirWalk( modified_types_filepath) else: default_ontology_exists = path.exists(constants.ONTOLOGY_ROOT) if not default_ontology_exists: print( 'Specified filepath [{0}] for default ontology does not exist'. format(constants.ONTOLOGY_ROOT)) return None # use default location for ontology files yaml_files = external_file_lib.RecursiveDirWalk( constants.ONTOLOGY_ROOT) config = presubmit_validate_types_lib.SeparateConfigFiles(yaml_files) universe = presubmit_validate_types_lib.BuildUniverse(config) namespace_validation = namespace_validator.NamespaceValidator( universe.GetEntityTypeNamespaces()) if not namespace_validation.IsValid(): print('Universe is not valid') return None return universe
def testNonexistentParent(self): entity_type = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F(['/woof']), parents=[('/nonexistent')]) type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace.InsertType(entity_type) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace])) self.assertTrue( namespace_validate.HasFindingTypes( [findings_lib.NonexistentParentError]))
def setUp(self): super(ModelTest).setUp() self.yaml = RecursiveDirWalk(test_constants.ONTOLOGY_ROOT) self.config = presubmit_validate_types_lib.SeparateConfigFiles(self.yaml) self.universe = presubmit_validate_types_lib.BuildUniverse(self.config) nv.NamespaceValidator(self.universe.GetEntityTypeNamespaces()) test_entity_type = self.universe.entity_type_universe.GetEntityType( namespace_name='HVAC', typename='ZONE_HVAC') test_field_list = [ EntityTypeField(optwrapper.field.namespace, optwrapper.field.field[0:], optwrapper.optional, optwrapper.field.increment) for optwrapper in test_entity_type.GetAllFields().values() ] self.test_match = Match( field_list=test_field_list, entity_type=test_entity_type, match_type='EXACT')
def Build(ontology_path: str) -> OntologyWrapper: """A constructor for the ontology explorer. Args: ontology_path: A path for an alternative ontology extended from DBO. Returns: ontology: An instance of the OntologyWrapper class. """ if ontology_path is not None: yaml_file_path = ontology_path else: yaml_file_path = constants.ONTOLOGY_ROOT yaml_files = external_file_lib.RecursiveDirWalk(yaml_file_path) config = presubmit_validate_types_lib.SeparateConfigFiles(yaml_files) universe = presubmit_validate_types_lib.BuildUniverse(config) nv.NamespaceValidator(universe.GetEntityTypeNamespaces()) ontology = OntologyWrapper(universe) return ontology
def BuildUniverse(modified_types_filepath=None): """Generates the ontology universe. Args: modified_types_filepath: filepath to the modified ontology types Returns: Generated universe object. """ if modified_types_filepath: modified_ontology_exists = path.exists(modified_types_filepath) if not modified_ontology_exists: print('Specified filepath for modified ontology does not exist') return None modified_types_filepath = path.expanduser(modified_types_filepath) external_file_lib.Validate(filter_text=None, changed_directory=modified_types_filepath, original_directory=path.join( '..', '..', '..', 'ontology', 'yaml', 'resources'), interactive=False) yaml_files = external_file_lib.RecursiveDirWalk( modified_types_filepath) else: # use default location for ontology files yaml_files = external_file_lib.RecursiveDirWalk( path.join('..', '..', '..', 'ontology', 'yaml', 'resources')) config = presubmit_validate_types_lib.SeparateConfigFiles(yaml_files) universe = presubmit_validate_types_lib.BuildUniverse(config) namespace_validation = namespace_validator.NamespaceValidator( universe.GetEntityTypeNamespaces()) if not namespace_validation.IsValid(): print('Universe is not valid') return None return universe
def testTypesWithEmptyFields(self): type1 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal') type2 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='wolf', description='canine animal') type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace.InsertType(type1) type_namespace.InsertType(type2) namespace_validator.NamespaceValidator([type_namespace]) self.assertFalse( type1.HasFindingTypes( [findings_lib.DuplicateLocalFieldSetsWarning])) self.assertFalse( type2.HasFindingTypes( [findings_lib.DuplicateLocalFieldSetsWarning]))
def testSetsInheritedOptionality(self): type1 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=[ _F1('/woof', optional=True), _F1('/growl', optional=True) ], parents=[]) type2 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='wolf', description='canine animal', local_field_tuples=[ _F1('/woof', optional=True), _F1('/growl', optional=False) ], parents=[]) type3 = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='dingo', description='canine animal', local_field_tuples=[_F1('/wag', optional=False)], parents=['ANIMAL/dog', 'ANIMAL/wolf']) type_namespace_1 = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace_1.InsertType(type1) type_namespace_1.InsertType(type2) type_namespace_1.InsertType(type3) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace_1])) self.assertTrue(namespace_validate.IsValid()) field_map = type3.GetAllFields() self.assertFalse(field_map['/growl'].optional) self.assertTrue(field_map['/woof'].optional) self.assertFalse(field_map['/wag'].optional)
def testInheritingFromPassthroughIsError(self): passthrough_type = entity_type_lib.EntityType( filepath='path/to/ANIMAL/gateway', typename='dog', description='a gateway to dogs', allow_undefined_fields=True) entity_type = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='dingo', description='canine animal', local_field_tuples=_F(['/woof']), parents=['ANIMAL/dog']) type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace.InsertType(passthrough_type) type_namespace.InsertType(entity_type) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace])) self.assertTrue( namespace_validate.HasFindingTypes( [findings_lib.PassthroughParentError]))
def testInheritTypesFromDifferentNamespaceError(self): type1 = entity_type_lib.EntityType(filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F( ['/woof', '/howl'])) type2 = entity_type_lib.EntityType(filepath='path/to/INSECT/fly', typename='moth', description='flying insect.', local_field_tuples=_F(['/wings']), parents=['ANIMALLLLL/dog']) type_namespace_1 = entity_type_lib.TypeNamespace(namespace='ANIMAL') type_namespace_1.InsertType(type1) type_namespace_2 = entity_type_lib.TypeNamespace(namespace='INSECT') type_namespace_2.InsertType(type2) namespace_validate = (namespace_validator.NamespaceValidator( [type_namespace_1, type_namespace_2])) self.assertTrue( namespace_validate.HasFindingTypes( [findings_lib.NonexistentParentError]))
def setUp(self): super(NamespaceValidatorTest, self).setUp() # Create a list of good TypeNamespace objects self.good_type1 = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='animal', description='member of the animal kingdom', local_field_tuples=_F(['/animalia'])) self.good_type2 = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='dog', description='canine animal', local_field_tuples=_F(['/woof', '/wag']), parents=['ANIMAL/animal']) self.good_type3 = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='puppy', description='baby dog', local_field_tuples=_F(['/cute']), parents=['ANIMAL/dog', 'ANIMAL/animal', 'ANIMAL/baby']) self.good_type4 = entity_type_lib.EntityType( filepath='path/to/ANIMAL/mammal', typename='baby', description='young animal', local_field_tuples=_F(['/young'])) self.type_namespace = entity_type_lib.TypeNamespace(namespace='ANIMAL') self.type_namespace.InsertType(self.good_type1) self.type_namespace.InsertType(self.good_type2) self.type_namespace.InsertType(self.good_type3) self.type_namespace.InsertType(self.good_type4) self.validate_good = (namespace_validator.NamespaceValidator( [self.type_namespace]))
def _ValidateConfigInner(unmodified, modified_base, modified_client, interactive=False): """Runs config validation and finding filtration. Args: unmodified: unchanged file paths to include in validation modified_base: paths to original versions of changed files in validation modified_client: paths to changed files in validation interactive: Set true for timing log messages. Returns: A tuple with a list of findings from validation and the universe """ # Separate different kinds of config files # Concatenate paths such that the changed files go last. This is important # because we always want conflicts between new and old files to show as # being caused by new changes and files are processed in order. start_time = time.time() cl_paths = unmodified + modified_client cl_config = SeparateConfigFiles(cl_paths) new_universe = BuildUniverse(cl_config) end_time = time.time() if interactive: print('New universe build: {0} seconds.\n'.format( str(end_time - start_time))) start_time = end_time if not new_universe.IsValid(): findings = [ f for f in new_universe.GetFindings() if isinstance(f, findings_lib.ValidationError) ] return findings, new_universe # Validate across the type namespaces. Short-circuit if validation fails and # return any breaking findings. cl_validator = namespace_validator.NamespaceValidator( new_universe.GetEntityTypeNamespaces()) if not cl_validator.IsValid(): universe_errors = [] universe_errors = [ f for f in new_universe.GetFindings() if isinstance(f, findings_lib.ValidationError) ] # Technically there should never be universe errors if we get here. Better # to be safe than sorry though. return cl_validator.GetFindings() + universe_errors, new_universe if interactive: end_time = time.time() print('New ns check: {0} seconds.\n'.format(str(end_time - start_time))) start_time = end_time base_paths = unmodified + modified_base base_config = SeparateConfigFiles(base_paths) old_universe = BuildUniverse(base_config) if interactive: end_time = time.time() print('Old uverse build: {0} seconds.\n'.format(str(end_time - start_time))) start_time = end_time # Run validation on base universe to expand types. namespace_validator.NamespaceValidator(old_universe.GetEntityTypeNamespaces()) if interactive: end_time = time.time() print('Old ns check: {0} seconds.\n'.format(str(end_time - start_time))) start_time = end_time mgr = entity_type_manager.EntityTypeManager(new_universe.entity_type_universe) mgr.Analyze() del mgr if interactive: end_time = time.time() print('Type analysis: {0} seconds.\n'.format(str(end_time - start_time))) start_time = end_time CheckBackwardsCompatibility(new_universe.entity_type_universe, old_universe.entity_type_universe) if interactive: end_time = time.time() print('Backwards compat: {0} seconds.\n'.format(str(end_time - start_time))) start_time = end_time # TODO(berkoben) pass this to methods below when fixing b/116850383? # cl_path_set = set(cl_paths) if new_universe.entity_type_universe: _SetTypeFolderChanges( list(new_universe.entity_type_universe.namespace_folder_map.values())) _SetFieldChanges(new_universe.field_universe, old_universe.field_universe) _SetSubfieldChanges(new_universe.subfield_universe, old_universe.subfield_universe) _SetStateChanges(new_universe.state_universe, old_universe.state_universe) _SetUnitChanges(new_universe.unit_universe, old_universe.unit_universe) filtered_findings = new_universe.GetFindings(True) all_findings = new_universe.GetFindings(False) diff = len(all_findings) - len(filtered_findings) if diff > 0: filtered_findings.append(findings_lib.SuppressedFindingsWarning(diff)) return filtered_findings, new_universe
def setUp(self): super().setUp() self.universe = create_simplified_universe() nv.NamespaceValidator(self.universe.GetEntityTypeNamespaces()) self.ontology = OntologyWrapper(self.universe)
def _GetEntityTypeUniverse(type_folders): namespace_validator.NamespaceValidator( [type_folder.local_namespace for type_folder in type_folders]) return entity_type_lib.EntityTypeUniverse(type_folders)
def create_simplified_universe( ) -> presubmit_validate_types_lib.ConfigUniverse: """Creates a simplified test universe with minimal configuration for testing. Returns: config_universe a partially defined ConfigUniverse """ # constant universes state_universe = states.STATE_UNIVERSE connection_universe = connections.CONNECTION_UNIVERSE subfield_universe = subfields.SUBFIELD_UNIVERSE # update unit folder with subfield_universe unit_folder = unit_lib.UnitFolder(folderpath='units') unit_folder.local_namespace.subfields = subfield_universe.GetSubfieldsMap( unit_folder.local_namespace.namespace) unit_folder.AddFromConfig(config_filename='units/units.yaml', documents=[units.UNIT_DOCUMENT]) unit_universe = unit_lib.UnitUniverse(folders=[unit_folder]) # subfield universe has to validate unit universe subfield_universe.ValidateUnits(unit_universe) # field universe depends on subfield and state universes field_folder = field_lib.FieldFolder(folderpath='fields') field_folder.local_namespace.subfields = subfield_universe.GetSubfieldsMap( field_folder.local_namespace.namespace) field_folder.local_namespace.states = state_universe.GetStatesMap( field_folder.local_namespace.namespace) field_folder.AddFromConfig(config_filename='fields/telemetry_fields.yaml', documents=[fields.TELEMETRY_FIELDS_DOCUMENT]) field_folder.AddFromConfig(config_filename='fields/metadata_fields.yaml', documents=[fields.METADATA_FIELDS_DOCUMENT]) field_universe = field_lib.FieldUniverse(folders=[field_folder]) # entity type universe depends on field universe global_type_folder = entity_type_lib.EntityTypeFolder( folderpath='entity_types', field_universe=field_universe) facilities_type_folder = entity_type_lib.EntityTypeFolder( folderpath='FACILITIES/entity_types', field_universe=field_universe) hvac_type_folder = entity_type_lib.EntityTypeFolder( folderpath='HVAC/entity_types', field_universe=field_universe) global_type_folder.AddFromConfig( config_filename='entity_types/global.yaml', documents=[types.GLOBAL_TYPES_DOCUMENT]) facilities_type_folder.AddFromConfig( config_filename='FACILITIES/entity_types/Facilities.yaml', documents=[types.FACILITIES_TYPES_DOCUMENT]) hvac_type_folder.AddFromConfig( config_filename='HVAC/entity_types/ANALYSIS.yaml', documents=[types.HVAC_ANALYSIS_TYPES_DOCUMENT]) hvac_type_folder.AddFromConfig( config_filename='HVAC/entity_types/ABSTRACT.yaml', documents=[types.HVAC_ABSTRACT_TYPES_DOCUMENT]) hvac_type_folder.AddFromConfig( config_filename='HVAC/entity_types/CHWS.yaml', documents=[types.HVAC_CHWS_TYPES_DOCUMENT]) hvac_type_folder.AddFromConfig( config_filename='HVAC/entity_types/GENERALTYPES.yaml', documents=[types.HVAC_GENERAL_TYPES_DOCUMENT]) hvac_type_folder.AddFromConfig( config_filename='HVAC/entity_types/SDC.yaml', documents=[types.HVAC_SDC_TYPES_DOCUMENT]) hvac_type_folder.AddFromConfig( config_filename='HVAC/entity_types/DMP.yaml', documents=[types.HVAC_DMP_TYPES_DOCUMENT]) entity_type_universe = entity_type_lib.EntityTypeUniverse( entity_type_folders=[ global_type_folder, facilities_type_folder, hvac_type_folder ]) config_universe = presubmit_validate_types_lib.ConfigUniverse( entity_type_universe=entity_type_universe, field_universe=field_universe, subfield_universe=subfield_universe, state_universe=state_universe, connection_universe=connection_universe, unit_universe=unit_universe) # call this to expand inherited fields namespace_validator.NamespaceValidator( config_universe.GetEntityTypeNamespaces()) return config_universe