def RecursiveDirWalk(directory): """Walks recursively a directory and returns a list of PathParts. Args: directory: a directory with the ontology yaml files. Returns: path_parts: a list of PathParts. """ if not directory: return [] path_parts = [] for dir_name, _, files in os.walk(directory, followlinks=False): tmp_relative_path = os.path.relpath(dir_name, directory) for yaml_file in files: if '.yaml' not in yaml_file: continue if tmp_relative_path != os.curdir: relative_path_yaml = os.path.join(tmp_relative_path, yaml_file) else: relative_path_yaml = yaml_file path_parts.append( base_lib.PathParts(root=directory, relative_path=relative_path_yaml)) return path_parts
def testSeparateConfigFiles(self): field1 = base_lib.PathParts(root='path/to/resources', relative_path='fields/field1') field2 = base_lib.PathParts(root='path/to/resources', relative_path='fields/field2') types1 = base_lib.PathParts(root='path/to/resources', relative_path='TEST/entity_types/types1') types2 = base_lib.PathParts(root='path/to/resources', relative_path='TEST/entity_types/types2') subfield1 = base_lib.PathParts(root='path/to/resources', relative_path='subfields/subfield1') state1 = base_lib.PathParts(root='path/to/resources', relative_path='states/state1') unit1 = base_lib.PathParts(root='path/to/resources', relative_path='units/unit1') config_list = [ field1, field2, types1, types2, subfield1, state1, unit1 ] config = presubmit_validate_types_lib.SeparateConfigFiles(config_list) self.assertIn(field1, config.fields) self.assertIn(field2, config.fields) self.assertIn(types1, config.type_defs) self.assertIn(types2, config.type_defs) self.assertIn(subfield1, config.subfields) self.assertIn(state1, config.states) self.assertIn(unit1, config.units)
def testParseFieldFoldersFromBadFileDuplicateKeys(self): bad_fields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/fields/duplicate_literal_fields.yaml') field_folders = parse.ParseFieldFoldersFromFiles([bad_fields]) # Error is added to the folder it is found in, even if fields may uplevel local_folder = field_folders[1] self.assertTrue( local_folder.HasFindingTypes([findings_lib.DuplicateKeyError]))
def testParseStateFoldersFromBadFileDuplicateKeys(self): bad_states = base_lib.PathParts( root=self.base_dir, relative_path='BAD/states/duplicate_states.yaml') state_folders = parse.ParseStateFoldersFromFiles([bad_states]) self.assertLen(state_folders, 1) state_folder = state_folders[0] self.assertTrue( state_folder.HasFindingTypes([findings_lib.DuplicateKeyError]))
def testParseSubfieldFoldersFromBadFileDuplicateFields(self): bad_subfields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/subfields/duplicate_subfield_keys.yaml') subfield_folders = parse.ParseSubfieldFoldersFromFiles([bad_subfields]) # should have 1 folder. self.assertLen(subfield_folders, 1) subfield_folder = subfield_folders[0] self.assertTrue( subfield_folder.HasFindingTypes([findings_lib.DuplicateKeyError]))
def testParseSubfieldFoldersFromBadFile(self): bad_subfields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/subfields/bad_local_subfields.yaml') subfield_folders = parse.ParseSubfieldFoldersFromFiles([bad_subfields]) # should have 1 folder. self.assertLen(subfield_folders, 1) subfield_folder = subfield_folders[0] self.assertTrue( subfield_folder.local_namespace.HasFindingTypes( [findings_lib.DuplicateSubfieldDefinitionError]))
def testParseFieldFoldersFromBadFile(self): bad_fields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/fields/bad_local_fields.yaml') field_folders = parse.ParseFieldFoldersFromFiles([bad_fields]) # should have 2 folders. (global folder is created automatically) self.assertLen(field_folders, 2) # Field in bad_local will be upleveled to global ns global_folder = field_folders[0] self.assertTrue( global_folder.local_namespace.HasFindingTypes( [findings_lib.DuplicateFieldDefinitionError]))
def testParseUnitFoldersFromBadFileWithSubfieldUniverse(self): subfield_folders = parse.ParseSubfieldFoldersFromFiles( [self.global_subfields_file, self.local_subfields_file]) subfield_universe = subfield_lib.SubfieldUniverse(subfield_folders) bad_units = base_lib.PathParts( root=self.base_dir, relative_path='BAD/units/bad_units.yaml') unit_folders = parse.ParseUnitFoldersFromFiles( [bad_units], subfield_universe=subfield_universe) local_folder = unit_folders[0] self.assertTrue( local_folder.HasFindingTypes( [findings_lib.UnknownMeasurementTypeError]))
def testParseUnitFoldersFromBadFile(self): bad_units = base_lib.PathParts( root=self.base_dir, relative_path='BAD/units/bad_units.yaml') unit_folders = parse.ParseUnitFoldersFromFiles([bad_units]) self.assertLen(unit_folders, 1) unit_folder = unit_folders[0] self.assertTrue( unit_folder.HasFindingTypes([findings_lib.StandardUnitCountError])) self.assertTrue( unit_folder.HasFindingTypes([findings_lib.UnknownUnitTagError])) self.assertTrue( unit_folder.HasFindingTypes( [findings_lib.DuplicateUnitDefinitionError]))
def testParseSubfieldFoldersFromBadFileWithUnitValidation(self): unit_folders = parse.ParseUnitFoldersFromFiles( [self.global_units_file]) unit_universe = unit_lib.UnitUniverse(unit_folders) bad_subfields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/subfields/missing_unit_subfields.yaml') subfield_folders = parse.ParseSubfieldFoldersFromFiles([bad_subfields]) local_folder = subfield_folders[0] local_folder.ValidateUnits(unit_universe) self.assertTrue(subfield_folders[0].HasFindingTypes( [findings_lib.MissingUnitError]))
def testParseStateFoldersFromBadFile(self): bad_states = base_lib.PathParts( root=self.base_dir, relative_path='BAD/states/bad_states.yaml') state_folders = parse.ParseStateFoldersFromFiles([bad_states]) self.assertLen(state_folders, 1) state_folder = state_folders[0] self.assertTrue( state_folder.HasFindingTypes( [findings_lib.MissingStateDescriptionWarning])) self.assertTrue( state_folder.HasFindingTypes([findings_lib.IllegalCharacterError])) self.assertTrue( state_folder.HasFindingTypes([findings_lib.IllegalKeyTypeError]))
def testParseTypeFoldersFromFileUndefinedFields(self): bad_type_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad3.yaml') fields_universe = field_lib.FieldUniverse([]) fields_universe._namespace_map = { '': ('current_sensor', 'fan_run_command') } type_folders = parse.ParseTypeFoldersFromFiles([bad_type_file], fields_universe) # should have 1 folder self.assertLen(type_folders, 1) self.assertTrue(type_folders[0].local_namespace.HasFindingTypes( [findings_lib.UndefinedFieldError])) valid_types = list( type_folders[0].local_namespace.valid_types_map.values()) self.assertFalse(valid_types)
def testParseFieldFoldersFromBadFileWithStateUniverse(self): state_folders = parse.ParseStateFoldersFromFiles( [self.global_states_file, self.local_states_file]) state_universe = state_lib.StateUniverse(state_folders) bad_fields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/fields/bad_state_fields.yaml') field_folders = parse.ParseFieldFoldersFromFiles( [bad_fields], state_universe=state_universe) local_folder = field_folders[1] self.assertTrue( local_folder.HasFindingTypes( [findings_lib.InvalidStateFormatError])) self.assertTrue( local_folder.HasFindingTypes([findings_lib.DuplicateStateError])) self.assertTrue( local_folder.HasFindingTypes([findings_lib.MissingStateError]))
def testParseTypeFoldersFromFileBadFormat(self): bad_types_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad1.yaml') fields_universe = field_lib.FieldUniverse([]) fields_universe._namespace_map = { '': ('dryer_run_status', 'fan_run_command') } type_folders = parse.ParseTypeFoldersFromFiles([bad_types_file], fields_universe) # should have 1 folder self.assertLen(type_folders, 1) type_folder = type_folders[0] # Should find 4 errors self.assertLen(type_folder.GetFindings(), 3) self.assertTrue( type_folder.HasFindingTypes([findings_lib.DuplicateFieldError])) self.assertTrue( type_folder.HasFindingTypes( [findings_lib.UnrecognizedFieldFormatError])) self.assertTrue( type_folder.HasFindingTypes([findings_lib.DuplicateParentError]))
def setUp(self): super(PresubmitValidateTypesTest, self).setUp() self.base_dir = RESOURCE_PATH # Paths to testing files # subfield files self.global_subfields = base_lib.PathParts( root=self.base_dir, relative_path='subfields/global_subfields.yaml') self.good_local_subfields = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/subfields/local_subfields.yaml') self.bad_local_subfields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/subfields/bad_local_subfields.yaml') # field files self.global_fields = base_lib.PathParts( root=self.base_dir, relative_path='fields/global_fields.yaml') self.good_local_fields = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/fields/local_fields.yaml') self.bad_local_fields = base_lib.PathParts( root=self.base_dir, relative_path='BAD/fields/bad_local_fields.yaml') # type files self.bad1_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad1.yaml') self.bad2_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad2.yaml') self.bad3_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad3.yaml') self.bad4_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad4.yaml') self.bad5_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad5.yaml') self.good1_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/entity_types/good1.yaml') self.good2_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/entity_types/good2.yaml') self.good1_depot_path = path.join('//depot/google3', RESOURCE_PATH, 'GOOD/entity_types/good1.yaml')
def testFileBadPath(self): bad_path = base_lib.PathParts(self.base_dir, 'bad_type_file') with self.assertRaises(ValueError): presubmit_validate_types_lib.RunPresubmit([], [], [bad_path])
def setUp(self): super(ParseConfigLibTest, self).setUp() self.base_dir = RESOURCE_PATH self.duplicate_types_file = base_lib.PathParts( root=self.base_dir, relative_path='BAD/entity_types/bad_duplicate_types.yaml') self.good_types_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/entity_types/good1.yaml') self.good_types_file_2 = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/entity_types/good2.yaml') self.empty_types_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/entity_types/empty.yaml') self.global_fields_file = base_lib.PathParts( root=self.base_dir, relative_path='fields/global_fields.yaml') self.local_fields_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/fields/local_fields.yaml') self.state_fields_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/fields/state_fields.yaml') self.global_subfields_file = base_lib.PathParts( root=self.base_dir, relative_path='subfields/global_subfields.yaml') self.local_subfields_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/subfields/local_subfields.yaml') self.global_states_file = base_lib.PathParts( root=self.base_dir, relative_path='states/global_states.yaml') self.local_states_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/states/local_states.yaml') self.global_units_file = base_lib.PathParts( root=self.base_dir, relative_path='units/global_units.yaml') self.local_units_file = base_lib.PathParts( root=self.base_dir, relative_path='GOOD/units/local_units.yaml')