def RecursiveDirWalk(directory):
"""Walks recursively a directory and returns a list of PathParts.
Args:
directory: a directory with the ontology yaml files.
Returns:
path_parts: a list of PathParts.
"""
if not directory:
return []
path_parts = []
for dir_name, _, files in os.walk(directory, followlinks=False):
tmp_relative_path = os.path.relpath(dir_name, directory)
for yaml_file in files:
if '.yaml' not in yaml_file:
continue
if tmp_relative_path != os.curdir:
relative_path_yaml = os.path.join(tmp_relative_path, yaml_file)
else:
relative_path_yaml = yaml_file
path_parts.append(
base_lib.PathParts(root=directory,
relative_path=relative_path_yaml))
return path_parts
def testSeparateConfigFiles(self):
field1 = base_lib.PathParts(root='path/to/resources',
relative_path='fields/field1')
field2 = base_lib.PathParts(root='path/to/resources',
relative_path='fields/field2')
types1 = base_lib.PathParts(root='path/to/resources',
relative_path='TEST/entity_types/types1')
types2 = base_lib.PathParts(root='path/to/resources',
relative_path='TEST/entity_types/types2')
subfield1 = base_lib.PathParts(root='path/to/resources',
relative_path='subfields/subfield1')
state1 = base_lib.PathParts(root='path/to/resources',
relative_path='states/state1')
unit1 = base_lib.PathParts(root='path/to/resources',
relative_path='units/unit1')
config_list = [
field1, field2, types1, types2, subfield1, state1, unit1
]
config = presubmit_validate_types_lib.SeparateConfigFiles(config_list)
self.assertIn(field1, config.fields)
self.assertIn(field2, config.fields)
self.assertIn(types1, config.type_defs)
self.assertIn(types2, config.type_defs)
self.assertIn(subfield1, config.subfields)
self.assertIn(state1, config.states)
self.assertIn(unit1, config.units)
def testParseFieldFoldersFromBadFileDuplicateKeys(self):
bad_fields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/fields/duplicate_literal_fields.yaml')
field_folders = parse.ParseFieldFoldersFromFiles([bad_fields])
# Error is added to the folder it is found in, even if fields may uplevel
local_folder = field_folders[1]
self.assertTrue(
local_folder.HasFindingTypes([findings_lib.DuplicateKeyError]))
def testParseStateFoldersFromBadFileDuplicateKeys(self):
bad_states = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/states/duplicate_states.yaml')
state_folders = parse.ParseStateFoldersFromFiles([bad_states])
self.assertLen(state_folders, 1)
state_folder = state_folders[0]
self.assertTrue(
state_folder.HasFindingTypes([findings_lib.DuplicateKeyError]))
def testParseSubfieldFoldersFromBadFileDuplicateFields(self):
bad_subfields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/subfields/duplicate_subfield_keys.yaml')
subfield_folders = parse.ParseSubfieldFoldersFromFiles([bad_subfields])
# should have 1 folder.
self.assertLen(subfield_folders, 1)
subfield_folder = subfield_folders[0]
self.assertTrue(
subfield_folder.HasFindingTypes([findings_lib.DuplicateKeyError]))
def testParseSubfieldFoldersFromBadFile(self):
bad_subfields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/subfields/bad_local_subfields.yaml')
subfield_folders = parse.ParseSubfieldFoldersFromFiles([bad_subfields])
# should have 1 folder.
self.assertLen(subfield_folders, 1)
subfield_folder = subfield_folders[0]
self.assertTrue(
subfield_folder.local_namespace.HasFindingTypes(
[findings_lib.DuplicateSubfieldDefinitionError]))
def testParseFieldFoldersFromBadFile(self):
bad_fields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/fields/bad_local_fields.yaml')
field_folders = parse.ParseFieldFoldersFromFiles([bad_fields])
# should have 2 folders. (global folder is created automatically)
self.assertLen(field_folders, 2)
# Field in bad_local will be upleveled to global ns
global_folder = field_folders[0]
self.assertTrue(
global_folder.local_namespace.HasFindingTypes(
[findings_lib.DuplicateFieldDefinitionError]))
def testParseUnitFoldersFromBadFileWithSubfieldUniverse(self):
subfield_folders = parse.ParseSubfieldFoldersFromFiles(
[self.global_subfields_file, self.local_subfields_file])
subfield_universe = subfield_lib.SubfieldUniverse(subfield_folders)
bad_units = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/units/bad_units.yaml')
unit_folders = parse.ParseUnitFoldersFromFiles(
[bad_units], subfield_universe=subfield_universe)
local_folder = unit_folders[0]
self.assertTrue(
local_folder.HasFindingTypes(
[findings_lib.UnknownMeasurementTypeError]))
def testParseUnitFoldersFromBadFile(self):
bad_units = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/units/bad_units.yaml')
unit_folders = parse.ParseUnitFoldersFromFiles([bad_units])
self.assertLen(unit_folders, 1)
unit_folder = unit_folders[0]
self.assertTrue(
unit_folder.HasFindingTypes([findings_lib.StandardUnitCountError]))
self.assertTrue(
unit_folder.HasFindingTypes([findings_lib.UnknownUnitTagError]))
self.assertTrue(
unit_folder.HasFindingTypes(
[findings_lib.DuplicateUnitDefinitionError]))
def testParseSubfieldFoldersFromBadFileWithUnitValidation(self):
unit_folders = parse.ParseUnitFoldersFromFiles(
[self.global_units_file])
unit_universe = unit_lib.UnitUniverse(unit_folders)
bad_subfields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/subfields/missing_unit_subfields.yaml')
subfield_folders = parse.ParseSubfieldFoldersFromFiles([bad_subfields])
local_folder = subfield_folders[0]
local_folder.ValidateUnits(unit_universe)
self.assertTrue(subfield_folders[0].HasFindingTypes(
[findings_lib.MissingUnitError]))
def testParseStateFoldersFromBadFile(self):
bad_states = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/states/bad_states.yaml')
state_folders = parse.ParseStateFoldersFromFiles([bad_states])
self.assertLen(state_folders, 1)
state_folder = state_folders[0]
self.assertTrue(
state_folder.HasFindingTypes(
[findings_lib.MissingStateDescriptionWarning]))
self.assertTrue(
state_folder.HasFindingTypes([findings_lib.IllegalCharacterError]))
self.assertTrue(
state_folder.HasFindingTypes([findings_lib.IllegalKeyTypeError]))
def testParseTypeFoldersFromFileUndefinedFields(self):
bad_type_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad3.yaml')
fields_universe = field_lib.FieldUniverse([])
fields_universe._namespace_map = {
'': ('current_sensor', 'fan_run_command')
}
type_folders = parse.ParseTypeFoldersFromFiles([bad_type_file],
fields_universe)
# should have 1 folder
self.assertLen(type_folders, 1)
self.assertTrue(type_folders[0].local_namespace.HasFindingTypes(
[findings_lib.UndefinedFieldError]))
valid_types = list(
type_folders[0].local_namespace.valid_types_map.values())
self.assertFalse(valid_types)
def testParseFieldFoldersFromBadFileWithStateUniverse(self):
state_folders = parse.ParseStateFoldersFromFiles(
[self.global_states_file, self.local_states_file])
state_universe = state_lib.StateUniverse(state_folders)
bad_fields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/fields/bad_state_fields.yaml')
field_folders = parse.ParseFieldFoldersFromFiles(
[bad_fields], state_universe=state_universe)
local_folder = field_folders[1]
self.assertTrue(
local_folder.HasFindingTypes(
[findings_lib.InvalidStateFormatError]))
self.assertTrue(
local_folder.HasFindingTypes([findings_lib.DuplicateStateError]))
self.assertTrue(
local_folder.HasFindingTypes([findings_lib.MissingStateError]))
def testParseTypeFoldersFromFileBadFormat(self):
bad_types_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad1.yaml')
fields_universe = field_lib.FieldUniverse([])
fields_universe._namespace_map = {
'': ('dryer_run_status', 'fan_run_command')
}
type_folders = parse.ParseTypeFoldersFromFiles([bad_types_file],
fields_universe)
# should have 1 folder
self.assertLen(type_folders, 1)
type_folder = type_folders[0]
# Should find 4 errors
self.assertLen(type_folder.GetFindings(), 3)
self.assertTrue(
type_folder.HasFindingTypes([findings_lib.DuplicateFieldError]))
self.assertTrue(
type_folder.HasFindingTypes(
[findings_lib.UnrecognizedFieldFormatError]))
self.assertTrue(
type_folder.HasFindingTypes([findings_lib.DuplicateParentError]))
def setUp(self):
super(PresubmitValidateTypesTest, self).setUp()
self.base_dir = RESOURCE_PATH
# Paths to testing files
# subfield files
self.global_subfields = base_lib.PathParts(
root=self.base_dir,
relative_path='subfields/global_subfields.yaml')
self.good_local_subfields = base_lib.PathParts(
root=self.base_dir,
relative_path='GOOD/subfields/local_subfields.yaml')
self.bad_local_subfields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/subfields/bad_local_subfields.yaml')
# field files
self.global_fields = base_lib.PathParts(
root=self.base_dir, relative_path='fields/global_fields.yaml')
self.good_local_fields = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/fields/local_fields.yaml')
self.bad_local_fields = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/fields/bad_local_fields.yaml')
# type files
self.bad1_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad1.yaml')
self.bad2_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad2.yaml')
self.bad3_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad3.yaml')
self.bad4_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad4.yaml')
self.bad5_file = base_lib.PathParts(
root=self.base_dir, relative_path='BAD/entity_types/bad5.yaml')
self.good1_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/entity_types/good1.yaml')
self.good2_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/entity_types/good2.yaml')
self.good1_depot_path = path.join('//depot/google3', RESOURCE_PATH,
'GOOD/entity_types/good1.yaml')
def testFileBadPath(self):
bad_path = base_lib.PathParts(self.base_dir, 'bad_type_file')
with self.assertRaises(ValueError):
presubmit_validate_types_lib.RunPresubmit([], [], [bad_path])
def setUp(self):
super(ParseConfigLibTest, self).setUp()
self.base_dir = RESOURCE_PATH
self.duplicate_types_file = base_lib.PathParts(
root=self.base_dir,
relative_path='BAD/entity_types/bad_duplicate_types.yaml')
self.good_types_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/entity_types/good1.yaml')
self.good_types_file_2 = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/entity_types/good2.yaml')
self.empty_types_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/entity_types/empty.yaml')
self.global_fields_file = base_lib.PathParts(
root=self.base_dir, relative_path='fields/global_fields.yaml')
self.local_fields_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/fields/local_fields.yaml')
self.state_fields_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/fields/state_fields.yaml')
self.global_subfields_file = base_lib.PathParts(
root=self.base_dir,
relative_path='subfields/global_subfields.yaml')
self.local_subfields_file = base_lib.PathParts(
root=self.base_dir,
relative_path='GOOD/subfields/local_subfields.yaml')
self.global_states_file = base_lib.PathParts(
root=self.base_dir, relative_path='states/global_states.yaml')
self.local_states_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/states/local_states.yaml')
self.global_units_file = base_lib.PathParts(
root=self.base_dir, relative_path='units/global_units.yaml')
self.local_units_file = base_lib.PathParts(
root=self.base_dir, relative_path='GOOD/units/local_units.yaml')
