def test_invalid_types(self): """ Check that configs are marked as invalid if elements in it are of the wrong type """ # Start with a valid config valid_config = { "suite_name": "hello", "checks": [{ "check_id": "one", "parameters": {}, "check_name": self.get_import_string("BasicTestCheck") }] } c1 = deepcopy(valid_config) c1["suite_name"] = ("this", "is", "not", "a", "string") c2 = deepcopy(valid_config) c2["checks"] = "oops" c3 = deepcopy(valid_config) c3["checks"][0]["check_id"] = {} c4 = deepcopy(valid_config) c4["checks"][0]["parameters"] = 0 for c in (c1, c2, c3, c4): with pytest.raises(TypeError): YamlParser.validate_config(c)
def test_include_absolute_path(self, tmpdir): yaml1 = tmpdir.join("yaml1.yml") yaml2 = tmpdir.join("yaml2.yml") yaml1.write( yaml.dump({ "suite_name": "first_suite", "checks": [{ "__INCLUDE__": str(yaml2) }] })) yaml2.write( yaml.dump({ "suite_name": "included_suite", "checks": [{ "check_id": "included_check", "check_name": self.get_import_string("BasicTestCheck"), "parameters": { "var": "value" } }] })) cls = YamlParser.get_checker_class(str(yaml1)) assert self.get_check_method_names(cls) == ["check_included_check"]
def test_invalid_config_in_included_file(self, tmpdir): """ Check that a validation error is raised if an included file is not valid """ yaml1 = tmpdir.join("yaml1.yml") yaml2 = tmpdir.join("yaml2.yml") yaml1.write( yaml.dump({ "suite_name": "first_suite", "checks": [{ "__INCLUDE__": "yaml2.yml" }] })) yaml2.write( yaml.dump({ "suite_name": "included_suite", "checks": [{ "cheque_id": "included_check", "check_nom": self.get_import_string("BasicTestCheck"), "purumuters": { "var": "value" } }] })) with pytest.raises(ValueError): cls = YamlParser.get_checker_class(str(yaml1))
def get_suites(cls, args): """ :param args: argparse arguments object :return: dictionary mapping check name to checker class """ suites = {} for filename in args.yaml_files: suite = YamlParser.get_checker_class(filename) suites[suite.__name__] = suite return suites
def test_recursive_include(self, tmpdir): """ Check that includes can be recursive, i.e 1 includes 2 which includes 3 will result in checks from 1 + 2 + 3 """ # Note: create 1 in different dir to 2 and 3, so that we check the # path to 3 is resolved relative to 2 (not relative to 1)... dir1 = tmpdir.mkdir("dir1") dir2 = tmpdir.mkdir("dir2") yaml1 = dir1.join("yaml1.yml") yaml2 = dir2.join("yaml2.yml") yaml3 = dir2.join("yaml3.yml") def get_check(name): return { "check_id": "from_{}".format(name), "check_name": self.get_import_string("BasicTestCheck"), "parameters": {} } yaml1.write( yaml.dump({ "suite_name": "one", "checks": [get_check("one"), { "__INCLUDE__": "../dir2/yaml2.yml" }] })) yaml2.write( yaml.dump({ "suite_name": "two", "checks": [get_check("two"), { "__INCLUDE__": "yaml3.yml" }] })) yaml3.write( yaml.dump({ "suite_name": "three", "checks": [ get_check("three"), ] })) cls = YamlParser.get_checker_class(str(yaml1)) assert self.get_check_method_names(cls) == [ "check_from_one", "check_from_three", "check_from_two" ]
def test_missing_keys(self): """ Check that a config with missing required fields is marked as invalid """ invalid_configs = [ {}, { "suite_name": "hello" }, # Missing checks { "checks": [] }, # Missing suite name # Missing parameters { "suite_name": "hello", "checks": [{ "check_id": "one" }] } ] valid_config = { "suite_name": "hello", "checks": [{ "check_id": "one", "parameters": {}, "check_name": "blah" }] } for c in invalid_configs: with pytest.raises(ValueError): YamlParser.validate_config(c) try: YamlParser.validate_config(valid_config) except ValueError: assert False, "Valid config was incorrectly marked as invalid"
def test_supported_ds(self): valid_config = { "suite_name": "test_suite", "checks": [{ "check_id": "one", "parameters": {}, "check_name": self.get_import_string("SupportDsTestCheckClass1") }, { "check_id": "one", "parameters": {}, "check_name": self.get_import_string("SupportDsTestCheckClass2") }] } check_cls = YamlParser.get_checker_class(valid_config) # Supported datasets for generated class should be types common to both # checks assert check_cls.supported_ds == [2, 3]
def test_class_gen(self): """ Check that a checker class is generated correctly """ check_cls = self.get_import_string("BasicTestCheck") config = { "suite_name": "test_suite", "checks": [{ "check_id": "one", "parameters": { "a": 42 }, "check_name": check_cls }, { "check_id": "two", "parameters": { "a": 19 }, "check_name": check_cls, "check_level": "LOW" }] } new_class = YamlParser.get_checker_class(config) # Check class inherits from BaseCheck assert BaseCheck in new_class.__bases__ # Check the expected methods are present assert self.get_check_method_names(new_class) == [ "check_one", "check_two" ] # Check name is correct assert new_class.__name__ == "test_suite" # Check the class methods return the expected values checker = new_class() assert checker.check_one("dataset") == ({"a": 42}, None) assert checker.check_two("dataset") == ({"a": 19}, "LOW")
def test_include_other_yaml_file(self, tmpdir): """ Check that checks from another YAML file can be included """ # Create YAML files in different directories so we can check that # included file is looked up relative to the first dir1 = tmpdir.mkdir("dir1") dir2 = tmpdir.mkdir("dir2") dir3 = dir2.mkdir("dir2") yaml1 = dir1.join("suite1.yml") yaml2 = dir2.join("suite2.yml") yaml1.write( yaml.dump({ "suite_name": "first_suite", "checks": [ { "check_id": "first_check", # The base check used here is not important... "check_name": self.get_import_string("BasicTestCheck"), "parameters": {} }, { "__INCLUDE__": "../dir2/suite2.yml" } ] })) yaml2.write( yaml.dump({ "suite_name": "this_should_not_be_seen", "checks": [{ "check_id": "included_check1", "check_name": self.get_import_string("BasicTestCheck"), "parameters": { "some_var": "some_value" } }, { "check_id": "included_check2", "check_name": self.get_import_string("BasicTestCheck"), "parameters": { "another_var": "another_value" } }] })) # Try generating the combined YAML from different directories to check # relative path lookup dirs = (tmpdir, dir1, dir2, dir3) for d in dirs: with d.as_cwd(): cls = YamlParser.get_checker_class(str(yaml1)) assert cls.__name__ == "first_suite" assert self.get_check_method_names(cls) == [ "check_first_check", "check_included_check1", "check_included_check2" ]
def test_no_checks(self): """ Check that a config with no checks is invalid """ with pytest.raises(ValueError): YamlParser.validate_config({"suite_name": "test", "checks": []})