def load(options, path, format=None, validate=True, vars=None): # should we do os.path.expanduser here? if not os.path.exists(path): raise IOError("Invalid path for config %r" % path) from c7n.schema import validate, StructureParser data = utils.load_file(path, format=format, vars=vars) structure = StructureParser() structure.validate(data) load_resources(structure.get_resource_types(data)) if isinstance(data, list): log.warning( 'yaml in invalid format. The "policies:" line is probably missing.' ) return None if validate: errors = validate(data) if errors: raise PolicyValidationError("Failed to validate policy %s \n %s" % (errors[1], errors[0])) # Test for empty policy file if not data or data.get('policies') is None: return None collection = PolicyCollection.from_data(data, options) if validate: # non schema validation of policies [p.validate() for p in collection] return collection
def test_bad_top_level_datastruct(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate([]) self.assertTrue( str(ecm.exception).startswith( 'Policy file top level data structure'))
def test_invalid_action(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({ 'policies': [{ 'name': 'foo', 'resource': 'ec2', 'actions': {} }] }) self.assertTrue( str(ecm.exception).startswith( 'policy:foo must use a list for actions found:dict')) with self.assertRaises(PolicyValidationError) as ecm: p.validate({ 'policies': [{ 'name': 'foo', 'resource': 'ec2', 'actions': [[]] }] }) self.assertTrue( str(ecm.exception).startswith( 'policy:foo action must be a mapping/dict found:list'))
def test_policies_not_list(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({'policies': {}}) self.assertTrue( str(ecm.exception).startswith( "`policies` key should be an array/list"))
def test_policy_not_mapping(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({'policies': [[]]}) self.assertTrue( str(ecm.exception).startswith( 'policy must be a dictionary/mapping found:list'))
def test_policy_extra_key(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({'policies': [{ 'name': 'foo', 'extra': 1, 'resource': 'aws.ec2'}]}) self.assertEqual(str(ecm.exception), "policy:foo has unknown keys: extra")
def validate(options): from c7n import schema if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() structure = StructureParser() errors = [] for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml', 'json'): data = yaml.load(fh.read(), Loader=DuplicateKeyCheckLoader) else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError("The config file must end in .json, .yml or .yaml.") try: structure.validate(data) except PolicyValidationError as e: log.error("Configuration invalid: {}".format(config_file)) log.error("%s" % e) errors.append(e) continue load_resources(structure.get_resource_types(data)) schm = schema.generate() errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ())} dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append(ValueError( "Only one policy with a given name allowed, duplicates: %s" % ( ", ".join(dupes) ) )) used_policy_names = used_policy_names.union(conf_policy_names) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() except Exception as e: msg = "Policy: %s is invalid: %s" % ( p.get('name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if errors: sys.exit(1)
def test_policy_missing_required(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({'policies': [{'resource': 'aws.ec2'}]}) self.assertTrue( str(ecm.exception).startswith("policy missing required keys"))
def test_policies_missing(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({}) self.assertTrue( str(ecm.exception).startswith("`policies` list missing"))
def test_extra_keys(self): p = StructureParser() with self.assertRaises(PolicyValidationError) as ecm: p.validate({'accounts': []}) self.assertTrue( str(ecm.exception).startswith('Policy files top level keys'))
def validate(options): from c7n import schema if len(options.configs) < 1: log.error('no config files specified') sys.exit(1) used_policy_names = set() structure = StructureParser() errors = [] found_deprecations = False footnotes = deprecated.Footnotes() for config_file in options.configs: config_file = os.path.expanduser(config_file) if not os.path.exists(config_file): raise ValueError("Invalid path for config %r" % config_file) options.dryrun = True fmt = config_file.rsplit('.', 1)[-1] with open(config_file) as fh: if fmt in ('yml', 'yaml', 'json'): # our loader is safe loader derived. data = yaml.load( fh.read(), Loader=DuplicateKeyCheckLoader) # nosec nosemgrep else: log.error("The config file must end in .json, .yml or .yaml.") raise ValueError( "The config file must end in .json, .yml or .yaml.") try: structure.validate(data) except PolicyValidationError as e: log.error("Configuration invalid: {}".format(config_file)) log.error("%s" % e) errors.append(e) continue load_resources(structure.get_resource_types(data)) schm = schema.generate() errors += schema.validate(data, schm) conf_policy_names = { p.get('name', 'unknown') for p in data.get('policies', ()) } dupes = conf_policy_names.intersection(used_policy_names) if len(dupes) >= 1: errors.append( ValueError( "Only one policy with a given name allowed, duplicates: %s" % (", ".join(dupes)))) used_policy_names = used_policy_names.union(conf_policy_names) source_locator = None if fmt in ('yml', 'yaml'): # For yaml files there is at least the expectation that the policy # name is on a line by itself. With JSON, the file could be one big # line. At this stage we are only attempting to find line number for # policies in yaml files. source_locator = SourceLocator(config_file) if not errors: null_config = Config.empty(dryrun=True, account_id='na', region='na') for p in data.get('policies', ()): try: policy = Policy(p, null_config, Bag()) policy.validate() # If the policy is invalid, there isn't much point checking # for deprecated usage as there is no guarantee as to the # state of the policy. if options.check_deprecations != deprecated.SKIP: report = deprecated.report(policy) if report: found_deprecations = True log.warning( "deprecated usage found in policy\n" + report.format(source_locator=source_locator, footnotes=footnotes)) except Exception as e: msg = "Policy: %s is invalid: %s" % (p.get( 'name', 'unknown'), e) errors.append(msg) if not errors: log.info("Configuration valid: {}".format(config_file)) continue log.error("Configuration invalid: {}".format(config_file)) for e in errors: log.error("%s" % e) if found_deprecations: notes = footnotes() if notes: log.warning("deprecation footnotes:\n" + notes) if options.check_deprecations == deprecated.STRICT: sys.exit(1) if errors: sys.exit(1)