def load(options, path, format=None, validate=True, vars=None):
# should we do os.path.expanduser here?
if not os.path.exists(path):
raise IOError("Invalid path for config %r" % path)
from c7n.schema import validate, StructureParser
data = utils.load_file(path, format=format, vars=vars)
structure = StructureParser()
structure.validate(data)
load_resources(structure.get_resource_types(data))
if isinstance(data, list):
log.warning(
'yaml in invalid format. The "policies:" line is probably missing.'
)
return None
if validate:
errors = validate(data)
if errors:
raise PolicyValidationError("Failed to validate policy %s \n %s" %
(errors[1], errors[0]))
# Test for empty policy file
if not data or data.get('policies') is None:
return None
collection = PolicyCollection.from_data(data, options)
if validate:
# non schema validation of policies
[p.validate() for p in collection]
return collection
def test_policy_not_mapping(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({'policies': [[]]})
self.assertTrue(
str(ecm.exception).startswith(
'policy must be a dictionary/mapping found:list'))
def test_policies_not_list(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({'policies': {}})
self.assertTrue(
str(ecm.exception).startswith(
"`policies` key should be an array/list"))
def test_bad_top_level_datastruct(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate([])
self.assertTrue(
str(ecm.exception).startswith(
'Policy file top level data structure'))
def test_policy_extra_key(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({'policies': [{
'name': 'foo', 'extra': 1, 'resource': 'aws.ec2'}]})
self.assertEqual(str(ecm.exception),
"policy:foo has unknown keys: extra")
def test_get_resource_types(self):
p = StructureParser()
self.assertEqual(
p.get_resource_types({
'policies': [{
'resource': 'ec2'
}, {
'resource': 'gcp.instance'
}]
}), set(('aws.ec2', 'gcp.instance')))
def get_validator(self, data):
# return a jsonschema validator for the policy data
# use the policy loader to load the resource types
self.policy_loader.load_data(
data, file_uri='memory://', validate=False)
rtypes = StructureParser().get_resource_types(data)
return self.policy_loader.validator.gen_schema(tuple(rtypes))
def test_invalid_action(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({
'policies': [{
'name': 'foo',
'resource': 'ec2',
'actions': {}
}]
})
self.assertTrue(
str(ecm.exception).startswith(
'policy:foo must use a list for actions found:dict'))
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({
'policies': [{
'name': 'foo',
'resource': 'ec2',
'actions': [[]]
}]
})
self.assertTrue(
str(ecm.exception).startswith(
'policy:foo action must be a mapping/dict found:list'))
def validate(options):
from c7n import schema
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
structure = StructureParser()
errors = []
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml', 'json'):
data = yaml.load(fh.read(), Loader=DuplicateKeyCheckLoader)
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError("The config file must end in .json, .yml or .yaml.")
try:
structure.validate(data)
except PolicyValidationError as e:
log.error("Configuration invalid: {}".format(config_file))
log.error("%s" % e)
errors.append(e)
continue
load_resources(structure.get_resource_types(data))
schm = schema.generate()
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown') for p in data.get('policies', ())}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(ValueError(
"Only one policy with a given name allowed, duplicates: %s" % (
", ".join(dupes)
)
))
used_policy_names = used_policy_names.union(conf_policy_names)
if not errors:
null_config = Config.empty(dryrun=True, account_id='na', region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
except Exception as e:
msg = "Policy: %s is invalid: %s" % (
p.get('name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if errors:
sys.exit(1)
def test_policy_missing_required(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({'policies': [{'resource': 'aws.ec2'}]})
self.assertTrue(
str(ecm.exception).startswith("policy missing required keys"))
def test_policies_missing(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({})
self.assertTrue(
str(ecm.exception).startswith("`policies` list missing"))
def test_extra_keys(self):
p = StructureParser()
with self.assertRaises(PolicyValidationError) as ecm:
p.validate({'accounts': []})
self.assertTrue(
str(ecm.exception).startswith('Policy files top level keys'))
def validate(options):
from c7n import schema
if len(options.configs) < 1:
log.error('no config files specified')
sys.exit(1)
used_policy_names = set()
structure = StructureParser()
errors = []
found_deprecations = False
footnotes = deprecated.Footnotes()
for config_file in options.configs:
config_file = os.path.expanduser(config_file)
if not os.path.exists(config_file):
raise ValueError("Invalid path for config %r" % config_file)
options.dryrun = True
fmt = config_file.rsplit('.', 1)[-1]
with open(config_file) as fh:
if fmt in ('yml', 'yaml', 'json'):
# our loader is safe loader derived.
data = yaml.load(
fh.read(),
Loader=DuplicateKeyCheckLoader) # nosec nosemgrep
else:
log.error("The config file must end in .json, .yml or .yaml.")
raise ValueError(
"The config file must end in .json, .yml or .yaml.")
try:
structure.validate(data)
except PolicyValidationError as e:
log.error("Configuration invalid: {}".format(config_file))
log.error("%s" % e)
errors.append(e)
continue
load_resources(structure.get_resource_types(data))
schm = schema.generate()
errors += schema.validate(data, schm)
conf_policy_names = {
p.get('name', 'unknown')
for p in data.get('policies', ())
}
dupes = conf_policy_names.intersection(used_policy_names)
if len(dupes) >= 1:
errors.append(
ValueError(
"Only one policy with a given name allowed, duplicates: %s"
% (", ".join(dupes))))
used_policy_names = used_policy_names.union(conf_policy_names)
source_locator = None
if fmt in ('yml', 'yaml'):
# For yaml files there is at least the expectation that the policy
# name is on a line by itself. With JSON, the file could be one big
# line. At this stage we are only attempting to find line number for
# policies in yaml files.
source_locator = SourceLocator(config_file)
if not errors:
null_config = Config.empty(dryrun=True,
account_id='na',
region='na')
for p in data.get('policies', ()):
try:
policy = Policy(p, null_config, Bag())
policy.validate()
# If the policy is invalid, there isn't much point checking
# for deprecated usage as there is no guarantee as to the
# state of the policy.
if options.check_deprecations != deprecated.SKIP:
report = deprecated.report(policy)
if report:
found_deprecations = True
log.warning(
"deprecated usage found in policy\n" +
report.format(source_locator=source_locator,
footnotes=footnotes))
except Exception as e:
msg = "Policy: %s is invalid: %s" % (p.get(
'name', 'unknown'), e)
errors.append(msg)
if not errors:
log.info("Configuration valid: {}".format(config_file))
continue
log.error("Configuration invalid: {}".format(config_file))
for e in errors:
log.error("%s" % e)
if found_deprecations:
notes = footnotes()
if notes:
log.warning("deprecation footnotes:\n" + notes)
if options.check_deprecations == deprecated.STRICT:
sys.exit(1)
if errors:
sys.exit(1)
