class TestInclude(BaseTestCase):
"""Used for Testing Rules"""
def setUp(self):
"""Setup"""
self.collection = RulesCollection()
self.collection.register(Configuration())
def tearDown(self):
"""Tear Down"""
# Reset the Spec override to prevent other tests to fail
cfnlint.helpers.initialize_specs()
def test_fail_run(self):
"""Failure test required"""
filename = 'test/fixtures/templates/bad/override/include.yaml'
template = self.load_template(filename)
with open('test/fixtures/templates/override_spec/include.json') as fp:
custom_spec = json.load(fp)
cfnlint.helpers.set_specs(custom_spec)
bad_runner = Runner(self.collection, filename, template, ['us-east-1'],
[])
errs = bad_runner.run()
self.assertEqual(2, len(errs))
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(include_experimental=True)
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
"config_rule": {
"filename": 'test/fixtures/templates/quickstart/config-rules.json',
"failures": 6
},
"iam": {
"filename": 'test/fixtures/templates/quickstart/iam.json',
"failures": 5
},
"nat_instance": {
"filename": 'test/fixtures/templates/quickstart/nat-instance.json',
"failures": 2
},
"vpc_management": {
"filename": 'test/fixtures/templates/quickstart/vpc-management.json',
"failures": 35
},
"vpc": {
"filename": 'test/fixtures/templates/quickstart/vpc.json',
"failures": 40
},
"poller": {
"filename": 'test/fixtures/templates/public/lambda-poller.json',
"failures": 1
}
}
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
self.rules.include_rules = ['I', 'W', 'E']
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
"generic_good": {
"filename": 'test/fixtures/templates/good/generic.yaml'
}
}
self.perfect_rule = 'test/fixtures/custom_rules/good/custom_rule_perfect.txt'
self.valid_boolean = 'test/fixtures/custom_rules/good/custom_rule_boolean.txt'
self.invalid_boolean = 'test/fixtures/custom_rules/bad/custom_rule_invalid_boolean.txt'
self.invalid_op = 'test/fixtures/custom_rules/bad/custom_rule_invalid_op.txt'
self.invalid_prop = 'test/fixtures/custom_rules/bad/custom_rule_invalid_prop.txt'
self.invalid_propkey = 'test/fixtures/custom_rules/bad/custom_rule_invalid_propkey.txt'
self.invalid_rt = 'test/fixtures/custom_rules/bad/custom_rule_invalid_rt.txt'
self.invalid_equal = 'test/fixtures/custom_rules/bad/custom_rule_invalid_equal.txt'
self.invalid_not_equal = 'test/fixtures/custom_rules/bad/custom_rule_invalid_not_equal.txt'
self.invalid_set = 'test/fixtures/custom_rules/bad/custom_rule_invalid_set.txt'
self.invalid_not_set = 'test/fixtures/custom_rules/bad/custom_rule_invalid_not_set.txt'
self.invalid_greater_than = 'test/fixtures/custom_rules/bad/custom_rule_invalid_greater_than.txt'
self.invalid_less_than = 'test/fixtures/custom_rules/bad/custom_rule_invalid_less_than.txt'
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(include_rules=['I'])
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
'generic': {
"filename": 'test/fixtures/templates/good/generic.yaml',
"failures": 0
},
'minimal': {
"filename": 'test/fixtures/templates/good/minimal.yaml',
"failures": 0
},
'transform': {
"filename": 'test/fixtures/templates/good/transform.yaml',
"failures": 0
},
'transform_bad': {
"filename": 'test/fixtures/templates/bad/transform_serverless_template.yaml',
"failures": 3
},
'conditions': {
"filename": 'test/fixtures/templates/good/conditions.yaml',
"failures": 0
},
'resources_codepipeline': {
'filename': 'test/fixtures/templates/good/resources_codepipeline.yaml',
'failures': 0
},
'transform_serverless_api': {
'filename': 'test/fixtures/templates/good/transform_serverless_api.yaml',
'failures': 0
},
'transform_serverless_function': {
'filename': 'test/fixtures/templates/good/transform_serverless_function.yaml',
'failures': 0
},
'transform_serverless_globals': {
'filename': 'test/fixtures/templates/good/transform_serverless_globals.yaml',
'failures': 1
},
'transform_list': {
'filename': 'test/fixtures/templates/good/transform/list_transform.yaml',
'failures': 0
},
'transform_list_many': {
'filename': 'test/fixtures/templates/good/transform/list_transform_many.yaml',
'failures': 0
},
'transform_list_not_sam': {
'filename': 'test/fixtures/templates/good/transform/list_transform_not_sam.yaml',
'failures': 0
}
}
class TestYamlParse(BaseTestCase):
"""Test YAML Parsing """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
"config_rule": {
"filename": 'test/fixtures/templates/public/lambda-poller.yaml',
"failures": 1
},
"generic_bad": {
"filename": 'test/fixtures/templates/bad/generic.yaml',
"failures": 29
}
}
def test_success_parse(self):
"""Test Successful YAML Parsing"""
for _, values in self.filenames.items():
filename = values.get('filename')
failures = values.get('failures')
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
assert len(matches) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), filename)
def test_success_parse_stdin(self):
"""Test Successful YAML Parsing through stdin"""
for _, values in self.filenames.items():
filename = '-'
failures = values.get('failures')
with open(values.get('filename'), 'r') as fp:
file_content = fp.read()
with patch('sys.stdin', StringIO(file_content)):
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
assert len(matches) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), values.get('filename'))
def test_map_failure(self):
"""Test a failure is passed on unhashable map"""
filename = 'test/fixtures/templates/bad/core/parse_invalid_map.yaml'
self.assertRaises(cfnlint.decode.cfn_yaml.CfnParseError, cfnlint.decode.cfn_yaml.load, filename)
def get_rules(rulesdir, ignore_rules, include_rules, configure_rules=None, include_experimental=False):
"""Get rules"""
rules = RulesCollection(ignore_rules, include_rules, configure_rules, include_experimental)
rules_dirs = [DEFAULT_RULESDIR] + rulesdir
try:
for rules_dir in rules_dirs:
rules.create_from_directory(rules_dir)
except OSError as e:
raise UnexpectedRuleException('Tried to append rules but got an error: %s' % str(e), 1)
return rules
def run_tests(self, rulename):
for _, values in self.filenames.items():
filename = values.get('filename')
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
rules = RulesCollection(None, None, None, False, None)
rules.create_from_custom_rules_file(rulename)
runner = cfnlint.runner.Runner(rules, filename, template, None,
None)
return runner.run()
class TestDuplicate(BaseTestCase):
"""Test Duplicates Parsing """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
def test_success_run(self):
"""Test success run"""
filename = 'test/fixtures/templates/good/generic.yaml'
try:
cfnlint.decode.cfn_yaml.load(filename)
except cfnlint.decode.cfn_yaml.CfnParseError:
assert(False)
return
assert(True)
def test_fail_json_run(self):
"""Test failure run"""
def test_fail_run(self):
"""Test failure run"""
filename = 'test/fixtures/templates/bad/duplicate.json'
try:
with open(filename) as fp:
json.load(fp, cls=cfnlint.decode.cfn_json.CfnJSONDecoder)
except cfnlint.decode.cfn_json.JSONDecodeError:
assert(True)
return
assert(False)
def test_fail_yaml_run(self):
"""Test failure run"""
filename = 'test/fixtures/templates/bad/duplicate.yaml'
try:
cfnlint.decode.cfn_yaml.load(filename)
except cfnlint.decode.cfn_yaml.CfnParseError:
assert(True)
return
assert(False)
class TestBaseTemplate(BaseRuleTestCase):
"""Test base template"""
def setUp(self):
"""Setup"""
self.collection = RulesCollection()
self.collection.register(Base())
def test_file_negative(self):
"""Failure test"""
failure = 'test/fixtures/templates/bad/template.yaml'
try:
Runner(self.collection, failure, True)
self.assertEqual(1, 0)
except Exception:
pass
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
"config_rule": {
"filename": 'test/fixtures/templates/public/lambda-poller.yaml',
"failures": 1
},
"generic_bad": {
"filename": 'test/fixtures/templates/bad/generic.yaml',
"failures": 29
}
}
def test_success_filtering_of_rules_exclude_longer(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_e0010(CloudFormationLintRule):
"""Error Rule"""
id = 'E0010'
class rule_e0002(CloudFormationLintRule):
"""Error Rule"""
id = 'E0002'
rules_to_add = [rule_e0000(), rule_e0010(), rule_e0002()]
rules = RulesCollection(ignore_rules=['E0002'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 2)
for rule in rules:
self.assertIn(rule.id, ['E0000', 'E0010'])
def test_success_filtering_of_rules_include_info(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
class rule_i0000(CloudFormationLintRule):
"""Info Rule"""
id = 'I0000'
rules_to_add = [rule_e0000(), rule_w0000(), rule_i0000()]
rules = RulesCollection(ignore_rules=None, include_rules=['I'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 3)
for rule in rules:
self.assertIn(rule.id, ['I0000', 'W0000', 'E0000'])
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(
include_rules=['I'],
include_experimental=True,
configure_rules={'E3012': {
'strict': 'false'
}})
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
'nist_high_master': {
'filename':
'test/fixtures/templates/quickstart/nist_high_master.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/nist_high_master.json'
},
'nist_application': {
'filename':
'test/fixtures/templates/quickstart/nist_application.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/nist_application.json'
},
'openshift': {
'filename':
'test/fixtures/templates/quickstart/openshift.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/openshift.json'
},
'cis_benchmark': {
'filename':
'test/fixtures/templates/quickstart/cis_benchmark.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/cis_benchmark.json'
}
}
def get_rules(append_rules, ignore_rules, include_rules, configure_rules=None, include_experimental=False,
mandatory_rules=None, custom_rules=None):
rules = RulesCollection(ignore_rules, include_rules, configure_rules,
include_experimental, mandatory_rules)
rules_paths = [DEFAULT_RULESDIR] + append_rules
try:
for rules_path in rules_paths:
if rules_path and os.path.isdir(os.path.expanduser(rules_path)):
rules.create_from_directory(rules_path)
else:
rules.create_from_module(rules_path)
rules.create_from_custom_rules_file(custom_rules)
except (OSError, ImportError) as e:
raise UnexpectedRuleException('Tried to append rules but got an error: %s' % str(e), 1)
return rules
class TestDirectives(BaseTestCase):
"""Test Directives """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(include_rules=['I'])
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
def test_templates(self):
"""Test ignoring certain rules"""
filename = 'test/fixtures/templates/bad/core/directives.yaml'
failures = 5
template = cfnlint.decode.cfn_yaml.load(filename)
runner = Runner(self.rules, filename, template, ['us-east-1'])
matches = []
matches.extend(runner.transform())
if not matches:
matches.extend(runner.run())
assert len(
matches) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), filename)
class TestNulls(BaseTestCase):
"""Test Null Value Parsing """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
def test_success_run(self):
"""Test success run"""
filename = 'test/fixtures/templates/good/generic.yaml'
try:
cfnlint.decode.cfn_yaml.load(filename)
except cfnlint.decode.cfn_yaml.CfnParseError:
assert (False)
return
assert (True)
def test_fail_json_run(self):
"""Test failure run"""
def test_success_filtering_of_rules_exclude_mandatory_long(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_e0010(CloudFormationLintRule):
"""Error Rule"""
id = 'E0010'
class rule_e0002(CloudFormationLintRule):
"""Error Rule"""
id = 'E0002'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
rules_to_add = [rule_e0000, rule_e0010, rule_e0002, rule_w0000]
rules = RulesCollection(ignore_rules=['E'], mandatory_rules=['E000'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 3)
for rule in rules:
self.assertIn(rule.id, ['E0000', 'E0002', 'W0000'])
def test_update_docs(self):
collection = RulesCollection(include_rules=['I'],
include_experimental=True)
if sys.version_info.major == 3:
builtin_module_name = 'builtins'
else:
builtin_module_name = '__builtin__'
mo = mock_open(read_data=self.TEST_TEXT)
mo.return_value.__iter__ = lambda self: self
mo.return_value.__iter__ = lambda self: iter(self.readline, '')
with patch('{}.open'.format(builtin_module_name),
mo) as mock_builtin_open:
cfnlint.maintenance.update_documentation(collection)
expected_calls = [
call('\n'),
call('Regular Text\n'),
call('## Rules\n'),
call(
'(_This documentation is generated by running `cfn-lint --update-documentation`, do not alter this manually_)\n\n'
),
call(
'The following **{}** rules are applied by this linter:\n\n'
.format(len(collection) + 3)),
call(
'| Rule ID | Title | Description | Config<br />(Name:Type:Default) | Source | Tags |\n'
),
call(
'| -------- | ----- | ----------- | ---------- | ------ | ---- |\n'
),
call(
'| [E0000<a name="E0000"></a>](../src/cfnlint/rules/__init__.py) | Parsing error found when parsing the template | Checks for JSON/YAML formatting errors in your template | | [Source](https://github.com/aws-cloudformation/cfn-python-lint) | `base` |\n'
),
call(
'| [E0001<a name="E0001"></a>](../src/cfnlint/rules/__init__.py) | Error found when transforming the template | Errors found when performing transformation on the template | | [Source](https://github.com/aws-cloudformation/cfn-python-lint) | `base`,`transform` |\n'
),
call(
'| [E0002<a name="E0002"></a>](../src/cfnlint/rules/__init__.py) | Error processing rule on the template | Errors found when processing a rule on the template | | [Source](https://github.com/aws-cloudformation/cfn-python-lint) | `base`,`rule` |\n'
),
call('\n\\* experimental rules\n'),
]
mock_builtin_open.return_value.write.assert_has_calls(
expected_calls)
self.assertEqual(len(expected_calls),
mock_builtin_open.return_value.write.call_count)
class TestQuickStartTemplates(BaseTestCase):
"""Test QuickStart Templates Parsing """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(include_rules=['I'])
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
'generic': {
"filename": 'test/fixtures/templates/good/generic.yaml',
"failures": 0
},
'minimal': {
"filename": 'test/fixtures/templates/good/minimal.yaml',
"failures": 0
},
'transform': {
"filename": 'test/fixtures/templates/good/transform.yaml',
"failures": 0
},
'transform_bad': {
"filename": 'test/fixtures/templates/bad/transform_serverless_template.yaml',
"failures": 3
},
'conditions': {
"filename": 'test/fixtures/templates/good/conditions.yaml',
"failures": 0
},
'resources_codepipeline': {
'filename': 'test/fixtures/templates/good/resources_codepipeline.yaml',
'failures': 0
},
'transform_serverless_api': {
'filename': 'test/fixtures/templates/good/transform_serverless_api.yaml',
'failures': 0
},
'transform_serverless_function': {
'filename': 'test/fixtures/templates/good/transform_serverless_function.yaml',
'failures': 0
},
'transform_serverless_globals': {
'filename': 'test/fixtures/templates/good/transform_serverless_globals.yaml',
'failures': 1
},
'transform_list': {
'filename': 'test/fixtures/templates/good/transform/list_transform.yaml',
'failures': 0
},
'transform_list_many': {
'filename': 'test/fixtures/templates/good/transform/list_transform_many.yaml',
'failures': 0
},
'transform_list_not_sam': {
'filename': 'test/fixtures/templates/good/transform/list_transform_not_sam.yaml',
'failures': 0
}
}
def test_templates(self):
"""Test Successful JSON Parsing"""
for _, values in self.filenames.items():
filename = values.get('filename')
failures = values.get('failures')
template = cfnlint.decode.cfn_yaml.load(filename)
runner = Runner(self.rules, filename, template, ['us-east-1'])
matches = []
matches.extend(runner.transform())
if not matches:
matches.extend(runner.run())
assert len(matches) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), filename)
def setUp(self):
"""Setup"""
self.collection = RulesCollection()
self.collection.register(Configuration())
def setUp(self):
"""Setup"""
self.collection = RulesCollection(include_rules=['I'],
include_experimental=True)
class TestCustomRuleParsing(BaseTestCase):
"""Test Node Objects """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
"generic_good": {
"filename": 'test/fixtures/templates/good/generic.yaml'
}
}
self.perfect_rule = 'test/fixtures/custom_rules/good/custom_rule_perfect.txt'
self.invalid_op = 'test/fixtures/custom_rules/bad/custom_rule_invalid_op.txt'
self.invalid_prop = 'test/fixtures/custom_rules/bad/custom_rule_invalid_prop.txt'
self.invalid_propkey = 'test/fixtures/custom_rules/bad/custom_rule_invalid_propkey.txt'
self.invalid_rt = 'test/fixtures/custom_rules/bad/custom_rule_invalid_rt.txt'
self.invalid_equal = 'test/fixtures/custom_rules/bad/custom_rule_invalid_equal.txt'
self.invalid_not_equal = 'test/fixtures/custom_rules/bad/custom_rule_invalid_not_equal.txt'
self.invalid_set = 'test/fixtures/custom_rules/bad/custom_rule_invalid_set.txt'
self.invalid_not_set = 'test/fixtures/custom_rules/bad/custom_rule_invalid_not_set.txt'
self.invalid_greater_than = 'test/fixtures/custom_rules/bad/custom_rule_invalid_greater_than.txt'
self.invalid_less_than = 'test/fixtures/custom_rules/bad/custom_rule_invalid_less_than.txt'
def test_perfect_parse(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(self.perfect_rule) == [])
def test_invalid_op(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_op)[0].message.find('not in supported') > -1)
def test_invalid_set(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_set)[0].message.find('In set check') > -1)
def test_invalid_not_set(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_not_set)[0].message.find('Not in set') > -1)
def test_invalid_equal(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_equal)[0].message.find('Must equal check') > -1)
def test_invalid_not_equal(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_not_equal)[0].message.find('Must not equal') > -1)
def test_invalid_greater_than(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_greater_than)[0].message.find('Greater than check') >
-1)
def test_invalid_less_than(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(
self.invalid_less_than)[0].message.find('Lesser than check') > -1)
def test_invalid_prop(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(self.invalid_prop) == [])
def test_invalid_propKey(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(self.invalid_propkey) == [])
def test_invalid_resource_type(self):
"""Test Successful Custom_Rule Parsing"""
assert (self.run_tests(self.invalid_rt) == [])
def run_tests(self, rulename):
for _, values in self.filenames.items():
filename = values.get('filename')
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
rules = RulesCollection(None, None, None, False, None)
rules.create_from_custom_rules_file(rulename)
runner = cfnlint.runner.Runner(rules, filename, template, None,
None)
return runner.run()
class TestQuickStartTemplatesNonStrict(BaseTestCase):
"""Test QuickStart Templates Parsing """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(
include_rules=['I'],
include_experimental=True,
configure_rules={'E3012': {
'strict': 'false'
}})
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
'nist_high_master': {
'filename':
'test/fixtures/templates/quickstart/nist_high_master.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/nist_high_master.json'
},
'nist_application': {
'filename':
'test/fixtures/templates/quickstart/nist_application.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/nist_application.json'
},
'openshift': {
'filename':
'test/fixtures/templates/quickstart/openshift.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/openshift.json'
},
'cis_benchmark': {
'filename':
'test/fixtures/templates/quickstart/cis_benchmark.yaml',
'results_filename':
'test/fixtures/results/quickstart/non_strict/cis_benchmark.json'
}
}
def test_templates(self):
"""Test Successful JSON Parsing"""
for _, values in self.filenames.items():
filename = values.get('filename')
failures = values.get('failures')
results_filename = values.get('results_filename')
template = cfnlint.decode.cfn_yaml.load(filename)
runner = Runner(self.rules, filename, template, ['us-east-1'])
matches = []
matches.extend(runner.transform())
if not matches:
matches.extend(runner.run())
if results_filename:
with open(results_filename) as json_data:
correct = json.load(json_data)
assert len(matches) == len(
correct), 'Expected {} failures, got {} on {}'.format(
len(correct), len(matches), filename)
for c in correct:
matched = False
for match in matches:
if c['Location']['Start']['LineNumber'] == match.linenumber and \
c['Location']['Start']['ColumnNumber'] == match.columnnumber and \
c['Rule']['Id'] == match.rule.id:
matched = True
assert matched is True, 'Expected error {} at line {}, column {} in matches for {}'.format(
c['Rule']['Id'], c['Location']['Start']['LineNumber'],
c['Location']['Start']['ColumnNumber'], filename)
else:
assert len(
matches
) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), filename)
class TestCfnJson(BaseTestCase):
"""Test JSON Parsing """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection(include_experimental=True)
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
self.filenames = {
"config_rule": {
"filename":
'test/fixtures/templates/quickstart/config-rules.json',
"failures": 4
},
"iam": {
"filename": 'test/fixtures/templates/quickstart/iam.json',
"failures": 5
},
"nat_instance": {
"filename":
'test/fixtures/templates/quickstart/nat-instance.json',
"failures": 2
},
"vpc_management": {
"filename":
'test/fixtures/templates/quickstart/vpc-management.json',
"failures": 8
},
"vpc": {
"filename": 'test/fixtures/templates/quickstart/vpc.json',
"failures": 0
},
"poller": {
"filename":
'test/fixtures/templates/public/lambda-poller.json',
"failures": 1
}
}
def test_success_parse(self):
"""Test Successful JSON Parsing"""
for _, values in self.filenames.items():
filename = values.get('filename')
failures = values.get('failures')
template = cfnlint.decode.cfn_json.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
assert len(
matches
) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), filename)
def test_success_escape_character(self):
"""Test Successful JSON Parsing"""
failures = 1
filename = 'test/fixtures/templates/good/decode/parsing.json'
template = cfnlint.decode.cfn_json.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
assert len(
matches) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), filename)
def test_success_parse_stdin(self):
"""Test Successful JSON Parsing through stdin"""
for _, values in self.filenames.items():
filename = '-'
failures = values.get('failures')
with open(values.get('filename'), 'r') as fp:
file_content = fp.read()
with patch('sys.stdin', StringIO(file_content)):
template = cfnlint.decode.cfn_json.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
assert len(
matches
) == failures, 'Expected {} failures, got {} on {}'.format(
failures, len(matches), values.get('filename'))
def test_fail_run(self):
"""Test failure run"""
filename = 'test/fixtures/templates/bad/json_parse.json'
try:
template = cfnlint.decode.cfn_json.load(filename)
except cfnlint.decode.cfn_json.JSONDecodeError:
assert (True)
return
assert (False)
def print_matches(self, matches, rules=None, filenames=None):
"""Output all the matches"""
if not rules:
rules = RulesCollection()
# These "base" rules are not passed into formatters
rules.extend([ParseError(), TransformError(), RuleError()])
results = []
for match in matches:
results.append(
sarif.Result(
rule_id=match.rule.id,
message=sarif.Message(text=match.message),
level=self._to_sarif_level(match.rule.severity),
locations=[
sarif.Location(
physical_location=sarif.PhysicalLocation(
artifact_location=sarif.ArtifactLocation(
uri=match.filename,
uri_base_id=self.uri_base_id,
),
region=sarif.Region(
start_column=match.columnnumber,
start_line=match.linenumber,
end_column=match.columnnumberend,
end_line=match.linenumberend,
),
))
],
))
# Output only the rules that have matches
matched_rules = set(r.rule_id for r in results)
rules_map = {r.id: r for r in list(rules)}
rules = [
sarif.ReportingDescriptor(
id=rule_id,
short_description=sarif.MultiformatMessageString(
text=rules_map[rule_id].shortdesc),
full_description=sarif.MultiformatMessageString(
text=rules_map[rule_id].description),
help_uri=rules_map[rule_id].source_url
if rules_map[rule_id] else None) for rule_id in matched_rules
]
run = sarif.Run(
tool=sarif.Tool(driver=sarif.ToolComponent(
name='cfn-lint',
short_description=sarif.MultiformatMessageString(
text=('Validates AWS CloudFormation templates against'
' the resource specification and additional'
' checks.')),
information_uri=
'https://github.com/aws-cloudformation/cfn-lint',
rules=rules,
version=cfnlint.version.__version__,
), ),
original_uri_base_ids={
self.uri_base_id:
sarif.ArtifactLocation(
description=sarif.MultiformatMessageString(
'The directory in which cfn-lint was run.'))
},
results=results,
)
log = sarif.SarifLog(version=self.version,
schema_uri=self.schema,
runs=[run])
# IMPORTANT: 'warning' is the default level in SARIF and will be
# stripped by serialization.
return to_json(log)
def setUp(self):
"""Setup"""
self.collection = RulesCollection()
self.collection.register(Base())
class TestRulesCollection(BaseTestCase):
"""Test Template RulesCollection in cfnlint """
def setUp(self):
""" SetUp template object"""
self.rules = RulesCollection()
self.rules.include_rules = ['I', 'W', 'E']
rulesdirs = [DEFAULT_RULESDIR]
for rulesdir in rulesdirs:
self.rules.create_from_directory(rulesdir)
def test_rule_ids_unique(self):
"""Test Rule IDs are Unique"""
existing_rules = []
for rule in self.rules:
self.assertFalse(rule.id in existing_rules)
existing_rules.append(rule.id)
def test_rule_ids_are_formatted_correctly(self):
"""Test Rule IDs are property formmated"""
for rule in self.rules:
self.assertIn(rule.id[0], ['W', 'I', 'E'])
self.assertEqual(len(rule.id), 5)
self.assertTrue(isinstance(int(rule.id[1:]), int))
def test_success_run(self):
""" Test Run Logic"""
filename = 'test/fixtures/templates/good/generic.yaml'
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
assert (matches == [])
def test_fail_run(self):
"""Test failure run"""
filename = 'test/fixtures/templates/bad/generic.yaml'
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
expected_err_count = 36
matches = []
matches.extend(self.rules.run(filename, cfn))
assert len(
matches
) == expected_err_count, 'Expected {} failures, got {}'.format(
expected_err_count, len(matches))
def test_fail_sub_properties_run(self):
"""Test failure run"""
filename = 'test/fixtures/templates/bad/resources/properties/onlyone.yaml'
template = cfnlint.decode.cfn_yaml.load(filename)
cfn = Template(filename, template, ['us-east-1'])
matches = []
matches.extend(self.rules.run(filename, cfn))
self.assertEqual(
5, len(matches),
'Expected {} failures, got {}'.format(5, len(matches)))
def test_success_filtering_of_rules_default(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
class rule_i0000(CloudFormationLintRule):
"""Info Rule"""
id = 'I0000'
rules_to_add = [rule_e0000(), rule_w0000(), rule_i0000()]
rules = RulesCollection(ignore_rules=None, include_rules=None)
rules.extend(rules_to_add)
self.assertEqual(len(rules), 2)
for rule in rules:
self.assertIn(rule.id, ['W0000', 'E0000'])
def test_success_filtering_of_rules_include_info(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
class rule_i0000(CloudFormationLintRule):
"""Info Rule"""
id = 'I0000'
rules_to_add = [rule_e0000(), rule_w0000(), rule_i0000()]
rules = RulesCollection(ignore_rules=None, include_rules=['I'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 3)
for rule in rules:
self.assertIn(rule.id, ['I0000', 'W0000', 'E0000'])
def test_success_filtering_of_rules_exclude(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
class rule_i0000(CloudFormationLintRule):
"""Info Rule"""
id = 'I0000'
rules_to_add = [rule_e0000(), rule_w0000(), rule_i0000()]
rules = RulesCollection(ignore_rules=['E'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 1)
for rule in rules:
self.assertIn(rule.id, ['W0000'])
def test_success_filtering_of_rules_exclude_long(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_e0010(CloudFormationLintRule):
"""Error Rule"""
id = 'E0010'
class rule_e0002(CloudFormationLintRule):
"""Error Rule"""
id = 'E0002'
rules_to_add = [rule_e0000(), rule_e0010(), rule_e0002()]
rules = RulesCollection(ignore_rules=['E000'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 1)
for rule in rules:
self.assertIn(rule.id, ['E0010'])
def test_success_filtering_of_rules_exclude_longer(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_e0010(CloudFormationLintRule):
"""Error Rule"""
id = 'E0010'
class rule_e0002(CloudFormationLintRule):
"""Error Rule"""
id = 'E0002'
rules_to_add = [rule_e0000(), rule_e0010(), rule_e0002()]
rules = RulesCollection(ignore_rules=['E0002'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 2)
for rule in rules:
self.assertIn(rule.id, ['E0000', 'E0010'])
def test_success_filtering_of_rules_exclude_mandatory(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
class rule_i0000(CloudFormationLintRule):
"""Info Rule"""
id = 'I0000'
rules_to_add = [rule_e0000(), rule_w0000(), rule_i0000()]
rules = RulesCollection(ignore_rules=['E'], mandatory_rules=['E'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 2)
for rule in rules:
self.assertIn(rule.id, ['E0000', 'W0000'])
def test_success_filtering_of_rules_exclude_mandatory_long(self):
"""Test extend function"""
class rule_e0000(CloudFormationLintRule):
"""Error Rule"""
id = 'E0000'
class rule_e0010(CloudFormationLintRule):
"""Error Rule"""
id = 'E0010'
class rule_e0002(CloudFormationLintRule):
"""Error Rule"""
id = 'E0002'
class rule_w0000(CloudFormationLintRule):
"""Warning Rule"""
id = 'W0000'
rules_to_add = [rule_e0000(), rule_e0010(), rule_e0002(), rule_w0000()]
rules = RulesCollection(ignore_rules=['E'], mandatory_rules=['E000'])
rules.extend(rules_to_add)
self.assertEqual(len(rules), 3)
for rule in rules:
self.assertIn(rule.id, ['E0000', 'E0002', 'W0000'])
def setUp(self):
"""Setup"""
self.collection = RulesCollection()
self.collection.register(Required())
def test_create_from_module(self):
"""Load rules from a module"""
rules = RulesCollection()
rules.create_from_module("cfnlint.rules.templates.Base")
self.assertIn('E1001', (r.id for r in rules))
