def test_missing_option(self):
c = config.YamlLintConfig('rules:\n' ' colons: enable\n')
self.assertEqual(c.rules['colons']['max-spaces-before'], 0)
self.assertEqual(c.rules['colons']['max-spaces-after'], 1)
c = config.YamlLintConfig('rules:\n'
' colons:\n'
' max-spaces-before: 9\n')
self.assertEqual(c.rules['colons']['max-spaces-before'], 9)
self.assertEqual(c.rules['colons']['max-spaces-after'], 1)
def test_extend_config_disable_rule(self):
old = config.YamlLintConfig('extends: default')
new = config.YamlLintConfig('extends: default\n'
'rules:\n'
' trailing-spaces: disable\n')
old.rules['trailing-spaces'] = False
self.assertEqual(sorted(new.rules.keys()), sorted(old.rules.keys()))
for rule in new.rules:
self.assertEqual(new.rules[rule], old.rules[rule])
def test_extend_config_override_rule_partly(self):
old = config.YamlLintConfig('extends: default')
new = config.YamlLintConfig('extends: default\n'
'rules:\n'
' empty-lines:\n'
' max-start: 42\n')
old.rules['empty-lines']['max-start'] = 42
self.assertEqual(sorted(new.rules.keys()), sorted(old.rules.keys()))
for rule in new.rules:
self.assertEqual(new.rules[rule], old.rules[rule])
def test_extend_remove_rule(self):
old = config.YamlLintConfig('rules:\n'
' colons:\n'
' max-spaces-before: 0\n'
' max-spaces-after: 1\n'
' hyphens:\n'
' max-spaces-after: 2\n')
new = config.YamlLintConfig('rules:\n' ' colons: disable\n')
new.extend(old)
self.assertEqual(sorted(new.rules.keys()), ['colons', 'hyphens'])
self.assertEqual(new.rules['colons'], False)
self.assertEqual(new.rules['hyphens']['max-spaces-after'], 2)
self.assertEqual(len(new.enabled_rules()), 1)
def test_extend_on_object(self):
old = config.YamlLintConfig('rules:\n'
' colons:\n'
' max-spaces-before: 0\n'
' max-spaces-after: 1\n')
new = config.YamlLintConfig('rules:\n'
' hyphens:\n'
' max-spaces-after: 2\n')
new.extend(old)
self.assertEqual(sorted(new.rules.keys()), ['colons', 'hyphens'])
self.assertEqual(new.rules['colons']['max-spaces-before'], 0)
self.assertEqual(new.rules['colons']['max-spaces-after'], 1)
self.assertEqual(new.rules['hyphens']['max-spaces-after'], 2)
self.assertEqual(len(new.enabled_rules(None)), 2)
def lint_file(filename):
return_code = 0
print(filename)
# validate filename
if not filename.endswith(".ksy"):
print("File names should end with .ksy")
return_code += 1
# validate encoding
with open(filename, 'rb') as f:
try:
f.read().decode('utf-8')
except UnicodeError:
print("string is not UTF-8")
return_code += 1
with open(filename) as f:
conf = config.YamlLintConfig(content=yamllint_config.YAMLLINT_RULES)
# lint file
for problem in linter.run(f, conf):
print(problem)
return_code += 1
f.seek(0, 0)
# validate schema
try:
inst = yaml.load(f, Loader=yaml.FullLoader)
# validate schema
return_code += lint_order(inst)
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(path.join(dir_path, "ksy_schema.json")) as schema_io:
schema = yaml.load(schema_io, Loader=yaml.FullLoader)
v = Draft7Validator(schema)
try:
for error in v.iter_errors(inst):
print(error.message)
return_code += 1
except Exception as e:
print("Validation failed. %s" % e)
return_code += 1
mname = inst['meta']['id'] + ".ksy"
if not mname == path.basename(filename):
print("meta.id must match filename %s %s" % (mname, filename))
return_code += 1
except yaml.composer.ComposerError:
pass
except Exception as e:
print("YAML parsing failed. %s" % e)
return_code += 1
return return_code
def test_missing_option(self):
with self.assertRaisesRegexp(
config.YamlLintConfigError,
'invalid config: missing option "max-spaces-before" '
'for rule "colons"'):
config.YamlLintConfig('rules:\n'
' colons:\n'
' max-spaces-after: 1\n')
def test_unknown_option(self):
with self.assertRaisesRegexp(
config.YamlLintConfigError,
'invalid config: unknown option "abcdef" for rule "colons"'):
config.YamlLintConfig('rules:\n'
' colons:\n'
' max-spaces-before: 0\n'
' max-spaces-after: 1\n'
' abcdef: yes\n')
def validate_markdown(args):
"""
Ensure the yaml encoded inside a YAML file is syntactically valid.
First attempt to strip the yaml from the .md file, second use the standard python yaml parser
to parse the embedded yaml. If it can't be passed then an error will be thrown and a stack
trace shown. In future we may try and catch this error and provide a user-friendly report).
In future we also perform additional structural validation on the yaml - check certain fields
are present etc. This could be done in various ways, e.g. jsonschema, programmatic checks. We
should also check translation -> jsonld -> rdf works as expected.
"""
def validate_structure(obj):
"""
Given an object, check to see if it has 'id', 'title', and 'layout' fields. If any are
missing, collect them in a list of errors which is then returned.
"""
errs = []
id = obj.get('id') or ''
if not id:
errs.append("No id: ")
if 'title' not in obj:
errs.append("No title: " + id)
if 'layout' not in obj:
errs.append("No layout tag: " + id +
" -- this is required for proper rendering")
# is_obsolete = ('is_obsolete' in obj)
# if 'description' not in obj:
# errs.append("No description: " + id + " " + ("OBS" if is_obsolete else ""))
return errs
errs = []
warn = []
for fn in args.files:
# we don't do anything with the results; an
# error is thrown
if not frontmatter.check(fn):
errs.append("%s does not contain frontmatter" % (fn))
yamltext = get_YAML_text(fn)
yaml_config = config.YamlLintConfig(file="util/config.yamllint")
for p in linter.run("---\n" + yamltext, yaml_config):
if p.level == "error":
errs.append(f"%s: {p}" % (fn))
elif p.level == "warning":
warn.append(f"%s: {p}" % (fn))
(obj, md) = load_md(fn)
errs += validate_structure(obj)
if len(warn) > 0:
print("WARNINGS:", file=sys.stderr)
for w in warn:
print("WARN: " + w, file=sys.stderr)
if len(errs) > 0:
print("FAILURES:", file=sys.stderr)
for e in errs:
print("ERROR:" + e, file=sys.stderr)
sys.exit(1)
def test_enable_disable_keywords(self):
c = config.YamlLintConfig('rules:\n'
' colons: enable\n'
' hyphens: disable\n')
self.assertEqual(c.rules['colons'], {
'level': 'error',
'max-spaces-after': 1,
'max-spaces-before': 0
})
self.assertEqual(c.rules['hyphens'], False)
def test_extend_config_override_whole_rule(self):
old = config.YamlLintConfig('extends: default')
new = config.YamlLintConfig('extends: default\n'
'rules:\n'
' empty-lines:\n'
' max: 42\n'
' max-start: 43\n'
' max-end: 44\n')
old.rules['empty-lines']['max'] = 42
old.rules['empty-lines']['max-start'] = 43
old.rules['empty-lines']['max-end'] = 44
self.assertEqual(sorted(new.rules.keys()), sorted(old.rules.keys()))
for rule in new.rules:
self.assertEqual(new.rules[rule], old.rules[rule])
self.assertEqual(new.rules['empty-lines']['max'], 42)
self.assertEqual(new.rules['empty-lines']['max-start'], 43)
self.assertEqual(new.rules['empty-lines']['max-end'], 44)
def test_parse_config(self):
new = config.YamlLintConfig('rules:\n'
' colons:\n'
' max-spaces-before: 0\n'
' max-spaces-after: 1\n')
self.assertEqual(list(new.rules.keys()), ['colons'])
self.assertEqual(new.rules['colons']['max-spaces-before'], 0)
self.assertEqual(new.rules['colons']['max-spaces-after'], 1)
self.assertEqual(len(new.enabled_rules()), 1)
def test_yes_no_for_booleans(self):
c = config.YamlLintConfig('rules:\n'
' indentation:\n'
' spaces: 2\n'
' indent-sequences: true\n'
' check-multi-line-strings: false\n')
self.assertTrue(c.rules['indentation']['indent-sequences'])
self.assertEqual(c.rules['indentation']['check-multi-line-strings'],
False)
c = config.YamlLintConfig('rules:\n'
' indentation:\n'
' spaces: 2\n'
' indent-sequences: yes\n'
' check-multi-line-strings: false\n')
self.assertTrue(c.rules['indentation']['indent-sequences'])
self.assertEqual(c.rules['indentation']['check-multi-line-strings'],
False)
c = config.YamlLintConfig('rules:\n'
' indentation:\n'
' spaces: 2\n'
' indent-sequences: whatever\n'
' check-multi-line-strings: false\n')
self.assertEqual(c.rules['indentation']['indent-sequences'],
'whatever')
self.assertEqual(c.rules['indentation']['check-multi-line-strings'],
False)
with self.assertRaisesRegexp(
config.YamlLintConfigError,
'invalid config: option "indent-sequences" of "indentation" '
'should be in '):
c = config.YamlLintConfig('rules:\n'
' indentation:\n'
' spaces: 2\n'
' indent-sequences: YES!\n'
' check-multi-line-strings: false\n')
def test_extend_recursive_default_values(self):
with tempfile.NamedTemporaryFile('w') as f:
f.write('rules:\n' ' braces:\n' ' max-spaces-inside: 1248\n')
f.flush()
c = config.YamlLintConfig('extends: ' + f.name + '\n'
'rules:\n'
' braces:\n'
' min-spaces-inside-empty: 2357\n')
self.assertEqual(c.rules['braces']['min-spaces-inside'], 0)
self.assertEqual(c.rules['braces']['max-spaces-inside'], 1248)
self.assertEqual(c.rules['braces']['min-spaces-inside-empty'], 2357)
self.assertEqual(c.rules['braces']['max-spaces-inside-empty'], -1)
with tempfile.NamedTemporaryFile('w') as f:
f.write('rules:\n' ' colons:\n' ' max-spaces-before: 1337\n')
f.flush()
c = config.YamlLintConfig('extends: ' + f.name + '\n'
'rules:\n'
' colons: enable\n')
self.assertEqual(c.rules['colons']['max-spaces-before'], 1337)
self.assertEqual(c.rules['colons']['max-spaces-after'], 1)
with tempfile.NamedTemporaryFile('w') as f1, \
tempfile.NamedTemporaryFile('w') as f2:
f1.write('rules:\n' ' colons:\n' ' max-spaces-before: 1337\n')
f1.flush()
f2.write('extends: ' + f1.name + '\n'
'rules:\n'
' colons: disable\n')
f2.flush()
c = config.YamlLintConfig('extends: ' + f2.name + '\n'
'rules:\n'
' colons: enable\n')
self.assertEqual(c.rules['colons']['max-spaces-before'], 0)
self.assertEqual(c.rules['colons']['max-spaces-after'], 1)
def _does_yaml_lint(self, yaml_file):
conf = utils.path_join(constants.DEFAULT_BASE_CONF,
'default_yaml_lint.yaml')
yaml_conf = config.YamlLintConfig(file=conf)
with open(yaml_file, 'r') as stream:
lint_problems = linter.run(stream, yaml_conf)
problems = 0
for problem in lint_problems:
LOG.warning(problem)
problems += 1
if problems > 0:
LOG.warning("%d problems detected with YAML (%s)" %
(problems, yaml_file))
return False
return True
def test_extend_remove_rule(self):
with tempfile.NamedTemporaryFile('w') as f:
f.write('rules:\n'
' colons:\n'
' max-spaces-before: 0\n'
' max-spaces-after: 1\n'
' hyphens:\n'
' max-spaces-after: 2\n')
f.flush()
c = config.YamlLintConfig('extends: ' + f.name + '\n'
'rules:\n'
' colons: disable\n')
self.assertEqual(sorted(c.rules.keys()), ['colons', 'hyphens'])
self.assertFalse(c.rules['colons'])
self.assertEqual(c.rules['hyphens']['max-spaces-after'], 2)
self.assertEqual(len(c.enabled_rules(None)), 1)
def test_unknown_rule(self):
with self.assertRaisesRegexp(
config.YamlLintConfigError,
'invalid config: no such rule: "this-one-does-not-exist"'):
config.YamlLintConfig('rules:\n' ' this-one-does-not-exist: {}\n')
def test_find_files_recursively(self):
conf = config.YamlLintConfig('extends: default')
self.assertEqual(
sorted(cli.find_files_recursively([self.wd], conf)),
[
os.path.join(self.wd, 'a.yaml'),
os.path.join(self.wd, 'dos.yml'),
os.path.join(self.wd, 'empty.yml'),
os.path.join(self.wd,
's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
os.path.join(self.wd, 'sub/ok.yaml'),
os.path.join(self.wd, 'warn.yaml')
],
)
items = [
os.path.join(self.wd, 'sub/ok.yaml'),
os.path.join(self.wd, 'empty-dir')
]
self.assertEqual(
sorted(cli.find_files_recursively(items, conf)),
[os.path.join(self.wd, 'sub/ok.yaml')],
)
items = [
os.path.join(self.wd, 'empty.yml'),
os.path.join(self.wd, 's')
]
self.assertEqual(
sorted(cli.find_files_recursively(items, conf)),
[
os.path.join(self.wd, 'empty.yml'),
os.path.join(self.wd,
's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml')
],
)
items = [
os.path.join(self.wd, 'sub'),
os.path.join(self.wd, '/etc/another/file')
]
self.assertEqual(
sorted(cli.find_files_recursively(items, conf)),
[
os.path.join(self.wd, '/etc/another/file'),
os.path.join(self.wd, 'sub/ok.yaml')
],
)
conf = config.YamlLintConfig('extends: default\n'
'yaml-files:\n'
' - \'*.yaml\' \n')
self.assertEqual(sorted(cli.find_files_recursively([self.wd], conf)), [
os.path.join(self.wd, 'a.yaml'),
os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
os.path.join(self.wd, 'sub/ok.yaml'),
os.path.join(self.wd, 'warn.yaml')
])
conf = config.YamlLintConfig('extends: default\n'
'yaml-files:\n'
' - \'*.yml\'\n')
self.assertEqual(sorted(cli.find_files_recursively([self.wd], conf)), [
os.path.join(self.wd, 'dos.yml'),
os.path.join(self.wd, 'empty.yml')
])
conf = config.YamlLintConfig('extends: default\n'
'yaml-files:\n'
' - \'*.json\'\n')
self.assertEqual(sorted(cli.find_files_recursively([self.wd], conf)),
[os.path.join(self.wd, 'no-yaml.json')])
conf = config.YamlLintConfig('extends: default\n'
'yaml-files:\n'
' - \'*\'\n')
self.assertEqual(sorted(cli.find_files_recursively([self.wd], conf)), [
os.path.join(self.wd, 'a.yaml'),
os.path.join(self.wd, 'dos.yml'),
os.path.join(self.wd, 'empty.yml'),
os.path.join(self.wd, 'no-yaml.json'),
os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8'),
os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
os.path.join(self.wd, 'sub/ok.yaml'),
os.path.join(self.wd, 'warn.yaml')
])
conf = config.YamlLintConfig('extends: default\n'
'yaml-files:\n'
' - \'*.yaml\'\n'
' - \'*\'\n'
' - \'**\'\n')
self.assertEqual(sorted(cli.find_files_recursively([self.wd], conf)), [
os.path.join(self.wd, 'a.yaml'),
os.path.join(self.wd, 'dos.yml'),
os.path.join(self.wd, 'empty.yml'),
os.path.join(self.wd, 'no-yaml.json'),
os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8'),
os.path.join(self.wd, 's/s/s/s/s/s/s/s/s/s/s/s/s/s/s/file.yaml'),
os.path.join(self.wd, 'sub/ok.yaml'),
os.path.join(self.wd, 'warn.yaml')
])
conf = config.YamlLintConfig('extends: default\n'
'yaml-files:\n'
' - \'s/**\'\n'
' - \'**/utf-8\'\n')
self.assertEqual(sorted(cli.find_files_recursively([self.wd], conf)),
[os.path.join(self.wd, 'non-ascii/éçäγλνπ¥/utf-8')])
def merge_files(context):
"""
Given a context containing path to template, env, and service:
merge config into template and output the result to stdout
Args:
context: a populated context object
"""
resolver = EFTemplateResolver(profile=context.profile,
region=context.region,
env=context.env,
service=context.service)
try:
with open(context.template_path, 'r') as f:
template_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading template file: {} {}".format(
context.template_path, repr(error)))
if context.no_params is False:
try:
with open(context.param_path, 'r') as f:
param_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading param file: {} {}".format(
context.param_path, repr(error)))
dest = yaml.safe_load(param_body)["dest"]
# if 'dest' for the current object contains an 'environments' list, check it
if "environments" in dest:
if not resolver.resolved["ENV_SHORT"] in dest["environments"]:
print("Environment: {} not enabled for {}".format(
resolver.resolved["ENV_SHORT"], context.template_path))
return
# Process the template_body - apply context + parameters
resolver.load(template_body, param_body)
else:
resolver.load(template_body)
rendered_body = resolver.render()
if not resolver.resolved_ok():
raise RuntimeError(
"Couldn't resolve all symbols; template has leftover {{ or }}: {}".
format(resolver.unresolved_symbols()))
if context.lint:
if context.template_path.endswith(".json"):
try:
json.loads(rendered_body, strict=False)
print("JSON passed linting process.")
except ValueError as e:
fail("JSON failed linting process.", e)
elif context.template_path.endswith((".yml", ".yaml")):
conf = yamllint_config.YamlLintConfig(content='extends: relaxed')
lint_output = yamllinter.run(rendered_body, conf)
lint_level = 'error'
lint_errors = [
issue for issue in lint_output if issue.level == lint_level
]
if lint_errors:
split_body = rendered_body.splitlines()
for error in lint_errors:
print(error)
# printing line - 1 because lists start at 0, but files at 1
print("\t", split_body[error.line - 1])
fail("YAML failed linting process.")
if context.verbose:
print(context)
if context.no_params:
print('no_params flag set to true!')
print(
'Inline template resolution based on external symbol lookup only and no destination for file write.\n'
)
else:
dir_path = normpath(dirname(dest["path"]))
print("make directories: {} {}".format(dir_path, dest["dir_perm"]))
print("chmod file to: " + dest["file_perm"])
user, group = dest["user_group"].split(":")
print("chown last directory in path to user: {}, group: {}".format(
user, group))
print("chown file to user: {}, group: {}\n".format(user, group))
print("template body:\n{}\nrendered body:\n{}\n".format(
template_body, rendered_body))
elif context.silent:
print("Config template rendered successfully.")
else:
print(rendered_body)
def test_invalid_conf(self):
with self.assertRaises(config.YamlLintConfigError):
config.YamlLintConfig('not: valid: yaml')