def check(self, paths): # type: (t.List[str]) -> None
"""Check the specified paths."""
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'config')
yaml_conf = YamlLintConfig(
file=os.path.join(config_path, 'default.yml'))
module_conf = YamlLintConfig(
file=os.path.join(config_path, 'modules.yml'))
plugin_conf = YamlLintConfig(
file=os.path.join(config_path, 'plugins.yml'))
for path in paths:
extension = os.path.splitext(path)[1]
with open(path, encoding='utf-8') as file:
contents = file.read()
if extension in ('.yml', '.yaml'):
self.check_yaml(yaml_conf, path, contents)
elif extension == '.py':
if path.startswith('lib/ansible/modules/') or path.startswith(
'plugins/modules/'):
conf = module_conf
else:
conf = plugin_conf
self.check_module(conf, path, contents)
else:
raise Exception('unsupported extension: %s' % extension)
def check(self, paths):
"""
:type paths: str
"""
config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'config')
yaml_conf = YamlLintConfig(
file=os.path.join(config_path, 'default.yml'))
module_conf = YamlLintConfig(
file=os.path.join(config_path, 'modules.yml'))
plugin_conf = YamlLintConfig(
file=os.path.join(config_path, 'plugins.yml'))
for path in paths:
extension = os.path.splitext(path)[1]
with open(path) as f:
contents = f.read()
if extension in ('.yml', '.yaml'):
self.check_yaml(yaml_conf, path, contents)
elif extension == '.py':
if path.startswith('lib/ansible/modules/') or path.startswith(
'plugins/modules/'):
conf = module_conf
else:
conf = plugin_conf
self.check_module(conf, path, contents)
else:
raise Exception('unsupported extension: %s' % extension)
def check(self, paths):
"""
:type paths: str
"""
yaml_conf = YamlLintConfig(
file='test/sanity/yamllint/config/default.yml')
module_conf = YamlLintConfig(
file='test/sanity/yamllint/config/modules.yml')
plugin_conf = YamlLintConfig(
file='test/sanity/yamllint/config/plugins.yml')
for path in paths:
extension = os.path.splitext(path)[1]
with open(path) as f:
contents = f.read()
if extension in ('.yml', '.yaml'):
self.check_yaml(yaml_conf, path, contents)
elif extension == '.py':
if path.startswith('lib/ansible/plugins/'):
conf = plugin_conf
else:
conf = module_conf
self.check_module(conf, path, contents)
else:
raise Exception('unsupported extension: %s' % extension)
class YamllintRule(AnsibleLintRule):
id = 'yaml'
shortdesc = 'Violations reported by yamllint'
description = DESCRIPTION
severity = 'VERY_LOW'
tags = ['formatting', 'yaml']
version_added = 'v5.0.0'
config = None
has_dynamic_tags = True
if "yamllint.config" in sys.modules:
config = YamlLintConfig(content=YAMLLINT_CONFIG)
# if we detect local yamllint config we use it but raise a warning
# as this is likely to get out of sync with our internal config.
for file in ['.yamllint', '.yamllint.yaml', '.yamllint.yml']:
if os.path.isfile(file):
_logger.warning(
"Loading custom %s config file, this extends our "
"internal yamllint config.",
file,
)
config_override = YamlLintConfig(file=file)
config_override.extend(config)
break
_logger.debug("Effective yamllint rules used: %s", config.rules)
def __init__(self) -> None:
"""Construct a rule instance."""
# customize id by adding the one reported by yamllint
self.id = self.__class__.id
def matchyaml(self, file: Lintable) -> List["MatchError"]:
"""Return matches found for a specific YAML text."""
matches: List["MatchError"] = []
filtered_matches = []
if file.kind == 'role':
return matches
if YamllintRule.config:
for p in run_yamllint(file.content, YamllintRule.config):
matches.append(
self.create_matcherror(
message=p.desc,
linenumber=p.line,
details="",
filename=str(file.path),
tag=p.rule,
)
)
if matches:
lines = file.content.splitlines()
for match in matches:
# rule.linenumber starts with 1, not zero
skip_list = get_rule_skips_from_line(lines[match.linenumber - 1])
# print(skip_list)
if match.rule.id not in skip_list and match.tag not in skip_list:
filtered_matches.append(match)
return filtered_matches
def test_disabled_with_changes(self):
config_content = '{"extends": "default", "rules": {"key-ordering": "disable"}}'
expected = """---
key 3:
something: 456
else: 45
key 2: v
key 1: val
lst:
- {first_name: something, age: 42, last_name: else}
- {age: 41, first_name: already, last_name: in_order}
"""
content = """---
key 3:
something: 456
else: 45
key 2: v
key 1: val
lst:
- {first_name: something, age: 42, last_name: else}
- {age: 41, first_name: already, last_name: in_order}
"""
output = read_and_format_text(content, YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_dos_lines_in_the_middle_with_comments(self):
config_content = (
'{"extends": "default", '
'"rules": {"empty-lines": {"max": 5}, "new-lines": {"type": "dos"}}}'
)
expected = """---\r
test:\r
key: value\r
\r
\r
# some test\r
\r
# some test\r
\r
lst:\r
- item1\r
"""
content = """---\r
test:\r
key: value\r
\r
\r
# some test\r
\r
# some test\r
\r
lst:\r
- item1\r
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def lint_yaml_files(input_filepaths, file=sys.stderr):
"""
Performs glinter YAML lint on a set of files.
:param input_filepaths: List of input files to lint.
:param file: The stream to write errors to.
:returns: List of nits.
"""
nits = []
for path in input_filepaths:
# yamllint needs both the file content and the path.
file_content = None
with path.open("r") as fd:
file_content = fd.read()
problems = linter.run(file_content, YamlLintConfig("extends: default"), path)
nits.extend(p for p in problems)
if len(nits):
print("Sorry, Glean found some glinter nits:", file=file)
for p in nits:
print("{} ({}:{}) - {}".format(path, p.line, p.column, p.message))
print("", file=file)
print("Please fix the above nits to continue.", file=file)
return nits
def build_fake_config(self, conf):
if conf is None:
conf = {}
else:
conf = yaml.safe_load(conf)
conf = {'extends': 'default', 'rules': conf}
return YamlLintConfig(yaml.safe_dump(conf))
def run(self):
''' run command '''
if self.excludes is not None:
print("Excludes:\n{0}".format(
yaml.dump(self.excludes, default_flow_style=False)))
config = YamlLintConfig(file=self.config_file)
has_errors = False
has_warnings = False
if self.format == 'parsable':
format_method = Format.parsable
else:
format_method = Format.standard_color
for yaml_file in find_files(os.getcwd(), self.excludes, None,
r'^[^\.].*\.ya?ml$'):
first = True
with open(yaml_file, 'r') as contents:
for problem in linter.run(contents, config):
if first and self.format != 'parsable':
print('\n{0}:'.format(os.path.relpath(yaml_file)))
first = False
print(format_method(problem, yaml_file))
if problem.level == linter.PROBLEM_LEVELS[2]:
has_errors = True
elif problem.level == linter.PROBLEM_LEVELS[1]:
has_warnings = True
if has_errors or has_warnings:
print('yamllint issues found')
raise SystemExit(1)
def test_disabled_present(self):
config_content = '{"extends": "default", "rules": {"empty-lines": "disable"}}'
expected = """
---
test:
key: value
lst:
- item1
"""
content = """
---
test:
key: value
lst:
- item1
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_ignore_file(self):
config = YamlLintConfig('{"extends": "default", "ignore": "something.yml"}')
self.assertTrue(is_matching_path("something.yaml", yaml_config=config))
self.assertFalse(is_matching_path("folder/something.yml", yaml_config=config))
self.assertTrue(is_matching_path(".yamllint", yaml_config=config))
self.assertTrue(is_matching_path(".somefolder/test.yaml", yaml_config=config))
self.assertFalse(is_matching_path("ghdfshs.fadsg", yaml_config=config))
def test_any_quote_if_needed(self):
config_content = "{extends: default, rules: {quoted-strings: {quote-type: any, required: only-when-needed}}}"
expected = """---
test42: {something469: value}
test43:
hey879: someothervalue
hey880: somevalue
required: "{this: needsquotes}"
lst:
- someitem
- someotheritem
- somefinalitem
- item4
"""
content = """---
test42: {something469: 'value'}
test43:
hey879: "someothervalue"
hey880: somevalue
required: "{this: needsquotes}"
lst:
- 'someitem'
- someotheritem
- "somefinalitem"
- item4
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_disabled(self):
config_content = '{extends: default, rules: {quoted-strings: "disable"}}'
expected = """---
test42: {something469: 'value'}
test43:
hey879: "someothervalue"
hey880: somevalue
lst:
- 'someitem'
- someotheritem
- "somefinalitem"
- item4
"""
content = """---
test42: {something469: 'value'}
test43:
hey879: "someothervalue"
hey880: somevalue
lst:
- 'someitem'
- someotheritem
- "somefinalitem"
- item4
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def lint_yaml_files(input_filepaths: Iterable[Path], file=sys.stderr) -> List:
"""
Performs glinter YAML lint on a set of files.
:param input_filepaths: List of input files to lint.
:param file: The stream to write errors to.
:returns: List of nits.
"""
# Generic type since the actual type comes from yamllint, which we don't
# control.
nits: List = []
for path in input_filepaths:
# yamllint needs both the file content and the path.
file_content = None
with path.open("r", encoding="utf-8") as fd:
file_content = fd.read()
problems = linter.run(file_content, YamlLintConfig("extends: default"),
path)
nits.extend((path, p) for p in problems)
if len(nits):
print("Sorry, Glean found some glinter nits:", file=file)
for (path, p) in nits:
print(f"{path} ({p.line}:{p.column}) - {p.message}")
print("", file=file)
print("Please fix the above nits to continue.", file=file)
return [x[1] for x in nits]
def main():
yaml_path = os.environ["INPUT_PATH"]
strict = os.environ["INPUT_STRICT"] == "true"
conf = YamlLintConfig("extends: default")
warning_count = 0
with open(yaml_path) as f:
problems = linter.run(f, conf, yaml_path)
for problem in problems:
if problem.level == "warning" and strict:
problem.level = "error"
print(
f"::{problem.level} file={yaml_path},line={problem.line},"
f"col={problem.column}::{problem.desc} ({problem.rule})"
)
if problem.level == "warning":
warning_count = warning_count + 1
if problem.level == "error":
sys.exit(1)
print(f"::set-output name=warnings::{warning_count}")
sys.exit(0)
def test_allowed(self):
config_content = (
'{"extends": "default",'
'"rules": {"empty-values": {"forbid-in-block-mappings": false, "forbid-in-flow-mappings": false}}}'
)
expected = """---
test42: {something469: }
test43:
hey879:
hey880:
lst: [key: ]
lst2:
- key2:
"""
content = """---
test42: {something469: }
test43:
hey879:
hey880: null
lst: [key: null]
lst2:
- key2: null
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_default_enabled(self):
config_content = '{"extends": "default", "rules": {"empty-values": "enable"}}'
expected = """---
test42: {something469: null}
test43:
hey879: null
hey880: null
lst: [key: null]
lst2:
- key2: null
"""
content = """---
test42: {something469: }
test43:
hey879:
hey880: null
lst: [key: null]
lst2:
- key2: null
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_double_quote(self):
config_content = (
"{extends: default,"
" rules: {truthy: enable, quoted-strings: {quote-type: double, required: false}}}"
)
expected = """---
test42: {something469: true}
test43:
noquote: TRuE
hey879: "TRuE"
"off": "FALSE"
"""
content = """---
test42: {something469: true}
test43:
noquote: TRuE
hey879: 'TRuE'
off: FALSE
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_any_quote_extra_required(self):
config_content = (
"{extends: default,"
" rules: {quoted-strings: {quote-type: any, required: false, extra-required: [some]}}}"
)
expected = """---
test42: {something469: 'value'}
test43:
hey879: "someothervalue"
hey880: "somevalue"
required: "{this: needsquotes}"
lst:
- 'someitem'
- "someotheritem"
- "somefinalitem"
- item4
"""
content = """---
test42: {something469: 'value'}
test43:
hey879: "someothervalue"
hey880: somevalue
required: "{this: needsquotes}"
lst:
- 'someitem'
- someotheritem
- "somefinalitem"
- item4
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_enabled(self):
config_content = '{extends: default, rules: {colons: "enable"}}'
expected = """---
test42: {something469: true}
test43:
'with_colon:': TRuE
hey879: 'TRuE'
"off": "FALSE"
lst:
- somelist
- something_else
"""
content = """---
test42 : {something469 : true}
test43 :
'with_colon:' : TRuE
hey879 : 'TRuE'
"off" : "FALSE"
lst:
- somelist
- something_else
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_double_quote_required(self):
config_content = "{extends: default, rules: {quoted-strings: {quote-type: double, required: true}}}"
expected = """---
test42: {something469: "value"}
test43:
hey879: "someothervalue"
hey880: "somevalue"
lst:
- "someitem"
- "someotheritem"
- "somefinalitem"
- "item4"
"""
content = """---
test42: {something469: 'value'}
test43:
hey879: "someothervalue"
hey880: somevalue
lst:
- 'someitem'
- someotheritem
- "somefinalitem"
- item4
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_disabled_dos_newline(self):
config_content = (
"{extends: default, rules: {colons: disable, new-lines: {type: dos}}}"
)
expected = """---\r
test42 : {something469 : true}\r
test43 :\r
'with_colon:' : TRuE\r
hey879 : 'TRuE'\r
"off" : "FALSE"\r
lst:\r
- somelist\r
- something_else\r
"""
content = """---
test42 : {something469 : true}
test43 :
'with_colon:' : TRuE
hey879 : 'TRuE'
"off" : "FALSE"
lst:
- somelist
- something_else
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_max_space_after_is_3(self):
config_content = "{extends: default, rules: {colons: {max-spaces-after: 3}}}"
expected = """---
test42: {something469: true}
test43:
'with_colon:': TRuE
hey879: 'TRuE'
"off": "FALSE"
lst:
- somelist
- something_else
"""
content = """---
test42 : {something469 : true}
test43 :
'with_colon:' : TRuE
hey879 : 'TRuE'
"off" : "FALSE"
lst:
- somelist
- something_else
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_default_no_seq_indent(self):
config_content = ('{"extends": "default", "rules": {'
'"comments-indentation": "enable",'
'"indentation": {"indent-sequences": false}'
"}}")
expected = """---
##############################
## Some text
# Fix me
test: # First key
# thing
key: value
# valid
lst:
- item1
# here too
"""
content = """---
##############################
## Some text
# Fix me
test: # First key
# thing
key: value
# valid
lst:
- item1
# here too
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def test_yaml_files_and_ignore(self):
config = YamlLintConfig('{"yaml-files": ["*.yaml"], "ignore": "test.yaml"}')
self.assertTrue(is_matching_path("something.yaml", yaml_config=config))
self.assertFalse(is_matching_path("folder/something.yml", yaml_config=config))
self.assertFalse(is_matching_path(".yamllint", yaml_config=config))
self.assertFalse(is_matching_path(".somefolder/test.yaml", yaml_config=config))
self.assertFalse(is_matching_path("ghdfshs.fadsg", yaml_config=config))
def test_default_enabled_present(self):
config_content = (
'{"extends": "default", "rules": {"comments-indentation": "enable"}}'
)
expected = """---
##############################
## Some text
# Fix me
test: # First key
# thing
key: value
# valid
lst:
- item1
"""
content = """---
##############################
## Some text
# Fix me
test: # First key
# thing
key: value
# valid
lst:
- item1
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
def read_and_format_text(
text: str,
config: Optional[YamlLintConfig] = YamlLintConfig("extends: default")
) -> str:
"""
Using getlocale and setlocale makes this function thread-unsafe
"""
default_locale = getlocale(LC_COLLATE)
if config and config.locale:
setlocale(LC_COLLATE, config.locale)
dumping_config = {}
loading_config = {}
rules = config.rules if config else {}
for rule_id, rule in RULES.items():
result = rule.apply_before_load(text, rules.get(rule_id), rules)
text = result.text
dumping_config.update(result.dumping_config)
loading_config.update(result.loading_config)
data = load(text, Loader, **loading_config)
for rule_id, rule in RULES.items():
data = rule.apply_before_dump(data, rules.get(rule_id), text, rules)
result = dump(data, Dumper=RoundTripDumper, **dumping_config)
for rule_id, rule in RULES.items():
result = rule.apply_on_result(result, text, rules.get(rule_id), rules)
if config and config.locale:
setlocale(LC_COLLATE, default_locale)
return result
def yaml_checker(path):
"""Read all yaml files in the directory and verify structure.
:param str path: Path to test cases.
:return: Number of errors.
:rtype: int
"""
path_for_check = Path(path)
if path_for_check.is_dir():
file_list = chain(path_for_check.glob('**/*.yaml'),
path_for_check.glob('**/*.yml'))
else:
file_list = [path_for_check]
error_count = 0
for file_name in file_list:
with open(str(file_name)) as file:
text = file.read()
conf = YamlLintConfig(
'document-start:\n'
' present: false\n'
'rules:\n'
' line-length:\n'
' max: 250\n'
' allow-non-breakable-words: false\n'
' allow-non-breakable-inline-mappings: true\n')
for err in linter.run(text, conf):
print(Format.parsable(err, file_name))
error_count += 1
try:
test_case = yaml.load(text)
except Exception:
error_count += 1
print('{}: Error load YAML with PyYAML library'.format(file_name))
else:
error_count += check_section(test_case,
'Description',
file_name,
is_markdown=True)
error_count += check_section(test_case,
'Requirements',
file_name,
section_type=list)
if not check_section(
test_case, 'Steps', file_name, section_type=list):
for step_count, step in enumerate(test_case['Steps'], 1):
error_count += check_section(step,
'Description',
file_name,
step_count,
is_markdown=True)
error_count += check_section(step,
'Expected result',
file_name,
step_count,
is_markdown=True)
else:
error_count += 1
print('\n\t{} errors were found'.format(error_count))
return error_count
def validate_yaml(yaml_in, yaml_fn):
"""Check with yamllint the yaml syntaxes
Looking for duplicate keys."""
try:
import yamllint.linter as linter
from yamllint.config import YamlLintConfig
except ImportError:
return
conf = """{"extends": "relaxed",
"rules": {"trailing-spaces": {"level": "warning"},
"new-lines": {"level": "warning"},
"new-line-at-end-of-file": {"level": "warning"}}}"""
if utils.file_exists(yaml_in):
with open(yaml_in) as in_handle:
yaml_in = in_handle.read()
out = linter.run(yaml_in, YamlLintConfig(conf))
for problem in out:
msg = '%(fn)s:%(line)s:%(col)s: [%(level)s] %(msg)s' % {
'fn': yaml_fn,
'line': problem.line,
'col': problem.column,
'level': problem.level,
'msg': problem.message
}
if problem.level == "error":
raise ValueError(msg)
def test_dos_max_start_end(self):
config_content = (
'{"extends": "default", '
'"rules": {"empty-lines": {"max-start": 2, "max-end": 1}, "new-lines": {"type": "dos"}}}'
)
expected = """\r
\r
---\r
test:\r
key: value\r
lst:\r
- item1\r
\r
"""
content = """\r
\r
\r
\r
---\r
test:\r
key: value\r
lst:\r
- item1\r
\r
\r
\r
\r
\r
"""
output = read_and_format_text(content,
YamlLintConfig(content=config_content))
self.assertEqual(expected, output)
