def __init__(self, args):
"""Initialize snapshot of environment configuration.
:type args: IntegrationConfig
"""
self.args = args
if self.args.explain:
self.data = {}
return
versions = ['']
versions += SUPPORTED_PYTHON_VERSIONS
versions += list(set(v.split('.')[0] for v in SUPPORTED_PYTHON_VERSIONS))
python_paths = dict((v, find_executable('python%s' % v, required=False)) for v in sorted(versions))
python_versions = dict((v, self.get_version([python_paths[v], '-V'])) for v in sorted(python_paths) if python_paths[v])
pip_paths = dict((v, find_executable('pip%s' % v, required=False)) for v in sorted(versions))
pip_versions = dict((v, self.get_version([pip_paths[v], '--version'])) for v in sorted(pip_paths) if pip_paths[v])
pip_interpreters = dict((v, self.get_shebang(pip_paths[v])) for v in sorted(pip_paths) if pip_paths[v])
known_hosts_hash = self.get_hash(os.path.expanduser('~/.ssh/known_hosts'))
self.data = dict(
python_paths=python_paths,
python_versions=python_versions,
pip_paths=pip_paths,
pip_versions=pip_versions,
pip_interpreters=pip_interpreters,
known_hosts_hash=known_hosts_hash,
)
def __init__(self):
"""Initialize snapshot of environment configuration."""
versions = ['']
versions += SUPPORTED_PYTHON_VERSIONS
versions += list(
set(v.split('.')[0] for v in SUPPORTED_PYTHON_VERSIONS))
python_paths = dict(
(v, find_executable('python%s' % v, required=False))
for v in sorted(versions))
python_versions = dict((v, self.get_version([python_paths[v], '-V']))
for v in sorted(python_paths)
if python_paths[v])
pip_paths = dict((v, find_executable('pip%s' % v, required=False))
for v in sorted(versions))
pip_versions = dict((v, self.get_version([pip_paths[v], '--version']))
for v in sorted(pip_paths) if pip_paths[v])
pip_interpreters = dict((v, self.get_shebang(pip_paths[v]))
for v in sorted(pip_paths) if pip_paths[v])
self.data = dict(
python_paths=python_paths,
python_versions=python_versions,
pip_paths=pip_paths,
pip_versions=pip_versions,
pip_interpreters=pip_interpreters,
)
def __init__(self, args):
"""Initialize snapshot of environment configuration.
:type args: IntegrationConfig
"""
self.args = args
if self.args.explain:
self.data = {}
return
versions = ['']
versions += SUPPORTED_PYTHON_VERSIONS
versions += list(set(v.split('.')[0] for v in SUPPORTED_PYTHON_VERSIONS))
python_paths = dict((v, find_executable('python%s' % v, required=False)) for v in sorted(versions))
python_versions = dict((v, self.get_version([python_paths[v], '-V'])) for v in sorted(python_paths) if python_paths[v])
pip_paths = dict((v, find_executable('pip%s' % v, required=False)) for v in sorted(versions))
pip_versions = dict((v, self.get_version([pip_paths[v], '--version'])) for v in sorted(pip_paths) if pip_paths[v])
pip_interpreters = dict((v, self.get_shebang(pip_paths[v])) for v in sorted(pip_paths) if pip_paths[v])
known_hosts_hash = self.get_hash(os.path.expanduser('~/.ssh/known_hosts'))
self.data = dict(
python_paths=python_paths,
python_versions=python_versions,
pip_paths=pip_paths,
pip_versions=pip_versions,
pip_interpreters=pip_interpreters,
known_hosts_hash=known_hosts_hash,
)
def inject_httptester(args):
"""
:type args: CommonConfig
"""
comment = ' # ansible-test httptester\n'
append_lines = ['127.0.0.1 %s%s' % (host, comment) for host in HTTPTESTER_HOSTS]
with open('/etc/hosts', 'r+') as hosts_fd:
original_lines = hosts_fd.readlines()
if not any(line.endswith(comment) for line in original_lines):
hosts_fd.writelines(append_lines)
# determine which forwarding mechanism to use
pfctl = find_executable('pfctl', required=False)
iptables = find_executable('iptables', required=False)
if pfctl:
kldload = find_executable('kldload', required=False)
if kldload:
try:
run_command(args, ['kldload', 'pf'], capture=True)
except SubprocessError:
pass # already loaded
rules = '''
rdr pass inet proto tcp from any to any port 80 -> 127.0.0.1 port 8080
rdr pass inet proto tcp from any to any port 443 -> 127.0.0.1 port 8443
'''
cmd = ['pfctl', '-ef', '-']
try:
run_command(args, cmd, capture=True, data=rules)
except SubprocessError:
pass # non-zero exit status on success
elif iptables:
ports = [
(80, 8080),
(443, 8443),
]
for src, dst in ports:
rule = ['-o', 'lo', '-p', 'tcp', '--dport', str(src), '-j', 'REDIRECT', '--to-port', str(dst)]
try:
# check for existing rule
cmd = ['iptables', '-t', 'nat', '-C', 'OUTPUT'] + rule
run_command(args, cmd, capture=True)
except SubprocessError:
# append rule when it does not exist
cmd = ['iptables', '-t', 'nat', '-A', 'OUTPUT'] + rule
run_command(args, cmd, capture=True)
else:
raise ApplicationError('No supported port forwarding mechanism detected.')
def get_docker_details(args):
"""
:type args: CommonConfig
:rtype: dict[str, any]
"""
docker = find_executable('docker', required=False)
info = None
version = None
if docker:
try:
info = docker_info(args)
except SubprocessError as ex:
display.warning('Failed to collect docker info:\n%s' % ex)
try:
version = docker_version(args)
except SubprocessError as ex:
display.warning('Failed to collect docker version:\n%s' % ex)
docker_details = dict(
executable=docker,
info=info,
version=version,
)
return docker_details
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
if self.vmware_test_platform is None or 'govcsim':
docker = find_executable('docker', required=False)
if docker:
return
skip = 'cloud/%s/' % self.platform
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which require the "docker" command: %s'
% (skip.rstrip('/'), ', '.join(skipped)))
else:
if os.path.isfile(self.config_static_path):
return
aci = self._create_ansible_core_ci()
if os.path.isfile(aci.ci_key):
return
if is_shippable():
return
super(VcenterProvider, self).filter(targets, exclude)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: SanityResult
"""
paths = sorted(i.path for i in targets.include
if os.path.splitext(i.path)[1] in ('.yml', '.yaml'))
if not paths:
return SanitySkipped(self.name)
cmd = [
'python%s' % args.python_version,
find_executable('yamllint'),
'--format',
'parsable',
] + paths
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
raise SubprocessError(cmd=cmd,
status=status,
stderr=stderr,
stdout=stdout)
if args.explain:
return SanitySuccess(self.name)
pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): \[(?P<level>warning|error)\] (?P<message>.*)$'
results = [
re.search(pattern, line).groupdict()
for line in stdout.splitlines()
]
results = [
SanityMessage(
message=r['message'],
path=r['path'],
line=int(r['line']),
column=int(r['column']),
level=r['level'],
) for r in results
]
if results:
return SanityFailure(self.name, messages=results)
return SanitySuccess(self.name)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: SanityResult
"""
with open('test/sanity/rstcheck/ignore-substitutions.txt', 'r') as ignore_fd:
ignore_substitutions = sorted(set(ignore_fd.read().splitlines()))
paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',))
if not paths:
return SanitySkipped(self.name)
cmd = [
'python%s' % args.python_version,
find_executable('rstcheck'),
'--report', 'warning',
'--ignore-substitutions', ','.join(ignore_substitutions),
] + paths
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stdout:
raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
if args.explain:
return SanitySuccess(self.name)
pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$'
results = [parse_to_dict(pattern, line) for line in stderr.splitlines()]
results = [SanityMessage(
message=r['message'],
path=r['path'],
line=int(r['line']),
column=0,
level=r['level'],
) for r in results]
if results:
return SanityFailure(self.name, messages=results)
return SanitySuccess(self.name)
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
if os.path.isfile(self.config_static_path):
return
docker = find_executable('docker')
if docker:
return
super(CsCloudProvider, self).filter(targets, exclude)
def intercept_command(args,
cmd,
target_name,
capture=False,
env=None,
data=None,
cwd=None,
python_version=None,
path=None):
"""
:type args: TestConfig
:type cmd: collections.Iterable[str]
:type target_name: str
:type capture: bool
:type env: dict[str, str] | None
:type data: str | None
:type cwd: str | None
:type python_version: str | None
:type path: str | None
:rtype: str | None, str | None
"""
if not env:
env = common_environment()
cmd = list(cmd)
inject_path = get_coverage_path(args)
config_path = os.path.join(inject_path, 'injector.json')
version = python_version or args.python_version
interpreter = find_executable('python%s' % version, path=path)
coverage_file = os.path.abspath(
os.path.join(
inject_path, '..', 'output', '%s=%s=%s=%s=coverage' %
(args.command, target_name, args.coverage_label
or 'local-%s' % version, 'python-%s' % version)))
env['PATH'] = inject_path + os.pathsep + env['PATH']
env['ANSIBLE_TEST_PYTHON_VERSION'] = version
env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter
config = dict(
python_interpreter=interpreter,
coverage_file=coverage_file if args.coverage else None,
)
if not args.explain:
with open(config_path, 'w') as config_fd:
json.dump(config, config_fd, indent=4, sort_keys=True)
return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd)
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
docker = find_executable('docker', required=False)
if docker:
return
skip = 'cloud/%s/' % self.platform
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which require the "docker" command: %s'
% (skip.rstrip('/'), ', '.join(skipped)))
def filter(self, targets, exclude):
"""Filter out the cloud tests when the necessary config and resources are not available.
:type targets: tuple[TestTarget]
:type exclude: list[str]
"""
docker = find_executable('docker', required=False)
if docker:
return
skip = 'cloud/%s/' % self.platform
skipped = [target.name for target in targets if skip in target.aliases]
if skipped:
exclude.append(skip)
display.warning('Excluding tests marked "%s" which require the "docker" command: %s'
% (skip.rstrip('/'), ', '.join(skipped)))
def pylint(self, args, context, paths):
"""
:type args: SanityConfig
:param context: str
:param paths: list[str]
:return: list[dict[str, str]]
"""
rcfile = 'test/sanity/pylint/config/%s' % context
if not os.path.exists(rcfile):
rcfile = 'test/sanity/pylint/config/default'
cmd = [
'python%s' % args.python_version,
find_executable('pylint'),
'--jobs', '0',
'--reports', 'n',
'--max-line-length', '160',
'--rcfile', rcfile,
'--output-format', 'json',
] + paths
env = ansible_environment(args)
if paths:
try:
stdout, stderr = run_command(args, cmd, env=env, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr or status >= 32:
raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
else:
stdout = None
if not args.explain and stdout:
messages = json.loads(stdout)
else:
messages = []
return messages
def intercept_command(args, cmd, target_name, capture=False, env=None, data=None, cwd=None, python_version=None, path=None):
"""
:type args: TestConfig
:type cmd: collections.Iterable[str]
:type target_name: str
:type capture: bool
:type env: dict[str, str] | None
:type data: str | None
:type cwd: str | None
:type python_version: str | None
:type path: str | None
:rtype: str | None, str | None
"""
if not env:
env = common_environment()
cmd = list(cmd)
inject_path = get_coverage_path(args)
config_path = os.path.join(inject_path, 'injector.json')
version = python_version or args.python_version
interpreter = find_executable('python%s' % version, path=path)
coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % (
args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)))
env['PATH'] = inject_path + os.pathsep + env['PATH']
env['ANSIBLE_TEST_PYTHON_VERSION'] = version
env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter
config = dict(
python_interpreter=interpreter,
coverage_file=coverage_file if args.coverage else None,
)
if not args.explain:
with open(config_path, 'w') as config_fd:
json.dump(config, config_fd, indent=4, sort_keys=True)
return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
skip_paths = read_lines_without_comments(PSLINT_SKIP_PATH)
invalid_ignores = []
ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH)
ignore = collections.defaultdict(dict)
line = 0
for ignore_entry in ignore_entries:
line += 1
if not ignore_entry:
continue
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
path, code = ignore_entry.split(' ', 1)
if not os.path.exists(path):
invalid_ignores.append(
(line, 'Remove "%s" since it does not exist' % path))
continue
ignore[path][code] = line
paths = sorted(
i.path for i in targets.include if os.path.splitext(i.path)[1] in (
'.ps1', '.psm1', '.psd1') and i.path not in skip_paths)
if not paths:
return SanitySkipped(self.name)
if not find_executable('pwsh', required='warning'):
return SanitySkipped(self.name)
# Make sure requirements are installed before running sanity checks
cmds = [['test/runner/requirements/sanity.ps1'],
['test/sanity/pslint/pslint.ps1'] + paths]
for cmd in cmds:
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
raise SubprocessError(cmd=cmd,
status=status,
stderr=stderr,
stdout=stdout)
if args.explain:
return SanitySuccess(self.name)
severity = [
'Information',
'Warning',
'Error',
'ParseError',
]
cwd = os.getcwd() + '/'
# replace unicode smart quotes and ellipsis with ascii versions
stdout = re.sub(u'[\u2018\u2019]', "'", stdout)
stdout = re.sub(u'[\u201c\u201d]', '"', stdout)
stdout = re.sub(u'[\u2026]', '...', stdout)
messages = json.loads(stdout)
errors = [
SanityMessage(
code=m['RuleName'],
message=m['Message'],
path=m['ScriptPath'].replace(cwd, ''),
line=m['Line'] or 0,
column=m['Column'] or 0,
level=severity[m['Severity']],
) for m in messages
]
line = 0
filtered = []
for error in errors:
if error.code in ignore[error.path]:
ignore[error.path][
error.
code] = None # error ignored, clear line number of ignore entry to track usage
else:
filtered.append(error) # error not ignored
errors = filtered
for invalid_ignore in invalid_ignores:
errors.append(
SanityMessage(
code='A201',
message=invalid_ignore[1],
path=PSLINT_IGNORE_PATH,
line=invalid_ignore[0],
column=1,
confidence=calculate_confidence(PSLINT_IGNORE_PATH, line,
args.metadata)
if args.metadata.changes else None,
))
for path in skip_paths:
line += 1
if not path:
continue
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(
SanityMessage(
code='A101',
message='Remove "%s" since it does not exist' % path,
path=PSLINT_SKIP_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(
((PSLINT_SKIP_PATH, line), (path, 0)),
args.metadata) if args.metadata.changes else None,
))
for path in paths:
if path not in ignore:
continue
for code in ignore[path]:
line = ignore[path][code]
if not line:
continue
errors.append(
SanityMessage(
code='A102',
message='Remove since "%s" passes "%s" test' %
(path, code),
path=PSLINT_IGNORE_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(
((PSLINT_IGNORE_PATH, line), (path, 0)),
args.metadata) if args.metadata.changes else None,
))
if errors:
return SanityFailure(self.name, messages=errors)
return SanitySuccess(self.name)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
with open(PSLINT_SKIP_PATH, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
invalid_ignores = []
with open(PSLINT_IGNORE_PATH, 'r') as ignore_fd:
ignore_entries = ignore_fd.read().splitlines()
ignore = collections.defaultdict(dict)
line = 0
for ignore_entry in ignore_entries:
line += 1
if ' ' not in ignore_entry:
invalid_ignores.append((line, 'Invalid syntax'))
continue
path, code = ignore_entry.split(' ', 1)
if not os.path.exists(path):
invalid_ignores.append((line, 'Remove "%s" since it does not exist' % path))
continue
ignore[path][code] = line
paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.ps1', '.psm1', '.psd1') and i.path not in skip_paths)
if not paths:
return SanitySkipped(self.name)
if not find_executable('pwsh', required='warning'):
return SanitySkipped(self.name)
cmd = ['test/sanity/pslint/pslint.ps1'] + paths
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
if args.explain:
return SanitySuccess(self.name)
severity = [
'Information',
'Warning',
'Error',
]
cwd = os.getcwd() + '/'
# replace unicode smart quotes with ascii versions
stdout = re.sub(u'[\u2018\u2019]', "'", stdout)
stdout = re.sub(u'[\u201c\u201d]', '"', stdout)
messages = json.loads(stdout)
errors = [SanityMessage(
code=m['RuleName'],
message=m['Message'],
path=m['ScriptPath'].replace(cwd, ''),
line=m['Line'] or 0,
column=m['Column'] or 0,
level=severity[m['Severity']],
) for m in messages]
line = 0
filtered = []
for error in errors:
if error.code in ignore[error.path]:
ignore[error.path][error.code] = None # error ignored, clear line number of ignore entry to track usage
else:
filtered.append(error) # error not ignored
errors = filtered
for invalid_ignore in invalid_ignores:
errors.append(SanityMessage(
code='A201',
message=invalid_ignore[1],
path=PSLINT_IGNORE_PATH,
line=invalid_ignore[0],
column=1,
confidence=calculate_confidence(PSLINT_IGNORE_PATH, line, args.metadata) if args.metadata.changes else None,
))
for path in skip_paths:
line += 1
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(SanityMessage(
code='A101',
message='Remove "%s" since it does not exist' % path,
path=PSLINT_SKIP_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(((PSLINT_SKIP_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None,
))
for path in paths:
if path not in ignore:
continue
for code in ignore[path]:
line = ignore[path][code]
if not line:
continue
errors.append(SanityMessage(
code='A102',
message='Remove since "%s" passes "%s" test' % (path, code),
path=PSLINT_IGNORE_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(((PSLINT_IGNORE_PATH, line), (path, 0)), args.metadata) if args.metadata.changes else None,
))
if errors:
return SanityFailure(self.name, messages=errors)
return SanitySuccess(self.name)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: SanityResult
"""
with open(PEP8_SKIP_PATH, 'r') as skip_fd:
skip_paths = skip_fd.read().splitlines()
with open(PEP8_LEGACY_PATH, 'r') as legacy_fd:
legacy_paths = legacy_fd.read().splitlines()
with open('test/sanity/pep8/legacy-ignore.txt', 'r') as ignore_fd:
legacy_ignore = set(ignore_fd.read().splitlines())
with open('test/sanity/pep8/current-ignore.txt', 'r') as ignore_fd:
current_ignore = sorted(ignore_fd.read().splitlines())
skip_paths_set = set(skip_paths)
legacy_paths_set = set(legacy_paths)
paths = sorted(
i.path for i in targets.include
if (os.path.splitext(i.path)[1] == '.py'
or i.path.startswith('bin/')) and i.path not in skip_paths_set)
cmd = [
'python%s' % args.python_version,
find_executable('pycodestyle'),
'--max-line-length',
'160',
'--config',
'/dev/null',
'--ignore',
','.join(sorted(current_ignore)),
] + paths
if paths:
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
raise SubprocessError(cmd=cmd,
status=status,
stderr=stderr,
stdout=stdout)
else:
stdout = None
if args.explain:
return SanitySuccess(self.name)
if stdout:
pattern = '^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): (?P<code>[WE][0-9]{3}) (?P<message>.*)$'
results = [
re.search(pattern, line).groupdict()
for line in stdout.splitlines()
]
else:
results = []
results = [
SanityMessage(
message=r['message'],
path=r['path'],
line=int(r['line']),
column=int(r['column']),
level='warning' if r['code'].startswith('W') else 'error',
code=r['code'],
) for r in results
]
failed_result_paths = set([result.path for result in results])
used_paths = set(paths)
errors = []
summary = {}
line = 0
for path in legacy_paths:
line += 1
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(
SanityMessage(
code='A101',
message='Remove "%s" since it does not exist' % path,
path=PEP8_LEGACY_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(
((PEP8_LEGACY_PATH, line), (path, 0)),
args.metadata) if args.metadata.changes else None,
))
if path in used_paths and path not in failed_result_paths:
# Keep files out of the list which no longer require the relaxed rule set.
errors.append(
SanityMessage(
code='A201',
message=
'Remove "%s" since it passes the current rule set' %
path,
path=PEP8_LEGACY_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(
((PEP8_LEGACY_PATH, line), (path, 0)),
args.metadata) if args.metadata.changes else None,
))
line = 0
for path in skip_paths:
line += 1
if not os.path.exists(path):
# Keep files out of the list which no longer exist in the repo.
errors.append(
SanityMessage(
code='A101',
message='Remove "%s" since it does not exist' % path,
path=PEP8_SKIP_PATH,
line=line,
column=1,
confidence=calculate_best_confidence(
((PEP8_SKIP_PATH, line), (path, 0)), args.metadata)
if args.metadata.changes else None,
))
for result in results:
if result.path in legacy_paths_set and result.code in legacy_ignore:
# Files on the legacy list are permitted to have errors on the legacy ignore list.
# However, we want to report on their existence to track progress towards eliminating these exceptions.
display.info('PEP 8: %s (legacy)' % result, verbosity=3)
key = '%s %s' % (result.code,
re.sub('[0-9]+', 'NNN', result.message))
if key not in summary:
summary[key] = 0
summary[key] += 1
else:
# Files not on the legacy list and errors not on the legacy ignore list are PEP 8 policy errors.
errors.append(result)
if summary:
lines = []
count = 0
for key in sorted(summary):
count += summary[key]
lines.append('PEP 8: %5d %s' % (summary[key], key))
display.info(
'PEP 8: There were %d different legacy issues found (%d total):'
% (len(summary), count),
verbosity=1)
display.info('PEP 8: Count Code Message', verbosity=1)
for line in lines:
display.info(line, verbosity=1)
if errors:
return SanityFailure(self.name, messages=errors)
return SanitySuccess(self.name)
def docker_available():
"""
:rtype: bool
"""
return find_executable('docker', required=False)
def docker_available():
"""
:rtype: bool
"""
return find_executable('docker', required=False)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
settings = self.load_processor(args)
paths = [target.path for target in targets.include]
if not find_executable('pwsh', required='warning'):
return SanitySkipped(self.name)
cmds = []
if args.requirements:
cmds.append([
os.path.join(ANSIBLE_ROOT,
'test/runner/requirements/sanity.ps1')
])
cmds.append(
[os.path.join(ANSIBLE_ROOT, 'test/sanity/pslint/pslint.ps1')] +
paths)
stdout = ''
for cmd in cmds:
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr:
raise SubprocessError(cmd=cmd,
status=status,
stderr=stderr,
stdout=stdout)
if args.explain:
return SanitySuccess(self.name)
severity = [
'Information',
'Warning',
'Error',
'ParseError',
]
cwd = data_context().content.root + '/'
# replace unicode smart quotes and ellipsis with ascii versions
stdout = re.sub(u'[\u2018\u2019]', "'", stdout)
stdout = re.sub(u'[\u201c\u201d]', '"', stdout)
stdout = re.sub(u'[\u2026]', '...', stdout)
messages = json.loads(stdout)
errors = [
SanityMessage(
code=m['RuleName'],
message=m['Message'],
path=m['ScriptPath'].replace(cwd, ''),
line=m['Line'] or 0,
column=m['Column'] or 0,
level=severity[m['Severity']],
) for m in messages
]
errors = settings.process_errors(errors, paths)
if errors:
return SanityFailure(self.name, messages=errors)
return SanitySuccess(self.name)
def test(self, args, targets):
"""
:type args: SanityConfig
:type targets: SanityTargets
:rtype: TestResult
"""
exclude_file = os.path.join(ANSIBLE_ROOT,
'test/sanity/shellcheck/exclude.txt')
exclude = set(
read_lines_without_comments(exclude_file,
remove_blank_lines=True,
optional=True))
settings = self.load_processor(args)
paths = [target.path for target in targets.include]
if not find_executable('shellcheck', required='warning'):
return SanitySkipped(self.name)
cmd = [
'shellcheck',
'-e',
','.join(sorted(exclude)),
'--format',
'checkstyle',
] + paths
try:
stdout, stderr = run_command(args, cmd, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr or status > 1:
raise SubprocessError(cmd=cmd,
status=status,
stderr=stderr,
stdout=stdout)
if args.explain:
return SanitySuccess(self.name)
# json output is missing file paths in older versions of shellcheck, so we'll use xml instead
root = fromstring(stdout) # type: Element
results = []
for item in root: # type: Element
for entry in item: # type: Element
results.append(
SanityMessage(
message=entry.attrib['message'],
path=item.attrib['name'],
line=int(entry.attrib['line']),
column=int(entry.attrib['column']),
level=entry.attrib['severity'],
code=entry.attrib['source'].replace('ShellCheck.', ''),
))
results = settings.process_errors(results, paths)
if results:
return SanityFailure(self.name, messages=results)
return SanitySuccess(self.name)
def pylint(self, args, context, paths):
"""
:type args: SanityConfig
:type context: str
:type paths: list[str]
:rtype: list[dict[str, str]]
"""
rcfile = 'test/sanity/pylint/config/%s' % context
if not os.path.exists(rcfile):
rcfile = 'test/sanity/pylint/config/default'
parser = configparser.SafeConfigParser()
parser.read(rcfile)
if parser.has_section('ansible-test'):
config = dict(parser.items('ansible-test'))
else:
config = dict()
disable_plugins = set(
i.strip() for i in config.get('disable-plugins', '').split(',')
if i)
load_plugins = set(self.plugin_names) - disable_plugins
cmd = [
'python%s' % args.python_version,
find_executable('pylint'),
'--jobs',
'0',
'--reports',
'n',
'--max-line-length',
'160',
'--rcfile',
rcfile,
'--output-format',
'json',
'--load-plugins',
','.join(load_plugins),
] + paths
env = ansible_environment(args)
env['PYTHONPATH'] += '%s%s' % (os.pathsep, self.plugin_dir)
if paths:
try:
stdout, stderr = run_command(args, cmd, env=env, capture=True)
status = 0
except SubprocessError as ex:
stdout = ex.stdout
stderr = ex.stderr
status = ex.status
if stderr or status >= 32:
raise SubprocessError(cmd=cmd,
status=status,
stderr=stderr,
stdout=stdout)
else:
stdout = None
if not args.explain and stdout:
messages = json.loads(stdout)
else:
messages = []
return messages
