def check(self, files_or_modules):
"""main checking entry: check a list of files or modules from their name.
files_or_modules is either a string or list of strings presenting modules to check.
"""
self.initialize()
if not isinstance(files_or_modules, (list, tuple)):
files_or_modules = (files_or_modules, )
if self.config.from_stdin:
if len(files_or_modules) != 1:
raise exceptions.InvalidArgsError(
"Missing filename required for --from-stdin")
filepath = files_or_modules[0]
with fix_import_path(files_or_modules):
self._check_files(
functools.partial(self.get_ast, data=_read_stdin()),
[self._get_file_descr_from_stdin(filepath)],
)
elif self.config.jobs == 1:
with fix_import_path(files_or_modules):
self._check_files(self.get_ast,
self._iterate_file_descrs(files_or_modules))
else:
check_parallel(
self,
self.config.jobs,
self._iterate_file_descrs(files_or_modules),
files_or_modules,
)
def test_map_reduce(self, num_files, num_jobs, num_checkers):
"""Compares the 3 key parameters for check_parallel() produces the same results.
The intent here is to validate the reduce step: no stats should be lost.
Checks regression of https://github.com/PyCQA/pylint/issues/4118
"""
# define the stats we expect to get back from the runs, these should only vary
# with the number of files.
file_infos = _gen_file_datas(num_files)
# Loop for single-proc and mult-proc, so we can ensure the same linter-config
for do_single_proc in range(2):
linter = PyLinter(reporter=Reporter())
# Assign between 1 and 3 checkers to the linter, they should not change the
# results of the lint
linter.register_checker(ParallelTestChecker(linter))
if num_checkers > 1:
linter.register_checker(ExtraParallelTestChecker(linter))
if num_checkers > 2:
linter.register_checker(ThirdParallelTestChecker(linter))
if do_single_proc:
# establish the baseline
assert (linter.namespace.jobs == 1
), "jobs>1 are ignored when calling _check_files"
linter._check_files(linter.get_ast, file_infos)
stats_single_proc = linter.stats
else:
check_parallel(
linter,
jobs=num_jobs,
files=file_infos,
arguments=None,
)
stats_check_parallel = linter.stats
assert str(stats_single_proc.by_msg) == str(
stats_check_parallel.by_msg
), "Single-proc and check_parallel() should return the same thing"
def test_invoke_single_job(self):
"""Tests basic checkers functionality using just a single workderdo
This is *not* the same -j1 and does not happen under normal operation"""
linter = PyLinter(reporter=Reporter())
linter.register_checker(SequentialTestChecker(linter))
# Create a dummy file, the actual contents of which will be ignored by the
# register test checkers, but it will trigger at least a single-job to be run.
single_file_container = _gen_file_datas(count=1)
# Invoke the lint process in a multiprocess way, although we only specify one
# job.
check_parallel(linter,
jobs=1,
files=single_file_container,
arguments=None)
assert {
"by_module": {
"--test-file_data-name-0--": {
"convention": 0,
"error": 0,
"fatal": 0,
"info": 0,
"refactor": 0,
"statement": 18,
"warning": 0,
}
},
"by_msg": collections.Counter(),
"convention": 0,
"error": 0,
"fatal": 0,
"info": 0,
"refactor": 0,
"statement": 18,
"warning": 0,
} == linter.stats
assert linter.msg_status == 0, "We expect a single-file check to exit cleanly"
def test_compare_workers_to_single_proc(self, num_files, num_jobs,
num_checkers):
"""Compares the 3 key parameters for check_parallel() produces the same results.
The intent here is to ensure that the check_parallel() operates on each file,
without ordering issues, irrespective of the number of workers used and the
number of checkers applied.
This test becomes more important if we want to change how we parametrise the
checkers, for example if we aim to batch the files across jobs.
"""
# define the stats we expect to get back from the runs, these should only vary
# with the number of files.
expected_stats = LinterStats(
by_module={
# pylint: disable-next=consider-using-f-string
"--test-file_data-name-%d--" % idx: ModuleStats(
convention=0,
error=0,
fatal=0,
info=0,
refactor=0,
statement=18,
warning=0,
)
for idx in range(num_files)
})
expected_stats.by_msg = {}
expected_stats.convention = 0
expected_stats.error = 0
expected_stats.fatal = 0
expected_stats.info = 0
expected_stats.refactor = 0
expected_stats.statement = 18 * num_files
expected_stats.warning = 0
file_infos = _gen_file_datas(num_files)
# Loop for single-proc and mult-proc, so we can ensure the same linter-config
for do_single_proc in range(2):
linter = PyLinter(reporter=Reporter())
# Assign between 1 and 3 checkers to the linter, they should not change the
# results of the lint
linter.register_checker(SequentialTestChecker(linter))
if num_checkers > 1:
linter.register_checker(ExtraSequentialTestChecker(linter))
if num_checkers > 2:
linter.register_checker(ThirdSequentialTestChecker(linter))
if do_single_proc:
# establish the baseline
assert (linter.namespace.jobs == 1
), "jobs>1 are ignored when calling _check_files"
linter._check_files(linter.get_ast, file_infos)
assert linter.msg_status == 0, "We should not fail the lint"
stats_single_proc = linter.stats
else:
check_parallel(
linter,
jobs=num_jobs,
files=file_infos,
arguments=None,
)
stats_check_parallel = linter.stats
assert linter.msg_status == 0, "We should not fail the lint"
assert str(stats_single_proc) == str(
stats_check_parallel
), "Single-proc and check_parallel() should return the same thing"
assert str(stats_check_parallel) == str(
expected_stats
), "The lint is returning unexpected results, has something changed?"
def test_sequential_checkers_work(self) -> None:
"""Tests original basic types of checker works as expected in -jN.
This means that a sequential checker should return the same data for a given
file-stream irrespective of whether it's run in -j1 or -jN
"""
linter = PyLinter(reporter=Reporter())
# Add a sequential checker to ensure it records data against some streams
linter.register_checker(SequentialTestChecker(linter))
# Create a dummy file, the actual contents of which will be ignored by the
# register test checkers, but it will trigger at least a single-job to be run.
single_file_container = _gen_file_datas(count=1)
# Invoke the lint process in a multiprocess way, although we only specify one
# job.
check_parallel(
linter,
jobs=1,
files=iter(single_file_container),
arguments=["--enable", "R9999"],
)
assert len(linter.get_checkers()) == 2, (
"We should only have the 'master' and 'sequential-checker' "
"checkers registered")
assert {
"--test-file_data-name-0--": {
"convention": 0,
"error": 0,
"fatal": 0,
"info": 0,
"refactor": 0,
"statement": 18,
"warning": 0,
}
} == linter.stats.by_module
assert not linter.stats.by_msg
assert linter.stats.convention == 0
assert linter.stats.error == 0
assert linter.stats.fatal == 0
assert linter.stats.info == 0
assert linter.stats.refactor == 0
assert linter.stats.statement == 18
assert linter.stats.warning == 0
# now run the regular mode of checking files and check that, in this proc, we
# collect the right data
filepath = [single_file_container[0][1]] # get the filepath element
linter.check(filepath)
assert {
"input.similar1": { # module is the only change from previous
"convention": 0,
"error": 0,
"fatal": 0,
"info": 0,
"refactor": 0,
"statement": 18,
"warning": 0,
}
} == linter.stats.by_module
assert not linter.stats.by_msg
assert linter.stats.convention == 0
assert linter.stats.error == 0
assert linter.stats.fatal == 0
assert linter.stats.info == 0
assert linter.stats.refactor == 0
assert linter.stats.statement == 18
assert linter.stats.warning == 0
