def test_pylint():
"""Test codebase for pylint errors."""
files_to_check = get_relevant_files()
log.info("{} changed files detected".format(len(files_to_check)))
rcfile, reps = (os.path.join(CODEBASE, '.pylintrc'), files_to_check)
pylint_args = [
'--rcfile={}'.format(rcfile),
]
log.info('applying pylint to repository {}'.format(reps))
pylint_args += reps
pylint_output = _WritableOutput()
pylint_reporter = TextReporter(pylint_output)
lint.Run(pylint_args, reporter=pylint_reporter, do_exit=False)
pylint_outputs = pylint_output.content
errors = []
for output in pylint_outputs:
if not any([i in output for i in IGNORE_LIST]):
errors.append(output)
if "Your code has been rated at" in output:
print("\n" + output)
if errors:
raise AssertionError(
'{} Pylint errors found. '
'For quick resolution, consider running this test locally before you push. '
'Scroll down for hyperlinks to errors.\n{}'.format(
len(errors), '\n'.join(errors)))
def lint_py_files_for_python3_compatibility(self):
"""Prints a list of Python 3 compatibility errors in the given list of
Python files.
Returns:
TaskResult. A TaskResult object representing the result of the lint
check.
"""
files_to_lint = self.all_filepaths
any_errors = False
error_messages = []
full_error_messages = []
name = 'Pylint for Python 3 compatibility'
files_to_lint_for_python3_compatibility = [
file_name for file_name in files_to_lint
if not re.match(r'^.*python_utils.*\.py$', file_name)
]
if not files_to_lint_for_python3_compatibility:
return [
concurrent_task_utils.TaskResult(name, False, [], [
'There are no Python files to lint for Python 3 '
'compatibility.'
])
]
_batch_size = 50
current_batch_start_index = 0
while current_batch_start_index < len(
files_to_lint_for_python3_compatibility):
# Note that this index is an exclusive upper bound -- i.e.,
# the current batch of files ranges from 'start_index' to
# 'end_index - 1'.
current_batch_end_index = min(
current_batch_start_index + _batch_size,
len(files_to_lint_for_python3_compatibility))
current_files_to_lint = files_to_lint_for_python3_compatibility[
current_batch_start_index:current_batch_end_index]
pylint_report = python_utils.string_io()
pylinter_for_python3 = lint.Run(
current_files_to_lint + ['--py3k'],
reporter=text.TextReporter(pylint_report),
exit=False).linter
if pylinter_for_python3.msg_status != 0:
lint_message = pylint_report.getvalue()
pylint_error_messages = (
self.get_trimmed_error_output(lint_message))
error_messages.append(pylint_error_messages)
full_error_messages.append('Messages for Python 3 support:')
full_error_messages.append(lint_message)
any_errors = True
current_batch_start_index = current_batch_end_index
return concurrent_task_utils.TaskResult(name, any_errors,
error_messages,
full_error_messages)
def run_pylint(filename, rcfile=None):
"""run pylint check."""
disable = ["E1103", # maybe-no-member
"W0142", # star-args
"W1201", # logging-not-lazy
"I0011", # locally-disabled
"I0012", # locally-enabled
"R0801", # duplicate-code
"R0901", # too-many-ancestors
"R0902", # too-many-instance-attributes
"R0903", # too-few-public-methods
"R0904", # too-many-public-methods
"R0921", # abstract-class-not-used
"R0922"] # abstract-class-little-used
enable = ["W0511"] # fixme
args = [
"-r", "n", "--persistent=n",
"-d", ",".join(disable),
"-e", ",".join(enable)]
if rcfile:
args.append("--rcfile=%s" % rcfile)
kwargs = dict(exit=False)
try:
kwargs['reporter'] = text.TextReporter(sys.stdout)
kwargs['reporter'].line_format # pylint: disable=pointless-statement
args += ["--msg-template",
"{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}"]
except AttributeError:
kwargs['reporter'] = text.ParseableTextReporter(sys.stdout)
args += ["-f", "parseable", "-i", "y"]
lint.Run(args + [filename], **kwargs)
def dolint(self, filename):
exceptions = set()
buffer = csio()
reporter = ParseableTextReporter(output=buffer)
options = list(self.config.get('options'))
options.append(filename)
lint.Run(options, reporter=reporter, exit=False)
output = buffer.getvalue()
buffer.close()
for line in output.splitlines():
if self.idline.match(line):
continue
if self.detail.match(line):
mo = self.detail.search(line)
tokens = mo.groups()
fn = tokens[0]
ln = tokens[1]
code = tokens[2]
codename = tokens[3]
func = tokens[4]
message = tokens[5]
if not self.config.ignore(fn, code, codename, message):
exceptions.add((fn, ln, code, codename, func, message))
return exceptions
def run_pylint():
"""Runs pylint on the module supplied via command line arguments.
Usage:
>>> bin/python tests/pylint.py path_to_module_or_package
where path_to_module is the relative or absolute path to the module
or package which you want to test with pylint.
The format of the message output by pylint is:
MESSAGE_TYPE: LINE_NUM:[OBJECT:] MESSAGE
where MESSAGE_TYPE can be C(convention), R(refactor), W(warning),
E(Error), F(Fatal)
Reports generation is disabled by default.
Ids are included with message types by default.
These settings can be changed in the args variable below.
For a full list of command line options pass --help .
For more information please refer to the pyline manual at
http://www.logilab.org/card/pylint_manual
"""
args = ['--reports=n', '--include-ids=y']
sys.argv.extend(args)
lint.Run(sys.argv[1:])
def _lint_py_files(config_pylint, files_to_lint, result):
"""Prints a list of lint errors in the given list of Python files.
Args:
- config_pylint: str. Path to the .pylintrc file.
- files_to_lint: list of str. A list of filepaths to lint.
- result: multiprocessing.Queue. A queue to put results of test
Returns:
None
"""
start_time = time.time()
are_there_errors = False
num_py_files = len(files_to_lint)
if not files_to_lint:
result.put('')
print 'There are no Python files to lint.'
return
try:
# This prints output to the console.
lint.Run(files_to_lint + [config_pylint])
except SystemExit as e:
if str(e) != '0':
are_there_errors = True
if are_there_errors:
result.put('%s Python linting failed' % _MESSAGE_TYPE_FAILED)
else:
result.put('%s %s Python files linted (%.1f secs)' % (
_MESSAGE_TYPE_SUCCESS, num_py_files, time.time() - start_time))
def review_pull_request(repository,
pull_request,
pylintrc,
assignees=None,
inline=False,
**credentials):
"""Creates inline comments on the given pull request with the
errors given by pylint"""
github = GithubPullReviewClient(repository, pull_request, **credentials)
if assignees and github.get_assignee_name() not in assignees:
return
files = [f.filename for f in github.get_files()]
files = [f for f in files if f.endswith(".py")]
handler = GithubInlineWriter(github) if inline else GithubCommentWriter(
github)
args = PYLINT_ARGS
if pylintrc is not None:
# Do not append to the existing constant
args = args + ["--rcfile=%s" % pylintrc]
if not files:
return
lint.Run(args + files, reporter=TextReporter(handler), exit=False)
handler.flush()
def _run_pylint(projroot: Path, pylintrc: Union[Path, str],
cache: Optional[FileCache], dirtyfiles: List[str],
allfiles: Optional[List[str]]) -> Dict[str, Any]:
import time
from pylint import lint
from efro.error import CleanError
from efro.terminal import Clr
start_time = time.time()
args = ['--rcfile', str(pylintrc), '--output-format=colorized']
args += dirtyfiles
name = f'{len(dirtyfiles)} file(s)'
run = lint.Run(args, do_exit=False)
if cache is not None:
assert allfiles is not None
result = _apply_pylint_run_to_cache(projroot, run, dirtyfiles,
allfiles, cache)
if result != 0:
raise CleanError(f'Pylint failed for {result} file(s).')
# Sanity check: when the linter fails we should always be failing too.
# If not, it means we're probably missing something and incorrectly
# marking a failed file as clean.
if run.linter.msg_status != 0 and result == 0:
raise RuntimeError('Pylint linter returned non-zero result'
' but we did not; this is probably a bug.')
else:
if run.linter.msg_status != 0:
raise CleanError('Pylint failed.')
duration = time.time() - start_time
print(f'{Clr.GRN}Pylint passed for {name}'
f' in {duration:.1f} seconds.{Clr.RST}')
sys.stdout.flush()
return {'f': dirtyfiles, 't': duration}
def test_pylint(self):
""" PyLint conformance """
class WritableObject(object):
"dummy output stream for pylint"
def __init__(self):
self.content = []
def write(self, st):
"dummy write"
self.content.append(st)
def read(self):
"dummy read"
wholetext = ''.join(self.content)
return re.findall(r"^[^I]:[\s\d]+,[\s\d]+:.*\n", wholetext, re.MULTILINE)
from pylint import lint
from pylint.reporters.text import TextReporter
ARGS = ["-d I,C0301,R0901,R0902,R0903,R0904,R0913,R0915,W0141,W0142,W0232,W0613"]
pylint_output = WritableObject()
with capture() as (dummy_out, dummy_err):
lint.Run([filename] + ARGS, reporter=TextReporter(pylint_output), exit=False)
msg = pylint_output.read()
self.assertEqual(len(msg), 0, "Pylint detected %d errors (and warnings) (first 10 shown):\n%s" % (len(msg), ''.join(msg[:10])))
def _run_script_lint(projroot: Path, pylintrc: Union[Path, str],
cache: FileCache, dirtyfiles: List[str],
allfiles: List[str]) -> Dict[str, Any]:
import time
from pylint import lint
start_time = time.time()
args = ['--rcfile', str(pylintrc)]
args += dirtyfiles
name = f'{len(dirtyfiles)} file(s)'
run = lint.Run(args, do_exit=False)
result = _apply_pylint_run_to_cache(projroot, run, dirtyfiles, allfiles,
cache)
if result != 0:
raise Exception(f'Linting failed for {result} file(s).')
# Sanity check: when the linter fails we should always be failing too.
# If not, it means we're probably missing something and incorrectly
# marking a failed file as clean.
if run.linter.msg_status != 0 and result == 0:
raise Exception('linter returned non-zero result but we did not;'
' this is probably a bug.')
# result = run.linter.msg_status
# we can run
duration = time.time() - start_time
print(f'Pylint passed for {name} in {duration:.1f} seconds.')
sys.stdout.flush()
return {'f': dirtyfiles, 't': duration}
def run_pylint(self, directory):
pylint_rc = os.path.join(ROOT_PATH, 'core', 'controllers', 'tests',
'pylint.rc')
pylint_args = [directory, '-E', '--rcfile=%s' % pylint_rc]
pylint_output = WritableObject()
lint.Run(pylint_args, reporter=TextReporter(pylint_output), exit=False)
return pylint_output
def test_lint(self):
"""Process modules for pylint tests
"""
txtreporter = TextReporter(self.pylint_output_file)
for root, _, files in os.walk(self.dir_path):
if (['connector', 'django'
] == [os.path.basename(fld) for fld in os.path.split(root)]):
continue
for name in files:
if name.endswith('.py') and name not in IGNORE_LIST:
current_path = os.path.join(root, name)
lint_args = [
current_path,
"--rcfile={0}".format(self.pylint_rc_path)
]
lint_run = lint.Run(lint_args,
reporter=txtreporter,
exit=False)
if lint_run.linter.stats['by_msg']:
rel_file_path = os.path.join(
os.path.relpath(root, _BASE_PATH), name)
self.failed_files.append(rel_file_path)
if self.failed_files:
file_names = ''
for file in self.failed_files:
file_names += file + '\n'
self.fail('Lint tests failed on following files\n{0}\n'
'For more information check {1}.'.format(
file_names,
os.path.relpath(self.output_file_path, _BASE_PATH)))
def run_pylint(module_path, pylint_options, ignore_prefixes=tuple()):
"""Runs Pylint. Returns a boolean indicating success"""
pylint_stats = Path('/run/user/{}/pylint_stats'.format(os.getuid()))
if not pylint_stats.parent.is_dir(): #pylint: disable=no-member
pylint_stats = Path('/run/shm/pylint_stats')
os.environ['PYLINTHOME'] = str(pylint_stats)
input_paths = list()
if not module_path.exists():
print('ERROR: Cannot find', module_path)
exit(1)
if module_path.is_dir():
for path in module_path.rglob('*.py'):
ignore_matched = False
for prefix in ignore_prefixes:
if path.parts[:len(prefix)] == prefix:
ignore_matched = True
break
if ignore_matched:
continue
input_paths.append(str(path))
else:
input_paths.append(str(module_path))
runner = lint.Run((*input_paths, *pylint_options), do_exit=False)
if pylint_stats.is_dir():
shutil.rmtree(str(pylint_stats))
if runner.linter.msg_status != 0:
print('WARNING: Non-zero exit status:', runner.linter.msg_status)
return False
return True
def _lint_py_files(config_pylint, config_pycodestyle, files_to_lint, result):
"""Prints a list of lint errors in the given list of Python files.
Args:
config_pylint: str. Path to the .pylintrc file.
config_pycodestyle: str. Path to the tox.ini file.
files_to_lint: list(str). A list of filepaths to lint.
result: multiprocessing.Queue. A queue to put results of test.
Returns:
None
"""
start_time = time.time()
are_there_errors = False
num_py_files = len(files_to_lint)
if not files_to_lint:
result.put('')
print 'There are no Python files to lint.'
return
print 'Linting %s Python files' % num_py_files
_BATCH_SIZE = 50
current_batch_start_index = 0
while current_batch_start_index < len(files_to_lint):
# Note that this index is an exclusive upper bound -- i.e., the current
# batch of files ranges from 'start_index' to 'end_index - 1'.
current_batch_end_index = min(
current_batch_start_index + _BATCH_SIZE, len(files_to_lint))
current_files_to_lint = files_to_lint[
current_batch_start_index: current_batch_end_index]
print 'Linting Python files %s to %s...' % (
current_batch_start_index + 1, current_batch_end_index)
# This line invokes Pylint and prints its output to the console.
pylinter = lint.Run(
current_files_to_lint + [config_pylint],
exit=False).linter
# These lines invoke Pycodestyle.
style_guide = pycodestyle.StyleGuide(config_file=config_pycodestyle)
pycodestyle_report = style_guide.check_files(current_files_to_lint)
# This line prints Pycodestyle's output to the console.
pycodestyle_report.print_statistics()
if pylinter.msg_status != 0 or pycodestyle_report.get_count() != 0:
are_there_errors = True
current_batch_start_index = current_batch_end_index
if are_there_errors:
result.put('%s Python linting failed' % _MESSAGE_TYPE_FAILED)
else:
result.put('%s %s Python files linted (%.1f secs)' % (
_MESSAGE_TYPE_SUCCESS, num_py_files, time.time() - start_time))
print 'Python linting finished.'
def run_pylint(filename):
ARGS = [] # put your own here
pylint_output = WritableObject()
lint.Run([filename]+ARGS, reporter=TextReporter(pylint_output), exit=False)
fullText = ""
for l in pylint_output.read():
fullText = fullText + l + "\n"
return fullText
def check_commit(c):
"""
runs tasks all after another
:param :
:return: None
"""
pylint_opts = ["src"]
lint.Run(pylint_opts)
def report_version():
try:
from pylint import lint
rc_fn = os.path.abspath(os.path.join('leo', 'test', 'pylint-leo-rc.txt'))
rc_fn = rc_fn.replace('\\', '/')
lint.Run(["--rcfile=%s" % (rc_fn), '--version',])
except ImportError:
g.trace('can not import pylint')
def run(self, argv):
output = _FilteredStringIO(self.FALSE_POSITIVES)
with OutputCapture():
from pylint import lint
from pylint.reporters.text import ParseableTextReporter
lint.Run(['--rcfile', self._pylintrc] + argv, reporter=ParseableTextReporter(output=output), exit=False)
return output
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["--include-ids=y", "--errors-only", "rbd_iscsi_client"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint():
buff = StringIO()
reporter = text.ParseableTextReporter(output=buff)
args = ["--include-ids=y", "-E", "savanna"]
lint.Run(args, reporter=reporter, exit=False)
val = buff.getvalue()
buff.close()
return val
def run_pylint(filename):
print "RUNNING PyLint FOR::::"+filename
pylint_output = WritableObject()
lint.Run([filename]+ARGS, reporter=JSONReporter(pylint_output), exit=False)
for l in pylint_output.read():
if "trailing-newlines" not in l:
print l
return pylint_output.content
def module_passes_lint_score_limit(path, limit, other_args=()) -> bool:
run = lint.Run([path, *other_args], do_exit=False)
score = run.linter.stats.get("global_note", -1.0)
if score < limit:
print(f"Score for {path} was {score:.03f}; less than limit {limit}")
return False
return True
def run(self):
"""run lint to analyst the script """
args = ["-r", "n"]
lint.Run([self.script] + args,
reporter=TextReporter(pylint_output),
exit=False)
self.result = stringclean(pylint_output.read())
return None
def run(self):
args = ['--rcfile', self.rcfile]
if not self.report:
args.append('-rn')
##pkgs = list(set(p.split(".")[0] for p in options.setup.packages))
pkgs = ['snakeguice']
linter.Run(args + pkgs)
def run_lint(path, pylintrc_path):
"""Run pylint and return exit code."""
try:
print 'Linting: %s' % path
lint.Run(['--rcfile=%s' % pylintrc_path, path])
return 0
except SystemExit as e:
return e.code
def test_pylint(self):
stdout = StringIO()
reporter = text.TextReporter(stdout)
opts = ["--score=no", "chatwatch"]
pylint.Run(opts, reporter=reporter, do_exit=False)
out = reporter.out.getvalue()
failed = bool(out)
assert not failed, "Pylint found violations"
def main():
DESC = ("PyLint wrapper that add the --fail-under option."
" All other arguments are passed to pylint.")
parser = argparse.ArgumentParser(description=DESC)
parser.add_argument(
"--path",
dest="path",
type=str,
help="path where the pylint validation are going to run",
)
parser.add_argument("--rcfile",
dest="rcfile",
type=str,
help="path for pylint configuration")
parser.add_argument(
"--fail-under",
dest="threshold",
type=float,
default=8,
help="If the final score is more than THRESHOLD, exit with"
" exitcode 0, and pylint's exitcode otherwise.",
)
parser.add_argument("--service",
dest="service",
type=str,
help="service to evaluate")
parser.add_argument("--version",
dest="version",
type=str,
help="python version to test")
args, remaining_args = parser.parse_known_args()
file_object, threshold = get_threshold("./private/pylint_score.json",
args.threshold, args.service,
args.version)
path = args.path
rcfile = args.rcfile
run = lint.Run(["--rcfile=%s" % rcfile, path], exit=False)
score = run.linter.stats["global_note"]
print("score {}".format(score))
print("threshold {}".format(threshold))
print("msg_status {}".format(run.linter.msg_status))
if score < threshold:
sys.exit(run.linter.msg_status)
else:
store_threshold(
file_object,
"./private/pylint_score.json",
score,
args.service,
args.version,
)
sys.exit(0)
def run(config, *args):
""" Run pylint """
try:
from pylint import lint
from pylint.reporters import text
except ImportError:
return 2
if config is None:
config = _shell.native('pylintrc')
argv = [
'--rcfile',
config,
'--reports',
'no',
]
stream = FilterStream(_term.terminfo())
old_stderr = _sys.stderr
try:
# pylint: disable = E1101
_sys.stderr = stream
from pylint import __pkginfo__
if __pkginfo__.numversion >= (1, 0, 0):
reporter = text.TextReporter(stream)
argv.extend([
'--msg-template',
'{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}'
])
else:
argv.extend(
['--output-format', 'parseable', '--include-ids', 'yes'])
if __pkginfo__.numversion < (0, 13):
lint.REPORTER_OPT_MAP['parseable'] = \
lambda: text.TextReporter2(stream)
reporter = text.TextReporter2(stream)
else:
reporter = text.ParseableTextReporter(stream)
lint.REPORTER_OPT_MAP['parseable'] = lambda: reporter
for path in args:
try:
try:
lint.Run(argv + [path], reporter=reporter)
except SystemExit:
pass # don't accept the exit. strange errors happen...
if stream.written:
print()
stream.written = False
except KeyboardInterrupt:
print()
raise
finally:
_sys.stderr = old_stderr
return 0
def teardown_test_environment(self, **kwargs):
args = ["--rcfile=%s" % self.config_path]
if self.errors_only:
args += ['--errors-only']
args += get_apps_under_test(self.test_labels, self.test_all)
lint.Run(args, reporter=ParseableTextReporter(output=self.output), exit=False)
return True
def run_pylint():
buff = StringIO()
args = ["--msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg}",
"-E",
"ceilometer"]
lint.Run(args, exit=False)
val = buff.getvalue()
buff.close()
return val
