def run(self):
messages = []
for code_file in self._code_files:
try:
tree = ast.parse(
open(code_file, 'r').read(),
filename=code_file,
)
except (SyntaxError, TypeError):
location = Location(
path=code_file,
module=None,
function=None,
line=1,
character=0,
)
message = Message(
source='mccabe',
code='MC0000',
location=location,
message='Could not parse file',
)
messages.append(message)
continue
visitor = PathGraphingAstVisitor()
visitor.preorder(tree, visitor)
for graph in visitor.graphs.values():
complexity = graph.complexity()
if complexity > self.max_complexity:
location = Location(
path=code_file,
module=None,
function=graph.entity,
line=graph.lineno,
character=0,
)
message = Message(
source='mccabe',
code='MC0001',
location=location,
message='%s is too complex (%s)' % (
graph.entity,
complexity,
),
)
messages.append(message)
return self.filter_messages(messages)
def run(self, found_files):
warnings = []
for filepath in found_files.iter_file_paths():
mimetype = mimetypes.guess_type(filepath)
if mimetype[0] is None or not mimetype[0].startswith("text/") or mimetype[1] is not None:
continue
try:
contents = read_py_file(filepath)
except CouldNotHandleEncoding:
continue
for line, code, message in check_file_contents(contents):
warnings.append({"line": line, "code": code, "message": message, "path": filepath})
messages = []
for warning in warnings:
path = warning["path"]
prefix = os.path.commonprefix([found_files.rootpath, path])
loc = Location(
path,
module_from_path(path[len(prefix) :]),
"",
warning["line"],
0,
absolute_path=True,
)
msg = Message("dodgy", warning["code"], loc, warning["message"])
messages.append(msg)
return messages
def run(self, found_files):
warnings = []
for filepath in found_files.iter_file_paths():
mimetype = mimetypes.guess_type(filepath)
if mimetype[0] is None or not mimetype[0].startswith('text/'):
continue
for line, code, message in check_file(filepath):
warnings.append({
'line': line,
'code': code,
'message': message,
'path': filepath
})
messages = []
for warning in warnings:
path = warning['path']
prefix = os.path.commonprefix([found_files.rootpath, path])
loc = Location(path,
module_from_path(path[len(prefix):]),
'',
warning['line'],
0,
absolute_path=True)
msg = Message('dodgy', warning['code'], loc, warning['message'])
messages.append(msg)
return messages
def get_messages(self):
all_items = (
("unused-function", "Unused function %s", self.unused_funcs),
("unused-property", "Unused property %s", self.unused_props),
("unused-variable", "Unused variable %s", self.unused_vars),
("unused-attribute", "Unused attribute %s", self.unused_attrs),
)
vulture_messages = []
for code, template, items in all_items:
for item in items:
try:
filename = item.file
except AttributeError:
filename = item.filename
if hasattr(item, "lineno"):
lineno = item.lineno # for older versions of vulture
else:
lineno = item.first_lineno
loc = Location(filename, None, None, lineno, -1)
message_text = template % item
message = Message("vulture", code, loc, message_text)
vulture_messages.append(message)
return self._internal_messages + vulture_messages
def run(self, found_files):
messages = []
for code_file in found_files.iter_module_paths():
try:
contents = read_py_file(code_file)
tree = ast.parse(
contents,
filename=code_file,
)
except CouldNotHandleEncoding as err:
messages.append(
make_tool_error_message(
code_file,
'mccabe',
'MC0000',
message='Could not handle the encoding of this file: %s'
% err.encoding))
continue
except SyntaxError as err:
messages.append(
make_tool_error_message(code_file,
'mccabe',
'MC0000',
line=err.lineno,
character=err.offset,
message='Syntax Error'))
continue
except TypeError:
messages.append(
make_tool_error_message(code_file,
'mccabe',
'MC0000',
message='Unable to parse file'))
continue
visitor = PathGraphingAstVisitor()
visitor.preorder(tree, visitor)
for graph in visitor.graphs.values():
complexity = graph.complexity()
if complexity > self.max_complexity:
location = Location(path=code_file,
module=None,
function=graph.entity,
line=graph.lineno,
character=0,
absolute_path=True)
message = Message(
source='mccabe',
code='MC0001',
location=location,
message='%s is too complex (%s)' % (
graph.entity,
complexity,
),
)
messages.append(message)
return self.filter_messages(messages)
def record_message(self,
filename=None,
line=None,
character=None,
code=None,
message=None):
code = code or 'FL0000'
if code in self.ignore:
return
location = Location(
path=filename,
module=None,
function=None,
line=line,
character=character,
)
message = Message(
source='pyflakes',
code=code,
location=location,
message=message,
)
self._messages.append(message)
def run(self, found_files):
paths = [path for path in found_files.iter_module_paths()]
paths.extend(self.options)
result = self.checker.run(paths)
report, _ = result[0], result[1:]
messages = []
for message in report.splitlines():
iter_message = iter(message.split(':'))
(path, line, char, err_type), err_msg = islice(iter_message, 4), list(message)
location = Location(
path=path,
module=None,
function=None,
line=line,
character=char,
absolute_path=True
)
message = Message(
source='mypy',
code=err_type,
location=location,
message=''.join(err_msg).strip()
)
messages.append(message)
return messages
def error(self, line_number, offset, text, check):
code = super().error(
line_number,
offset,
text,
check,
)
if code is None:
# The error pycodestyle found is being ignored, let's move on.
return
# Get a clean copy of the message text without the code embedded.
text = text[5:]
# mixed indentation (E101) is a file global message
if code == "E101":
line_number = None
# Record the message using prospector's data structures.
location = Location(
path=self.filename,
module=None,
function=None,
line=line_number,
character=(offset + 1),
)
message = Message(
# TODO: legacy output naming
source="pycodestyle",
code=code,
location=location,
message=text,
)
self._prospector_messages.append(message)
def format_message(message):
try:
(path, line, char, err_type, err_msg) = message.split(":", 4)
line = int(line)
character = int(char)
except ValueError:
try:
(path, line, err_type, err_msg) = message.split(":", 3)
line = int(line)
character = None
except ValueError:
(path, err_type, err_msg) = message.split(":", 2)
line = 0
character = None
location = Location(
path=path,
module=None,
function=None,
line=line,
character=character,
)
return Message(
source="mypy",
code=err_type.lstrip(" "),
location=location,
message=err_msg.lstrip(" "),
)
def _run_markdownlint(matched_filenames, show_lint_files):
"""Run markdownlint on matched_filenames."""
from prospector.message import Message, Location
for filename in matched_filenames:
_debug_linter_status("mdl", filename, show_lint_files)
try:
proc = subprocess.Popen(["mdl"] + matched_filenames,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
lines = proc.communicate()[0].decode().splitlines()
except OSError as error:
if error.errno == errno.ENOENT:
return []
lines = [
re.match(r"([\w\-.\/\\ ]+)\:([0-9]+)\: (\w+) (.+)", l).groups(1)
for l in lines
]
return_dict = dict()
for filename, lineno, code, msg in lines:
key = _Key(filename, int(lineno), code)
loc = Location(filename, None, None, int(lineno), 0)
return_dict[key] = Message("markdownlint", code, loc, msg)
return return_dict
def _combine_w0614(self, messages):
"""
For the "unused import from wildcard import" messages,
we want to combine all warnings about the same line into
a single message.
"""
by_loc = defaultdict(list)
out = []
for message in messages:
if message.code == 'unused-wildcard-import':
by_loc[message.location].append(message)
else:
out.append(message)
for location, message_list in by_loc.items():
names = []
for msg in message_list:
names.append(
_UNUSED_WILDCARD_IMPORT_RE.match(msg.message).group(1))
msgtxt = 'Unused imports from wildcard import: %s' % ', '.join(
names)
combined_message = Message('pylint', 'unused-wildcard-import',
location, msgtxt)
out.append(combined_message)
return out
def _custom_reporter(error, file_path):
"""Reporter for polysquare-generic-file-linter."""
line = error.line_offset + 1
key = _Key(file_path, line, "file/spelling_error")
loc = Location(file_path, None, None, line, 0)
# suppress(protected-access)
desc = lint._SPELLCHECK_MESSAGES[error.error_type].format(error.word)
return_dict[key] = Message("spellcheck-linter", "file/spelling_error",
loc, desc)
def run(self, found_files):
messages = []
checker = PEP257Checker()
for code_file in found_files.iter_module_paths():
try:
for error in checker.check_source(
open(code_file, 'r').read(),
code_file,
):
location = Location(
path=code_file,
module=None,
function='',
line=error.line,
character=0,
absolute_path=True,
)
message = Message(
source='pep257',
code=error.code,
location=location,
message=error.message.partition(':')[2].strip(),
)
messages.append(message)
except AllError as exc:
location = Location(
path=code_file,
module=None,
function=None,
line=1,
character=0,
absolute_path=True,
)
message = Message(
source='pep257',
code='D000',
location=location,
message=exc.message,
)
messages.append(message)
return self.filter_messages(messages)
def error(self, line, offset, text, check):
"""Record error and store in return_dict."""
code = super(Flake8MergeReporter,
self).error(line, offset, text, check)
key = _Key(self._current_file, line, code)
return_dict[key] = Message(
code, code,
Location(self._current_file, None, None, line, offset),
text[5:])
def add_message(code, message, setting):
if code in self.ignore_codes:
return
line = -1
for number, fileline in enumerate(raw_contents):
if setting in fileline:
line = number + 1
break
location = Location(relative_filepath, None, None, line, 0, False)
message = Message('profile-validator', code, location, message)
messages.append(message)
def test_format_message_without_character(self):
location = Location(path="file.py",
module=None,
function=None,
line=17,
character=None)
expected = Message(source="mypy",
code="error",
location=location,
message="Important error")
self.assertEqual(format_message("file.py:17: error: Important error"),
expected)
def test_format_message_without_character_and_columns_in_message(self):
location = Location(path="file.py",
module=None,
function=None,
line=17,
character=None)
expected = Message(source="mypy",
code="note",
location=location,
message="Important error")
self.assertEqual(
format_message('file.py:17: note: unused "type: ignore" comment'),
expected)
def run(self, found_files):
messages = []
checker = ConventionChecker()
for code_file in found_files.iter_module_paths():
try:
for error in checker.check_source(read_py_file(code_file),
code_file, None):
location = Location(
path=code_file,
module=None,
function="",
line=error.line,
character=0,
absolute_path=True,
)
message = Message(
source="pydocstyle",
code=error.code,
location=location,
message=error.message.partition(":")[2].strip(),
)
messages.append(message)
except CouldNotHandleEncoding as err:
messages.append(
make_tool_error_message(
code_file,
"pydocstyle",
"D000",
message=
f"Could not handle the encoding of this file: {err.encoding}",
))
continue
except AllError as exc:
# pydocstyle's Parser.parse_all method raises AllError when an
# attempt to analyze the __all__ definition has failed. This
# occurs when __all__ is too complex to be parsed.
messages.append(
make_tool_error_message(
code_file,
"pydocstyle",
"D000",
line=1,
character=0,
message=exc.args[0],
))
continue
return self.filter_messages(messages)
def execute(self):
summary = {
'started': datetime.now(),
'libraries': self.libraries,
'strictness': self.config.strictness,
'profiles': self.profiles,
'adaptors': [adaptor.name for adaptor in self.adaptors],
'tools': self.config.tools,
}
# Prep the tools.
for tool in self.tool_runners:
tool.prepare(self.path, self.ignores, self.config, self.adaptors)
# Run the tools
messages = []
for tool in self.tool_runners:
try:
messages += tool.run()
except Exception: # pylint: disable=W0703
if self.config.die_on_tool_error:
raise
else:
for name, cls in tools.TOOLS.items():
if cls == tool.__class__:
toolname = name
break
else:
toolname = 'Unknown'
loc = Location(self.path, None, None, None, None)
msg = 'Tool %s failed to run (exception was raised)' % (
toolname, )
message = Message(
toolname,
'failure',
loc,
message=msg,
)
messages.append(message)
messages = self.process_messages(messages)
summary['message_count'] = len(messages)
summary['completed'] = datetime.now()
delta = (summary['completed'] - summary['started'])
summary['time_taken'] = '%0.2f' % delta.total_seconds()
return summary, messages
def run(self, found_files):
self.manager.files_list = sorted(found_files.iter_file_paths())
self.manager.exclude_files = []
if not self.manager.b_ts.tests:
raise ValueError("No test will run for bandit")
self.manager.run_tests()
results = self.manager.get_issue_list(sev_level=RANKING[self.severity], conf_level=RANKING[self.confidence])
messages = []
for result in results:
loc = Location(os.path.abspath(result.fname), None, "", int(result.lineno), 0)
msg = Message("bandit", result.test_id, loc, result.text)
messages.append(msg)
return messages
def run(self):
warnings = run_checks(self.rootpath, self.ignore)
messages = []
for warning in warnings:
path = warning['path']
loc = Location(path,
module_from_path(path),
'',
warning['line'],
0,
absolute_path=False)
msg = Message('dodgy', warning['code'], loc, warning['message'])
messages.append(msg)
return messages
def get_messages(self):
all_items = (
('unused-function', 'Unused function %s', self.unused_funcs),
('unused-property', 'Unused property %s', self.unused_props),
('unused-variable', 'Unused variable %s', self.unused_vars),
('unused-attribute', 'Unused attribute %s', self.unused_attrs)
)
messages = []
for code, template, items in all_items:
for item in items:
loc = Location(item.file, None, None, item.lineno, -1)
message_text = template % item
message = Message('vulture', code, loc, message_text)
messages.append(message)
return messages
def add_message(self, msg_id, location, msg):
# (* magic is acceptable here)
loc = Location(*location)
# At this point pylint will give us the code but we want the
# more user-friendly symbol
try:
msg_data = self._message_store.get_message_definitions(msg_id)
except UnknownMessageError:
# this shouldn't happen, as all pylint errors should be
# in the message store, but just in case we'll fall back
# to using the code.
msg_symbol = msg_id
else:
msg_symbol = msg_data[0].symbol
message = Message("pylint", msg_symbol, loc, msg)
self._messages.append(message)
def test_format_dupplicated_module_linux(self):
location = Location(path="file.py",
module=None,
function=None,
line=0,
character=None)
expected = Message(
source="mypy",
code="error",
location=location,
message=
"Duplicate module named 'file' (also at '/Repositories/file.py')",
)
self.assertEqual(
format_message(
"file.py: error: Duplicate module named 'file' (also at '/Repositories/file.py')"
),
expected,
)
def _run_pyroma(setup_file, show_lint_files):
"""Run pyroma."""
from pyroma import projectdata, ratings
from prospector.message import Message, Location
_debug_linter_status("pyroma", setup_file, show_lint_files)
return_dict = dict()
data = projectdata.get_data(os.getcwd())
all_tests = ratings.ALL_TESTS
for test in [mod() for mod in [t.__class__ for t in all_tests]]:
if test.test(data) is False:
class_name = test.__class__.__name__
key = _Key(setup_file, 0, class_name)
loc = Location(setup_file, None, None, 0, 0)
msg = test.message()
return_dict[key] = Message("pyroma", class_name, loc, msg)
return return_dict
def get_messages(self):
all_items = (
('unused-function', 'Unused function %s', self.unused_funcs),
('unused-property', 'Unused property %s', self.unused_props),
('unused-variable', 'Unused variable %s', self.unused_vars),
('unused-attribute', 'Unused attribute %s', self.unused_attrs)
)
vulture_messages = []
for code, template, items in all_items:
for item in items:
try:
filename = item.file
except AttributeError:
filename = item.filename
loc = Location(filename, None, None, item.lineno, -1)
message_text = template % item
message = Message('vulture', code, loc, message_text)
vulture_messages.append(message)
return self._internal_messages + vulture_messages
def run(self, found_files):
messages = []
for module in found_files.iter_module_paths(include_ignored=True):
dirname, filename = os.path.split(module)
if filename != 'setup.py':
continue
data = projectdata.get_data(dirname)
all_tests = [m() for m in PYROMA_TEST_CLASSES]
for test in all_tests:
code = PYROMA_CODES.get(test.__class__, 'PYRUNKNOWN')
if code in self.ignore_codes:
continue
passed = test.test(data)
if passed is False: # passed can be True, False or None...
loc = Location(module, 'setup', None, -1, -1)
msg = Message('pyroma', code, loc, test.message())
messages.append(msg)
return messages
def _error_message(self, filepath, message):
location = Location(filepath, None, None, 0, 0)
return Message('prospector', 'config-problem', location, message)
def execute(self):
deprecated_names = self.config.replace_deprecated_tool_names()
summary = {
"started": datetime.now(),
}
summary.update(self.config.get_summary_information())
found_files = find_python(
self.config.ignores,
self.config.paths,
self.config.explicit_file_mode,
self.config.workdir,
)
messages = []
# see if any old tool names are run
for deprecated_name in deprecated_names:
loc = Location(self.config.workdir, None, None, None, None)
new_name = DEPRECATED_TOOL_NAMES[deprecated_name]
msg = (
f"Tool {deprecated_name} has been renamed to {new_name}. "
f"The old name {deprecated_name} is now deprecated and will be removed in Prospector 2.0. "
f"Please update your prospector configuration.")
message = Message(
"prospector",
"Deprecation",
loc,
message=msg,
)
messages.append(message)
warnings.warn(msg, category=DeprecationWarning)
# Run the tools
for tool in self.config.get_tools(found_files):
for name, cls in tools.TOOLS.items():
if cls == tool.__class__:
toolname = name
break
else:
toolname = "Unknown"
try:
# Tools can output to stdout/stderr in unexpected places, for example,
# pydocstyle emits warnings about __all__ and as pyroma exec's the setup.py
# file, it will execute any print statements in that, etc etc...
with CaptureOutput(
hide=not self.config.direct_tool_stdout) as capture:
messages += tool.run(found_files)
if self.config.include_tool_stdout:
loc = Location(self.config.workdir, None, None, None,
None)
if capture.get_hidden_stderr():
msg = f"stderr from {toolname}:\n{capture.get_hidden_stderr()}"
messages.append(
Message(toolname,
"hidden-output",
loc,
message=msg))
if capture.get_hidden_stdout():
msg = f"stdout from {toolname}:\n{capture.get_hidden_stdout()}"
messages.append(
Message(toolname,
"hidden-output",
loc,
message=msg))
except FatalProspectorException as fatal:
sys.stderr.write(fatal.message)
sys.exit(2)
except Exception as ex: # pylint:disable=broad-except
if self.config.die_on_tool_error:
raise FatalProspectorException from ex
loc = Location(self.config.workdir, None, None, None, None)
msg = (
f"Tool {toolname} failed to run "
f"(exception was raised, re-run prospector with -X to see the stacktrace)"
)
message = Message(
toolname,
"failure",
loc,
message=msg,
)
messages.append(message)
messages = self.process_messages(found_files, messages)
summary["message_count"] = len(messages)
summary["completed"] = datetime.now()
# Timedelta.total_seconds() is not available
# on Python<=2.6 so we calculate it ourselves
# See issue #60 and http://stackoverflow.com/a/3694895
delta = summary["completed"] - summary["started"]
total_seconds = (delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6
summary["time_taken"] = "%0.2f" % total_seconds
external_config = []
for tool, configured_by in self.config.configured_by.items():
if configured_by is not None:
external_config.append((tool, configured_by))
if len(external_config) > 0:
summary["external_config"] = ", ".join(
["%s: %s" % info for info in external_config])
self.summary = summary
self.messages = self.messages + messages
def execute(self):
summary = {
'started': datetime.now(),
}
summary.update(self.config.get_summary_information())
found_files = find_python(self.config.ignores, self.config.paths,
self.config.explicit_file_mode,
self.config.workdir)
# Run the tools
messages = []
for tool in self.config.get_tools(found_files):
for name, cls in tools.TOOLS.items():
if cls == tool.__class__:
toolname = name
break
else:
toolname = 'Unknown'
try:
# Tools can output to stdout/stderr in unexpected places, for example,
# pep257 emits warnings about __all__ and as pyroma exec's the setup.py
# file, it will execute any print statements in that, etc etc...
with capture_output(
hide=not self.config.direct_tool_stdout) as capture:
messages += tool.run(found_files)
if self.config.include_tool_stdout:
loc = Location(self.config.workdir, None, None, None,
None)
if capture.get_hidden_stderr():
msg = 'stderr from %s:\n%s' % (
toolname, capture.get_hidden_stderr())
messages.append(
Message(toolname,
'hidden-output',
loc,
message=msg))
if capture.get_hidden_stdout():
msg = 'stdout from %s:\n%s' % (
toolname, capture.get_hidden_stdout())
messages.append(
Message(toolname,
'hidden-output',
loc,
message=msg))
except FatalProspectorException as fatal:
sys.stderr.write(fatal.message)
sys.exit(2)
except Exception: # pylint: disable=broad-except
if self.config.die_on_tool_error:
raise
else:
loc = Location(self.config.workdir, None, None, None, None)
msg = 'Tool %s failed to run (exception was raised)' % (
toolname, )
message = Message(
toolname,
'failure',
loc,
message=msg,
)
messages.append(message)
messages = self.process_messages(found_files, messages)
summary['message_count'] = len(messages)
summary['completed'] = datetime.now()
# Timedelta.total_seconds() is not available
# on Python<=2.6 so we calculate it ourselves
# See issue #60 and http://stackoverflow.com/a/3694895
delta = (summary['completed'] - summary['started'])
total_seconds = (delta.microseconds +
(delta.seconds + delta.days * 24 * 3600) * 1e6) / 1e6
summary['time_taken'] = '%0.2f' % total_seconds
external_config = []
for tool, configured_by in self.config.configured_by.items():
if configured_by is not None:
external_config.append((tool, configured_by))
if len(external_config) > 0:
summary['external_config'] = ', '.join(
['%s: %s' % info for info in external_config])
self.summary = summary
self.messages = self.messages + messages