def configure(self, prospector_config, found_files):
extra_sys_path = found_files.get_minimal_syspath()
check_paths = self._get_pylint_check_paths(found_files)
pylint_options = prospector_config.tool_options("pylint")
self._set_path_finder(extra_sys_path, pylint_options)
linter = ProspectorLinter(found_files)
config_messages, configured_by = self._get_pylint_configuration(
check_paths, linter, prospector_config, pylint_options
)
# we don't want similarity reports right now
linter.disable("similarities")
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
if linter.config.jobs == 0:
linter.config.jobs = _cpu_count()
self._linter = linter
return configured_by, config_messages
def prepare(self, rootpath, ignore, args, adaptors):
linter = ProspectorLinter(ignore, rootpath)
linter.load_default_plugins()
extra_sys_path, check_paths = _find_package_paths(ignore, rootpath)
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
for adaptor in adaptors:
adaptor.adapt_pylint(linter)
self._args = linter.load_command_line_configuration(check_paths)
# disable the warnings about disabling warnings...
linter.disable('I0011')
linter.disable('I0012')
linter.disable('I0020')
linter.disable('I0021')
# disable the 'mixed indentation' warning, since it actually will only allow
# the indentation specified in the pylint configuration file; we replace it
# instead with our own version which is more lenient and configurable
linter.disable('W0312')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector()
linter.set_reporter(self._collector)
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if args.max_line_length is not None:
if option[0] == 'max-line-length':
checker.set_option('max-line-length',
args.max_line_length)
self._linter = linter
def configure(self, prospector_config, found_files):
config_messages = []
extra_sys_path = found_files.get_minimal_syspath()
check_paths = self._get_pylint_check_paths(found_files)
pylint_options = prospector_config.tool_options("pylint")
self._set_path_finder(extra_sys_path, pylint_options)
linter = ProspectorLinter(found_files)
ext_found = False
configured_by = None
config_messages, configured_by = self._get_pylint_configuration(
check_paths, config_messages, configured_by, ext_found, linter,
prospector_config, pylint_options)
# Pylint 1.4 introduced the idea of explicitly specifying which
# C-extensions to load. This is because doing so allows them to
# execute any code whatsoever, which is considered to be unsafe.
# The following option turns off this, allowing any extension to
# load again, since any setup.py can execute arbitrary code and
# the safety gained through this approach seems minimal. Leaving
# it on means that the inference engine cannot do much inference
# on modules with C-extensions which is a bit useless.
linter.set_option("unsafe-load-any-extension", True)
# we don't want similarity reports right now
linter.disable("similarities")
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
if linter.config.jobs == 0:
linter.config.jobs = _cpu_count()
self._linter = linter
return configured_by, config_messages
def prepare(self, rootpath, ignore, args, adaptors):
linter = ProspectorLinter(ignore, rootpath)
linter.load_default_plugins()
extra_sys_path, check_paths = _find_package_paths(ignore, rootpath)
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
for adaptor in adaptors:
adaptor.adapt_pylint(linter)
self._args = linter.load_command_line_configuration(check_paths)
# disable the warnings about disabling warnings...
linter.disable('I0011')
linter.disable('I0012')
linter.disable('I0020')
linter.disable('I0021')
# disable the 'mixed indentation' warning, since it actually will only allow
# the indentation specified in the pylint configuration file; we replace it
# instead with our own version which is more lenient and configurable
linter.disable('W0312')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector()
linter.set_reporter(self._collector)
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if args.max_line_length is not None:
if option[0] == 'max-line-length':
checker.set_option('max-line-length', args.max_line_length)
self._linter = linter
class PylintTool(ToolBase):
def __init__(self):
self._args = self._extra_sys_path = None
self._collector = self._linter = None
self._orig_sys_path = []
def prepare(self, rootpath, ignore, args, adaptors):
linter = ProspectorLinter(ignore)
linter.load_default_plugins()
extra_sys_path, check_paths = _find_package_paths(ignore, rootpath)
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
for adaptor in adaptors:
adaptor.adapt_pylint(linter)
self._args = linter.load_command_line_configuration(check_paths)
# disable the warnings about disabling warnings...
linter.disable('I0011')
linter.disable('I0012')
linter.disable('I0020')
linter.disable('I0021')
# disable the 'mixed indentation' warning, since it actually will only allow
# the indentation specified in the pylint configuration file; we replace it
# instead with our own version which is more lenient and configurable
linter.disable('W0312')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector()
linter.set_reporter(self._collector)
self._linter = linter
def _combine_w0614(self, messages):
"""
For the "unused import from wildcard import" messages, we want to combine all warnings about
the same line into a single message
"""
by_loc = defaultdict(list)
out = []
for message in messages:
if message.code == 'W0614':
by_loc[message.location].append(message)
else:
out.append(message)
for location, message_list in by_loc.iteritems():
names = []
for msg in message_list:
names.append(_W0614_RE.match(msg.message).group(1))
msgtxt = 'Unused imports from wildcard import: %s' % ', '.join(names)
combined_message = Message('pylint', 'W0614', location, msgtxt)
out.append(combined_message)
return out
def _combine_(self, messages):
pass
def combine(self, messages):
"""
Some error messages are repeated, causing many errors where only one is strictly necessary. For
example, having a wildcard import will result in one 'Unused Import' warning for every unused import.
This method will combine these into a single warning.
"""
combined = self._combine_w0614(messages)
return sorted(combined)
def run(self):
# note: Pylint will exit with a status code indicating the health of
# the code it was checking. Prospector will not mimic this behaviour,
# as it interferes with scripts which depend on and expect the exit
# code of the code checker to match whether the check itself was
# successful
self._linter.check(self._args)
sys.path = self._orig_sys_path
messages = self._collector.get_messages()
return self.combine(messages)
class PylintTool(ToolBase):
# There are several methods on this class which could technically
# be functions (they don't use the 'self' argument) but that would
# make this module/class a bit ugly.
# pylint:disable=no-self-use
def __init__(self):
self._args = self._extra_sys_path = None
self._collector = self._linter = None
self._orig_sys_path = []
def _prospector_configure(self, prospector_config, linter):
linter.load_default_plugins()
linter.load_plugin_modules(['pylint_common'])
if 'django' in prospector_config.libraries:
linter.load_plugin_modules(['pylint_django'])
if 'celery' in prospector_config.libraries:
linter.load_plugin_modules(['pylint_celery'])
if 'flask' in prospector_config.libraries:
linter.load_plugin_modules(['pylint_flask'])
for msg_id in prospector_config.get_disabled_messages('pylint'):
try:
linter.disable(msg_id)
# pylint: disable=pointless-except
except UnknownMessageError:
# If the msg_id doesn't exist in PyLint any more,
# don't worry about it.
pass
options = prospector_config.tool_options('pylint')
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if option[0] in options:
checker.set_option(option[0], options[option[0]])
# The warnings about disabling warnings are useful for figuring out
# with other tools to suppress messages from. For example, an unused
# import which is disabled with 'pylint disable=unused-import' will
# still generate an 'FL0001' unused import warning from pyflakes.
# Using the information from these messages, we can figure out what
# was disabled.
linter.disable(
'locally-disabled') # notification about disabling a message
linter.disable(
'locally-enabled') # notification about enabling a message
linter.enable(
'file-ignored') # notification about disabling an entire file
linter.enable('suppressed-message'
) # notification about a message being supressed
linter.disable(
'useless-suppression'
) # notification about message supressed which was not raised
linter.disable(
'deprecated-pragma'
) # notification about use of deprecated 'pragma' option
# disable the 'mixed indentation' warning, since it actually will only
# allow the indentation specified in the pylint configuration file; we
# replace it instead with our own version which is more lenient and
# configurable
linter.disable('mixed-indentation')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
max_line_length = prospector_config.max_line_length
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if max_line_length is not None:
if option[0] == 'max-line-length':
checker.set_option('max-line-length', max_line_length)
def _error_message(self, filepath, message):
location = Location(filepath, None, None, 0, 0)
return Message('prospector', 'config-problem', location, message)
def _pylintrc_configure(self, pylintrc, linter):
errors = []
linter.load_default_plugins()
are_plugins_loaded = linter.config_from_file(pylintrc)
if not are_plugins_loaded and hasattr(linter.config, 'load_plugins'):
for plugin in linter.config.load_plugins:
try:
linter.load_plugin_modules([plugin])
except ImportError:
errors.append(
self._error_message(
pylintrc, "Could not load plugin %s" % plugin))
return errors
def configure(self, prospector_config, found_files):
config_messages = []
extra_sys_path = found_files.get_minimal_syspath()
# create a list of packages, but don't include packages which are
# subpackages of others as checks will be duplicated
packages = [
os.path.split(p)
for p in found_files.iter_package_paths(abspath=False)
]
packages.sort(key=len)
check_paths = set()
for package in packages:
package_path = os.path.join(*package)
if len(package) == 1:
check_paths.add(package_path)
continue
for i in range(1, len(package)):
if os.path.join(*package[:-i]) in check_paths:
break
else:
check_paths.add(package_path)
for filepath in found_files.iter_module_paths(abspath=False):
package = os.path.dirname(filepath).split(os.path.sep)
for i in range(0, len(package)):
if os.path.join(*package[:i + 1]) in check_paths:
break
else:
check_paths.add(filepath)
check_paths = [found_files.to_absolute_path(p) for p in check_paths]
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
ext_found = False
configured_by = None
linter = ProspectorLinter(found_files)
if prospector_config.use_external_config('pylint'):
# try to find a .pylintrc
pylintrc = prospector_config.external_config_location('pylint')
if pylintrc is None:
pylintrc = find_pylintrc()
if pylintrc is None:
pylintrc_path = os.path.join(prospector_config.workdir,
'.pylintrc')
if os.path.exists(pylintrc_path):
pylintrc = pylintrc_path
if pylintrc is not None:
# load it!
configured_by = pylintrc
ext_found = True
self._args = linter.load_command_line_configuration(
check_paths)
config_messages += self._pylintrc_configure(pylintrc, linter)
if not ext_found:
linter.reset_options()
self._args = linter.load_command_line_configuration(check_paths)
self._prospector_configure(prospector_config, linter)
# Pylint 1.4 introduced the idea of explicitly specifying which
# C-extensions to load. This is because doing so allows them to
# execute any code whatsoever, which is considered to be unsafe.
# The following option turns off this, allowing any extension to
# load again, since any setup.py can execute arbitrary code and
# the safety gained through this approach seems minimal. Leaving
# it on means that the inference engine cannot do much inference
# on modules with C-extensions which is a bit useless.
linter.set_option('unsafe-load-any-extension', True)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
self._linter = linter
return configured_by, config_messages
def _combine_w0614(self, messages):
"""
For the "unused import from wildcard import" messages,
we want to combine all warnings about the same line into
a single message.
"""
by_loc = defaultdict(list)
out = []
for message in messages:
if message.code == 'unused-wildcard-import':
by_loc[message.location].append(message)
else:
out.append(message)
for location, message_list in by_loc.items():
names = []
for msg in message_list:
names.append(
_UNUSED_WILDCARD_IMPORT_RE.match(msg.message).group(1))
msgtxt = 'Unused imports from wildcard import: %s' % ', '.join(
names)
combined_message = Message('pylint', 'unused-wildcard-import',
location, msgtxt)
out.append(combined_message)
return out
def combine(self, messages):
"""
Combine repeated messages.
Some error messages are repeated, causing many errors where
only one is strictly necessary.
For example, having a wildcard import will result in one
'Unused Import' warning for every unused import.
This method will combine these into a single warning.
"""
combined = self._combine_w0614(messages)
return sorted(combined)
def run(self, found_files):
self._linter.check(self._args)
sys.path = self._orig_sys_path
messages = self._collector.get_messages()
return self.combine(messages)
def configure(self, prospector_config, found_files):
config_messages = []
extra_sys_path = found_files.get_minimal_syspath()
# create a list of packages, but don't include packages which are
# subpackages of others as checks will be duplicated
packages = [
os.path.split(p)
for p in found_files.iter_package_paths(abspath=False)
]
packages.sort(key=len)
check_paths = set()
for package in packages:
package_path = os.path.join(*package)
if len(package) == 1:
check_paths.add(package_path)
continue
for i in range(1, len(package)):
if os.path.join(*package[:-i]) in check_paths:
break
else:
check_paths.add(package_path)
for filepath in found_files.iter_module_paths(abspath=False):
package = os.path.dirname(filepath).split(os.path.sep)
for i in range(0, len(package)):
if os.path.join(*package[:i + 1]) in check_paths:
break
else:
check_paths.add(filepath)
check_paths = [found_files.to_absolute_path(p) for p in check_paths]
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
ext_found = False
configured_by = None
linter = ProspectorLinter(found_files)
if prospector_config.use_external_config('pylint'):
# try to find a .pylintrc
pylintrc = prospector_config.external_config_location('pylint')
if pylintrc is None:
pylintrc = find_pylintrc()
if pylintrc is None:
pylintrc_path = os.path.join(prospector_config.workdir,
'.pylintrc')
if os.path.exists(pylintrc_path):
pylintrc = pylintrc_path
if pylintrc is not None:
# load it!
configured_by = pylintrc
ext_found = True
self._args = linter.load_command_line_configuration(
check_paths)
config_messages += self._pylintrc_configure(pylintrc, linter)
if not ext_found:
linter.reset_options()
self._args = linter.load_command_line_configuration(check_paths)
self._prospector_configure(prospector_config, linter)
# Pylint 1.4 introduced the idea of explicitly specifying which
# C-extensions to load. This is because doing so allows them to
# execute any code whatsoever, which is considered to be unsafe.
# The following option turns off this, allowing any extension to
# load again, since any setup.py can execute arbitrary code and
# the safety gained through this approach seems minimal. Leaving
# it on means that the inference engine cannot do much inference
# on modules with C-extensions which is a bit useless.
linter.set_option('unsafe-load-any-extension', True)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
self._linter = linter
return configured_by, config_messages
class PylintTool(ToolBase):
# There are several methods on this class which could technically
# be functions (they don't use the 'self' argument) but that would
# make this module/class a bit ugly.
# pylint:disable=no-self-use
def __init__(self):
self._args = self._extra_sys_path = None
self._collector = self._linter = None
self._orig_sys_path = []
def _prospector_configure(self, prospector_config, linter):
linter.load_default_plugins()
linter.load_plugin_modules(['pylint_common'])
if 'django' in prospector_config.libraries:
linter.load_plugin_modules(['pylint_django'])
if 'celery' in prospector_config.libraries:
linter.load_plugin_modules(['pylint_celery'])
if 'flask' in prospector_config.libraries:
linter.load_plugin_modules(['pylint_flask'])
for msg_id in prospector_config.get_disabled_messages('pylint'):
try:
linter.disable(msg_id)
# pylint: disable=pointless-except
except UnknownMessageError:
# If the msg_id doesn't exist in PyLint any more,
# don't worry about it.
pass
options = prospector_config.tool_options('pylint')
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if option[0] in options:
checker.set_option(option[0], options[option[0]])
# The warnings about disabling warnings are useful for figuring out
# with other tools to suppress messages from. For example, an unused
# import which is disabled with 'pylint disable=unused-import' will still
# generate an 'FL0001' unused import warning from pyflakes. Using the
# information from these messages, we can figure out what was disabled.
linter.disable('locally-disabled') # notification about disabling a message
linter.disable('locally-enabled') # notification about enabling a message
linter.enable('file-ignored') # notification about disabling an entire file
linter.enable('suppressed-message') # notification about a message being supressed
linter.disable('useless-suppression') # notification about message supressed which was not raised
linter.disable('deprecated-pragma') # notification about use of deprecated 'pragma' option
# disable the 'mixed indentation' warning, since it actually will only allow
# the indentation specified in the pylint configuration file; we replace it
# instead with our own version which is more lenient and configurable
linter.disable('mixed-indentation')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
max_line_length = prospector_config.max_line_length
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if max_line_length is not None:
if option[0] == 'max-line-length':
checker.set_option('max-line-length', max_line_length)
def _error_message(self, filepath, message):
location = Location(filepath, None, None, 0, 0)
return Message(
'prospector',
'config-problem',
location,
message
)
def _pylintrc_configure(self, pylintrc, linter):
errors = []
linter.load_default_plugins()
linter.config_from_file(pylintrc)
if hasattr(linter.config, 'load_plugins'):
for plugin in linter.config.load_plugins:
try:
linter.load_plugin_modules([plugin])
except ImportError:
errors.append(self._error_message(pylintrc, "Could not load plugin %s" % plugin))
return errors
def configure(self, prospector_config, found_files):
config_messages = []
extra_sys_path = found_files.get_minimal_syspath()
# create a list of packages, but don't include packages which are
# subpackages of others as checks will be duplicated
packages = [p.split(os.path.sep) for p in found_files.iter_package_paths(abspath=False)]
packages.sort(key=len)
check_paths = set()
for package in packages:
package_path = os.path.join(*package)
if len(package) == 1:
check_paths.add(package_path)
continue
for i in range(1, len(package)):
if os.path.join(*package[:-i]) in check_paths:
break
else:
check_paths.add(package_path)
for filepath in found_files.iter_module_paths(abspath=False):
package = os.path.dirname(filepath).split(os.path.sep)
for i in range(0, len(package)):
if os.path.join(*package[:i + 1]) in check_paths:
break
else:
check_paths.add(filepath)
check_paths = [found_files.to_absolute_path(p) for p in check_paths]
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
ext_found = False
configured_by = None
linter = ProspectorLinter(found_files)
if prospector_config.use_external_config('pylint'):
# try to find a .pylintrc
pylintrc = prospector_config.external_config_location('pylint')
if pylintrc is None:
pylintrc = find_pylintrc()
if pylintrc is None:
pylintrc_path = os.path.join(prospector_config.workdir, '.pylintrc')
if os.path.exists(pylintrc_path):
pylintrc = pylintrc_path
if pylintrc is not None:
# load it!
configured_by = pylintrc
ext_found = True
self._args = linter.load_command_line_configuration(check_paths)
config_messages += self._pylintrc_configure(pylintrc, linter)
if not ext_found:
linter.reset_options()
self._args = linter.load_command_line_configuration(check_paths)
self._prospector_configure(prospector_config, linter)
# Pylint 1.4 introduced the idea of explicitly specifying which C-extensions
# to load. This is because doing so allows them to execute any code whatsoever,
# which is considered to be unsafe. The following option turns off this, allowing
# any extension to load again, since any setup.py can execute arbitrary code and
# the safety gained through this approach seems minimal. Leaving it on means
# that the inference engine cannot do much inference on modules with C-extensions
# which is a bit useless.
linter.set_option('unsafe-load-any-extension', True)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
self._linter = linter
return configured_by, config_messages
def _combine_w0614(self, messages):
"""
For the "unused import from wildcard import" messages,
we want to combine all warnings about the same line into a single message.
"""
by_loc = defaultdict(list)
out = []
for message in messages:
if message.code == 'unused-wildcard-import':
by_loc[message.location].append(message)
else:
out.append(message)
for location, message_list in by_loc.items():
names = []
for msg in message_list:
names.append(_UNUSED_WILDCARD_IMPORT_RE.match(msg.message).group(1))
msgtxt = 'Unused imports from wildcard import: %s' % ', '.join(names)
combined_message = Message('pylint', 'unused-wildcard-import', location, msgtxt)
out.append(combined_message)
return out
def combine(self, messages):
"""
Some error messages are repeated, causing many errors where only one is strictly necessary.
For example, having a wildcard import will result in one 'Unused Import' warning for every unused import.
This method will combine these into a single warning.
"""
combined = self._combine_w0614(messages)
return sorted(combined)
def run(self, found_files):
self._linter.check(self._args)
sys.path = self._orig_sys_path
messages = self._collector.get_messages()
return self.combine(messages)
def configure(self, prospector_config, found_files):
config_messages = []
extra_sys_path = found_files.get_minimal_syspath()
# create a list of packages, but don't include packages which are
# subpackages of others as checks will be duplicated
packages = [p.split(os.path.sep) for p in found_files.iter_package_paths(abspath=False)]
packages.sort(key=len)
check_paths = set()
for package in packages:
package_path = os.path.join(*package)
if len(package) == 1:
check_paths.add(package_path)
continue
for i in range(1, len(package)):
if os.path.join(*package[:-i]) in check_paths:
break
else:
check_paths.add(package_path)
for filepath in found_files.iter_module_paths(abspath=False):
package = os.path.dirname(filepath).split(os.path.sep)
for i in range(0, len(package)):
if os.path.join(*package[:i + 1]) in check_paths:
break
else:
check_paths.add(filepath)
check_paths = [found_files.to_absolute_path(p) for p in check_paths]
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
ext_found = False
configured_by = None
linter = ProspectorLinter(found_files)
if prospector_config.use_external_config('pylint'):
# try to find a .pylintrc
pylintrc = prospector_config.external_config_location('pylint')
if pylintrc is None:
pylintrc = find_pylintrc()
if pylintrc is None:
pylintrc_path = os.path.join(prospector_config.workdir, '.pylintrc')
if os.path.exists(pylintrc_path):
pylintrc = pylintrc_path
if pylintrc is not None:
# load it!
configured_by = pylintrc
ext_found = True
self._args = linter.load_command_line_configuration(check_paths)
config_messages += self._pylintrc_configure(pylintrc, linter)
if not ext_found:
linter.reset_options()
self._args = linter.load_command_line_configuration(check_paths)
self._prospector_configure(prospector_config, linter)
# Pylint 1.4 introduced the idea of explicitly specifying which C-extensions
# to load. This is because doing so allows them to execute any code whatsoever,
# which is considered to be unsafe. The following option turns off this, allowing
# any extension to load again, since any setup.py can execute arbitrary code and
# the safety gained through this approach seems minimal. Leaving it on means
# that the inference engine cannot do much inference on modules with C-extensions
# which is a bit useless.
linter.set_option('unsafe-load-any-extension', True)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
self._linter = linter
return configured_by, config_messages
class PylintTool(ToolBase):
def __init__(self):
self._args = self._extra_sys_path = None
self._collector = self._linter = None
self._orig_sys_path = []
def prepare(self, rootpath, ignore, args, adaptors):
linter = ProspectorLinter(ignore, rootpath)
linter.load_default_plugins()
extra_sys_path, check_paths = _find_package_paths(ignore, rootpath)
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
for adaptor in adaptors:
adaptor.adapt_pylint(linter)
self._args = linter.load_command_line_configuration(check_paths)
# disable the warnings about disabling warnings...
linter.disable('I0011')
linter.disable('I0012')
linter.disable('I0020')
linter.disable('I0021')
# disable the 'mixed indentation' warning, since it actually will only allow
# the indentation specified in the pylint configuration file; we replace it
# instead with our own version which is more lenient and configurable
linter.disable('W0312')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector()
linter.set_reporter(self._collector)
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if args.max_line_length is not None:
if option[0] == 'max-line-length':
checker.set_option('max-line-length', args.max_line_length)
self._linter = linter
def _combine_w0614(self, messages):
"""
For the "unused import from wildcard import" messages, we want to combine all warnings about
the same line into a single message
"""
by_loc = defaultdict(list)
out = []
for message in messages:
if message.code == 'W0614':
by_loc[message.location].append(message)
else:
out.append(message)
for location, message_list in by_loc.items():
names = []
for msg in message_list:
names.append(_W0614_RE.match(msg.message).group(1))
msgtxt = 'Unused imports from wildcard import: %s' % ', '.join(names)
combined_message = Message('pylint', 'W0614', location, msgtxt)
out.append(combined_message)
return out
def _combine_(self, messages):
pass
def combine(self, messages):
"""
Some error messages are repeated, causing many errors where only one is strictly necessary. For
example, having a wildcard import will result in one 'Unused Import' warning for every unused import.
This method will combine these into a single warning.
"""
combined = self._combine_w0614(messages)
return sorted(combined)
def hide_stdout(self):
self._streams = [sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__]
sys.stdout, sys.stderr = DummyStream(), DummyStream()
sys.__stdout__, sys.__stderr__ = DummyStream(), DummyStream()
def restore_stdout(self):
sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__ = self._streams
del self._streams
def run(self):
# note: Pylint will exit with a status code indicating the health of
# the code it was checking. Prospector will not mimic this behaviour,
# as it interferes with scripts which depend on and expect the exit
# code of the code checker to match whether the check itself was
# successful.
# Additionally, pylint has occasional print statements which can be triggered
# in exceptional cases. If this happens, then the output formatting of
# prospector will be broken (for example, JSON format). Therefore we will
# override stdout to neutralise these errant statements.
# For an example, see
# https://bitbucket.org/logilab/pylint/src/3f8ededd0b1637396937da8fe136f51f2bafb047/checkers/variables.py?at=default#cl-617
# TODO: it'd be nice in the future to do something with this data in case it's useful!
self.hide_stdout()
try:
self._linter.check(self._args)
finally:
self.restore_stdout()
sys.path = self._orig_sys_path
messages = self._collector.get_messages()
return self.combine(messages)
def prepare(self, found_files, args, adaptors):
linter = ProspectorLinter(found_files)
linter.load_default_plugins()
extra_sys_path = found_files.get_minimal_syspath()
# create a list of packages, but don't include packages which are
# subpackages of others as checks will be duplicated
packages = [p.split(os.path.sep) for p in found_files.iter_package_paths(abspath=False)]
packages.sort(key=len)
check_paths = set()
for package in packages:
package_path = os.path.join(*package)
if len(package) == 1:
check_paths.add(package_path)
continue
for i in range(1, len(package)):
if os.path.join(*package[:-i]) in check_paths:
break
else:
check_paths.add(package_path)
for filepath in found_files.iter_module_paths(abspath=False):
package = os.path.dirname(filepath).split(os.path.sep)
for i in range(0, len(package)):
if os.path.join(*package[:i+1]) in check_paths:
break
else:
check_paths.add(filepath)
check_paths = [found_files.to_absolute_path(p) for p in check_paths]
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(extra_sys_path) + sys.path
for adaptor in adaptors:
adaptor.adapt_pylint(linter)
self._args = linter.load_command_line_configuration(check_paths)
# The warnings about disabling warnings are useful for figuring out
# with other tools to suppress messages from. For example, an unused
# import which is disabled with 'pylint:disable=W0611' will still
# generate an 'FL0001' unused import warning from pyflakes. Using the
# information from these messages, we can figure out what was disabled.
linter.disable('I0011') # notification about disabling a message
linter.disable('I0012') # notification about enabling a message
linter.enable('I0013') # notification about disabling an entire file
linter.enable('I0020') # notification about a message being supressed
linter.disable('I0021') # notification about message supressed which was not raised
linter.disable('I0022') # notification about use of deprecated 'pragma' option
# disable the 'mixed indentation' warning, since it actually will only allow
# the indentation specified in the pylint configuration file; we replace it
# instead with our own version which is more lenient and configurable
linter.disable('W0312')
indent_checker = IndentChecker(linter)
linter.register_checker(indent_checker)
# we don't want similarity reports right now
linter.disable('similarities')
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector()
linter.set_reporter(self._collector)
for checker in linter.get_checkers():
if not hasattr(checker, 'options'):
continue
for option in checker.options:
if args.max_line_length is not None:
if option[0] == 'max-line-length':
checker.set_option('max-line-length', args.max_line_length)
self._linter = linter
class PylintTool(ToolBase):
# There are several methods on this class which could technically
# be functions (they don't use the 'self' argument) but that would
# make this module/class a bit ugly.
# pylint:disable=no-self-use
def __init__(self):
self._args = None
self._collector = self._linter = None
self._orig_sys_path = []
def _prospector_configure(self, prospector_config, linter):
errors = []
if "django" in prospector_config.libraries:
linter.load_plugin_modules(["pylint_django"])
if "celery" in prospector_config.libraries:
linter.load_plugin_modules(["pylint_celery"])
if "flask" in prospector_config.libraries:
linter.load_plugin_modules(["pylint_flask"])
profile_path = os.path.join(prospector_config.workdir, prospector_config.profile.name)
for plugin in prospector_config.profile.pylint.get("load-plugins", []):
try:
linter.load_plugin_modules([plugin])
except ImportError:
errors.append(self._error_message(profile_path, f"Could not load plugin {plugin}"))
for msg_id in prospector_config.get_disabled_messages("pylint"):
try:
linter.disable(msg_id)
except UnknownMessageError:
# If the msg_id doesn't exist in PyLint any more,
# don't worry about it.
pass
options = prospector_config.tool_options("pylint")
for checker in linter.get_checkers():
if not hasattr(checker, "options"):
continue
for option in checker.options:
if option[0] in options:
checker.set_option(option[0], options[option[0]])
# The warnings about disabling warnings are useful for figuring out
# with other tools to suppress messages from. For example, an unused
# import which is disabled with 'pylint disable=unused-import' will
# still generate an 'FL0001' unused import warning from pyflakes.
# Using the information from these messages, we can figure out what
# was disabled.
linter.disable("locally-disabled") # notification about disabling a message
linter.enable("file-ignored") # notification about disabling an entire file
linter.enable("suppressed-message") # notification about a message being suppressed
linter.disable("deprecated-pragma") # notification about use of deprecated 'pragma' option
max_line_length = prospector_config.max_line_length
for checker in linter.get_checkers():
if not hasattr(checker, "options"):
continue
for option in checker.options:
if max_line_length is not None:
if option[0] == "max-line-length":
checker.set_option("max-line-length", max_line_length)
return errors
def _error_message(self, filepath, message):
location = Location(filepath, None, None, 0, 0)
return Message("prospector", "config-problem", location, message)
def _pylintrc_configure(self, pylintrc, linter):
errors = []
are_plugins_loaded = linter.config_from_file(pylintrc)
if not are_plugins_loaded and hasattr(linter.config, "load_plugins"):
for plugin in linter.config.load_plugins:
try:
linter.load_plugin_modules([plugin])
except ImportError:
errors.append(self._error_message(pylintrc, f"Could not load plugin {plugin}"))
return errors
def configure(self, prospector_config, found_files):
extra_sys_path = found_files.get_minimal_syspath()
check_paths = self._get_pylint_check_paths(found_files)
pylint_options = prospector_config.tool_options("pylint")
self._set_path_finder(extra_sys_path, pylint_options)
linter = ProspectorLinter(found_files)
config_messages, configured_by = self._get_pylint_configuration(
check_paths, linter, prospector_config, pylint_options
)
# we don't want similarity reports right now
linter.disable("similarities")
# use the collector 'reporter' to simply gather the messages
# given by PyLint
self._collector = Collector(linter.msgs_store)
linter.set_reporter(self._collector)
if linter.config.jobs == 0:
linter.config.jobs = _cpu_count()
self._linter = linter
return configured_by, config_messages
def _set_path_finder(self, extra_sys_path, pylint_options):
# insert the target path into the system path to get correct behaviour
self._orig_sys_path = sys.path
if not pylint_options.get("use_pylint_default_path_finder"):
# note: we prepend, so that modules are preferentially found in the
# path given as an argument. This prevents problems where we are
# checking a module which is already on sys.path before this
# manipulation - for example, if we are checking 'requests' in a local
# checkout, but 'requests' is already installed system wide, pylint
# will discover the system-wide modules first if the local checkout
# does not appear first in the path
sys.path = list(set(list(extra_sys_path) + sys.path))
def _get_pylint_check_paths(self, found_files):
# create a list of packages, but don't include packages which are
# subpackages of others as checks will be duplicated
check_paths = set()
# TODO: removing this janky str->Path->str stuff in 1.8, but for now...
packages = [Path(p).absolute() for p in found_files.iter_package_paths(abspath=True)]
modules = [Path(p).absolute() for p in found_files.iter_module_paths(abspath=True)]
# don't add modules that are in known packages
for module in modules:
for package in packages:
if _is_relative_to(module, package):
break
else:
check_paths.add(module)
# sort from earlier packages first...
packages.sort(key=lambda p: len(str(p)))
for idx, package in enumerate(packages):
# yuck o(n2) but... temporary
for prev_pkg in packages[:idx]:
if _is_relative_to(package, prev_pkg):
# this is a sub-package of a package we know about
break
else:
# we should care about this one
check_paths.add(package)
check_paths = [str(p) for p in check_paths]
return check_paths
def _get_pylint_configuration(self, check_paths, linter, prospector_config, pylint_options):
self._args = linter.load_command_line_configuration(check_paths)
linter.load_default_plugins()
config_messages = self._prospector_configure(prospector_config, linter)
configured_by = None
if prospector_config.use_external_config("pylint"):
# try to find a .pylintrc
pylintrc = pylint_options.get("config_file")
external_config = prospector_config.external_config_location("pylint")
pylintrc = pylintrc or external_config or find_pylintrc()
if pylintrc is None: # nothing explicitly configured
for possible in (".pylintrc", "pylintrc", "pyproject.toml", "setup.cfg"):
pylintrc_path = os.path.join(prospector_config.workdir, possible)
# TODO: pyproject and setup.cfg might not actually have any pylint config
# in, they should be skipped in that case
if os.path.exists(pylintrc_path):
pylintrc = pylintrc_path
break
if pylintrc is not None:
# load it!
configured_by = pylintrc
config_messages += self._pylintrc_configure(pylintrc, linter)
return config_messages, configured_by
def _combine_w0614(self, messages):
"""
For the "unused import from wildcard import" messages,
we want to combine all warnings about the same line into
a single message.
"""
by_loc = defaultdict(list)
out = []
for message in messages:
if message.code == "unused-wildcard-import":
by_loc[message.location].append(message)
else:
out.append(message)
for location, message_list in by_loc.items():
names = []
for msg in message_list:
names.append(_UNUSED_WILDCARD_IMPORT_RE.match(msg.message).group(1))
msgtxt = "Unused imports from wildcard import: %s" % ", ".join(names)
combined_message = Message("pylint", "unused-wildcard-import", location, msgtxt)
out.append(combined_message)
return out
def combine(self, messages):
"""
Combine repeated messages.
Some error messages are repeated, causing many errors where
only one is strictly necessary.
For example, having a wildcard import will result in one
'Unused Import' warning for every unused import.
This method will combine these into a single warning.
"""
combined = self._combine_w0614(messages)
return sorted(combined)
def run(self, found_files) -> List[Message]:
self._linter.check(self._args)
sys.path = self._orig_sys_path
messages = self._collector.get_messages()
return self.combine(messages)