class TestExcluder(TestBase): def setUp(self): super(TestExcluder, self).setUp() excludes_text = textwrap.dedent(""" # ignore C++ .*\.cpp::.* # ignore python .*\.py::Flake8 """) self.excluder = FileExcluder( self._create_scalastyle_excludes_file([excludes_text]), logger) def _create_scalastyle_excludes_file(self, exclude_patterns=None): return self.create_file( relpath='scalastyle_excludes.txt', contents='\n'.join(exclude_patterns) if exclude_patterns else '') def test_excludes_cpp_any(self): self.assertFalse(self.excluder.should_include('test/file.cpp', '.*')) def test_excludes_cpp_flake8(self): self.assertFalse( self.excluder.should_include('test/file.cpp', 'Flake8')) def test_excludes_python_flake8(self): self.assertFalse(self.excluder.should_include('test/file.py', 'Flake8')) def test_excludes_python_trailingws(self): self.assertTrue( self.excluder.should_include('test/file.py', 'TrailingWhiteSpace'))
class TestExcluder(BaseTest): def setUp(self): super(TestExcluder, self).setUp() excludes_text = textwrap.dedent(""" # ignore C++ .*\.cpp::.* # ignore python .*\.py::Flake8 """) self.excluder = FileExcluder( self._create_scalastyle_excludes_file([excludes_text]), logger) def _create_scalastyle_excludes_file(self, exclude_patterns=None): return self.create_file( relpath='scalastyle_excludes.txt', contents='\n'.join(exclude_patterns) if exclude_patterns else '') def test_excludes_cpp_any(self): self.assertFalse(self.excluder.should_include('test/file.cpp', '.*')) def test_excludes_cpp_flake8(self): self.assertFalse(self.excluder.should_include('test/file.cpp', 'Flake8')) def test_excludes_python_flake8(self): self.assertFalse(self.excluder.should_include('test/file.py', 'Flake8')) def test_excludes_python_trailingws(self): self.assertTrue(self.excluder.should_include('test/file.py', 'TrailingWhiteSpace'))
def get_nits(self, python_file): """Iterate over the instances style checker and yield Nits. :param python_file: PythonFile Object """ if noqa_file_filter(python_file): return if self.options.suppress: # Filter out any suppressed plugins excluder = FileExcluder(self.options.suppress, self.context.log) check_plugins = [ plugin for plugin in self._plugins if excluder.should_include(python_file.filename, plugin.name) ] else: check_plugins = self._plugins for plugin in check_plugins: for nit in plugin.checker(python_file): if nit._line_number is None: yield nit continue nit_slice = python_file.line_range(nit._line_number) for line_number in range(nit_slice.start, nit_slice.stop): if noqa_line_filter(python_file, line_number): break else: yield nit
def get_nits(self, python_file): """Iterate over the instances style checker and yield Nits. :param python_file: PythonFile Object """ if noqa_file_filter(python_file): return if self.options.suppress: # Filter out any suppressed plugins excluder = FileExcluder(self.options.suppress, self.context.log) check_plugins = [plugin for plugin in self._plugins if excluder.should_include(python_file.filename, plugin.name)] else: check_plugins = self._plugins for plugin in check_plugins: for nit in plugin.checker(python_file): if nit._line_number is None: yield nit continue nit_slice = python_file.line_range(nit._line_number) for line_number in range(nit_slice.start, nit_slice.stop): if noqa_line_filter(python_file, line_number): break else: yield nit
class PythonCheckStyleTask(PythonTask): _PYTHON_SOURCE_EXTENSION = '.py' _plugins = [] _subsystems = tuple() def __init__(self, *args, **kwargs): super(PythonCheckStyleTask, self).__init__(*args, **kwargs) self._plugins = [plugin for plugin in self._plugins if not plugin.skip()] self.options = self.get_options() self.excluder = FileExcluder(self.options.suppress, self.context.log) @classmethod def global_subsystems(cls): return super(PythonTask, cls).global_subsystems() + cls._subsystems @classmethod def register_options(cls, register): super(PythonCheckStyleTask, cls).register_options(register) register('--severity', fingerprint=True, default='COMMENT', type=str, help='Only messages at this severity or higher are logged. [COMMENT WARNING ERROR].') register('--strict', fingerprint=True, default=False, action='store_true', help='If enabled, have non-zero exit status for any nit at WARNING or higher.') # Skip short circuits before fingerprinting register('--skip', default=False, action='store_true', help='If enabled, skip this style checker.') register('--suppress', fingerprint=True, type=file_option, default=None, help='Takes a XML file where specific rules on specific files will be skipped.') register('--fail', fingerprint=True, default=True, action='store_true', help='Prevent test failure but still produce output for problems.') @classmethod def supports_passthru_args(cls): return True def _is_checked(self, target): return isinstance(target, PythonTarget) and target.has_sources(self._PYTHON_SOURCE_EXTENSION) @classmethod def clear_plugins(cls): """Clear all current plugins registered.""" cls._plugins = [] @classmethod def register_plugin(cls, name, subsystem): """Register plugin to be run as part of Python Style checks. :param string name: Name of the plugin. :param PluginSubsystemBase subsystem: Plugin subsystem subclass. """ plugin = LintPlugin(name=name, subsystem=subsystem) cls._plugins.append(plugin) cls._subsystems += (plugin.subsystem, ) def get_nits(self, python_file): """Iterate over the instances style checker and yield Nits. :param python_file: PythonFile Object """ if noqa_file_filter(python_file): return if self.options.suppress: # Filter out any suppressed plugins check_plugins = [plugin for plugin in self._plugins if self.excluder.should_include(python_file.filename, plugin.name)] else: check_plugins = self._plugins for plugin in check_plugins: for nit in plugin.checker(python_file): if nit._line_number is None: yield nit continue nit_slice = python_file.line_range(nit._line_number) for line_number in range(nit_slice.start, nit_slice.stop): if noqa_line_filter(python_file, line_number): break else: yield nit def check_file(self, filename): """Process python file looking for indications of problems. :param filename: (str) Python source filename :return: (int) number of failures """ try: python_file = PythonFile.parse(os.path.join(get_buildroot(), filename)) except SyntaxError as e: print('{filename}:SyntaxError: {error}'.format(filename=filename, error=e)) return 1 # If the user specifies an invalid severity use comment. severity = Nit.SEVERITY.get(self.options.severity, Nit.COMMENT) failure_count = 0 fail_threshold = Nit.WARNING if self.options.strict else Nit.ERROR for i, nit in enumerate(self.get_nits(python_file)): if i == 0: print() # Add an extra newline to clean up the output only if we have nits. if nit.severity >= severity: print('{nit}\n'.format(nit=nit)) if nit.severity >= fail_threshold: failure_count += 1 return failure_count def checkstyle(self, sources): """Iterate over sources and run checker on each file. Files can be suppressed with a --suppress option which takes an xml file containing file paths that have exceptions and the plugins they need to ignore. :param sources: iterable containing source file names. :return: (int) number of failures """ failure_count = 0 for filename in sources: failure_count += self.check_file(filename) if failure_count > 0 and self.options.fail: raise TaskError('{} Python Style issues found'.format(failure_count), exit_code=1) return failure_count def execute(self): """Run Checkstyle on all found source files.""" if self.options.skip: return with self.invalidated(self.context.targets(self._is_checked)) as invalidation_check: sources = self.calculate_sources([vt.target for vt in invalidation_check.invalid_vts]) if sources: return self.checkstyle(sources) def calculate_sources(self, targets): """Generate a set of source files from the given targets.""" sources = set() for target in targets: sources.update( source for source in target.sources_relative_to_buildroot() if source.endswith(self._PYTHON_SOURCE_EXTENSION) ) return sources
class PythonCheckStyleTask(PythonTask): _PYTHON_SOURCE_EXTENSION = '.py' _plugins = [] _subsystems = tuple() def __init__(self, *args, **kwargs): super(PythonCheckStyleTask, self).__init__(*args, **kwargs) self._plugins = [ plugin for plugin in self._plugins if not plugin.skip() ] self.options = self.get_options() self.excluder = FileExcluder(self.options.suppress, self.context.log) @classmethod def global_subsystems(cls): return super(PythonTask, cls).global_subsystems() + cls._subsystems @classmethod def register_options(cls, register): super(PythonCheckStyleTask, cls).register_options(register) register( '--severity', fingerprint=True, default='COMMENT', type=str, help= 'Only messages at this severity or higher are logged. [COMMENT WARNING ERROR].' ) register( '--strict', fingerprint=True, type=bool, help= 'If enabled, have non-zero exit status for any nit at WARNING or higher.' ) # Skip short circuits before fingerprinting register('--skip', type=bool, help='If enabled, skip this style checker.') register( '--suppress', fingerprint=True, type=file_option, default=None, help= 'Takes a XML file where specific rules on specific files will be skipped.' ) register( '--fail', fingerprint=True, default=True, type=bool, help='Prevent test failure but still produce output for problems.') @classmethod def supports_passthru_args(cls): return True def _is_checked(self, target): return isinstance(target, PythonTarget) and target.has_sources( self._PYTHON_SOURCE_EXTENSION) @classmethod def clear_plugins(cls): """Clear all current plugins registered.""" cls._plugins = [] @classmethod def register_plugin(cls, name, subsystem): """Register plugin to be run as part of Python Style checks. :param string name: Name of the plugin. :param PluginSubsystemBase subsystem: Plugin subsystem subclass. """ plugin = LintPlugin(name=name, subsystem=subsystem) cls._plugins.append(plugin) cls._subsystems += (plugin.subsystem, ) def get_nits(self, filename): """Iterate over the instances style checker and yield Nits. :param filename: str pointing to a file within the buildroot. """ try: python_file = PythonFile.parse(filename, root=get_buildroot()) except CheckSyntaxError as e: yield e.as_nit() return if noqa_file_filter(python_file): return if self.options.suppress: # Filter out any suppressed plugins check_plugins = [ plugin for plugin in self._plugins if self.excluder.should_include(filename, plugin.name) ] else: check_plugins = self._plugins for plugin in check_plugins: for i, nit in enumerate(plugin.checker(python_file)): if i == 0: # NB: Add debug log header for nits from each plugin, but only if there are nits from it. self.context.log.debug('Nits from plugin {} for {}'.format( plugin.name, filename)) if not nit.has_lines_to_display: yield nit continue if all(not line_contains_noqa(line) for line in nit.lines): yield nit def check_file(self, filename): """Process python file looking for indications of problems. :param filename: (str) Python source filename :return: (int) number of failures """ # If the user specifies an invalid severity use comment. log_threshold = Nit.SEVERITY.get(self.options.severity, Nit.COMMENT) failure_count = 0 fail_threshold = Nit.WARNING if self.options.strict else Nit.ERROR for i, nit in enumerate(self.get_nits(filename)): if i == 0: print( ) # Add an extra newline to clean up the output only if we have nits. if nit.severity >= log_threshold: print('{nit}\n'.format(nit=nit)) if nit.severity >= fail_threshold: failure_count += 1 return failure_count def checkstyle(self, sources): """Iterate over sources and run checker on each file. Files can be suppressed with a --suppress option which takes an xml file containing file paths that have exceptions and the plugins they need to ignore. :param sources: iterable containing source file names. :return: (int) number of failures """ failure_count = 0 for filename in sources: failure_count += self.check_file(filename) if failure_count > 0 and self.options.fail: raise TaskError( '{} Python Style issues found. For import order related issues, please try ' '`./pants fmt.isort <targets>`'.format(failure_count)) return failure_count def execute(self): """Run Checkstyle on all found source files.""" if self.options.skip: return with self.invalidated(self.context.targets( self._is_checked)) as invalidation_check: sources = self.calculate_sources( [vt.target for vt in invalidation_check.invalid_vts]) if sources: return self.checkstyle(sources) def calculate_sources(self, targets): """Generate a set of source files from the given targets.""" sources = set() for target in targets: sources.update( source for source in target.sources_relative_to_buildroot() if source.endswith(self._PYTHON_SOURCE_EXTENSION)) return sources
class PythonCheckStyleTask(LintTaskMixin, Task): _PYTHON_SOURCE_EXTENSION = '.py' _plugins = [] _subsystems = tuple() def __init__(self, *args, **kwargs): super(PythonCheckStyleTask, self).__init__(*args, **kwargs) self._plugins = [plugin for plugin in self._plugins if not plugin.skip()] self.options = self.get_options() self.excluder = FileExcluder(self.options.suppress, self.context.log) @classmethod def subsystem_dependencies(cls): return super(Task, cls).subsystem_dependencies() + cls._subsystems @classmethod def register_options(cls, register): super(PythonCheckStyleTask, cls).register_options(register) register('--severity', fingerprint=True, default='COMMENT', type=str, help='Only messages at this severity or higher are logged. [COMMENT WARNING ERROR].') register('--strict', fingerprint=True, type=bool, help='If enabled, have non-zero exit status for any nit at WARNING or higher.') register('--suppress', fingerprint=True, type=file_option, default=None, help='Takes a XML file where specific rules on specific files will be skipped.') register('--fail', fingerprint=True, default=True, type=bool, help='Prevent test failure but still produce output for problems.') @classmethod def supports_passthru_args(cls): return True def _is_checked(self, target): return (not target.is_synthetic and isinstance(target, PythonTarget) and target.has_sources(self._PYTHON_SOURCE_EXTENSION)) @classmethod def clear_plugins(cls): """Clear all current plugins registered.""" cls._plugins = [] @classmethod def register_plugin(cls, name, subsystem): """Register plugin to be run as part of Python Style checks. :param string name: Name of the plugin. :param PluginSubsystemBase subsystem: Plugin subsystem subclass. """ plugin = LintPlugin(name=name, subsystem=subsystem) cls._plugins.append(plugin) cls._subsystems += (plugin.subsystem, ) def get_nits(self, filename): """Iterate over the instances style checker and yield Nits. :param filename: str pointing to a file within the buildroot. """ try: python_file = PythonFile.parse(filename, root=get_buildroot()) except CheckSyntaxError as e: yield e.as_nit() return if noqa_file_filter(python_file): return if self.options.suppress: # Filter out any suppressed plugins check_plugins = [plugin for plugin in self._plugins if self.excluder.should_include(filename, plugin.name)] else: check_plugins = self._plugins for plugin in check_plugins: for i, nit in enumerate(plugin.checker(python_file)): if i == 0: # NB: Add debug log header for nits from each plugin, but only if there are nits from it. self.context.log.debug('Nits from plugin {} for {}'.format(plugin.name, filename)) if not nit.has_lines_to_display: yield nit continue if all(not line_contains_noqa(line) for line in nit.lines): yield nit def check_file(self, filename): """Process python file looking for indications of problems. :param filename: (str) Python source filename :return: (int) number of failures """ # If the user specifies an invalid severity use comment. log_threshold = Nit.SEVERITY.get(self.options.severity, Nit.COMMENT) failure_count = 0 fail_threshold = Nit.WARNING if self.options.strict else Nit.ERROR for i, nit in enumerate(self.get_nits(filename)): if i == 0: print() # Add an extra newline to clean up the output only if we have nits. if nit.severity >= log_threshold: print('{nit}\n'.format(nit=nit)) if nit.severity >= fail_threshold: failure_count += 1 return failure_count def checkstyle(self, sources): """Iterate over sources and run checker on each file. Files can be suppressed with a --suppress option which takes an xml file containing file paths that have exceptions and the plugins they need to ignore. :param sources: iterable containing source file names. :return: (int) number of failures """ failure_count = 0 for filename in sources: failure_count += self.check_file(filename) if failure_count > 0 and self.options.fail: raise TaskError( '{} Python Style issues found. You may try `./pants fmt <targets>`'.format(failure_count)) return failure_count def execute(self): """Run Checkstyle on all found non-synthetic source files.""" with self.invalidated(self.get_targets(self._is_checked)) as invalidation_check: sources = self.calculate_sources([vt.target for vt in invalidation_check.invalid_vts]) if sources: return self.checkstyle(sources) def calculate_sources(self, targets): """Generate a set of source files from the given targets.""" sources = set() for target in targets: sources.update( source for source in target.sources_relative_to_buildroot() if source.endswith(self._PYTHON_SOURCE_EXTENSION) ) return sources