def run_command(cls, file_list: List[str], **__):
style_guide = StyleGuide(paths=file_list, config_file=".pycodestyle")
report = style_guide.check_files()
if report.total_errors:
sys.stderr.write(str(report.total_errors) + "\n")
sys.exit(1)
def test_pep8():
report = StyleGuide(ignore=['E501', 'E402']).check_files(
[dirname(abspath(__file__)) + '/../'])
report.print_statistics()
if report.messages:
raise Exception("pep8")
def pycodestyle(ticket, **kwds):
"""
run ``pycodestyle --select=W605`` on the modified .py files
we do not check the files names "all.py" and "__init__.py" that
usually just contain unused import lines
same thing for files named "*catalog*.py"
"""
changed_files = list(
subprocess.Popen([
'git', 'diff', '--name-only',
'patchbot/base..patchbot/ticket_merged'
],
stdout=subprocess.PIPE).stdout)
changed_files = [f.decode('utf8').strip("\n") for f in changed_files]
style = StyleGuide(select=['W605'])
errors = 0
for a_file in changed_files:
if os.path.exists(a_file) and os.path.splitext(a_file)[1] == '.py':
filename = os.path.split(a_file)[1]
if not (filename == "all.py" or filename == "__init__.py"
or "catalog" in filename):
rep = style.check_files([a_file])
errors = rep.file_errors
# this should print the errors out, with files and lines
full_msg = "found {} invalid escape sequences in the modified files"
full_msg = full_msg.format(errors)
print(full_msg)
if errors:
raise ValueError(full_msg)
def cleopatra(path, name, language, **options):
"""
Cleopatra.
Checks for code style.
Requirements for passing:
- For the given file, the number of PEP 8 violations must not exceed
MAX_LINT.
- No more than 1000 lines long.
- By default, MAX_LINT is 5.
Language requirements:
- Only Python is currently supported.
"""
MAX_LINT = options.get('MAX_LINT', 5)
MAX_LOC = options.get('MAX_LOC', 1000)
sg = StyleGuide(paths=[str(path), '--count', '-qq'])
errs = sg.check_files().total_errors
del sg
assert errs <= MAX_LINT
with open(str(path)) as fp:
loc = len(fp.readlines())
assert loc <= MAX_LOC
def pycodestyle(ticket, **kwds):
"""
run ``pycodestyle --select=W605`` on the modified .py files
we do not check the files names "all.py" and "__init__.py" that
usually just contain unused import lines
same thing for files named "*catalog*.py"
"""
changed_files = list(subprocess.Popen(['git', 'diff', '--name-only', 'patchbot/base..patchbot/ticket_merged'], stdout=subprocess.PIPE).stdout)
changed_files = [f.decode('utf8').strip("\n") for f in changed_files]
style = StyleGuide(select=['W605'])
errors = 0
for a_file in changed_files:
if os.path.exists(a_file) and os.path.splitext(a_file)[1] == '.py':
filename = os.path.split(a_file)[1]
if not (filename == "all.py" or filename == "__init__.py" or
"catalog" in filename):
rep = style.check_files([a_file])
errors = rep.file_errors
# this should print the errors out, with files and lines
full_msg = "found {} invalid escape sequences in the modified files"
full_msg = full_msg.format(errors)
print(full_msg)
if errors:
raise ValueError(full_msg)
def test_pep8_conformance(self):
pep8style = StyleGuide(show_source=True)
# files = (['add_weeks.py', 'google_finance.py'])
from list_files import list_python_files
files = list_python_files('.')
print('Checking', files, 'for pep8 conformance.')
pep8style.check_files(files)
self.assertEqual(pep8style.options.report.total_errors, 0)
def process_module(self, node):
'''
process a module
the module's content is accessible via node.file_stream object
'''
nodepaths = []
if not isinstance(node.path, list):
nodepaths = [node.path]
else:
nodepaths = node.path
for node_path in nodepaths:
if node_path not in _PROCESSED_NODES:
stylechecker = StyleGuide(parse_argv=False,
config_file=False,
quiet=2,
reporter=PyLintPEP8Reporter)
_PROCESSED_NODES[node_path] = stylechecker.check_files(
[node_path])
for code, lineno, text in _PROCESSED_NODES[node_path].locations:
pylintcode = '{0}8{1}'.format(code[0], code[1:])
if pylintcode in self.msgs_map:
# This will be handled by PyLint itself, skip it
continue
if pylintcode not in _KNOWN_PEP8_IDS:
if pylintcode not in _UNHANDLED_PEP8_IDS:
_UNHANDLED_PEP8_IDS.append(pylintcode)
self.add_message('W8888',
line=lineno,
args=(code, text))
continue
if pylintcode not in self._msgs:
# Not for our class implementation to handle
continue
if code in ('E111', 'E113'):
if _PROCESSED_NODES[node_path].lines[
lineno - 1].strip().startswith('#'):
# If E111 is triggered in a comment I consider it, at
# least, bad judgement. See https://github.com/jcrocholl/pep8/issues/300
# If E113 is triggered in comments, which I consider a bug,
# skip it. See https://github.com/jcrocholl/pep8/issues/274
continue
try:
self.add_message(pylintcode,
line=lineno,
args=(code, text))
except TypeError as exc:
if 'not all arguments' not in str(exc):
raise
# Message does not support being passed the text arg
self.add_message(pylintcode, line=lineno, args=(code, ))
def test_pep8():
report = StyleGuide(ignore=['E126', 'E501', 'E402']).check_files([
os.path.dirname(os.path.abspath(__file__)) + '/../spark_helpers',
'/shared/fabfile.py'
])
report.print_statistics()
if report.messages:
raise Exception("pep8")
def test_codestyle_conformance(self):
# Ignore the following errors:
# E501 line too long (xxx > 79 characters)
codestyle = StyleGuide(show_source=True, ignore=['E501'])
# files = (['add_weeks.py', 'google_finance.py'])
from list_files import list_python_files
files = list_python_files('.')
print('Checking', files, 'for codestyle conformance.')
report = codestyle.check_files(files)
report.print_statistics()
self.assertEqual(codestyle.options.report.total_errors, 0)
def init_tests(pep8style: StyleGuide) -> None:
"""
Initialize testing framework.
A test file can provide many tests. Each test starts with a
declaration. This declaration is a single line starting with '#:'.
It declares codes of expected failures, separated by spaces or 'Okay'
if no failure is expected.
If the file does not contain such declaration, it should pass all
tests. If the declaration is empty, following lines are not checked,
until next declaration.
Examples:
* Only E224 and W701 are expected: #: E224 W701
* Following example is conform: #: Okay
* Don't check these lines: #:
"""
report = pep8style.init_report(TestReport)
runner = pep8style.input_file
def run_tests(filename: str) -> int:
"""Run all the tests from a file."""
lines = readlines(filename) + ['#:\n']
line_offset = 0
codes = ['Okay']
testcase = []
count_files = report.counters['files']
for index, line in enumerate(lines):
if not line.startswith('#:'):
if codes:
# Collect the lines of the test case
testcase.append(line)
continue
if codes and index:
if 'noeol' in codes:
testcase[-1] = testcase[-1].rstrip('\n')
codes = [c for c in codes if c not in ('Okay', 'noeol')]
# Run the checker
runner(filename,
testcase,
expected=codes,
line_offset=line_offset)
# output the real line numbers
line_offset = index + 1
# configure the expected errors
codes = line.split()[1:]
# empty the test case buffer
del testcase[:]
report.counters['files'] = count_files + 1
return report.counters['failed tests']
pep8style.runner = run_tests
def init_tests(pep8style: StyleGuide) -> None:
"""
Initialize testing framework.
A test file can provide many tests. Each test starts with a
declaration. This declaration is a single line starting with '#:'.
It declares codes of expected failures, separated by spaces or 'Okay'
if no failure is expected.
If the file does not contain such declaration, it should pass all
tests. If the declaration is empty, following lines are not checked,
until next declaration.
Examples:
* Only E224 and W701 are expected: #: E224 W701
* Following example is conform: #: Okay
* Don't check these lines: #:
"""
report = pep8style.init_report(TestReport)
runner = pep8style.input_file
def run_tests(filename: str) -> int:
"""Run all the tests from a file."""
lines = readlines(filename) + ['#:\n']
line_offset = 0
codes = ['Okay']
testcase = []
count_files = report.counters['files']
for index, line in enumerate(lines):
if not line.startswith('#:'):
if codes:
# Collect the lines of the test case
testcase.append(line)
continue
if codes and index:
if 'noeol' in codes:
testcase[-1] = testcase[-1].rstrip('\n')
codes = [c for c in codes
if c not in ('Okay', 'noeol')]
# Run the checker
runner(filename, testcase, expected=codes,
line_offset=line_offset)
# output the real line numbers
line_offset = index + 1
# configure the expected errors
codes = line.split()[1:]
# empty the test case buffer
del testcase[:]
report.counters['files'] = count_files + 1
return report.counters['failed tests']
pep8style.runner = run_tests
def test_format_pep8():
"""
Test if pep8 is respected.
"""
pep8_checker = StyleGuide()
files_to_check = []
for path in list_files(".py"):
rel_path = os.path.relpath(path, cleverhans.__path__[0])
if rel_path in whitelist_pep8:
continue
else:
files_to_check.append(path)
report = pep8_checker.check_files(files_to_check)
if report.total_errors > 0:
raise AssertionError("PEP8 Format not respected")
def run(path, code=None, params=None, **meta):
"""Check code with pycodestyle.
:return list: List of errors.
"""
parser = get_parser()
for option in parser.option_list:
if option.dest and option.dest in params:
value = params[option.dest]
if not isinstance(value, str):
continue
params[option.dest] = option.convert_value(option, params[option.dest])
P8Style = StyleGuide(reporter=_PycodestyleReport, **params)
buf = StringIO(code)
return P8Style.input_file(path, lines=buf.readlines())
def find_issues(check_files, dirnames):
"""
Finds all issues in the given directories (filtered by check_files).
"""
checker = StyleGuide()
checker.options.ignore = IGNORE_ERRORS
filenames = dirnames
if check_files is not None:
filenames = filter_file_list(check_files, dirnames)
report = checker.check_files(filenames)
if report.messages:
yield ("style issue", "python code violates pep8")
def run_tests(self):
from pycodestyle import StyleGuide
package_dir = os.path.dirname(os.path.abspath(__file__))
sources = glob(os.path.join(package_dir, 'texel_assignment', '*.py'))
style_guide = StyleGuide(paths=sources)
options = style_guide.options
report = style_guide.check_files()
report.print_statistics()
if report.total_errors:
if options.count:
sys.stderr.write(str(report.total_errors) + '\n')
sys.exit(1)
def test_format_pep8():
"""
Test if pep8 is respected.
"""
pep8_checker = StyleGuide()
files_to_check = []
for path in list_files(".py"):
rel_path = os.path.relpath(path, cleverhans.__path__[0])
if rel_path in whitelist_pep8:
continue
else:
files_to_check.append(path)
report = pep8_checker.check_files(files_to_check)
if report.total_errors > 0:
raise AssertionError("PEP8 Format not respected")
def find_issues(check_files, dirnames):
"""
Finds all issues in the given directories (filtered by check_files).
"""
checker = StyleGuide()
checker.options.ignore = IGNORE_ERRORS
filenames = dirnames
if check_files is not None:
filenames = filter_file_list(check_files, dirnames)
report = checker.check_files(filenames)
if len(report.messages) > 0:
yield ("style issue", "python code violates pep8")
def run(path, code=None, params=None, **meta):
"""Check code with pycodestyle.
:return list: List of errors.
"""
parser = get_parser()
for option in parser.option_list:
if option.dest and option.dest in params:
value = params[option.dest]
if not isinstance(value, str):
continue
params[option.dest] = option.convert_value(
option, params[option.dest])
P8Style = StyleGuide(reporter=_PycodestyleReport, **params)
buf = StringIO(code)
return P8Style.input_file(path, lines=buf.readlines())
class PyCodeStyle(Checker):
def __init__(self, options: Optional[dict] = None):
options = options or {}
self._style = StyleGuide(
select="E,W", reporter=PyCodeStyle.ErrorReport, **options
)
self._style.options.max_line_length = 88
def check(self, filepaths: List[str]) -> List[Message]:
report = self._style.check_files(filepaths)
return [self._to_msg(e) for e in report.errors]
@staticmethod
def _to_msg(err: Tuple) -> Message:
return Message(
code=err[3], description=err[4], filepath=err[0], line=err[1], column=err[2]
)
class ErrorReport(BaseReport):
def __init__(self, options):
super().__init__(options)
self.errors = []
def error(self, line_number, offset, text, check):
code = super().error(line_number, offset, text, check)
if code:
self.errors.append(
(self.filename, line_number, offset, code, text[5:], check.__doc__)
)
def run(path, code=None, params=None, **meta):
"""Check code with pycodestyle.
:return list: List of errors.
"""
parser = get_parser()
for option in parser.option_list:
if option.dest and option.dest in params:
value = params[option.dest]
if isinstance(value, str):
params[option.dest] = option.convert_value(option, value)
for key in ["filename", "exclude", "select", "ignore"]:
if key in params and isinstance(params[key], str):
params[key] = _parse_multi_options(params[key])
P8Style = StyleGuide(reporter=_PycodestyleReport, **params)
buf = StringIO(code)
return P8Style.input_file(path, lines=buf.readlines())
def run(path, code=None, params=None, **meta):
"""Check code with pycodestyle.
:return list: List of errors.
"""
parser = get_parser()
for option in parser.option_list:
if option.dest and option.dest in params:
value = params[option.dest]
if isinstance(value, str):
params[option.dest] = option.convert_value(option, value)
for key in ["filename", "exclude", "select", "ignore"]:
if key in params and isinstance(params[key], str):
params[key] = _parse_multi_options(params[key])
P8Style = StyleGuide(reporter=_PycodestyleReport, **params)
buf = StringIO(code)
return P8Style.input_file(path, lines=buf.readlines())
def check_easyconfigs_style(easyconfigs, verbose=False):
"""
Check the given list of easyconfigs for style
:param: easyconfigs list of file paths to easyconfigs
:param: verbose print our statistics and be verbose about the errors and warning
:return: the number of warnings and errors
"""
# importing autopep8 changes some pep8 functions.
# We reload it to be sure to get the real pep8 functions.
if 'pycodestyle' in sys.modules:
reload(pycodestyle)
else:
reload(pep8)
# register the extra checks before using pep8:
# any function in this module starting with `_eb_check_` will be used.
cands = globals()
for check_function in sorted([
cands[f] for f in cands
if callable(cands[f]) and f.startswith(EB_CHECK)
]):
_log.debug("Adding custom style check %s", check_function)
register_check(check_function)
styleguide = StyleGuide(quiet=False, config_file=None)
options = styleguide.options
# we deviate from standard pep8 and allow 120 chars
# on a line: the default of 79 is too narrow.
options.max_line_length = MAX_LINE_LENGTH
# we ignore some tests
# note that W291 has been replaced by our custom W299
options.ignore = (
'W291', # replaced by W299
)
options.verbose = int(verbose)
result = styleguide.check_files(easyconfigs)
if verbose:
result.print_statistics()
return result.total_errors
def _main():
"""Parse options and run checks on Python source."""
import signal
# Handle "Broken pipe" gracefully
try:
signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1))
except AttributeError:
pass # not supported on Windows
parser = get_parser()
parser.add_option("--grade-config", action="append", dest="grade_config")
parser.add_option("--max-points", type="float", dest="max_points")
style_guide = StyleGuide(parser=parser, parse_argv=True)
options = style_guide.options
style_guide.init_report(TapErrorReport)
report = style_guide.check_files()
report.print_results()
def _main():
"""Parse options and run checks on Python source."""
import signal
# Handle "Broken pipe" gracefully
try:
signal.signal(signal.SIGPIPE, lambda signum, frame: sys.exit(1))
except AttributeError:
pass # not supported on Windows
parser = get_parser()
parser.add_option("--grade-config", action="append", dest="grade_config")
parser.add_option("--max-points", type="float", dest="max_points")
style_guide = StyleGuide(parser=parser, parse_argv=True)
options = style_guide.options
style_guide.init_report(TapErrorReport)
report = style_guide.check_files()
report.print_results()
def check_easyconfigs_style(easyconfigs, verbose=False):
"""
Check the given list of easyconfigs for style
:param: easyconfigs list of file paths to easyconfigs
:param: verbose print our statistics and be verbose about the errors and warning
:return: the number of warnings and errors
"""
# importing autopep8 changes some pep8 functions.
# We reload it to be sure to get the real pep8 functions.
if 'pycodestyle' in sys.modules:
reload(pycodestyle)
else:
reload(pep8)
# register the extra checks before using pep8:
# any function in this module starting with `_eb_check_` will be used.
cands = globals()
for check_function in sorted([cands[f] for f in cands if callable(cands[f]) and f.startswith(EB_CHECK)]):
_log.debug("Adding custom style check %s", check_function)
register_check(check_function)
styleguide = StyleGuide(quiet=False, config_file=None)
options = styleguide.options
# we deviate from standard pep8 and allow 120 chars
# on a line: the default of 79 is too narrow.
options.max_line_length = MAX_LINE_LENGTH
# we ignore some tests
# note that W291 has been replaced by our custom W299
options.ignore = (
'W291', # replaced by W299
)
options.verbose = int(verbose)
result = styleguide.check_files(easyconfigs)
if verbose:
result.print_statistics()
return result.total_errors
def check_pycodestyle(code):
"""
Given some code, uses the PyCodeStyle module (was PEP8) to return a list
of items describing issues of coding style. See:
https://pycodestyle.readthedocs.io/en/latest/intro.html
"""
# PyCodeStyle reads input from files, so make a temporary file containing
# the code.
code_fd, code_filename = tempfile.mkstemp()
os.close(code_fd)
with open(code_filename, 'w', newline='') as code_file:
write_and_flush(code_file, code)
# Configure which PEP8 rules to ignore.
ignore = ('E121', 'E123', 'E126', 'E226', 'E302', 'E305', 'E24', 'E704',
'W291', 'W292', 'W293', 'W391', 'W503', )
style = StyleGuide(parse_argv=False, config_file=False)
style.options.ignore = ignore
checker = Checker(code_filename, options=style.options)
# Re-route stdout to a temporary buffer to be parsed below.
temp_out = io.StringIO()
sys.stdout = temp_out
# Check the code.
checker.check_all()
# Put stdout back and read the content of the buffer. Remove the temporary
# file created at the start.
sys.stdout = sys.__stdout__
temp_out.seek(0)
results = temp_out.read()
temp_out.close()
code_file.close()
os.remove(code_filename)
# Parse the output from the tool into a dictionary of structured data.
style_feedback = {}
for result in results.split('\n'):
matcher = STYLE_REGEX.match(result)
if matcher:
line_no, col, msg = matcher.groups()
line_no = int(line_no) - 1
code, description = msg.split(' ', 1)
if code == 'E303':
description += _(' above this line')
if line_no not in style_feedback:
style_feedback[line_no] = []
style_feedback[line_no].append({
'line_no': line_no,
'column': int(col) - 1,
'message': description.capitalize(),
'code': code,
})
return style_feedback
def print_files_information_pep8():
"""
Print the list of files which can be removed from the whitelist and the
list of files which do not respect PEP8 formatting that aren't in the
whitelist
"""
infracting_files = []
non_infracting_files = []
pep8_checker = StyleGuide(quiet=True)
for path in list_files(".py"):
number_of_infractions = pep8_checker.input_file(path)
rel_path = os.path.relpath(path, cleverhans.__path__[0])
if number_of_infractions > 0:
if rel_path not in whitelist_pep8:
infracting_files.append(path)
else:
if rel_path in whitelist_pep8:
non_infracting_files.append(path)
print("Files that must be corrected or added to whitelist:")
for file in infracting_files:
print(file)
print("Files that can be removed from whitelist:")
for file in non_infracting_files:
print(file)
def print_files_information_pep8():
"""
Print the list of files which can be removed from the whitelist and the
list of files which do not respect PEP8 formatting that aren't in the
whitelist
"""
infracting_files = []
non_infracting_files = []
pep8_checker = StyleGuide(quiet=True)
for path in list_files(".py"):
number_of_infractions = pep8_checker.input_file(path)
rel_path = os.path.relpath(path, cleverhans.__path__[0])
if number_of_infractions > 0:
if rel_path not in whitelist_pep8:
infracting_files.append(path)
else:
if rel_path in whitelist_pep8:
non_infracting_files.append(path)
print("Files that must be corrected or added to whitelist:")
for file in infracting_files:
print(file)
print("Files that can be removed from whitelist:")
for file in non_infracting_files:
print(file)
def run_tests(style: StyleGuide) -> BaseReport:
options = style.options
if options.doctest:
import doctest
fail_d, done_d = doctest.testmod(report=False, verbose=options.verbose)
fail_s, done_s = selftest(options)
count_failed = fail_s + fail_d
if not options.quiet:
count_passed = done_d + done_s - count_failed
print("%d passed and %d failed." % (count_passed, count_failed))
print("Test failed." if count_failed else "Test passed.")
if count_failed:
sys.exit(1)
if options.testsuite:
init_tests(style)
return style.check_files()
def run_tests(style: StyleGuide) -> BaseReport:
options = style.options
if options.doctest:
import doctest
fail_d, done_d = doctest.testmod(report=False, verbose=options.verbose)
fail_s, done_s = selftest(options)
count_failed = fail_s + fail_d
if not options.quiet:
count_passed = done_d + done_s - count_failed
print("%d passed and %d failed." % (count_passed, count_failed))
print("Test failed." if count_failed else "Test passed.")
if count_failed:
sys.exit(1)
if options.testsuite:
init_tests(style)
return style.check_files()
def run_check(self, ctx: RunContext): # noqa
"""Check code with pycodestyle."""
params = ctx.get_params("pycodestyle")
options = ctx.options
if options:
params.setdefault("max_line_length", options.max_line_length)
if params:
parser = get_parser()
for option in parser.option_list:
if option.dest and option.dest in params:
value = params[option.dest]
if isinstance(value, str):
params[option.dest] = option.convert_value(
option, value)
style = StyleGuide(reporter=_PycodestyleReport, **params)
options = style.options
options.report.ctx = ctx # type: ignore
checker = Checker(ctx.filename, lines=ctx.lines, options=options)
checker.check_all()
except ImportError:
Checker = None
while True:
size = stdin.buffer.read(10)
size = int(size)
if not size > 0:
continue
buf = bytes()
while len(buf) < size:
buf += stdin.buffer.read(min(1024, size - len(buf)))
lines = buf.decode("utf-8").splitlines()
opts, text = lines[:3], [l + "\n" for l in lines[3:]]
if Checker is not None:
style_guide = StyleGuide()
options = style_guide.options
select = [x for x in opts[0].strip().split(',') if len(x) > 0]
ignore = [x for x in opts[1].strip().split(',') if len(x) > 0]
options.select = tuple(select)
options.ignore = tuple(ignore)
options.max_line_length = int(opts[2])
stderr.flush()
c = Checker(lines=text, options=options)
output = StringIO()
with redirect_stdout(output):
# writes directly to stdout, so catch it ...
c.check_all()
output = output.getvalue()
output = output[:2**15]
def __init__(self, options: Optional[dict] = None):
options = options or {}
self._style = StyleGuide(
select="E,W", reporter=PyCodeStyle.ErrorReport, **options
)
self._style.options.max_line_length = 88
#!/usr/bin/env python3
"""
Does pylint linting and pycodestyle (PEP8) code style checks on the modules.
"""
import os
import sys
from pycodestyle import Checker, StyleGuide
import pylint.lint
ROOT_PATH = os.path.join(os.path.dirname(__file__), '..')
MODULES_TO_CHECK = ['cbor', 'examples', 'scripts', 'tests']
PEP8_OPTIONS = StyleGuide(
config_file=os.path.join(ROOT_PATH, 'setup.cfg')).options
def pep8_file(filename):
"""return whether the given file passes PEP8 standards"""
checker = Checker(filename=filename, options=PEP8_OPTIONS)
return checker.check_all() == 0
def pep8_module(module_name):
"""return whether the given module passes PEP8 standards"""
success = True
for subdir, dirs, files in os.walk(os.path.join(ROOT_PATH, module_name)):
for file in files:
if file.endswith('.py'):
if not pep8_file(os.path.join(subdir, file)):
success = False
return success
