def run_linter():
# type: () -> int
targets = [] # type: List[str]
if len(target_langs) != 0:
targets = [
target for lang in target_langs
for target in self.by_lang[lang]
]
if len(targets) == 0:
# If this linter has a list of languages, and
# no files in those languages are to be checked,
# then we can safely return success without
# invoking the external linter.
return 0
if self.args.fix and fix_arg:
command.append(fix_arg)
if pass_targets:
full_command = command + targets
else:
full_command = command
p = subprocess.Popen(full_command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
assert p.stdout # use of subprocess.PIPE indicates non-None
for line in iter(p.stdout.readline, b''):
print_err(name, color, line)
return p.wait() # Linter exit code
def print_error(
self,
rule: Rule,
line: str,
identifier: str,
color: str,
fn: str,
line_number: int,
) -> None:
print_err(
identifier, color,
'{} {}at {} line {}:'.format(YELLOW + rule['description'], BLUE,
fn, line_number))
print_err(identifier, color, line)
if self.verbose:
if rule.get('good_lines'):
print_err(
identifier, color, GREEN + " Good code: {}{}".format(
(YELLOW + " | " + GREEN).join(
rule['good_lines']), ENDC))
if rule.get('bad_lines'):
print_err(
identifier, color, MAGENTA + " Bad code: {}{}".format(
(YELLOW + " | " + MAGENTA).join(
rule['bad_lines']), ENDC))
print_err(identifier, color, "")
def run_linter():
# type: () -> int
targets = [] # type: List[str]
if len(target_langs) != 0:
targets = [target for lang in target_langs for target in self.by_lang[lang]]
if len(targets) == 0:
# If this linter has a list of languages, and
# no files in those languages are to be checked,
# then we can safely return success without
# invoking the external linter.
return 0
if pass_targets:
full_command = command + targets
else:
full_command = command
p = subprocess.Popen(full_command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
assert p.stdout # use of subprocess.PIPE indicates non-None
for line in iter(p.stdout.readline, b''):
print_err(name, color, line)
return p.wait() # Linter exit code
def run_pyflakes(
files: Sequence[str],
options: argparse.Namespace,
suppress_patterns: Sequence[Tuple[str, str]] = [],
) -> bool:
if len(files) == 0:
return False
failed = False
color = next(colors)
with subprocess.Popen(
['pyflakes', '--', *files],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
) as pyflakes:
# Implied by use of subprocess.PIPE
assert pyflakes.stdout is not None
assert pyflakes.stderr is not None
def suppress_line(line: str) -> bool:
for file_pattern, line_pattern in suppress_patterns:
if file_pattern in line and line_pattern in line:
return True
return False
for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines():
if not suppress_line(ln):
print_err('pyflakes', color, ln)
failed = True
return failed
def custom_check_file(fn: str,
identifier: str,
rules: RuleList,
color: Optional[Iterable[str]],
max_length: Optional[int]=None) -> bool:
failed = False
line_tups = get_line_info_from_file(fn=fn)
rules_to_apply = get_rules_applying_to_fn(fn=fn, rules=rules)
for rule in rules_to_apply:
ok = check_file_for_pattern(
fn=fn,
line_tups=line_tups,
identifier=identifier,
color=color,
rule=rule,
)
if not ok:
failed = True
# TODO: Move the below into more of a framework.
firstline = None
lastLine = None
if line_tups:
firstline = line_tups[0][3] # line_fully_stripped for the first line.
lastLine = line_tups[-1][1]
if max_length is not None:
ok = check_file_for_long_lines(
fn=fn,
max_length=max_length,
line_tups=line_tups,
)
if not ok:
failed = True
if firstline:
if os.path.splitext(fn)[1] and 'zerver/' in fn:
shebang_rules = [{'pattern': '^#!',
'description': "zerver library code shouldn't have a shebang line."}]
else:
shebang_rules = [{'pattern': '#!/usr/bin/python',
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`"},
{'pattern': '#!/usr/bin/env python$',
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}]
for rule in shebang_rules:
if re.search(rule['pattern'], firstline):
print_err(identifier, color,
'{} at {} line 1:'.format(rule['description'], fn))
print_err(identifier, color, firstline)
failed = True
if lastLine and ('\n' not in lastLine):
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
failed = True
return failed
def custom_check_file(fn: str,
identifier: str,
rules: RuleList,
color: Optional[Iterable[str]],
max_length: Optional[int]=None) -> bool:
failed = False
line_tups = get_line_info_from_file(fn=fn)
rules_to_apply = get_rules_applying_to_fn(fn=fn, rules=rules)
for rule in rules_to_apply:
ok = check_file_for_pattern(
fn=fn,
line_tups=line_tups,
identifier=identifier,
color=color,
rule=rule,
)
if not ok:
failed = True
# TODO: Move the below into more of a framework.
firstline = None
lastLine = None
if line_tups:
firstline = line_tups[0][3] # line_fully_stripped for the first line.
lastLine = line_tups[-1][1]
if max_length is not None:
ok = check_file_for_long_lines(
fn=fn,
max_length=max_length,
line_tups=line_tups,
)
if not ok:
failed = True
if firstline:
if os.path.splitext(fn)[1] and 'zerver/' in fn:
shebang_rules = [{'pattern': '^#!',
'description': "zerver library code shouldn't have a shebang line."}]
else:
shebang_rules = [{'pattern': '#!/usr/bin/python',
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`"},
{'pattern': '#!/usr/bin/env python$',
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}]
for rule in shebang_rules:
if re.search(rule['pattern'], firstline):
print_err(identifier, color,
'{} at {} line 1:'.format(rule['description'], fn))
print_err(identifier, color, firstline)
failed = True
if lastLine and ('\n' not in lastLine):
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
failed = True
return failed
def check_file_for_pattern(fn: str,
line_tups: List[LineTup],
identifier: str,
color: Optional[Iterable[str]],
rule: Rule) -> bool:
'''
DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING.
This function gets called ~40k times, once per file per regex.
Inside it's doing a regex check for every line in the file, so
it's important to do things like pre-compiling regexes.
DO NOT INLINE THIS FUNCTION.
We need to see it show up in profiles, and the function call
overhead will never be a bottleneck.
'''
exclude_lines = {
line for
(exclude_fn, line) in rule.get('exclude_line', set())
if exclude_fn == fn
}
pattern = re.compile(rule['pattern'])
strip_rule = rule.get('strip') # type: Optional[str]
ok = True
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
if line_fully_stripped in exclude_lines:
exclude_lines.remove(line_fully_stripped)
continue
try:
line_to_check = line_fully_stripped
if strip_rule is not None:
if strip_rule == '\n':
line_to_check = line_newline_stripped
else:
raise Exception("Invalid strip rule")
if pattern.search(line_to_check):
if rule.get("exclude_pattern"):
if re.search(rule['exclude_pattern'], line_to_check):
continue
print_err(identifier, color, '{} at {} line {}:'.format(
rule['description'], fn, i+1))
print_err(identifier, color, line)
ok = False
except Exception:
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
traceback.print_exc()
if exclude_lines:
print('Please remove exclusions for file %s: %s' % (fn, exclude_lines))
return ok
def check_file_for_pattern(fn: str,
line_tups: List[LineTup],
identifier: str,
color: Optional[Iterable[str]],
rule: Rule) -> bool:
'''
DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING.
This function gets called ~40k times, once per file per regex.
Inside it's doing a regex check for every line in the file, so
it's important to do things like pre-compiling regexes.
DO NOT INLINE THIS FUNCTION.
We need to see it show up in profiles, and the function call
overhead will never be a bottleneck.
'''
exclude_lines = {
line for
(exclude_fn, line) in rule.get('exclude_line', set())
if exclude_fn == fn
}
pattern = re.compile(rule['pattern'])
strip_rule = rule.get('strip') # type: Optional[str]
ok = True
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
if line_fully_stripped in exclude_lines:
exclude_lines.remove(line_fully_stripped)
continue
try:
line_to_check = line_fully_stripped
if strip_rule is not None:
if strip_rule == '\n':
line_to_check = line_newline_stripped
else:
raise Exception("Invalid strip rule")
if pattern.search(line_to_check):
if rule.get("exclude_pattern"):
if re.search(rule['exclude_pattern'], line_to_check):
continue
print_err(identifier, color, '{} at {} line {}:'.format(
rule['description'], fn, i+1))
print_err(identifier, color, line)
ok = False
except Exception:
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
traceback.print_exc()
if exclude_lines:
print('Please remove exclusions for file %s: %s' % (fn, exclude_lines))
return ok
def custom_check_file(self, fn, identifier, color, max_length=None):
# type: (str, str, Optional[Iterable[str]], Optional[int]) -> bool
failed = False
line_tups = self.get_line_info_from_file(fn=fn)
rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.rules)
for rule in rules_to_apply:
ok = self.check_file_for_pattern(
fn=fn,
line_tups=line_tups,
identifier=identifier,
color=color,
rule=rule,
)
if not ok:
failed = True
# TODO: Move the below into more of a framework.
firstline = None
lastLine = None
if line_tups:
# line_fully_stripped for the first line.
firstline = line_tups[0][3]
lastLine = line_tups[-1][1]
if max_length is not None:
ok = self.check_file_for_long_lines(
fn=fn,
max_length=max_length,
line_tups=line_tups,
)
if not ok:
failed = True
if firstline:
shebang_rules_to_apply = self.get_rules_applying_to_fn(
fn=fn, rules=self.shebang_rules)
for rule in shebang_rules_to_apply:
if re.search(rule['pattern'], firstline):
print_err(
identifier, color,
'{} at {} line 1:'.format(rule['description'], fn))
print_err(identifier, color, firstline)
failed = True
if lastLine and ('\n' not in lastLine):
print(
"No newline at the end of file. Fix with `sed -i '$a\\' %s`" %
(fn, ))
failed = True
return failed
def print_error(self, rule, line, identifier, color, fn, line_number):
# type: (Dict[str, Any], str, str, Optional[Iterable[str]], str, int) -> None
print_err(identifier, color, '{} {}at {} line {}:'.format(
YELLOW + rule['description'], BLUE, fn, line_number))
print_err(identifier, color, line)
if self.verbose:
if rule.get('good_lines'):
print_err(identifier, color, GREEN + " Good code: {}{}".format(
(YELLOW + " | " + GREEN).join(rule['good_lines']), ENDC))
if rule.get('bad_lines'):
print_err(identifier, color, MAGENTA + " Bad code: {}{}".format(
(YELLOW + " | " + MAGENTA).join(rule['bad_lines']), ENDC))
print_err(identifier, color, "")
def run_pycodestyle(files, ignored_rules):
# type: (List[str], List[str]) -> bool
failed = False
color = next(colors)
pep8 = subprocess.Popen(
['pycodestyle'] + files +
['--ignore={rules}'.format(rules=','.join(ignored_rules))],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
for line in iter(pep8.stdout.readline, b''):
print_err('pep8', color, line)
failed = True
return failed
def check_pyflakes(files, options):
# type: (List[str], argparse.Namespace) -> bool
if len(files) == 0:
return False
failed = False
color = next(colors)
pyflakes = subprocess.Popen(['pyflakes'] + files,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE
for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines():
if options.full or not suppress_line(ln):
print_err('pyflakes', color, ln)
failed = True
return failed
def run_pycodestyle(files, ignored_rules):
# type: (List[str], List[str]) -> bool
if len(files) == 0:
return False
failed = False
color = next(colors)
pep8 = subprocess.Popen(
['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
assert pep8.stdout is not None # Implied by use of subprocess.PIPE
for line in iter(pep8.stdout.readline, b''):
print_err('pep8', color, line)
failed = True
return failed
def check_pyflakes(files, options):
# type: (List[str], argparse.Namespace) -> bool
if len(files) == 0:
return False
failed = False
color = next(colors)
pyflakes = subprocess.Popen(['pyflakes'] + files,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE
for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines():
if options.full or not suppress_line(ln):
print_err('pyflakes', color, ln)
failed = True
return failed
def check_pyflakes(options, by_lang):
# type: (Any, Dict[str, List[str]]) -> bool
if len(by_lang['py']) == 0:
return False
failed = False
color = next(colors)
pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE
for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines():
if options.full or not suppress_line(ln):
print_err('pyflakes', color, ln)
failed = True
return failed
def run_command(
name: str,
color: str,
command: Sequence[str],
suppress_line: Callable[[str], bool] = lambda line: False,
) -> int:
with subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as p:
assert p.stdout is not None
for line in iter(p.stdout.readline, ""):
if not suppress_line(line):
print_err(name, color, line)
return p.wait()
def run_pyflakes(files, options, suppress_patterns=[]):
# type: (List[str], argparse.Namespace, List[Tuple[str, str]]) -> bool
if len(files) == 0:
return False
failed = False
color = next(colors)
pyflakes = subprocess.Popen(['pyflakes'] + files,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE
def suppress_line(line: str) -> bool:
for file_pattern, line_pattern in suppress_patterns:
if file_pattern in line and line_pattern in line:
return True
return False
for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines():
if options.full or not suppress_line(ln):
print_err('pyflakes', color, ln)
failed = True
return failed
def run_command(
name: str,
color: str,
command: Sequence[str],
suppress_line: Callable[[str], bool] = lambda line: False,
) -> int:
with subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as p:
assert p.stdout is not None
for line in iter(p.stdout.readline, ""):
if not suppress_line(line):
print_err(name, color, line)
if p.wait() < 0:
try:
signal_name = signal.Signals(-p.returncode).name
except (AttributeError, ValueError):
signal_name = "signal {}".format(-p.returncode)
print_err(name, color,
"{} terminated by {}".format(command[0], signal_name))
return p.returncode
def custom_check_file(fn,
identifier,
rules,
color,
skip_rules=None,
max_length=None):
# type: (str, str, RuleList, str, Optional[Iterable[str]], Optional[int]) -> bool
failed = False
line_tups = []
for i, line in enumerate(open(fn)):
line_newline_stripped = line.strip('\n')
line_fully_stripped = line_newline_stripped.strip()
skip = False
for skip_rule in skip_rules or []:
if re.match(skip_rule, line):
skip = True
if line_fully_stripped.endswith(' # nolint'):
continue
if skip:
continue
tup = (i, line, line_newline_stripped, line_fully_stripped)
line_tups.append(tup)
rules_to_apply = []
for rule in rules:
excluded = False
for item in rule.get('exclude', set()):
if fn.startswith(item):
excluded = True
break
if excluded:
continue
if rule.get("include_only"):
found = False
for item in rule.get("include_only", set()):
if item in fn:
found = True
if not found:
continue
rules_to_apply.append(rule)
for rule in rules_to_apply:
exclude_lines = {
line
for (exclude_fn, line) in rule.get('exclude_line', set())
if exclude_fn == fn
}
pattern = rule['pattern']
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
if line_fully_stripped in exclude_lines:
exclude_lines.remove(line_fully_stripped)
continue
try:
line_to_check = line_fully_stripped
if rule.get('strip') is not None:
if rule['strip'] == '\n':
line_to_check = line_newline_stripped
else:
raise Exception("Invalid strip rule")
if re.search(pattern, line_to_check):
if rule.get("exclude_pattern"):
if re.search(rule['exclude_pattern'], line_to_check):
continue
print_err(
identifier, color,
'{} at {} line {}:'.format(rule['description'], fn,
i + 1))
print_err(identifier, color, line)
failed = True
except Exception:
print("Exception with %s at %s line %s" %
(rule['pattern'], fn, i + 1))
traceback.print_exc()
if exclude_lines:
print('Please remove exclusions for file %s: %s' %
(fn, exclude_lines))
# TODO: Move the below into more of a framework.
firstline = None
if line_tups:
firstline = line_tups[0][3] # line_fully_stripped for the first line.
lastLine = None
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
if isinstance(line, bytes):
line_length = len(line.decode("utf-8"))
else:
line_length = len(line)
if (max_length is not None and line_length > max_length
and '# type' not in line and 'test' not in fn
and 'example' not in fn and
# Don't throw errors for markdown format URLs
not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and
# Don't throw errors for URLs in code comments
not re.search(r"[#].*http.*", line)
and not re.search(r"`\{\{ api_url \}\}[^`]+`", line)
and "# ignorelongline" not in line and 'migrations' not in fn):
print("Line too long (%s) at %s line %s: %s" %
(len(line), fn, i + 1, line_newline_stripped))
failed = True
lastLine = line
if firstline:
if os.path.splitext(fn)[1] and 'zerver/' in fn:
shebang_rules = [{
'pattern':
'^#!',
'description':
"zerver library code shouldn't have a shebang line."
}]
else:
shebang_rules = [{
'pattern':
'#!/usr/bin/python',
'description':
"Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`"
}, {
'pattern':
'#!/usr/bin/env python$',
'description':
"Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."
}]
for rule in shebang_rules:
if re.search(rule['pattern'], firstline):
print_err(identifier, color,
'{} at {} line 1:'.format(rule['description'], fn))
print_err(identifier, color, firstline)
failed = True
if lastLine and ('\n' not in lastLine):
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" %
(fn, ))
failed = True
return failed
def custom_check_file(fn, identifier, rules, color, skip_rules=None, max_length=None):
# type: (str, str, RuleList, str, Optional[Iterable[str]], Optional[int]) -> bool
failed = False
line_tups = []
for i, line in enumerate(open(fn)):
line_newline_stripped = line.strip('\n')
line_fully_stripped = line_newline_stripped.strip()
skip = False
for skip_rule in skip_rules or []:
if re.match(skip_rule, line):
skip = True
if line_fully_stripped.endswith(' # nolint'):
continue
if skip:
continue
tup = (i, line, line_newline_stripped, line_fully_stripped)
line_tups.append(tup)
rules_to_apply = []
for rule in rules:
excluded = False
for item in rule.get('exclude', set()):
if fn.startswith(item):
excluded = True
break
if excluded:
continue
if rule.get("include_only"):
found = False
for item in rule.get("include_only", set()):
if item in fn:
found = True
if not found:
continue
rules_to_apply.append(rule)
for rule in rules_to_apply:
exclude_lines = {
line for
(exclude_fn, line) in rule.get('exclude_line', set())
if exclude_fn == fn
}
pattern = rule['pattern']
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
if line_fully_stripped in exclude_lines:
exclude_lines.remove(line_fully_stripped)
continue
try:
line_to_check = line_fully_stripped
if rule.get('strip') is not None:
if rule['strip'] == '\n':
line_to_check = line_newline_stripped
else:
raise Exception("Invalid strip rule")
if re.search(pattern, line_to_check):
if rule.get("exclude_pattern"):
if re.search(rule['exclude_pattern'], line_to_check):
continue
print_err(identifier, color, '{} at {} line {}:'.format(
rule['description'], fn, i+1))
print_err(identifier, color, line)
failed = True
except Exception:
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
traceback.print_exc()
if exclude_lines:
print('Please remove exclusions for file %s: %s' % (fn, exclude_lines))
# TODO: Move the below into more of a framework.
firstline = None
if line_tups:
firstline = line_tups[0][3] # line_fully_stripped for the first line.
lastLine = None
for (i, line, line_newline_stripped, line_fully_stripped) in line_tups:
if isinstance(line, bytes):
line_length = len(line.decode("utf-8"))
else:
line_length = len(line)
if (max_length is not None and line_length > max_length and
'# type' not in line and 'test' not in fn and 'example' not in fn and
# Don't throw errors for markdown format URLs
not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and
# Don't throw errors for URLs in code comments
not re.search(r"[#].*http.*", line) and
not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and
"# ignorelongline" not in line and 'migrations' not in fn):
print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped))
failed = True
lastLine = line
if firstline:
if os.path.splitext(fn)[1] and 'zerver/' in fn:
shebang_rules = [{'pattern': '^#!',
'description': "zerver library code shouldn't have a shebang line."}]
else:
shebang_rules = [{'pattern': '#!/usr/bin/python',
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`"},
{'pattern': '#!/usr/bin/env python$',
'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}]
for rule in shebang_rules:
if re.search(rule['pattern'], firstline):
print_err(identifier, color,
'{} at {} line 1:'.format(rule['description'], fn))
print_err(identifier, color, firstline)
failed = True
if lastLine and ('\n' not in lastLine):
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
failed = True
return failed
