def run_linter(): # type: () -> int targets = [] # type: List[str] if len(target_langs) != 0: targets = [ target for lang in target_langs for target in self.by_lang[lang] ] if len(targets) == 0: # If this linter has a list of languages, and # no files in those languages are to be checked, # then we can safely return success without # invoking the external linter. return 0 if self.args.fix and fix_arg: command.append(fix_arg) if pass_targets: full_command = command + targets else: full_command = command p = subprocess.Popen(full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) assert p.stdout # use of subprocess.PIPE indicates non-None for line in iter(p.stdout.readline, b''): print_err(name, color, line) return p.wait() # Linter exit code
def print_error( self, rule: Rule, line: str, identifier: str, color: str, fn: str, line_number: int, ) -> None: print_err( identifier, color, '{} {}at {} line {}:'.format(YELLOW + rule['description'], BLUE, fn, line_number)) print_err(identifier, color, line) if self.verbose: if rule.get('good_lines'): print_err( identifier, color, GREEN + " Good code: {}{}".format( (YELLOW + " | " + GREEN).join( rule['good_lines']), ENDC)) if rule.get('bad_lines'): print_err( identifier, color, MAGENTA + " Bad code: {}{}".format( (YELLOW + " | " + MAGENTA).join( rule['bad_lines']), ENDC)) print_err(identifier, color, "")
def run_linter(): # type: () -> int targets = [] # type: List[str] if len(target_langs) != 0: targets = [target for lang in target_langs for target in self.by_lang[lang]] if len(targets) == 0: # If this linter has a list of languages, and # no files in those languages are to be checked, # then we can safely return success without # invoking the external linter. return 0 if pass_targets: full_command = command + targets else: full_command = command p = subprocess.Popen(full_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) assert p.stdout # use of subprocess.PIPE indicates non-None for line in iter(p.stdout.readline, b''): print_err(name, color, line) return p.wait() # Linter exit code
def run_pyflakes( files: Sequence[str], options: argparse.Namespace, suppress_patterns: Sequence[Tuple[str, str]] = [], ) -> bool: if len(files) == 0: return False failed = False color = next(colors) with subprocess.Popen( ['pyflakes', '--', *files], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, ) as pyflakes: # Implied by use of subprocess.PIPE assert pyflakes.stdout is not None assert pyflakes.stderr is not None def suppress_line(line: str) -> bool: for file_pattern, line_pattern in suppress_patterns: if file_pattern in line and line_pattern in line: return True return False for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines(): if not suppress_line(ln): print_err('pyflakes', color, ln) failed = True return failed
def custom_check_file(fn: str, identifier: str, rules: RuleList, color: Optional[Iterable[str]], max_length: Optional[int]=None) -> bool: failed = False line_tups = get_line_info_from_file(fn=fn) rules_to_apply = get_rules_applying_to_fn(fn=fn, rules=rules) for rule in rules_to_apply: ok = check_file_for_pattern( fn=fn, line_tups=line_tups, identifier=identifier, color=color, rule=rule, ) if not ok: failed = True # TODO: Move the below into more of a framework. firstline = None lastLine = None if line_tups: firstline = line_tups[0][3] # line_fully_stripped for the first line. lastLine = line_tups[-1][1] if max_length is not None: ok = check_file_for_long_lines( fn=fn, max_length=max_length, line_tups=line_tups, ) if not ok: failed = True if firstline: if os.path.splitext(fn)[1] and 'zerver/' in fn: shebang_rules = [{'pattern': '^#!', 'description': "zerver library code shouldn't have a shebang line."}] else: shebang_rules = [{'pattern': '#!/usr/bin/python', 'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`"}, {'pattern': '#!/usr/bin/env python$', 'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}] for rule in shebang_rules: if re.search(rule['pattern'], firstline): print_err(identifier, color, '{} at {} line 1:'.format(rule['description'], fn)) print_err(identifier, color, firstline) failed = True if lastLine and ('\n' not in lastLine): print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,)) failed = True return failed
def check_file_for_pattern(fn: str, line_tups: List[LineTup], identifier: str, color: Optional[Iterable[str]], rule: Rule) -> bool: ''' DO NOT MODIFY THIS FUNCTION WITHOUT PROFILING. This function gets called ~40k times, once per file per regex. Inside it's doing a regex check for every line in the file, so it's important to do things like pre-compiling regexes. DO NOT INLINE THIS FUNCTION. We need to see it show up in profiles, and the function call overhead will never be a bottleneck. ''' exclude_lines = { line for (exclude_fn, line) in rule.get('exclude_line', set()) if exclude_fn == fn } pattern = re.compile(rule['pattern']) strip_rule = rule.get('strip') # type: Optional[str] ok = True for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: if line_fully_stripped in exclude_lines: exclude_lines.remove(line_fully_stripped) continue try: line_to_check = line_fully_stripped if strip_rule is not None: if strip_rule == '\n': line_to_check = line_newline_stripped else: raise Exception("Invalid strip rule") if pattern.search(line_to_check): if rule.get("exclude_pattern"): if re.search(rule['exclude_pattern'], line_to_check): continue print_err(identifier, color, '{} at {} line {}:'.format( rule['description'], fn, i+1)) print_err(identifier, color, line) ok = False except Exception: print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1)) traceback.print_exc() if exclude_lines: print('Please remove exclusions for file %s: %s' % (fn, exclude_lines)) return ok
def custom_check_file(self, fn, identifier, color, max_length=None): # type: (str, str, Optional[Iterable[str]], Optional[int]) -> bool failed = False line_tups = self.get_line_info_from_file(fn=fn) rules_to_apply = self.get_rules_applying_to_fn(fn=fn, rules=self.rules) for rule in rules_to_apply: ok = self.check_file_for_pattern( fn=fn, line_tups=line_tups, identifier=identifier, color=color, rule=rule, ) if not ok: failed = True # TODO: Move the below into more of a framework. firstline = None lastLine = None if line_tups: # line_fully_stripped for the first line. firstline = line_tups[0][3] lastLine = line_tups[-1][1] if max_length is not None: ok = self.check_file_for_long_lines( fn=fn, max_length=max_length, line_tups=line_tups, ) if not ok: failed = True if firstline: shebang_rules_to_apply = self.get_rules_applying_to_fn( fn=fn, rules=self.shebang_rules) for rule in shebang_rules_to_apply: if re.search(rule['pattern'], firstline): print_err( identifier, color, '{} at {} line 1:'.format(rule['description'], fn)) print_err(identifier, color, firstline) failed = True if lastLine and ('\n' not in lastLine): print( "No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn, )) failed = True return failed
def print_error(self, rule, line, identifier, color, fn, line_number): # type: (Dict[str, Any], str, str, Optional[Iterable[str]], str, int) -> None print_err(identifier, color, '{} {}at {} line {}:'.format( YELLOW + rule['description'], BLUE, fn, line_number)) print_err(identifier, color, line) if self.verbose: if rule.get('good_lines'): print_err(identifier, color, GREEN + " Good code: {}{}".format( (YELLOW + " | " + GREEN).join(rule['good_lines']), ENDC)) if rule.get('bad_lines'): print_err(identifier, color, MAGENTA + " Bad code: {}{}".format( (YELLOW + " | " + MAGENTA).join(rule['bad_lines']), ENDC)) print_err(identifier, color, "")
def run_pycodestyle(files, ignored_rules): # type: (List[str], List[str]) -> bool failed = False color = next(colors) pep8 = subprocess.Popen( ['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) for line in iter(pep8.stdout.readline, b''): print_err('pep8', color, line) failed = True return failed
def check_pyflakes(files, options): # type: (List[str], argparse.Namespace) -> bool if len(files) == 0: return False failed = False color = next(colors) pyflakes = subprocess.Popen(['pyflakes'] + files, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines(): if options.full or not suppress_line(ln): print_err('pyflakes', color, ln) failed = True return failed
def run_pycodestyle(files, ignored_rules): # type: (List[str], List[str]) -> bool if len(files) == 0: return False failed = False color = next(colors) pep8 = subprocess.Popen( ['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) assert pep8.stdout is not None # Implied by use of subprocess.PIPE for line in iter(pep8.stdout.readline, b''): print_err('pep8', color, line) failed = True return failed
def check_pyflakes(options, by_lang): # type: (Any, Dict[str, List[str]]) -> bool if len(by_lang['py']) == 0: return False failed = False color = next(colors) pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines(): if options.full or not suppress_line(ln): print_err('pyflakes', color, ln) failed = True return failed
def run_command( name: str, color: str, command: Sequence[str], suppress_line: Callable[[str], bool] = lambda line: False, ) -> int: with subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) as p: assert p.stdout is not None for line in iter(p.stdout.readline, ""): if not suppress_line(line): print_err(name, color, line) return p.wait()
def run_pyflakes(files, options, suppress_patterns=[]): # type: (List[str], argparse.Namespace, List[Tuple[str, str]]) -> bool if len(files) == 0: return False failed = False color = next(colors) pyflakes = subprocess.Popen(['pyflakes'] + files, stdout=subprocess.PIPE, stderr=subprocess.PIPE) assert pyflakes.stdout is not None # Implied by use of subprocess.PIPE def suppress_line(line: str) -> bool: for file_pattern, line_pattern in suppress_patterns: if file_pattern in line and line_pattern in line: return True return False for ln in pyflakes.stdout.readlines() + pyflakes.stderr.readlines(): if options.full or not suppress_line(ln): print_err('pyflakes', color, ln) failed = True return failed
def run_command( name: str, color: str, command: Sequence[str], suppress_line: Callable[[str], bool] = lambda line: False, ) -> int: with subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, ) as p: assert p.stdout is not None for line in iter(p.stdout.readline, ""): if not suppress_line(line): print_err(name, color, line) if p.wait() < 0: try: signal_name = signal.Signals(-p.returncode).name except (AttributeError, ValueError): signal_name = "signal {}".format(-p.returncode) print_err(name, color, "{} terminated by {}".format(command[0], signal_name)) return p.returncode
def custom_check_file(fn, identifier, rules, color, skip_rules=None, max_length=None): # type: (str, str, RuleList, str, Optional[Iterable[str]], Optional[int]) -> bool failed = False line_tups = [] for i, line in enumerate(open(fn)): line_newline_stripped = line.strip('\n') line_fully_stripped = line_newline_stripped.strip() skip = False for skip_rule in skip_rules or []: if re.match(skip_rule, line): skip = True if line_fully_stripped.endswith(' # nolint'): continue if skip: continue tup = (i, line, line_newline_stripped, line_fully_stripped) line_tups.append(tup) rules_to_apply = [] for rule in rules: excluded = False for item in rule.get('exclude', set()): if fn.startswith(item): excluded = True break if excluded: continue if rule.get("include_only"): found = False for item in rule.get("include_only", set()): if item in fn: found = True if not found: continue rules_to_apply.append(rule) for rule in rules_to_apply: exclude_lines = { line for (exclude_fn, line) in rule.get('exclude_line', set()) if exclude_fn == fn } pattern = rule['pattern'] for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: if line_fully_stripped in exclude_lines: exclude_lines.remove(line_fully_stripped) continue try: line_to_check = line_fully_stripped if rule.get('strip') is not None: if rule['strip'] == '\n': line_to_check = line_newline_stripped else: raise Exception("Invalid strip rule") if re.search(pattern, line_to_check): if rule.get("exclude_pattern"): if re.search(rule['exclude_pattern'], line_to_check): continue print_err( identifier, color, '{} at {} line {}:'.format(rule['description'], fn, i + 1)) print_err(identifier, color, line) failed = True except Exception: print("Exception with %s at %s line %s" % (rule['pattern'], fn, i + 1)) traceback.print_exc() if exclude_lines: print('Please remove exclusions for file %s: %s' % (fn, exclude_lines)) # TODO: Move the below into more of a framework. firstline = None if line_tups: firstline = line_tups[0][3] # line_fully_stripped for the first line. lastLine = None for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: if isinstance(line, bytes): line_length = len(line.decode("utf-8")) else: line_length = len(line) if (max_length is not None and line_length > max_length and '# type' not in line and 'test' not in fn and 'example' not in fn and # Don't throw errors for markdown format URLs not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and # Don't throw errors for URLs in code comments not re.search(r"[#].*http.*", line) and not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and "# ignorelongline" not in line and 'migrations' not in fn): print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i + 1, line_newline_stripped)) failed = True lastLine = line if firstline: if os.path.splitext(fn)[1] and 'zerver/' in fn: shebang_rules = [{ 'pattern': '^#!', 'description': "zerver library code shouldn't have a shebang line." }] else: shebang_rules = [{ 'pattern': '#!/usr/bin/python', 'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`" }, { 'pattern': '#!/usr/bin/env python$', 'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`." }] for rule in shebang_rules: if re.search(rule['pattern'], firstline): print_err(identifier, color, '{} at {} line 1:'.format(rule['description'], fn)) print_err(identifier, color, firstline) failed = True if lastLine and ('\n' not in lastLine): print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn, )) failed = True return failed
def custom_check_file(fn, identifier, rules, color, skip_rules=None, max_length=None): # type: (str, str, RuleList, str, Optional[Iterable[str]], Optional[int]) -> bool failed = False line_tups = [] for i, line in enumerate(open(fn)): line_newline_stripped = line.strip('\n') line_fully_stripped = line_newline_stripped.strip() skip = False for skip_rule in skip_rules or []: if re.match(skip_rule, line): skip = True if line_fully_stripped.endswith(' # nolint'): continue if skip: continue tup = (i, line, line_newline_stripped, line_fully_stripped) line_tups.append(tup) rules_to_apply = [] for rule in rules: excluded = False for item in rule.get('exclude', set()): if fn.startswith(item): excluded = True break if excluded: continue if rule.get("include_only"): found = False for item in rule.get("include_only", set()): if item in fn: found = True if not found: continue rules_to_apply.append(rule) for rule in rules_to_apply: exclude_lines = { line for (exclude_fn, line) in rule.get('exclude_line', set()) if exclude_fn == fn } pattern = rule['pattern'] for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: if line_fully_stripped in exclude_lines: exclude_lines.remove(line_fully_stripped) continue try: line_to_check = line_fully_stripped if rule.get('strip') is not None: if rule['strip'] == '\n': line_to_check = line_newline_stripped else: raise Exception("Invalid strip rule") if re.search(pattern, line_to_check): if rule.get("exclude_pattern"): if re.search(rule['exclude_pattern'], line_to_check): continue print_err(identifier, color, '{} at {} line {}:'.format( rule['description'], fn, i+1)) print_err(identifier, color, line) failed = True except Exception: print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1)) traceback.print_exc() if exclude_lines: print('Please remove exclusions for file %s: %s' % (fn, exclude_lines)) # TODO: Move the below into more of a framework. firstline = None if line_tups: firstline = line_tups[0][3] # line_fully_stripped for the first line. lastLine = None for (i, line, line_newline_stripped, line_fully_stripped) in line_tups: if isinstance(line, bytes): line_length = len(line.decode("utf-8")) else: line_length = len(line) if (max_length is not None and line_length > max_length and '# type' not in line and 'test' not in fn and 'example' not in fn and # Don't throw errors for markdown format URLs not re.search(r"^\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and # Don't throw errors for URLs in code comments not re.search(r"[#].*http.*", line) and not re.search(r"`\{\{ api_url \}\}[^`]+`", line) and "# ignorelongline" not in line and 'migrations' not in fn): print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped)) failed = True lastLine = line if firstline: if os.path.splitext(fn)[1] and 'zerver/' in fn: shebang_rules = [{'pattern': '^#!', 'description': "zerver library code shouldn't have a shebang line."}] else: shebang_rules = [{'pattern': '#!/usr/bin/python', 'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/python`"}, {'pattern': '#!/usr/bin/env python$', 'description': "Use `#!/usr/bin/env python3` instead of `#!/usr/bin/env python`."}] for rule in shebang_rules: if re.search(rule['pattern'], firstline): print_err(identifier, color, '{} at {} line 1:'.format(rule['description'], fn)) print_err(identifier, color, firstline) failed = True if lastLine and ('\n' not in lastLine): print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,)) failed = True return failed