def python(script): """ Validates a Python script using pyflakes. Arguments: script -- The script to validate. Returns: A list of (line number, message) tuples contain all warnings. """ l = [] try: c = compile(script, u'<string>', u'exec', _ast.PyCF_ONLY_AST) except SyntaxError as e: return [ (e.lineno-1, e.args[0]) ] else: if Checker is None: return [] # Older version of PyFlakes don't support the builtins keyword, and # we don't want to crash on that. if pyflakesBuiltins: messages = Checker(c, builtins=_builtins).messages else: messages = Checker(c).messages for msg in messages: if _filter is None or not _filter(msg): l.append((msg.lineno-1, msg.message % msg.message_args)) return l
def _pyflakes_no_migrations(self, tree, filename='(none)', builtins=None): if os.path.basename(os.path.dirname(filename)) in ('migrations', 'south_migrations', 'migrations_django'): self.messages = [] else: Checker.___init___(self, tree, filename, builtins)
def test_star_imports(): for code in [ code_mod1, code_mod2, code_mod3, code_mod8, code_submod3, code_submod_init, code_submod_recursive_submod1 ]: stars = star_imports(Checker(ast.parse(code))) assert stars == [] stars = star_imports(Checker(ast.parse(code_mod4))) assert stars == ['.mod1', '.mod2'] stars = star_imports(Checker(ast.parse(code_mod5))) assert stars == ['module.mod1', 'module.mod2'] stars = star_imports(Checker(ast.parse(code_mod6))) assert stars == ['os.path'] stars = star_imports(Checker(ast.parse(code_mod7))) assert stars == ['.mod6'] stars = star_imports(Checker(ast.parse(code_mod9))) assert stars == ['.mod8'] stars = star_imports(Checker(ast.parse(code_submod1))) assert stars == ['..mod1', '..mod2', '.submod3'] stars = star_imports(Checker(ast.parse(code_submod2))) assert stars == ['module.mod1', 'module.mod2', 'module.submod.submod3'] for code in [code_submod4, code_submod_recursive_submod2]: stars = star_imports(Checker(ast.parse(code))) assert stars == ['.'] stars = star_imports(Checker(ast.parse(code_submod_recursive_init))) assert stars == ['.submod1']
def _pyflakes_no_migrations(self, tree, filename='(none)', builtins=None, file_tokens=()): if os.path.basename(os.path.dirname(filename)) == 'migrations': self.messages = [] else: Checker.___init___(self, tree, filename, builtins, file_tokens)
def _pyflakes_checker(codeblock): from pyflakes.checker import Checker codeblock = PythonBlock(codeblock) version = _pyflakes_version() if version <= 0.4: # pyflakes 0.4 uses the 'compiler' module. return Checker(codeblock.parse_tree) elif version >= 0.5: # pyflakes 0.5 uses the 'ast' module. return Checker(codeblock.ast_node) else: raise Exception("Unknown pyflakes version %r" % (version, ))
def run_pyflakes_for_package(package_name, extra_ignore=None): """ If pyflakes is installed, run it across the given package name returning any warnings found. """ ignore_strings = PYFLAKES_IGNORE if extra_ignore: ignore_strings += extra_ignore try: from pyflakes.checker import Checker except ImportError: return [] warnings = [] for (root, dirs, files) in os.walk(path_for_import(package_name)): for f in files: # Ignore migrations. directory = root.split(os.sep)[-1] if not f.endswith(".py") or directory == "migrations": continue path = os.path.join(root, f) with open(path, "U") as source_file: source = source_file.read() try: compile(source, f, "exec") except (SyntaxError, IndentationError), value: info = (path, value.lineno, value.args[0]) warnings.append("Invalid syntax in %s:%d: %s" % info) result = Checker(parse(source), path) for warning in result.messages: message = unicode(warning) for ignore in ignore_strings: if ignore in message: break else: warnings.append(message)
def check_syntax(self, code): try: tree = parse(code) except (SyntaxError, IndentationError, ValueError) as e: return {"lineno": e.lineno, "offset": e.offset, "text": e.text} else: return Checker(tree).messages
def get_fitness(tokens, check_warnings=True): separator = " " code = separator.join(tokens) fitness = 0 error_indexes = list() try: tree = parse(code) except SyntaxError: error = exc_info()[1] lineno, offset, text = error.lineno - 1, error.offset - 1, error.text index = get_sequence_index(tokens, offset, separator) fitness += 50 error_indexes = [index] return fitness, error_indexes if check_warnings: # Okay, it's syntactically valid. Now check for warnings. result = Checker(tree, '<input>') fitness += len(result.messages) error_indexes = [ get_sequence_index(tokens, m.col, separator) for m in result.messages ] return fitness, error_indexes
def pyflakes(codeString, filename): """ Check the Python source given by C{codeString} for flakes. """ # First, compile into an AST and handle syntax errors. try: tree = compile(codeString, filename, "exec", _ast.PyCF_ONLY_AST) except SyntaxError as value: msg = value.args[0] lineno = value.lineno # If there's an encoding problem with the file, the text is None. if value.text is None: # Avoid using msg, since for the only known case, it contains a # bogus message that claims the encoding the file declared was # unknown. msg = "Problem decoding source" lineno = 1 error = Message(filename, lineno) error.message = msg + "%s" error.message_args = "" return [error] else: # Okay, it's syntactically valid. Now check it. w = Checker(tree, filename) w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno)) return w.messages
def run(code, filename, reporter=None): """ Check the Python source given by C{codeString} for flakes. @param codeString: The Python source to check. @type codeString: C{str} @param filename: The name of the file the source came from, used to report errors. @type filename: C{str} @param reporter: A L{Reporter} instance, where errors and warnings will be reported. @return: The number of warnings emitted. @rtype: C{int} """ if reporter is None: reporter = default_reporter() # First, compile into an AST and handle syntax errors. try: tree = compile(code, filename, "exec", _ast.PyCF_ONLY_AST) except SyntaxError: value = sys.exc_info()[1] msg = value.args[0] (lineno, offset, text) = value.lineno, value.offset, value.text if PYPY: if text is None: lines = code.splitlines() if len(lines) >= lineno: text = lines[lineno - 1] if sys.version_info >= (3, ) and isinstance(text, bytes): try: text = text.decode('ascii') except UnicodeDecodeError: text = None offset -= 1 # If there's an encoding problem with the file, the text is None. if text is None: # Avoid using msg, since for the only known case, it contains a # bogus message that claims the encoding the file declared was # unknown. reporter.unexpectedError(filename, 'problem decoding source') else: reporter.syntaxError(filename, msg, lineno, offset, text) return None, reporter except Exception: reporter.unexpectedError(filename, 'problem decoding source') return None, reporter # Okay, it's syntactically valid. Now check it. w = Checker(tree, filename) w.messages.sort(key=lambda m: m.lineno) for warning in w.messages: reporter.flake(warning) return w, reporter
def test_names_to_replace(): for code in [ code_mod1, code_mod2, code_mod3, code_mod7, code_mod8, code_submod3, code_submod_init, code_submod_recursive_init, code_submod_recursive_submod1 ]: names = names_to_replace(Checker(ast.parse(code))) assert names == set() for code in [code_mod4, code_mod5]: names = names_to_replace(Checker(ast.parse(code))) assert names == {'a', 'b', 'c', 'd'} for code in [code_submod1, code_submod2]: names = names_to_replace(Checker(ast.parse(code))) assert names == {'a', 'b', 'c', 'd', 'e'} names = names_to_replace(Checker(ast.parse(code_submod4))) assert names == {'func'} names = names_to_replace(Checker(ast.parse(code_mod6))) assert names == {'isfile', 'join'} names = names_to_replace(Checker(ast.parse(code_submod_recursive_submod2))) assert names == {'a'} names = names_to_replace(Checker(ast.parse(code_mod9))) assert names == {'a', 'b'}
def check_code(code): try: tree = compile(code, 'test', 'exec', _ast.PyCF_ONLY_AST) except SyntaxError as syn_error: error = pyflakes.messages.Message('test', syn_error) error.message = 'Syntax Error' return [error] else: warnings = Checker(tree, 'test') return warnings.messages
def pyflakes_checker(path): with open(path, "U") as source_file: source = source_file.read() try: tree = compile(source, path, "exec", PyCF_ONLY_AST) except (SyntaxError, IndentationError) as value: info = (path, value.lineno, value.args[0]) yield "Invalid syntax in %s:%d: %s" % info else: result = Checker(tree, path) for warning in result.messages: yield str(warning)
def get_names(code, filename='<unknown>'): # TODO: Make the doctests work """ Get a set of defined top-level names from code. Example: >>> get_names(''' ... import mod ... a = 1 ... def func(): ... b = 2 ... ''') # doctest: +SKIP {'a', 'func', 'mod'} Star imports in code are returned like >>> get_names(''' ... from .mod1 import * ... from module.mod2 import * ... ''') # doctest: +SKIP {'.mod1.*', 'module.mod2.*'} __all__ is respected. Constructs supported by pyflakes like __all__ += [...] work. >>> get_names(''' ... a = 1 ... b = 2 ... c = 3 ... __all__ = ['a'] ... __all__ += ['b'] ... ''') # doctest: +SKIP {'a', 'b'} Returns a set of names, or raises SyntaxError if the code is not valid syntax. """ tree = ast.parse(code, filename=filename) checker = Checker(tree) for scope in checker.deadScopes: if isinstance(scope, ModuleScope): names = scope.keys() - set(dir(builtins)) - set(MAGIC_GLOBALS) break else: raise RuntimeError(f"Could not parse the names") if '__all__' in names: return set(scope['__all__'].names) return names
def test_external_initialization(self): """ Assert if provision for external initialization is present. Ensures that flake8 can initialize plugin with pyflakes instance. """ file = ['import sys'] tree = ast.parse(''.join(file)) checker = Checker(tree, self.filename) no_future_instance = NoFutureImport(tree, self.filename) no_future_instance.checker = checker self.assertTrue(no_future_instance.module_scope) self.assertFalse(len(list(no_future_instance.run())))
def check_with_pyflakes(source_code, filename=None): """Check source code with pyflakes Returns an empty list if pyflakes is not installed""" try: if filename is None: filename = '<string>' try: source_code += '\n' except TypeError: # Python 3 source_code += to_binary_string('\n') import _ast from pyflakes.checker import Checker # First, compile into an AST and handle syntax errors. try: tree = compile(source_code, filename, "exec", _ast.PyCF_ONLY_AST) except SyntaxError as value: # If there's an encoding problem with the file, the text is None. if value.text is None: results = [] else: results = [(value.args[0], value.lineno)] except (ValueError, TypeError): # Example of ValueError: file contains invalid \x escape character # (see http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674797) # Example of TypeError: file contains null character # (see http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674796) results = [] else: # Okay, it's syntactically valid. Now check it. w = Checker(tree, filename) w.messages.sort(key=lambda x: x.lineno) results = [] coding = encoding.get_coding(source_code) lines = source_code.splitlines() for warning in w.messages: if 'analysis:ignore' not in \ to_text_string(lines[warning.lineno-1], coding): results.append((warning.message % warning.message_args, warning.lineno)) except Exception: # Never return None to avoid lock in spyder/widgets/editor.py # See Issue 1547 results = [] if DEBUG_EDITOR: traceback.print_exc() # Print exception in internal console return results
def fix_code(code, *, file, max_line_length=100, verbose=False, quiet=False, allow_dynamic=True): """ Return a fixed version of the code `code` from the file `file` Raises RuntimeError if it is is not valid Python. See the docstring of replace_imports() for the meaning of the keyword arguments to this function. If allow_dynamic=True, then external modules will be dynamically imported. """ directory, filename = os.path.split(file) try: tree = ast.parse(code, filename=file) except SyntaxError as e: raise RuntimeError(f"SyntaxError: {e}") checker = Checker(tree) stars = star_imports(checker) names = names_to_replace(checker) mod_names = {} for mod in stars: mod_names[mod] = get_module_names(mod, directory, allow_dynamic=allow_dynamic) repls = {i: [] for i in stars} for name in names: mods = [mod for mod in mod_names if name in mod_names[mod]] if not mods: if not quiet: print(f"Warning: {file}: could not find import for '{name}'", file=sys.stderr) continue if len(mods) > 1: if not quiet: print(f"Warning: {file}: '{name}' comes from multiple modules: {', '.join(map(repr, mods))}. Using '{mods[-1]}'.", file=sys.stderr) repls[mods[-1]].append(name) new_code = replace_imports(code, repls, file=file, verbose=verbose, quiet=quiet, max_line_length=max_line_length) return new_code
def pyflakes_check(code): try: tree = compile(code, "ignored.py", "exec", _ast.PyCF_ONLY_AST) except SyntaxError as syntax_error: return [ dict(text='Syntax Error', row=syntax_error.lineno - 1, type='error') ] except ValueError as value_error: message = 'Error: {} somewhere in file'.format(value_error.message) return [dict(text=message, row=0, type='error')] w = Checker(tree, '') return [ dict(text=m.message % m.message_args, row=m.lineno - 1, type='warning') for m in w.messages ]
def check_file(path, flakesignore): if not hasattr(tokenize, 'open'): codeString = path.read() else: with tokenize.open(path.strpath) as f: codeString = f.read() filename = py.builtin._totext(path) errors = [] try: tree = compile(codeString, filename, "exec", _ast.PyCF_ONLY_AST) except SyntaxError: value = sys.exc_info()[1] (lineno, offset, text) = value.lineno, value.offset, value.text if text is None: errors.append("%s: problem decoding source" % filename) else: line = text.splitlines()[-1] if offset is not None: offset = offset - (len(text) - len(line)) msg = '%s:%d: %s' % (filename, lineno, value.args[0]) msg = "%s\n%s" % (msg, line) if offset is not None: msg = "%s\n%s" % (msg, "%s^" % (" " * offset)) errors.append(msg) return 1, errors else: # Okay, it's syntactically valid. Now check it. w = Checker(tree, filename) w.messages.sort(key=lambda m: m.lineno) lines = codeString.split('\n') for warning in w.messages: if warning.__class__.__name__ in flakesignore or is_ignored_line(lines[warning.lineno - 1].strip()): continue errors.append( '%s:%s: %s\n%s' % ( warning.filename, warning.lineno, warning.__class__.__name__, warning.message % warning.message_args)) return len(errors), errors
def check(filename): from pyflakes import reporter as mod_reporter from pyflakes.checker import Checker codeString = open(filename).read() reporter = mod_reporter._makeDefaultReporter() # First, compile into an AST and handle syntax errors. try: tree = compile(codeString, filename, "exec", _ast.PyCF_ONLY_AST) except SyntaxError: value = sys.exc_info()[1] msg = value.args[0] (lineno, offset, text) = value.lineno, value.offset, value.text # If there's an encoding problem with the file, the text is None. if text is None: # Avoid using msg, since for the only known case, it contains a # bogus message that claims the encoding the file declared was # unknown. reporter.unexpectedError(filename, 'problem decoding source') else: reporter.syntaxError(filename, msg, lineno, offset, text) return 1 except Exception: reporter.unexpectedError(filename, 'problem decoding source') return 1 else: # Okay, it's syntactically valid. Now check it. lines = codeString.splitlines() warnings = Checker(tree, filename) warnings.messages.sort(key=lambda m: m.lineno) real_messages = [] for m in warnings.messages: line = lines[m.lineno - 1] if 'pyflakes:ignore' in line.rsplit('#', 1)[-1]: # ignore lines with pyflakes:ignore pass else: real_messages.append(m) reporter.flake(m) return len(real_messages)
def getBufferErrors(sourceCode): """Provides a list of warnings/errors for the given source code""" sourceCode += '\n' # First, compile into an AST and handle syntax errors. try: tree = compile(sourceCode, "<string>", "exec", PyCF_ONLY_AST) except SyntaxError as value: # If there's an encoding problem with the file, the text is None. if value.text is None: return {}, [] return {value.lineno: [value.args[0]]}, [] except (ValueError, TypeError) as value: # ValueError may happened in case of invalid \x escape character # E.g. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674797 # TypeError may happened in case of null characters in a file # E.g. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674796 msg = str(value) if msg == "": return {-1: ["Could not compile buffer: unknown error"]}, [] return {-1: ["Could not compile buffer: " + msg]}, [] # Okay, it's syntactically valid. Now check it. check = Checker(tree, "<string>") results = {} lines = sourceCode.splitlines() for warning in check.messages: if isinstance(warning.lineno, int): lineno = warning.lineno else: # By some reasons I see ast NAME node here (pyflakes 0.7.3) lineno = warning.lineno.lineno if not IGNORE_REGEXP.search(lines[lineno - 1]): if lineno in results: results[lineno].append(warning.message % warning.message_args) else: results[lineno] = [warning.message % warning.message_args] # Radon: CC complexity as the second value return results, sorted_results(cc_visit_ast(tree))
def __syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, ignoreStarImportWarnings=False): """ Function to compile one Python source file to Python bytecode and to perform a pyflakes check. @param filename source filename (string) @param codestring string containing the code to compile (string) @keyparam checkFlakes flag indicating to do a pyflakes check (boolean) @keyparam ignoreStarImportWarnings flag indicating to ignore 'star import' warnings (boolean) @return dictionary with the keys 'error' and 'warnings' which hold a list containing details about the error/ warnings (file name, line number, column, codestring (only at syntax errors), the message, a list with arguments for the message) """ try: import builtins except ImportError: import __builtin__ as builtins # __IGNORE_WARNING__ try: if sys.version_info[0] == 2: file_enc = filename.encode(sys.getfilesystemencoding()) else: file_enc = filename # It also encode the code back to avoid 'Encoding declaration in # unicode string' exception on Python2 codestring = normalizeCode(codestring) # Check for VCS conflict markers conflict = VcsConflictMarkerRe.search(codestring) if conflict is not None: start, i = conflict.span() lineindex = 1 + codestring.count("\n", 0, start) return [{ 'error': (file_enc, lineindex, 0, "", "VCS conflict marker found") }] if filename.endswith('.ptl'): try: import quixote.ptl_compile except ImportError: return [{ 'error': (filename, 0, 0, '', 'Quixote plugin not found.') }] template = quixote.ptl_compile.Template(codestring, file_enc) template.compile() else: module = builtins.compile(codestring, file_enc, 'exec', ast.PyCF_ONLY_AST) except SyntaxError as detail: index = 0 code = "" error = "" lines = traceback.format_exception_only(SyntaxError, detail) if sys.version_info[0] == 2: lines = [x.decode(sys.getfilesystemencoding()) for x in lines] match = re.match('\s*File "(.+)", line (\d+)', lines[0].replace('<string>', filename)) if match is not None: fn, line = match.group(1, 2) if lines[1].startswith('SyntaxError:'): error = re.match('SyntaxError: (.+)', lines[1]).group(1) else: code = re.match('(.+)', lines[1]).group(1) for seLine in lines[2:]: if seLine.startswith('SyntaxError:'): error = re.match('SyntaxError: (.+)', seLine).group(1) elif seLine.rstrip().endswith('^'): index = len(seLine.rstrip()) - 4 else: fn = detail.filename line = detail.lineno or 1 error = detail.msg return [{'error': (fn, int(line), index, code.strip(), error)}] except ValueError as detail: try: fn = detail.filename line = detail.lineno error = detail.msg except AttributeError: fn = filename line = 1 error = str(detail) return [{'error': (fn, line, 0, "", error)}] except Exception as detail: try: fn = detail.filename line = detail.lineno error = detail.msg return [{'error': (fn, line, 0, "", error)}] except: # this catchall is intentional pass # pyflakes if not checkFlakes: return [{}] results = [] lines = codestring.splitlines() try: warnings = Checker(module, filename, withDoctest=True) warnings.messages.sort(key=lambda a: a.lineno) for warning in warnings.messages: if ignoreStarImportWarnings and \ isinstance(warning, ImportStarUsed): continue _fn, lineno, col, message, msg_args = warning.getMessageData() if "__IGNORE_WARNING__" not in extractLineFlags(lines[lineno - 1].strip()): results.append((_fn, lineno, col, "", message, msg_args)) except SyntaxError as err: if err.text.strip(): msg = err.text.strip() else: msg = err.msg results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, [])) return [{'warnings': results}]
tree = compile(source_code, filename, "exec", _ast.PyCF_ONLY_AST) except SyntaxError, value: # If there's an encoding problem with the file, the text is None. if value.text is None: return [] else: return [(value.args[0], value.lineno)] except (ValueError, TypeError): # Example of ValueError: file contains invalid \x escape character # (see http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674797) # Example of TypeError: file contains null character # (see http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674796) return [] else: # Okay, it's syntactically valid. Now check it. w = Checker(tree, filename) w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno)) results = [] lines = source_code.splitlines() for warning in w.messages: if 'analysis:ignore' not in lines[warning.lineno - 1]: results.append( (warning.message % warning.message_args, warning.lineno)) return results # Required versions # Why >=0.5.0? Because it's based on _ast (thread-safe) PYFLAKES_REQVER = '>=0.5.0' dependencies.add("pyflakes", _("Real-time code analysis on the Editor"),
def pep8_checker(path): for line_number, text in Checker( path, options=style_guide.options).check_all(): yield "%s:%s: %s" % (path, line_number, text)
def pep8_checker(path): for line_number, text in Checker(path).check_all(): yield "%s:%s: %s" % (path, line_number, text)
tree = compile(sourceCode, "<string>", "exec", PyCF_ONLY_AST) except SyntaxError, value: # If there's an encoding problem with the file, the text is None. if value.text is None: return [] return [(value.args[0], value.lineno)] except (ValueError, TypeError), value: # ValueError may happened in case of invalid \x escape character # E.g. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674797 # TypeError may happened in case of null characters in a file # E.g. http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=674796 msg = str(value) if msg == "": return [("Could not compile buffer: unknown error", -1)] return [("Could not compile buffer: " + msg, -1)] # Okay, it's syntactically valid. Now check it. w = Checker(tree, "<string>") results = [] lines = sourceCode.splitlines() for warning in w.messages: if type(warning.lineno) == int: lineno = warning.lineno else: # By some reasons I see ast NAME node here (pyflakes 0.7.3) lineno = warning.lineno.lineno if 'analysis:ignore' not in lines[lineno - 1]: results.append((warning.message % warning.message_args, lineno)) results.sort(key=lambda x: x[1]) return results
def run(self, filename, file): tree = ast.parse(''.join(file)) result = Checker(tree, filename=filename, withDoctest=True) yield PyFlakesResult(self, result.deadScopes, result.messages)
""" If pyflakes is installed, run it across the given package name returning any warnings found. """ from pyflakes.checker import Checker def pyflakes_checker(path): with open(path, "U") as source_file: source = source_file.read() try: tree = compile(source, path, "exec", PyCF_ONLY_AST) except (SyntaxError, IndentationError), value: info = (path, value.lineno, value.args[0]) yield "Invalid syntax in %s:%d: %s" % info else: result = Checker(tree, path) for warning in result.messages: yield unicode(warning) args = (pyflakes_checker, package_name, extra_ignore) return _run_checker_for_package(*args) def run_pep8_for_package(package_name, extra_ignore=None): """ If pep8 is installed, run it across the given package name returning any warnings or errors found. """ import pep8 class Checker(pep8.Checker):
QProgEdit is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with QProgEdit. If not, see <http://www.gnu.org/licenses/>. """ import _ast try: from pyflakes.checker import Checker # Check whether PyFlakes supports the builtins keyword try: Checker(compile(u'', u'<string>', u'exec', _ast.PyCF_ONLY_AST), \ builtins=None) pyflakesBuiltins = True except TypeError: pyflakesBuiltins = False except: Checker = None _builtins = [] _filter = None def addPythonBuiltins(builtins): """ Adds a number of names that should be interpreted as builtins, and not trigger a warning.
def checker(self): if self._checker is None: self._checker = Checker(self.tree, self.filename) return self._checker