Example #1
0
def test(modules=None):
    import os
    import ast as ast_mod
    if not modules:
        #d = 'C:\\Python25\\Lib\\'
        d = '/usr/lib/python2.7/'
        modules = [d + f for f in os.listdir(d) if f.endswith('.py')]
    for f in modules:
        shutil.copy(f, 'input.py')
        msg = '%s...' % os.path.basename(f)
        ast, comments = parse_with_comments(f)
        print_code_and_ast(f, ast, comments)
        print
        print
        print ast_mod.dump(ast_mod.parse(file(f).read()), 0)
        try:
            ast2py(ast, comments)
        except:
            raise
            msg += 'failed to converting AST back to python code'
        try:
            if f == 'parser_test_source.py':
                print;print
                print file('output.py').read()   # see the results
            parse(file('output.py').read())
            msg += 'OK'
        except:
            raise
            msg += 'failed to parse generated code'
        print msg

        break
def main(argv):
    args = argv[1:]
    for path in args:
        if path.endswith('.zip'):
            zf = zipfile.ZipFile(path)
            for name in zf.namelist():
                sys.stderr.write('parsing: %r\n' % name)
                sys.stderr.flush()
                data = zf.read(name)
                try:
                    tree = compiler.parse(data)
                    traverse(tree)
                except SyntaxError, e:
                    print 'error:', name, e
            zf.close()
        else:
            sys.stderr.write('parsing: %r\n' % path)
            sys.stderr.flush()
            fp = open(path, 'rb')
            data = fp.read()
            try:
                tree = compiler.parse(data)
                traverse(tree)
            except SyntaxError, e:
                print 'error:', name, e
            fp.close()
Example #3
0
	def run(self, grammar):
		self.message = self.name + ': '
		if self.code == '':
			self.message = self.message + 'No test set'
			return
		try:
			try:
				tree = str(compiler.parse(self.code))
			except SyntaxError:
				self.message = self.message + """Error in test.\n""" + self.code
				raise EscapeException()
			try:
				matcher = grammar(tree)
				generated = matcher.apply('any')
			except ParseError:
				self.message = self.message + """Error in grammar.\n""" + self.code + """\n\n""" + tree
				raise EscapeException()
			try:
				assert str(compiler.parse(generated)) == tree
			except AssertionError:
				self.message = self.message + """Error, generated code does not match original.\n""" + self.code + """\n\n""" + tree + """\n\n""" + generated
				raise EscapeException()
			except SyntaxError:
				self.message = self.message + """Error in generated code.\n""" + self.code + """\n\n""" + tree + """\n\n""" + generated
				raise EscapeException()
			self.message = self.message + "OK"
			self.result = True
		except EscapeException:
			pass
    def _safeEval(self, s):
        """
        Evaluate strings that only contain the following structures:
        const,  tuple,  list,   dict
        Taken from c.l.py newsgroup posting Nov 5, 2003 by Huaiyu Zhu at IBM Almaden
        ??? this may need to be expanded to support complex numbers in lists, etc.
        """
        #print "in _safeEval. input: '%s'" % s
        node1 = compiler.parse(s)

        # !!! special case of attempting to compile a lone string
        if node1.doc is not None and len(node1.node.nodes) == 0:
            #print "in _safeEval. string: '%s' found as docstring" % node1.doc
            return node1.doc

        #print "in _safeEval. nodes: '%s'" % (node1,)
        stmts = node1.node.nodes
        assert len(stmts) == 1
        node = compiler.parse(s).node.nodes[0]
        assert node.__class__ == compiler.ast.Discard
        nodes = node.getChildNodes()
        assert len(nodes) == 1
        result = self._safeAssemble(nodes[0])
        #print "in _safeEval result: '%s'" % (result,)
        return result
Example #5
0
def get_ast():
    if options.filename:
        with open(options.filename, 'r') as f:
            return compiler.parse("".join(f.readlines()))
    else:
        import sys
        return compiler.parse("".join(sys.stdin))
Example #6
0
    def _safeEval(self, s):
        """
        Evaluate strings that only contain the following structures:
        const,  tuple,  list,   dict
        Taken from c.l.py newsgroup posting Nov 5, 2003 by Huaiyu Zhu at IBM Almaden
        ??? this may need to be expanded to support complex numbers in lists, etc.
        """
        #print "in _safeEval. input: '%s'" % s
        node1 = compiler.parse(s)

        # !!! special case of attempting to compile a lone string
        if node1.doc is not None and len(node1.node.nodes) == 0:
            #print "in _safeEval. string: '%s' found as docstring" % node1.doc
            return node1.doc

        #print "in _safeEval. nodes: '%s'" % (node1,)
        stmts = node1.node.nodes
        assert len(stmts) == 1
        node = compiler.parse(s).node.nodes[0]
        assert node.__class__ == compiler.ast.Discard
        nodes = node.getChildNodes()
        assert len(nodes) == 1
        result = self._safeAssemble(nodes[0])
        #print "in _safeEval result: '%s'" % (result,)
        return result
Example #7
0
def verify_source(source):
    tree = fix_tree(parse(source, 'exec'))
    code = pycodegen.ModuleCodeGenerator(tree).getCode()
    generator = ModuleSourceCodeGenerator(tree)
    source = generator.getSourceCode()
    tree = fix_tree(parse(source, 'exec'))
    if code.co_code != pycodegen.ModuleCodeGenerator(tree).getCode().co_code:
        return source
Example #8
0
def get_ast():
    if options.filename:
        with open(options.filename, "r") as f:
            return compiler.parse("".join(f.readlines()))
    else:
        import sys

        return compiler.parse("".join(sys.stdin))
Example #9
0
def errorPythonSource(source):
    """ 检查python代码的合理性 """
    try:
        source = source.replace('\r', '')
        compiler.parse(source)
    except:
        return "语法错误"
    return ''
Example #10
0
 def check_syntax(self, cmd_string):
     "Checks syntax for syntax-errors"
     try:
         compiler.parse(cmd_string)
         return True
     except Exception, e:
         self.gui.show_error_dialog("Syntax Error In Experiment Script", "Experiment Script: " + str(e))
         self.gui.new_log_message("Syntax Error: " + str(e), "ES")
         return False
Example #11
0
    def has_changed(self, fpath):
        """ answer whether has_changed(fpath) since list mirror """
        before       = open(fpath,'r').read()
        after        = open((self^fpath),'r').read()
        file_changed = not( before == after )
        ast_changed  = compiler.parse(before,'exec')==compiler.parse(after,'exec')

        out = dict( file_changed=file_changed,
                    ast_changed=ast_changed )
        return out
Example #12
0
def _analyzeGens(top, absnames):
    genlist = []
    for g in top:
        if isinstance(g, _UserCode):
            ast = g
        elif isinstance(g, (_AlwaysComb, _Always)):
            f = g.func
            s = inspect.getsource(f)
            # remove decorators
            s = re.sub(r"@.*", "", s)
            s = s.lstrip()
            ast = compiler.parse(s)
            # print ast
            ast.sourcefile = inspect.getsourcefile(f)
            ast.lineoffset = inspect.getsourcelines(f)[1]-1
            ast.symdict = f.func_globals.copy()
            ast.callstack = []
            # handle free variables
            if f.func_code.co_freevars:
                for n, c in zip(f.func_code.co_freevars, f.func_closure):
                    obj = _cell_deref(c)
                    if isinstance(g, _AlwaysComb):
                        # print type(obj)
                        assert isinstance(obj, (int, long, Signal)) or \
                               _isMem(obj) or _isTupleOfInts(obj)
                    ast.symdict[n] = obj
            ast.name = absnames.get(id(g), str(_Label("BLOCK"))).upper()
            v = _NotSupportedVisitor(ast)
            compiler.walk(ast, v)
            if isinstance(g, _AlwaysComb):
                v = _AnalyzeAlwaysCombVisitor(ast, g.senslist)
            else:
                v = _AnalyzeAlwaysDecoVisitor(ast, g.senslist)
            compiler.walk(ast, v)
        else: # @instance
            f = g.gen.gi_frame
            s = inspect.getsource(f)
            # remove decorators
            s = re.sub(r"@.*", "", s)
            s = s.lstrip()
            ast = compiler.parse(s)
            # print ast
            ast.sourcefile = inspect.getsourcefile(f)
            ast.lineoffset = inspect.getsourcelines(f)[1]-1
            ast.symdict = f.f_globals.copy()
            ast.symdict.update(f.f_locals)
            ast.callstack = []
            ast.name = absnames.get(id(g), str(_Label("BLOCK"))).upper()
            v = _NotSupportedVisitor(ast)
            compiler.walk(ast, v)
            v = _AnalyzeBlockVisitor(ast)
            compiler.walk(ast, v)
        genlist.append(ast)
    return genlist
 def pythonCodeContainsErrorOnParse(self, code):
     flag = False
     ls = []
     try:
         compiler.parse(code, mode = "exec")
     except Exception as e:
         flag = True
         # traceback.print_exc()
         s = traceback.format_exc()
         ls = s.split("\n")
     return flag,ls
Example #14
0
def fetch_git(repo_dir):

    #create new user object
    users = Users()
    os.chdir(repo_dir)
    os.system('git checkout master -q')
    proc = subprocess.Popen(['git', 'log', '--pretty=format:%H %ct'],stdout=subprocess.PIPE)
    commit_log = proc.stdout.readlines()
    commit_log.reverse()
    for log in commit_log:
        log = log.strip()
        (commit,date) = log.split(' ')
        os.system('git checkout ' + commit + ' -q')
    #    print(commit,date)
        proc = subprocess.Popen(['git', 'diff-tree', '--no-commit-id', '--name-only', '-r', commit],stdout=subprocess.PIPE)
        changes = proc.stdout.readlines()
        #print(changes)
        for change in changes:
            path = change.strip()
            dir,filename = os.path.split(path)
            host,name = os.path.split(dir)
    
            user = users.get_user(host,name)
            if user == None:
                user = users.add_user(host,name,filename)

            try:
                with open(path) as fh:
                    data = fh.read()
                    try:
                        compiler.parse(data)
                        syntax = 1
                    except Exception as e:
                        #print("exception!")
                        #print(e)
                        syntax = 0
                    lines = len(data.splitlines())
            except IOError:
                continue

            #times 1000 for javascript flot
            time = int(date) * 1000 
            history = History(commit,time,lines,syntax)

            user.add_version(filename,history)

    #reset to head
    os.system('git checkout master -q')
    os.chdir('..')
    return users
Example #15
0
def verify_source(source):
    tree = fix_tree(parse(source, 'exec'))
    code = pycodegen.ModuleCodeGenerator(tree).getCode()
    generator = ModuleSourceCodeGenerator(tree)
    generated = generator.getSourceCode()
    try:
        new = fix_tree(parse(generated, 'exec'))
    except SyntaxError:
        return generated

    old = code.co_code
    new = pycodegen.ModuleCodeGenerator(new).getCode().co_code

    if old != new:
        return generated
Example #16
0
 def canonize(self, student_answer, state=None):
     if re.match('.*;[ \t]*$', student_answer):
         return (False,
                 'On ne met pas de <tt>;</tt> en fin de ligne en Python')
     if '...' in student_answer:
         return (
             False,
             "Enlevez les «...» en début de ligne, ils ne font pas parti du langage. C'est l'invite de commande de l'interpréteur"
         )
     try:
         compiler.parse(student_answer)
     except SyntaxError as e:
         return (False, 'Message de Python : <b>' + cgi.escape(str(e)) +
                 '</b><br>')
     return P_clean(student_answer)
Example #17
0
def verify_source(source):
    tree = fix_tree(parse(source, 'exec'))
    code = pycodegen.ModuleCodeGenerator(tree).getCode()
    generator = ModuleSourceCodeGenerator(tree)
    generated = generator.getSourceCode()
    try:
        new = fix_tree(parse(generated, 'exec'))
    except SyntaxError:
        return generated

    old = code.co_code
    new = pycodegen.ModuleCodeGenerator(new).getCode().co_code

    if old != new:
        return generated
Example #18
0
def searchImportedModulesForDefinition(scope,node):
    lines = scope.module.getSourceNode().getLines()
    for lineno in scope.getImportLineNumbers():
        logicalline = getLogicalLine(lines,lineno)
        logicalline = makeLineParseable(logicalline)
        ast = compiler.parse(logicalline)
        class ImportVisitor:
            def __init__(self,node):
                self.target = node
                self.match = None
                assert isinstance(self.target,Name), \
                       "Getattr not supported"
                
            def visitFrom(self, node):
                module = resolveImportedModuleOrPackage(scope,node.modname)
                if module is None: # couldn't find module
                    return 
                
                if node.names[0][0] == '*': # e.g. from foo import *
                    match = findDefinitionFromASTNode(module,self.target)
                    if match is not None:
                        self.match = match
                    return
                    
                for name, alias in node.names:
                    if alias is None and name == self.target.name:
                        match = findDefinitionFromASTNode(module,self.target)
                        if match is not None:
                            self.match = match
                        return


        match = visitor.walk(ast, ImportVisitor(node)).match
        if match:
            return match
    def literal_eval(node_or_string):
        """
        Safely evaluate an expression node or a string containing a Python
        expression.  The string or node provided may only consist of the  following
        Python literal structures: strings, numbers, tuples, lists, dicts,  booleans,
        and None.
        """
        _safe_names = {'None': None, 'True': True, 'False': False}
        if isinstance(node_or_string, string_types):
            node_or_string = parse(node_or_string, mode='eval')
        if isinstance(node_or_string, ast.Expression):
            node_or_string = node_or_string.node

        def _convert(node):
            if isinstance(node, ast.Const) and isinstance(node.value, (text_type, binary_type, float, complex) + integer_types):
                return node.value
            elif isinstance(node, ast.Tuple):
                return tuple(map(_convert, node.nodes))
            elif isinstance(node, ast.List):
                return list(map(_convert, node.nodes))
            elif isinstance(node, ast.Dict):
                return dict((_convert(k), _convert(v)) for k, v in node.items())
            elif isinstance(node, ast.Name):
                if node.name in _safe_names:
                    return _safe_names[node.name]
            elif isinstance(node, ast.UnarySub):
                return -_convert(node.expr)  # pylint: disable=invalid-unary-operand-type
            raise ValueError('malformed string')
        return _convert(node_or_string)
Example #20
0
 def test_visitLambda(self):
     from common import MatchFinder
     finder = MatchFinder()
     src = '''x = lambda a, b, c=None, d=None: (a + b) and c or d'''
     ast = compiler.parse(src)
     finder.reset(src)
     compiler.walk(ast, finder)
Example #21
0
    def split_commands(self, python):
        """ Split multiple lines of code into discrete commands that can be
        executed singly.

        Parameters
        ----------
        python : str
            Pure, exec'able Python code.

        Returns
        -------
        commands : list of str
            Separate commands that can be exec'ed independently.
        """

        # The compiler module will parse the code into an abstract syntax tree.
        ast = compiler.parse(python)

        # Each separate command is available by iterating over ast.node. The
        # lineno attribute is the line number (1-indexed) beginning the commands
        # suite.
        linenos = [x.lineno-1 for x in ast.node]

        # When we finally get the slices, we will need to slice all the way to
        # the end even though we don't have a line number for it. Fortunately,
        # None does the job nicely.
        linenos.append(None)
        lines = python.split('\n')

        # Hooray for incomprehensible list comprehensions!
        commands = ['\n'.join(lines[i:j]) for i, j in 
            zip(linenos[:-1], linenos[1:])]

        return commands
Example #22
0
def makejson(config):
    #config=json.load('make_path')
    with open(config['source'], encoding='utf8') as f:
        python_code = f.read()
    tree = ast.parse(python_code)

    js_code_body = compiler.parse(tree)

    # import include code
    if 'include' in config:
        js_code_head_list = []
        include = config['include'] if type(
            config['include']) == list else [config['include']]
        for _include in include:
            with open(_include, encoding="utf8") as f:
                js_code_head_list.append(f.read())
        js_code_head = '\n'.join(js_code_head_list)
    else:
        js_code_head = ''

    # direct export
    if 'module_name' not in config:
        return output(config, js_code_head + '\n' + js_code_body)

    #module export
    module_name = config['module_name']
    exports = config['export']
    comment = config.get('comment', '')
    content = module_template.render(js_code_head=tab(js_code_head),
                                     js_code_body=tab(js_code_body),
                                     module_name=module_name,
                                     exports=exports,
                                     comment=comment)
    return output(config, content)
Example #23
0
	def updateModel(self, buf):
		"""
		To call whenever the model has been updated in a way
		that could impact the outline.
		
		We do not modify the current outline in case of a
		syntax error.
		"""
		try:
			mod = compiler.parse(buf)
		except SyntaxError:
			log("Syntax error, no outline updated")
			return
			
		self.clear()
		self._items = []
		visitor = OutlineAstVisitor(self)
		try:
			visitor.preorder(mod, visitor, None)
			visitor.endCurrentVisitedItem()
			del mod, visitor
		except SyntaxError:
			log("Syntax error, outline not created")
		finally:
			gc.collect()
def getAssignments(lines):
    class AssignVisitor:
        def __init__(self):
            self.assigns = []

        def visitAssTuple(self, node):
            for a in node.nodes:
                if a.name not in self.assigns:
                    self.assigns.append(a.name)

        def visitAssName(self, node):
            if node.name not in self.assigns:
                self.assigns.append(node.name)

        def visitAugAssign(self, node):
            if isinstance(node.node, compiler.ast.Name):
                if node.node.name not in self.assigns:
                    self.assigns.append(node.node.name)

    assignfinder = AssignVisitor()
    for line in lines:
        doctoredline = makeLineParseable(line)
        try:
            ast = compiler.parse(doctoredline)
        except ParserError:
            raise ParserException("couldnt parse:"+doctoredline)
        visitor.walk(ast, assignfinder)
    return assignfinder.assigns
Example #25
0
 def check(code_string, filename):
     """check a code string"""
     try:
         tree = compiler.parse(code_string)
     except (SyntaxError, IndentationError):
         value = sys.exc_info()[1]
         try:
             (lineno, offset, line) = value[1][1:]
         except IndexError:
             print('could not compile %r' % (filename,), file=sys.stderr)
             return 1
         if line.endswith("\n"):
             line = line[:-1]
         print('%s:%d: could not compile' % (filename, lineno), file=sys.stderr)
         print(line, file=sys.stderr)
         print(" " * (offset-2), "^", file=sys.stderr)
         return 1
     else:
         import locale
         try:
             locale.setlocale(locale.LC_ALL, 'C')
         except locale.Error as e:
             print('setlocale failed: %s' % (e,), file=sys.stderr)
         w = FlakeChecker(tree, filename)
         w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
         for warning in w.messages:
             print(warning)
         return len(w.messages)
Example #26
0
    def get_pysmell_code_walk_to_text(self, text):
        code = compiler.parse(text)

        class GlobalCodeFinder(CodeFinder):
            def visitFunction(self, func):
                self.enterScope(func)
                if self.inClassFunction:
                    if func.name != '__init__':
                        if func.decorators and 'property' in [
                                getName(n) for n in func.decorators
                        ]:
                            self.modules.addProperty(self.currentClass,
                                                     func.name)
                        else:
                            self.modules.addMethod(self.currentClass,
                                                   func.name,
                                                   getFuncArgs(func), func.doc
                                                   or "")
                    else:
                        self.modules.setConstructor(self.currentClass,
                                                    getFuncArgs(func))
                elif len(self.scope) == 1:
                    self.modules.addFunction(func.name,
                                             getFuncArgs(func, inClass=False),
                                             func.doc or "")

                #self.visit(func.code) Remove this line
                self.exitScope()

        if self.scope == SCOPE_GLOBAL:
            codefinder = GlobalCodeFinder()
        else:
            codefinder = CodeFinder()
        codefinder.modules = PyPleteModuleDict()
        return compiler.walk(code, codefinder)
Example #27
0
    def literal_eval(node_or_string):
        """Safely evaluate a node/string containing a Python expression.

Thestring or node provided may only consist of the following Python literal
structures: strings, numbers, tuples, lists, dicts,  booleans, and None.

Essentially a backport of the literal_eval function in python 2.6 onwards.
From: http://mail.python.org/pipermail/python-list/2009-September/1219992.html
        """
        _safe_names = {'None': None, 'True': True, 'False': False}
        if isinstance(node_or_string, str):
            node_or_string = parse(node_or_string, mode='eval')
        if isinstance(node_or_string, ast.Expression):
            node_or_string = node_or_string.node

        def _convert(node):
            '''Convert node/string to expression.'''
            if isinstance(node, ast.Const) and isinstance(
                    node.value, (str, int, float, complex)):
                return node.value
            elif isinstance(node, ast.Tuple):
                return tuple(_convert(element) for element in node.nodes)
            elif isinstance(node, ast.List):
                return list(_convert(element) for element in node.nodes)
            elif isinstance(node, ast.Dict):
                return dict((_convert(k), _convert(v)) for k, v in node.items)
            elif isinstance(node, ast.Name):
                if node.name in _safe_names:
                    return _safe_names[node.name]
            elif isinstance(node, ast.UnarySub):
                return -_convert(node.expr)
            raise ValueError('malformed string')

        return _convert(node_or_string)
def get_vars_in_expression(source):
    '''Get list of variable names in a python expression.'''

    import compiler
    from compiler.ast import Node

    ##
    # @brief Internal recursive function.
    # @param node An AST parse Node.
    # @param var_list Input list of variables.
    # @return An updated list of variables.
    def get_vars_body(node, var_list=[]):
        if isinstance(node, Node):
            if node.__class__.__name__ == 'Name':
                for child in node.getChildren():
                    if child not in var_list:
                        var_list.append(child)
            for child in node.getChildren():
                if isinstance(child, Node):
                    for child in node.getChildren():
                        var_list = get_vars_body(child, var_list)
                    break

        return var_list

    return get_vars_body(compiler.parse(source))
Example #29
0
File: lib.py Project: ypid/check_mk
        def literal_eval(node_or_string):
            _safe_names = {'none': none, 'true': true, 'false': false}

            if isinstance(node_or_string, basestring):
                node_or_string = parse(node_or_string, mode='eval')
            if isinstance(node_or_string, compiler.ast.expression):
                node_or_string = node_or_string.node

            def _convert(node):
                if isinstance(node, compiler.ast.const) and isinstance(
                        node.value, (basestring, int, float, long, complex)):
                    return node.value
                elif isinstance(node, compiler.ast.tuple):
                    return tuple(map(_convert, node.nodes))
                elif isinstance(node, compiler.ast.list):
                    return list(map(_convert, node.nodes))
                elif isinstance(node, compiler.ast.dict):
                    return dict(
                        (_convert(k), _convert(v)) for k, v in node.items)
                elif isinstance(node, compiler.ast.name):
                    if node.name in _safe_names:
                        return _safe_names[node.name]
                elif isinstance(node, compiler.ast.unarysub):
                    return -_convert(node.expr)
                raise valueerror('malformed string')

            return _convert(node_or_string)
Example #30
0
def get_code_complexity(code, min=7, filename='stdin'):
    complex = []
    try:
        ast = parse(code)
    except AttributeError:
        e = sys.exc_info()[1]
        sys.stderr.write("Unable to parse %s: %s\n" % (filename, e))
        return 0

    visitor = PathGraphingAstVisitor()
    visitor.preorder(ast, visitor)
    for graph in visitor.graphs.values():
        if graph is None:
            # ?
            continue
        if graph.complexity() >= min:
            complex.append(
                dict(type='W',
                     lnum=graph.lineno,
                     text='%s %r is too complex (%d)' % (
                         WARNING_CODE,
                         graph.entity,
                         graph.complexity(),
                     )))

    return complex
Example #31
0
def main():
    filename = 'tests/implicit_main.py'
    ast = parse(open(filename).read())
    import pprint
    pprint.pprint(ast)
    s = ast2py(ast)
    return s
Example #32
0
        def literal_eval(node_or_string):
            _safe_names = {'none': none, 'true': true, 'false': false}

            if isinstance(node_or_string, basestring):
                node_or_string = parse(node_or_string, mode='eval')
            if isinstance(node_or_string, compiler.ast.expression):
                node_or_string = node_or_string.node

            def _convert(node):
                if isinstance(node, compiler.ast.const) and isinstance(node.value,
                        (basestring, int, float, long, complex)):
                     return node.value
                elif isinstance(node, compiler.ast.tuple):
                    return tuple(map(_convert, node.nodes))
                elif isinstance(node, compiler.ast.list):
                    return list(map(_convert, node.nodes))
                elif isinstance(node, compiler.ast.dict):
                    return dict((_convert(k), _convert(v)) for k, v
                                in node.items)
                elif isinstance(node, compiler.ast.name):
                    if node.name in _safe_names:
                        return _safe_names[node.name]
                elif isinstance(node, compiler.ast.unarysub):
                    return -_convert(node.expr)
                raise valueerror('malformed string')

            return _convert(node_or_string)
    def checkBeforeAndAfter(self):
        from RestrictedPython.RCompile import RModule
        from RestrictedPython.tests import before_and_after
        from compiler import parse

        defre = re.compile(r'def ([_A-Za-z0-9]+)_(after|before)\(')

        beforel = [
            name for name in before_and_after.__dict__
            if name.endswith("_before")
        ]

        for name in beforel:
            before = getattr(before_and_after, name)
            before_src = get_source(before)
            before_src = re.sub(defre, r'def \1(', before_src)
            rm = RModule(before_src, '')
            tree_before = rm._get_tree()

            after = getattr(before_and_after, name[:-6] + 'after')
            after_src = get_source(after)
            after_src = re.sub(defre, r'def \1(', after_src)
            tree_after = parse(after_src)

            self.assertEqual(str(tree_before), str(tree_after))

            rm.compile()
            verify.verify(rm.getCode())
Example #34
0
    def find_executable_statements(self, text, exclude=None):
        # Find lines which match an exclusion pattern.
        excluded = {}
        suite_spots = {}
        if exclude:
            reExclude = re.compile(exclude)
            lines = text.split('\n')
            for i in range(len(lines)):
                if reExclude.search(lines[i]):
                    excluded[i + 1] = 1

        # Parse the code and analyze the parse tree to find out which statements
        # are multiline, and where suites begin and end.
        import parser
        tree = parser.suite(text + '\n\n').totuple(1)
        self.get_suite_spots(tree, suite_spots)
        #print "Suite spots:", suite_spots

        # Use the compiler module to parse the text and find the executable
        # statements.  We add newlines to be impervious to final partial lines.
        statements = {}
        ast = compiler.parse(text + '\n\n')
        visitor = StatementFindingAstVisitor(statements, excluded, suite_spots)
        compiler.walk(ast, visitor, walker=visitor)

        lines = statements.keys()
        lines.sort()
        excluded_lines = excluded.keys()
        excluded_lines.sort()
        return lines, excluded_lines, suite_spots
Example #35
0
def run_pyflakes_for_package(package_name, extra_ignore=None):
    """
    If pyflakes is installed, run it across the given package name 
    returning any warnings found.
    """
    ignore_strings = PYFLAKES_IGNORE
    if extra_ignore:
        ignore_strings += extra_ignore
    try:
        from pyflakes.checker import Checker
    except ImportError:
        return []
    warnings = []
    for (root, dirs, files) in os.walk(path_for_import(package_name)):
        for f in files:
            # Ignore migrations.
            directory = root.split(os.sep)[-1]
            if not f.endswith(".py") or directory == "migrations":
                continue
            path = os.path.join(root, f)
            with open(path, "U") as source_file:
                source = source_file.read()
            try:
                compile(source, f, "exec")
            except (SyntaxError, IndentationError), value:
                info = (path, value.lineno, value.args[0])
                warnings.append("Invalid syntax in %s:%d: %s" % info)
            result = Checker(parse(source), path)
            for warning in result.messages:
                message = unicode(warning)
                for ignore in ignore_strings:
                    if ignore in message:
                        break
                else:
                    warnings.append(message)
Example #36
0
def main(argv):
    opar = optparse.OptionParser()
    opar.add_option("-d",
                    "--dot",
                    dest="dot",
                    help="output a graphviz dot file",
                    action="store_true")
    opar.add_option("-m",
                    "--min",
                    dest="min",
                    help="minimum complexity for output",
                    type="int",
                    default=2)

    options, args = opar.parse_args(argv)

    text = open(args[0], "rU").read() + '\n\n'
    ast = parse(text)
    visitor = PathGraphingAstVisitor()
    visitor.preorder(ast, visitor)

    if options.dot:
        print('graph {')
        for graph in visitor.graphs.values():
            if graph.complexity() >= options.min:
                graph.to_dot()
        print('}')
    else:
        for graph in visitor.graphs.values():
            if graph.complexity() >= options.min:
                print(graph.name, graph.complexity())
Example #37
0
def parse_python_source(fn):
    """Parse the file 'fn' and return two things:

    1. The AST tree.
    2. A list of lines of the source line (typically used for verbose error
       messages).

    If the file has a syntax error in it, the first argument will be None.
    """
    # Read the file's contents to return it.
    # Note: we make sure to use universal newlines.
    try:
        contents = open(fn, 'rU').read()
        lines = contents.splitlines()
    except (IOError, OSError) as e:
        logging.error("Could not read file '%s'." % fn)
        return None, None

    # Convert the file to an AST.
    try:
        ast = compiler.parse(contents)
    except SyntaxError as e:
        err = '%s:%s: %s' % (fn, e.lineno or '--', e.msg)
        logging.error("Error processing file '%s':\n%s" % (fn, err))
        return None, lines
    except TypeError as e:
        # Note: this branch untested, applied from a user-submitted patch.
        err = '%s: %s' % (fn, str(e))
        logging.error("Error processing file '%s':\n%s" % (fn, err))
        return None, lines

    return ast, lines
Example #38
0
 def run(self):
     self.sleep(2)
     exts = settings.SYNTAX.get('python')['extension']
     file_ext = file_manager.get_file_extension(self._editor.ID)
     if file_ext in exts:
         try:
             self.reset()
             source = self._editor.get_text()
             if self._editor.encoding is not None:
                 source = source.encode(self._editor.encoding)
             parseResult = compiler.parse(source)
             self.checker = checker.Checker(parseResult, self._editor.ID)
             for m in self.checker.messages:
                 lineno = m.lineno - 1
                 if lineno not in self.errorsSummary:
                     message = [m.message % m.message_args]
                 else:
                     message = self.errorsSummary[lineno]
                     message += [m.message % m.message_args]
                 self.errorsSummary[lineno] = message
         except Exception, reason:
             if hasattr(reason, 'lineno'):
                 self.errorsSummary[reason.lineno - 1] = [reason.message]
             else:
                 self.errorsSummary[0] = [reason.message]
Example #39
0
def safeEval(s, namespace={'True': True, 'False': False, 'None': None}):
    """Evaluates s, safely.  Useful for turning strings into tuples/lists/etc.
    without unsafely using eval()."""
    try:
        node = compiler.parse(s)
    except SyntaxError, e:
        raise ValueError, 'Invalid string: %s.' % e
def check(codeString, filename):
    message = []
    try:
        codeString = codeString.replace('\r\n', '\n')
        tree = compiler.parse(codeString)
    except (SyntaxError, IndentationError):
        value = sys.exc_info()[1]
        if isinstance(value, Exception):
            lineno = value.lineno
            offset = value.offset
            line = value.text
            #            if line and line.endswith("\n"):
            #                line = line[:-1]
            message.append((filename, lineno, value.msg))
    except:
        error.traceback()
        message.append(
            (filename, -1,
             tr('There are some unknown errors, please check error.txt')))
    else:
        w = checker.Checker(tree, filename)
        w.messages.sort(lambda a, b: cmp(a.lineno, b.lineno))
        for warning in w.messages:
            message.append((filename, warning.lineno,
                            warning.message % warning.message_args))

    return message
Example #41
0
    def literal_eval(node_or_string):
        """
        Safely evaluate an expression node or a string containing a Python
        expression.  The string or node provided may only consist of the  following
        Python literal structures: strings, numbers, tuples, lists, dicts,  booleans,
        and None.
        """
        _safe_names = {'None': None, 'True': True, 'False': False}
        if isinstance(node_or_string, string_types):
            node_or_string = parse(node_or_string, mode='eval')
        if isinstance(node_or_string, ast.Expression):
            node_or_string = node_or_string.node

        def _convert(node):
            if isinstance(node, ast.Const) and isinstance(node.value, (text_type, binary_type, float, complex) + integer_types):
                return node.value
            elif isinstance(node, ast.Tuple):
                return tuple(map(_convert, node.nodes))
            elif isinstance(node, ast.List):
                return list(map(_convert, node.nodes))
            elif isinstance(node, ast.Dict):
                return dict((_convert(k), _convert(v)) for k, v in node.items())
            elif isinstance(node, ast.Name):
                if node.name in _safe_names:
                    return _safe_names[node.name]
            elif isinstance(node, ast.UnarySub):
                return -_convert(node.expr)
            raise ValueError('malformed string')
        return _convert(node_or_string)
Example #42
0
def get_code_complexity(code, min=7, filename='stdin'):
    complex = []
    try:
        ast = parse(code)
    except (AttributeError, SyntaxError):
        e = sys.exc_info()[1]
        sys.stderr.write("Unable to parse %s: %s\n" % (filename, e))
        return 0

    visitor = PathGraphingAstVisitor()
    visitor.preorder(ast, visitor)
    for graph in visitor.graphs.values():
        if graph is None:
            # ?
            continue
        if graph.complexity() >= min:
            graph.filename = filename
            if not skip_warning(graph):
                msg = '%s:%d:1: %s %r is too complex (%d)' % (
                    filename,
                    graph.lineno,
                    WARNING_CODE,
                    graph.entity,
                    graph.complexity(),
                )
                complex.append(msg)

    if len(complex) == 0:
        return 0

    print('\n'.join(complex))
    return len(complex)
Example #43
0
def get_code_complexity(code, min=7, filename='stdin'):
    complex = []
    try:
        ast = parse(code)
    except (AttributeError, SyntaxError):
        e = sys.exc_info()[1]
        sys.stderr.write("Unable to parse %s: %s\n" % (filename, e))
        return 0

    visitor = PathGraphingAstVisitor()
    visitor.preorder(ast, visitor)
    for graph in visitor.graphs.values():
        if graph is None:
            # ?
            continue
        if graph.complexity() >= min:
            graph.filename = filename
            if not skip_warning(graph):
                msg = '%s:%d:1: %s %r is too complex (%d)' % (
                    filename,
                    graph.lineno,
                    WARNING_CODE,
                    graph.entity,
                    graph.complexity(),
                )
                complex.append(msg)

    if len(complex) == 0:
        return 0

    print('\n'.join(complex))
    return len(complex)
Example #44
0
def parse_python_source(fn):
     contents = open(fn, 'rU').read()
     ast = compiler.parse(contents)
     vis = ImportVisitor() 

     compiler.walk(ast, vis, ImportWalker(vis))
     return vis.finalize()
Example #45
0
def parse_python_source(fn):
    contents = open(fn, 'rU').read()
    ast = compiler.parse(contents)
    vis = ImportVisitor()

    compiler.walk(ast, vis, ImportWalker(vis))
    return vis.finalize()
    def _checkBeforeAndAfter(self, mod):
            from RestrictedPython.RCompile import RModule
            from compiler import parse

            defre = re.compile(r'def ([_A-Za-z0-9]+)_(after|before)\(')

            beforel = [name for name in mod.__dict__
                       if name.endswith("_before")]

            for name in beforel:
                before = getattr(mod, name)
                before_src = get_source(before)
                before_src = re.sub(defre, r'def \1(', before_src)
                rm = RModule(before_src, '')
                tree_before = rm._get_tree()

                after = getattr(mod, name[:-6]+'after')
                after_src = get_source(after)
                after_src = re.sub(defre, r'def \1(', after_src)
                tree_after = parse(after_src)

                self.assertEqual(str(tree_before), str(tree_after))

                rm.compile()
                verify.verify(rm.getCode())
Example #47
0
def parse_n3_term(src):
    """
    Parse a Notation3 value into a RDFTerm object (IRI or Literal).

    This parser understands IRIs and quoted strings; basic non-string types
    (integers, decimals, booleans, etc) are not supported yet.
    """

    src = unicode(src)

    if src.startswith('<'):
        # `src` is an IRI
        if not src.endswith('>'):
            raise ValueError

        value = src[1:-1]
        if '<' in value or '>' in value:
            raise ValueError

        return IRI(value)

    else:
        datatype_match = _n3parser_datatype.search(src)
        if datatype_match is not None:
            datatype = datatype_match.group('datatype')
            src = _n3parser_datatype.sub('', src)
        else:
            datatype = None

        lang_match = _n3parser_lang.search(src)
        if lang_match is not None:
            lang = lang_match.group('lang')
            src = _n3parser_lang.sub('', src)
        else:
            lang = None

        # Python literals syntax is mostly compatible with N3.
        # We don't execute the code, just turn it into an AST.
        try:
            ast = compiler.parse("value = u" + src)
        except:
            raise ValueError

        # Don't allow any extra tokens in the AST
        if len(ast.node.getChildNodes()) != 1:
            raise ValueError
        assign_node = ast.node.getChildNodes()[0]
        if len(assign_node.getChildNodes()) != 2:
            raise ValueError
        value_node = assign_node.getChildNodes()[1]
        if value_node.getChildNodes():
            raise ValueError
        if value_node.__class__ != compiler.ast.Const:
            raise ValueError
        value = value_node.value

        if type(value) is not unicode:
            raise ValueError

        return Literal(value, datatype, lang)
Example #48
0
    def compile(self, viewcode_str, filename):
        """Compile the view code and return a code object
        and dictionary of globals needed by the code object.
        """
        viewcode = compile(viewcode_str, filename, 'exec')

        # scan top-level code only for "import foo" and
        # "from foo import *" and "from foo import bar, baz"
        viewglobals = {'__builtins__': __builtins__}
        for stmt in compiler.parse(viewcode_str).node:
            if isinstance(stmt, Import):
                modname, asname = stmt.names[0]
                if asname is None:
                    asname = modname
                viewglobals[asname] = __import__(modname)
            elif isinstance(stmt, From):
                fromlist = [x[0] for x in stmt.names]
                module = __import__(stmt.modname, {}, {}, fromlist)
                for name, asname in stmt.names:
                    if name == '*':
                        for starname in getattr(module, '__all__', dir(module)):
                            viewglobals[starname] = getattr(module, starname)
                    else:
                        if asname is None:
                            asname = name
                        viewglobals[asname] = getattr(module, name)

        return viewcode, viewglobals
Example #49
0
    def find_executable_statements(self, text, exclude=None):
        # Find lines which match an exclusion pattern.
        excluded = {}
        suite_spots = {}
        if exclude:
            reExclude = re.compile(exclude)
            lines = text.split('\n')
            for i in range(len(lines)):
                if reExclude.search(lines[i]):
                    excluded[i+1] = 1

        # Parse the code and analyze the parse tree to find out which statements
        # are multiline, and where suites begin and end.
        import parser
        tree = parser.suite(text+'\n\n').totuple(1)
        self.get_suite_spots(tree, suite_spots)
        #print "Suite spots:", suite_spots
        
        # Use the compiler module to parse the text and find the executable
        # statements.  We add newlines to be impervious to final partial lines.
        statements = {}
        ast = compiler.parse(text+'\n\n')
        visitor = StatementFindingAstVisitor(statements, excluded, suite_spots)
        compiler.walk(ast, visitor, walker=visitor)

        lines = statements.keys()
        lines.sort()
        excluded_lines = excluded.keys()
        excluded_lines.sort()
        return lines, excluded_lines, suite_spots
Example #50
0
def scan_python_file(filename, calls):
    """Scan a python file for gettext calls."""
    def scan(nodelist):
        for node in nodelist:
            if isinstance(node, ast.CallFunc):
                handle = False
                for pos, n in enumerate(node):
                    if pos == 0:
                        if isinstance(n, ast.Name) and n.name in calls:
                            handle = True
                    elif pos == 1:
                        if handle:
                            if n.__class__ is ast.Const and \
                               isinstance(n.value, basestring):
                                yield n.lineno, n.value
                            break
                        else:
                            for line in scan([n]):
                                yield line
            elif hasattr(node, '__iter__'):
                for n in scan(node):
                    yield n

    fp = file(filename)
    try:
        try:
            return scan(parse(fp.read()))
        except:
            print >> sys.stderr, 'Syntax Error in file %r' % filename
    finally:
        fp.close()
Example #51
0
def main(argv):
    opar = optparse.OptionParser()
    opar.add_option("-d", "--dot", dest="dot",
                    help="output a graphviz dot file", action="store_true")
    opar.add_option("-m", "--min", dest="min",
                    help="minimum complexity for output", type="int",
                    default=2)

    options, args = opar.parse_args(argv)

    text = open(args[0], "rU").read() + '\n\n'
    ast = parse(text)
    visitor = PathGraphingAstVisitor()
    visitor.preorder(ast, visitor)

    if options.dot:
        print('graph {')
        for graph in visitor.graphs.values():
            if graph.complexity() >= options.min:
                graph.to_dot()
        print('}')
    else:
        for graph in visitor.graphs.values():
            if graph.complexity() >= options.min:
                print(graph.name, graph.complexity())
Example #52
0
 def parse_python(self, lineno, gen, template):
     """
     Convert the passed generator into a flat string representing
     python sourcecode and return an ast node or raise a
     TemplateSyntaxError.
     """
     tokens = []
     for t_lineno, t_token, t_data in gen:
         if t_token == 'string':
             # because some libraries have problems with unicode
             # objects we do some lazy unicode processing here.
             # if a string is ASCII only we yield it as string
             # in other cases as unicode. This works around
             # problems with datetimeobj.strftime()
             # also escape newlines in strings
             t_data = t_data.replace('\n', '\\n')
             try:
                 str(t_data)
             except UnicodeError:
                 tokens.append('u' + t_data)
                 continue
         tokens.append(t_data)
     source = '\xef\xbb\xbf' + (template % (u' '.join(tokens)).
                                encode('utf-8'))
     try:
         ast = parse(source, 'exec')
     except SyntaxError, e:
         raise TemplateSyntaxError('invalid syntax in expression',
                                   lineno + (e.lineno or 0),
                                   self.filename)
Example #53
0
 def _literal_eval(node_or_string):
     """
     Safely evaluate an expression node or a string containing a Python
     expression.  The string or node provided may only consist of the  following
     Python literal structures: strings, numbers, tuples, lists, dicts,  booleans,
     and None.
     """
     _safe_names = {'None': None, 'True': True, 'False': False}
     if isinstance(node_or_string, basestring):
         node_or_string = parse(node_or_string, mode='eval')
     if isinstance(node_or_string, Expression):
         node_or_string = node_or_string.node
     def _convert(node):
         if isinstance(node, Const) and isinstance(node.value, (basestring, int, float, long, complex)):
              return node.value
         elif isinstance(node, Tuple):
             return tuple(map(_convert, node.nodes))
         elif isinstance(node, List):
             return list(map(_convert, node.nodes))
         elif isinstance(node, Dict):
             return dict((_convert(k), _convert(v)) for k, v in node.items)
         elif isinstance(node, Name):
             if node.name in _safe_names:
                 return _safe_names[node.name]
         elif isinstance(node, UnarySub):
             return -_convert(node.expr)
         raise ValueError('malformed string')
     return _convert(node_or_string)
Example #54
0
def scan_python_file(filename, calls):
    """Scan a python file for gettext calls."""
    def scan(nodelist):
        for node in nodelist:
            if isinstance(node, ast.CallFunc):
                handle = False
                for pos, n in enumerate(node):
                    if pos == 0:
                        if isinstance(n, ast.Name) and n.name in calls:
                            handle = True
                    elif pos == 1:
                        if handle:
                            if n.__class__ is ast.Const and \
                               isinstance(n.value, basestring):
                                yield n.lineno, n.value
                            break
                        else:
                            for line in scan([n]):
                                yield line
            elif hasattr(node, '__iter__'):
                for n in scan(node):
                    yield n

    fp = file(filename)
    try:
        try:
            return scan(parse(fp.read()))
        except:
            print >> sys.stderr, 'Syntax Error in file %r' % filename
    finally:
        fp.close()
Example #55
0
def test_with_pyflakes(path='.'):
    """
    Test code with Pyflakes
    """
    import pyflakes.checker
    exclude = 'ws-meta-in.py', 'test_syntax.py', 'test_point.py', 'test_pml.py'
    cwd = os.getcwd()
    top = os.path.join(os.path.dirname(__file__), '..')
    os.chdir(top)
    messages = []
    for dirpath, dirnames, filenames in os.walk(path):
        for filename in filenames:
            if filename.endswith('.py') and filename not in exclude:
                filename = os.path.join(dirpath, filename)
                code = open(filename, 'U').read()
                compile(code, filename, 'exec')
                tree = compiler.parse(code)
                checker = pyflakes.checker.Checker(tree, filename)
                for m in checker.messages:
                    if (not m.filename.endswith('__init__.py')
                            or m.message != '%r imported but unused'):
                        messages += [m]
                        print m
    os.chdir(cwd)
    assert messages == []
    return
Example #56
0
def get_code_complexity(code, min=7, filename='stdin'):
    complex = []
    try:
        ast = parse(code)
    except AttributeError:
        e = sys.exc_info()[1]
        sys.stderr.write("Unable to parse %s: %s\n" % (filename, e))
        return 0

    visitor = PathGraphingAstVisitor()
    visitor.preorder(ast, visitor)
    for graph in visitor.graphs.values():
        if graph is None:
            # ?
            continue
        if graph.complexity() >= min:
            complex.append(dict(
                type = 'W',
                lnum = graph.lineno,
                text = '%s %r is too complex (%d)' % (
                    WARNING_CODE,
                    graph.entity,
                    graph.complexity(),
                )
            ))

    return complex
Example #57
0
def parse_module(module_text, filename):
    """Return a module documentation tree from `module_text`."""
    ast = compiler.parse(module_text)
    token_parser = TokenParser(module_text)
    visitor = ModuleVisitor(filename, token_parser)
    compiler.walk(ast, visitor, walker=visitor)
    return visitor.module
Example #58
0
def test_with_pyflakes(path="."):
    """
    Test code with Pyflakes
    """
    import pyflakes.checker

    exclude = "ws-meta-in.py", "test_syntax.py", "test_point.py", "test_pml.py"
    cwd = os.getcwd()
    top = os.path.join(os.path.dirname(__file__), "..")
    os.chdir(top)
    messages = []
    for dirpath, dirnames, filenames in os.walk(path):
        for filename in filenames:
            if filename.endswith(".py") and filename not in exclude:
                filename = os.path.join(dirpath, filename)
                code = open(filename, "U").read()
                compile(code, filename, "exec")
                tree = compiler.parse(code)
                checker = pyflakes.checker.Checker(tree, filename)
                for m in checker.messages:
                    if not m.filename.endswith("__init__.py") or m.message != "%r imported but unused":
                        messages += [m]
                        print m
    os.chdir(cwd)
    assert messages == []
    return