def test_no_asserts(self): """bzr shouldn't use the 'assert' statement.""" # assert causes too much variation between -O and not, and tends to # give bad errors to the user def search(x): # scan down through x for assert statements, report any problems # this is a bit cheesy; it may get some false positives? if x[0] == symbol.assert_stmt: return True elif x[0] == token.NAME: # can't search further down return False for sub in x[1:]: if sub and search(sub): return True return False badfiles = [] for fname, text in self.get_source_file_contents(): if not self.is_our_code(fname): continue ast = parser.ast2tuple(parser.suite(''.join(text))) if search(ast): badfiles.append(fname) if badfiles: self.fail( "these files contain an assert statement and should not:\n%s" % '\n'.join(badfiles))
def parse2pom(inf): ast = parser.suite(inf.read()) t = parser.ast2tuple(ast, 1) # with line numbers del ast top = _get_class(t[0])() for i in t[1:]: _add_element(top, i) return top
def test_compile_empty_function(self): ast1 = parser.suite('def f(): pass') ast2 = cpy_parser.suite('def f(): pass') tup1 = parser.ast2tuple(ast1) tup2 = cpy_parser.ast2tuple(ast2) self.assertEquals(tup1, tup2)
def parse(expression_string): # To be forgiving, clean up the string a bit. expression_string = expression_string.replace("\n", " ").strip() # Now parse it. try: ast = parser.ast2tuple(parser.expr(expression_string)) except parser.ParserError, exception: raise ParseError, str(exception)
def python_parse(source, mode='exec', lineno=False): """parse python source using CPython's parser module and return nested tuples """ if mode == 'eval': tp = parser.expr(source) else: tp = parser.suite(source) return parser.ast2tuple(tp, line_info=lineno)
def test_compile_function(self): source = "def f(x, arg1=None): return x" ast1 = parser.suite(source) ast2 = cpy_parser.suite(source) tup1 = parser.ast2tuple(ast1) tup2 = cpy_parser.ast2tuple(ast2) self.assertEquals(tup1, tup2)
def __init__(self, fp): """ Count lines of code in 'fp'. """ self.lines = set() self.ast = parser.suite(fp.read()) self.tree = parser.ast2tuple(self.ast, True) self.find_terminal_nodes(self.tree)
def __init__(self, source_no_encoding, pubapi): # Our public API (__all__) self.pubapi = pubapi # Names of imported modules self.modnames = [] self.symtab = symtable.symtable(source_no_encoding, "-", "exec") cst = parser.suite(source_no_encoding) elements = parser.ast2tuple(cst, line_info=1) self.names = {} self.walk(elements, [self.symtab])
def test_full_expression(self): full_expr = "urbansim.gridcell.population" t = parser.ast2tuple(parser.suite(full_expr)) same1, vars1 = match(FULL_TREE_EXPRESSION, t) self.assert_(same1, msg="pattern did not match") expr_tree = vars1['expr'] same2, vars2 = match(EXPRESSION_IS_FULLY_QUALIFIED_VARIABLE, expr_tree) self.assert_(same2, msg="pattern did not match") self.assertEqual(len(vars2), 3, msg="wrong number of items in dictionary") self.assertEqual(vars2['package'], 'urbansim', msg="bad value in dictionary") self.assertEqual(vars2['dataset'], 'gridcell', msg="bad value in dictionary") self.assertEqual(vars2['shortname'], 'population', msg="bad value in dictionary")
def testChunk(t, fileName): global _numFailed print('----', fileName, end=' ') try: ast = parser.suite(t) tup = parser.ast2tuple(ast) # this discards the first AST; a huge memory savings when running # against a large source file like Tkinter.py. ast = None new = parser.tuple2ast(tup) except parser.ParserError as err: print() print('parser module raised exception on input file', fileName + ':') traceback.print_exc() _numFailed = _numFailed + 1 else: if tup != parser.ast2tuple(new): print() print('parser module failed on input file', fileName) _numFailed = _numFailed + 1 else: print('o.k.')
def get_docs(fileName): """Retrieve information from the parse tree of a source file. fileName Name of the file to read Python source code from. """ source = open(fileName).read() import os basename = os.path.basename(os.path.splitext(fileName)[0]) import parser ast = parser.suite(source) tup = parser.ast2tuple(ast) return ModuleInfo(tup, basename)
def testChunk(t, fileName): global _numFailed print "----", fileName, try: ast = parser.suite(t) tup = parser.ast2tuple(ast) # this discards the first AST; a huge memory savings when running # against a large source file like Tkinter.py. ast = None new = parser.tuple2ast(tup) except parser.ParserError, err: print print "parser module raised exception on input file", fileName + ":" traceback.print_exc() _numFailed = _numFailed + 1
def testChunk(t, fileName): global _numFailed print '----', fileName, try: ast = parser.suite(t) tup = parser.ast2tuple(ast) # this discards the first AST; a huge memory savings when running # against a large source file like Tkinter.py. ast = None new = parser.tuple2ast(tup) except parser.ParserError, err: print print 'parser module raised exception on input file', fileName + ':' traceback.print_exc() _numFailed = _numFailed + 1
def MASK_test_full_assignment_with_comment_and_newline(self): """ Parse an assignment and match it. In addition to test_full_assignment_with_comment, this checks if comments terminated by newline are supported for a variable. Currently broken for Python 2.6. """ full_expr = "myvar = urbansim.gridcell.population # comment\n" t = parser.ast2tuple(parser.suite(full_expr)) same1, vars1 = match(FULL_TREE_ASSIGNMENT, t) self.assert_(same1, msg="pattern did not match") expr_tree = vars1['expr'] same2, vars2 = match(EXPRESSION_IS_FULLY_QUALIFIED_VARIABLE, expr_tree) self.assert_(same2, msg="pattern did not match") self.assertEqual(len(vars2), 3, msg="wrong number of items in dictionary") self.assertEqual(vars2['package'], 'urbansim', msg="bad value in dictionary") self.assertEqual(vars2['dataset'], 'gridcell', msg="bad value in dictionary") self.assertEqual(vars2['shortname'], 'population', msg="bad value in dictionary")
def test_full_expression(self): """ Parse an expression and match it. This checks that this version of Python is producing parse trees like those that the patterns were constructed from. Not a complete check, but does some checking for changes between versions of Python. """ full_expr = "urbansim.gridcell.population" t = parser.ast2tuple(parser.suite(full_expr)) same1, vars1 = match(FULL_TREE_EXPRESSION, t) self.assert_(same1, msg="pattern did not match") expr_tree = vars1['expr'] same2, vars2 = match(EXPRESSION_IS_FULLY_QUALIFIED_VARIABLE, expr_tree) self.assert_(same2, msg="pattern did not match") self.assertEqual(len(vars2), 3, msg="wrong number of items in dictionary") self.assertEqual(vars2['package'], 'urbansim', msg="bad value in dictionary") self.assertEqual(vars2['dataset'], 'gridcell', msg="bad value in dictionary") self.assertEqual(vars2['shortname'], 'population', msg="bad value in dictionary")
def test_full_expression_with_comment(self): """ Parse an expression and match it. In addition to test_full_expression, this checks if comments are supported for a variable. This test used to fail for Python 2.7. """ full_expr = "urbansim.gridcell.population #comment" t = parser.ast2tuple(parser.suite(full_expr)) same1, vars1 = match(FULL_TREE_EXPRESSION, t) self.assert_(same1, msg="pattern did not match") expr_tree = vars1['expr'] same2, vars2 = match(EXPRESSION_IS_FULLY_QUALIFIED_VARIABLE, expr_tree) self.assert_(same2, msg="pattern did not match") self.assertEqual(len(vars2), 3, msg="wrong number of items in dictionary") self.assertEqual(vars2['package'], 'urbansim', msg="bad value in dictionary") self.assertEqual(vars2['dataset'], 'gridcell', msg="bad value in dictionary") self.assertEqual(vars2['shortname'], 'population', msg="bad value in dictionary")
def _parse_expr(self, expr): # Parse expr and return the parsetree and alias. # If expr is just an expression, then alias will be None. # If expr is an assignment v=e then alias will be v, and # expr_parsetree will be the parsetree for e. # If the parse raises a syntax error, just let that be handled # by the regular Python compiler's error handler. # Raise an exception if the expression doesn't match either an expression # or a statement (this would happen if the expression consists of multiple # statements, which parses correctly so wouldn't be caught by the Python compiler). full_tree = parser.ast2tuple(parser.suite(expr)) same, vars = match(FULL_TREE_EXPRESSION, full_tree) if same: return (vars['expr'], None) same, vars = match(FULL_TREE_ASSIGNMENT, full_tree) if same: return (vars['expr'], vars['alias']) raise ValueError, "invalid expression (perhaps multiple statements?): " + expr
def process_file(self, scope, sourcefile, sxr): self.scopes = list(scope) input = open(sourcefile.abs_name, 'r+') src = input.readlines() self.lines = len(`len(src) + 1`) ptree = parser.ast2tuple(parser.suite(''.join(src))) input.seek(0) self.lexer = tokenize.generate_tokens(input.readline) #self.lexer = LexerDebugger(tokenize.generate_tokens(input.readline)) self.sxr = open(sxr, 'w+') lineno_template = '%%%ds' % self.lines lineno = lineno_template % self.lineno self.sxr.write(header % {'filename': sourcefile.name}) try: self.handle(ptree) except StopIteration: raise self.sxr.write(trailer) self.scopes.pop()
def test_var_parsetree_to_string(self): expr = "x" t = parser.ast2tuple(parser.suite(expr)) s = parsetree_to_string(t) self.assertEqual(s, expr)
def test_adjacent_keywords_parsetree_to_string(self): expr = "x not in dict and y<3*z" t = parser.ast2tuple(parser.suite(expr)) s = parsetree_to_string(t) self.assertEqual(s, expr)
#! /usr/bin/env python
def test_expr1_parsetree_to_string(self): expr = "urbansim.gridcell.population" t = parser.ast2tuple(parser.suite(expr)) s = parsetree_to_string(t) self.assertEqual(s, expr)
def test_expr2_parsetree_to_string(self): expr = "myneighborhood.aggregate(10*myzone.my_variable,intermediates=[myfaz,myfazdistr],function=sum)" t = parser.ast2tuple(parser.suite(expr)) s = parsetree_to_string(t) self.assertEqual(s, expr)
def parse(expression_string): # To be forgiving, clean up the string a bit. expression_string = expression_string.replace("\n", " ").strip() # Now parse it. try: ast = parser.ast2tuple(parser.expr(expression_string)) except parser.ParserError, exception: raise ParseError, str(exception) expression = fromAst(ast) # Perform standard substitutions. expression = symbols.substituteConstants(expression) return expression if __name__ == "__main__": import sys expression = sys.argv[1] ast = parser.ast2tuple(parser.expr(expression)) import pprint pprint.pprint(_addNamesToAstList(ast)) result = fromAst(ast) print print repr(result) print str(result)
"""Parse tree transformation module.
def Show(): "Edit local config file." global AssignVar, Assignments, Config, LineNum, LastLine # Load the display template T = Template.Template("localconfig.html") T["ErrMsg"] = "Displaying requested view. Click on filename to toggle." ErrStr = None # Toggle modes? if Form.has_key("subcmd") and (Form["subcmd"].value == "toggle"): if PVars["LocalConfig"] == "Text": PVars["LocalConfig"] = "Form" else: PVars["LocalConfig"] = "Text" PVars.Save() # Get file T["FilePath"] = Defaults.TMDARC try: F = open(Defaults.TMDARC) FileContents = F.readlines() F.close() except IOError: FileContents = [] # Which view does the user want? Form or text? if PVars["LocalConfig"] == "Form" and Version.TMDA < 1.1: # User wants to view the config in form mode. First we need to dismantle # the current config file in a way that we can rebuild it, then we need to # verify that it is safe to proceed. # Extract any comments & blank lines Config = {} LastLine = 1 for LastLine in range(len(FileContents)): if CommentSearch.search(FileContents[LastLine]): Config[LastLine + 1] = FileContents[LastLine].strip() LastLine += 2 # Test to see if we can do this safely FileContents = "".join(FileContents) ASP = parser.ast2tuple(parser.suite(FileContents), 1) Assignments = {} try: # Capture each expression for Exp in ASP[1:]: AssignVar = None Line = Parse(Exp[1], ConfigSyms) # Track commands in dictionaries if Line: Config[LineNum] = Line if AssignVar: Assignments[AssignVar] = LineNum except (KeyError, parser.ParserError), ErrMsg: # Nope. Not safe! T["ErrMsg"] = """Reverted to text mode. Not safe to proceed in form mode. %s of current %s""" % (ErrMsg, Defaults.TMDARC) PVars["LocalConfig"] = "Text" PVars.Save() else: # We're in form view, hide text view T["TextView"] # Save changes? FormVars = re.split("[,\s]+", T["FormVars"]) if Form.has_key("subcmd") and (Form["subcmd"].value == "save"): try: # Go through each variable we're supporting and determine: # [1] Is the given value acceptable? # [2] Is the new value different than the old? for Var in FormVars: Parts = Var.split(":") # Provided a value? if Form.has_key(Parts[0]) and Form[Parts[0]].value.strip(): Value = Form[Parts[0]].value if len(Parts) == 2: # Lists and dictionaries must be tested for dangerous code ASP = parser.ast2tuple(parser.suite(Value), 1) if Parts[1] == "L": Value = Parse(ASP, ListSyms) else: Value = Parse(ASP, DictSyms) else: # Integers are okay as-is, strings must be escaped if (len(Parts) == 1) or (Parts[1] != "I"): Value = Escape(Value) # If no value is given, use None else: Value = "None" # Has value been changed? if vars(Defaults)[Parts[0]] != eval(Value): Set(Parts[0], Value) vars(Defaults)[Parts[0]] = eval(Value) # Check code for syntax errors BEFORE saving try: # Re-assemble config Temp = Config Config = "" Lines = Temp.keys() Lines.sort() for Line in Lines: Config += Temp[Line] + "\n" # Try to execute it try: exec(Config) except (ImportError, NameError): pass # Looks okay, so save it T["ErrRow"] F = open(Defaults.TMDARC, "w") F.write(Config) F.close() except SyntaxError, (ErrStr, Details): T["ErrStr"] = "SyntaxError: line %d, char %d<br>(%s)" % Details[1:4] except (KeyError, parser.ParserError), ErrStr: # Don't like the look of this var! T["ErrStr"] = """<nobr>Contents of variable %s look "unsafe".<br> %s</nobr>""" % (Parts[0], ErrStr) # Display current values for Var in FormVars: Parts = Var.split(":") Value = vars(Defaults)[Parts[0]] if Value == None: Value = "" else: if len(Parts) == 2: Value = repr(Value) else: Value = str(Value) T[Parts[0]] = HTMLEscSearch.sub(HTMLEscSub, Value) if len(Parts) > 2: for Part in Parts[1:]: if str(Value) == Part: T["%s%sSelected" % (Parts[0], Part)] = " selected" T["%s%sChecked" % (Parts[0], Part)] = " checked" else: T["%s%sSelected" % (Parts[0], Part)] = "" T["%s%sChecked" % (Parts[0], Part)] = ""
def transform(self, tree): """Transform an AST into a modified parse tree.""" if not (isinstance(tree, tuple) or isinstance(tree, list)): tree = parser.ast2tuple(tree, line_info=1) return self.compile_node(tree)
def get_parse_tree(fname): ast = parser.suite(open(fname).read()) return parser.ast2tuple(ast)
def test_constant_parsetree_to_string(self): expr = "42" t = parser.ast2tuple(parser.suite(expr)) s = parsetree_to_string(t) self.assertEqual(s, expr)
def print_exec_tree(expr): t = parser.ast2tuple(parser.suite(expr)) pprint(integer2symbolic(t))
def transform(self, tree): """Transform an AST into a modified parse tree.""" if type(tree) != type(()) and type(tree) != type([]): tree = parser.ast2tuple(tree, line_info=1) return self.compile_node(tree)
global _numFailed print '----', fileName, try: ast = parser.suite(t) tup = parser.ast2tuple(ast) # this discards the first AST; a huge memory savings when running # against a large source file like Tkinter.py. ast = None new = parser.tuple2ast(tup) except parser.ParserError, err: print print 'parser module raised exception on input file', fileName + ':' traceback.print_exc() _numFailed = _numFailed + 1 else: if tup != parser.ast2tuple(new): print print 'parser module failed on input file', fileName _numFailed = _numFailed + 1 else: print 'o.k.' def testFile(fileName): t = open(fileName).read() testChunk(t, fileName) def test(): import sys args = sys.argv[1:] if not args: import glob
def transform(self, tree): """Transform an AST into a modified parse tree.""" if type(tree) != type(()) and type(tree) != type([]): tree = parser.ast2tuple(tree,1) return self.compile_node(tree)
global _numFailed print '----', fileName, try: ast = parser.suite(t) tup = parser.ast2tuple(ast) # this discards the first AST; a huge memory savings when running # against a large source file like Tkinter.py. ast = None new = parser.tuple2ast(tup) except parser.ParserError, err: print print 'parser module raised exception on input file', fileName + ':' traceback.print_exc() _numFailed = _numFailed + 1 else: if tup != parser.ast2tuple(new): print print 'parser module failed on input file', fileName _numFailed = _numFailed + 1 else: print 'o.k.' def testFile(fileName): t = open(fileName).read() testChunk(t, fileName) def test(): import sys args = sys.argv[1:]
def Show(): "Edit local config file." global AssignVar, Assignments, Config, LineNum, LastLine # Load the display template T = Template.Template("localconfig.html") T["ErrMsg"] = "Displaying requested view. Click on filename to toggle." ErrStr = None # Toggle modes? if Form.has_key("subcmd") and (Form["subcmd"].value == "toggle"): if PVars["LocalConfig"] == "Text": PVars["LocalConfig"] = "Form" else: PVars["LocalConfig"] = "Text" PVars.Save() # Get file T["FilePath"] = Defaults.TMDARC try: F = open(Defaults.TMDARC) FileContents = F.readlines() F.close() except IOError: FileContents = [] # Which view does the user want? Form or text? if PVars["LocalConfig"] == "Form" and Version.TMDA < 1.1: # User wants to view the config in form mode. First we need to dismantle # the current config file in a way that we can rebuild it, then we need to # verify that it is safe to proceed. # Extract any comments & blank lines Config = {} LastLine = 1 for LastLine in range(len(FileContents)): if CommentSearch.search(FileContents[LastLine]): Config[LastLine + 1] = FileContents[LastLine].strip() LastLine += 2 # Test to see if we can do this safely FileContents = "".join(FileContents) ASP = parser.ast2tuple(parser.suite(FileContents), 1) Assignments = {} try: # Capture each expression for Exp in ASP[1:]: AssignVar = None Line = Parse(Exp[1], ConfigSyms) # Track commands in dictionaries if Line: Config[LineNum] = Line if AssignVar: Assignments[AssignVar] = LineNum except (KeyError, parser.ParserError), ErrMsg: # Nope. Not safe! T["ErrMsg"] = """Reverted to text mode. Not safe to proceed in form mode. %s of current %s""" % (ErrMsg, Defaults.TMDARC) PVars["LocalConfig"] = "Text" PVars.Save() else: # We're in form view, hide text view T["TextView"] # Save changes? FormVars = re.split("[,\s]+", T["FormVars"]) if Form.has_key("subcmd") and (Form["subcmd"].value == "save"): try: # Go through each variable we're supporting and determine: # [1] Is the given value acceptable? # [2] Is the new value different than the old? for Var in FormVars: Parts = Var.split(":") # Provided a value? if Form.has_key( Parts[0]) and Form[Parts[0]].value.strip(): Value = Form[Parts[0]].value if len(Parts) == 2: # Lists and dictionaries must be tested for dangerous code ASP = parser.ast2tuple(parser.suite(Value), 1) if Parts[1] == "L": Value = Parse(ASP, ListSyms) else: Value = Parse(ASP, DictSyms) else: # Integers are okay as-is, strings must be escaped if (len(Parts) == 1) or (Parts[1] != "I"): Value = Escape(Value) # If no value is given, use None else: Value = "None" # Has value been changed? if vars(Defaults)[Parts[0]] != eval(Value): Set(Parts[0], Value) vars(Defaults)[Parts[0]] = eval(Value) # Check code for syntax errors BEFORE saving try: # Re-assemble config Temp = Config Config = "" Lines = Temp.keys() Lines.sort() for Line in Lines: Config += Temp[Line] + "\n" # Try to execute it try: exec(Config) except (ImportError, NameError): pass # Looks okay, so save it T["ErrRow"] F = open(Defaults.TMDARC, "w") F.write(Config) F.close() except SyntaxError, (ErrStr, Details): T["ErrStr"] = "SyntaxError: line %d, char %d<br>(%s)" % Details[ 1:4] except (KeyError, parser.ParserError), ErrStr: # Don't like the look of this var! T["ErrStr"] = """<nobr>Contents of variable %s look "unsafe".<br> %s</nobr>""" % (Parts[0], ErrStr) # Display current values for Var in FormVars: Parts = Var.split(":") Value = vars(Defaults)[Parts[0]] if Value == None: Value = "" else: if len(Parts) == 2: Value = repr(Value) else: Value = str(Value) T[Parts[0]] = HTMLEscSearch.sub(HTMLEscSub, Value) if len(Parts) > 2: for Part in Parts[1:]: if str(Value) == Part: T["%s%sSelected" % (Parts[0], Part)] = " selected" T["%s%sChecked" % (Parts[0], Part)] = " checked" else: T["%s%sSelected" % (Parts[0], Part)] = "" T["%s%sChecked" % (Parts[0], Part)] = ""
def test(self): cases = [("""\ print """, """\ __print__() """), ("""\ print '' """, """\ __print__('') """), ("""\ print '', """, """\ __print_comma__('') """), ("""\ print 'hello' """, """\ __print__('hello') """), ("""\ print>>fh, 'hello' """, """\ __print_file__(fh, 'hello') """), ("""\ print>>fh, 'comma', """, """\ __print_file_comma__(fh, 'comma') """), ("""\ print a """, """\ __print__(a) """), ("""\ print a, 1, repr(z) """, """\ __print__(a, 1, repr(z)) """), # ("""\ # if __name__ == "__main__": # print a, 1, repr(z) # """, """\ # if __name__ == "__main__": # __print__(a, 1, repr(z)) # """), ] for before, after in cases: before_parsed = parser.ast2tuple(parser.suite(before), line_info=1) expected = parser.ast2tuple(parser.suite(after), line_info=1) actual = tutorial.convert_print_statments(before_parsed) try: self.assertEquals(actual, expected) except: print "BEFORE", before.strip() print "TEMPLATE", tutorial.EXPR_TEMPLATE print "AFTER", after.strip() print "PARSED", before_parsed print "EXPECTED", expected print "ACTUAL", actual meld(pformat(namify(actual)), pformat(namify(expected))) raise