def parsePythonFile(filename):
"""
Read a python source file and print the global symbols defined
in it.
The named file is opened and read. Newlines are canonicalized.
The whole file is handed to the internal python parser, which
generates an Abstract Syntax Tree. This is converted into a
list form which is then scanned by lookAt() for global symbol
definitions.
@param filename: name of a python source file.
@type filename: string
"""
file = open(filename)
codeString = file.read()
codeString = string.replace(codeString, "\r\n", "\n")
codeString = string.replace(codeString, "\r", "\n")
if codeString and codeString[-1] != '\n':
codeString = codeString + '\n'
# print "file: %s" % codeString
file.close()
try:
ast = parser.suite(codeString)
except SyntaxError:
return
parseTree = parser.ast2list(ast)
if (verbose):
printParseTree(parseTree, "")
lookAt(parseTree)
def main(fn="/Users/dwhall/Code/other/pyzeroconf-0.12/Browser.py"):
"""Returns a readable AST list for the given filename."""
src = open(fn,'r').read()
ast = parser.suite(src)
lis = parser.ast2list(ast, True)
replaceNodeType(lis)
return lis
def parsePythonFile(filename):
"""
Read a python source file and print the global symbols defined
in it.
The named file is opened and read. Newlines are canonicalized.
The whole file is handed to the internal python parser, which
generates an Abstract Syntax Tree. This is converted into a
list form which is then scanned by lookAt() for global symbol
definitions.
@param filename: name of a python source file.
@type filename: string
"""
file = open(filename)
codeString = file.read()
codeString = string.replace(codeString, "\r\n", "\n")
codeString = string.replace(codeString, "\r", "\n")
if codeString and codeString[-1] != '\n' :
codeString = codeString + '\n'
# print "file: %s" % codeString
file.close()
try:
ast = parser.suite(codeString)
except SyntaxError:
return
parseTree = parser.ast2list(ast)
if (verbose):
printParseTree(parseTree, "")
lookAt(parseTree)
def generate_dot_code(python_code):
'''internal parse tree representation'''
ast = parser.suite(python_code)
ast_list = parser.ast2list(ast)
#annotated_ast_list = annotate_ast_list(ast_list)
#pprint.pprint(annotated_ast_list)
call_graph = {}
construct_call_graph(ast_list, call_graph)
print call_graph
#pprint.pprint(call_graph)
dot = []
dot.append("digraph G {")
dot.append("rankdir=LR")
for from_fn, to_fns in call_graph.iteritems():
print from_fn, to_fns
if not to_fns:
dot.append('%s;' % from_fn)
for to_fn in to_fns:
if to_fn not in call_graph:
print "Here"
continue
dot.append('%s -> %s;' % (from_fn, to_fn))
dot.append("}")
return '\n'.join(dot)
def generate_dot_code(python_code):
'''internal parse tree representation'''
ast = parser.suite(python_code)
ast_list = parser.ast2list(ast)
#annotated_ast_list = annotate_ast_list(ast_list)
#pprint.pprint(annotated_ast_list)
call_graph = {}
construct_call_graph(ast_list, call_graph)
print call_graph
#pprint.pprint(call_graph)
dot = []
dot.append("digraph G {")
dot.append("rankdir=LR")
for from_fn, to_fns in call_graph.iteritems():
print from_fn, to_fns
if not to_fns:
dot.append('%s;' % from_fn)
for to_fn in to_fns:
if to_fn not in call_graph:
print "Here"
continue
dot.append('%s -> %s;' % (from_fn, to_fn))
dot.append("}")
return '\n'.join(dot)
def compile(src, file_name, ctype):
if ctype=='eval': ast=parser.expr(src)
elif ctype=='exec': ast=parser.suite(src)
l=ast2list(ast)
l=munge(l)
ast=sequence2ast(l)
return parser.compileast(ast, file_name)
def compile(src, file_name, ctype):
if ctype == 'eval': ast = parser.expr(src)
elif ctype == 'exec': ast = parser.suite(src)
l = ast2list(ast)
l = munge(l)
ast = sequence2ast(l)
return parser.compileast(ast, file_name)
def generate_dot_code(python_code):
ast = parser.suite(python_code)
ast_list = parser.ast2list(ast)
call_graph = {}
construct_call_graph(ast_list, call_graph)
dot = []
dot.append('digraph G {')
dot.append('rankdir=LR')
def close(self):
self.show("Colorizing Python source text - parsing...")
import parser
try:
nodes = parser.ast2list(parser.suite(self.__source), 1)
except parser.ParserError as err:
self.__viewer.context.message("Syntax error in Python source: %s" %
err)
self.setup_tags()
from types import IntType, ListType
ISTERMINAL = token.ISTERMINAL
wanted = self.__wanted_terminals.has_key
ws_width = self.__ws_width
tag_add = self.tag_add = self.__viewer.text.tag_add
colorize = self.colorize
prevline, prevcol = 0, 0
sourcetext = str.split(self.__source, "\n")
sourcetext.insert(0, '')
self.show("Colorizing Python source text - coloring...")
steps = 0
while nodes:
steps = steps + 1
if not (steps % 2000): self.show()
node = nodes[0]
del nodes[0]
if type(node) is ListType:
ntype = node[0]
if wanted(ntype):
[ntype, nstr, lineno] = node
# The parser spits out the line number the token ENDS on,
# not the line it starts on!
if ntype == token.STRING and "\n" in nstr:
strlines = str.split(nstr, "\n")
endpos = lineno, len(strlines[-1]), sourcetext[lineno]
lineno = lineno - len(strlines) + 1
else:
endpos = ()
if prevline != lineno:
tag_add('python:comment', "%d.%d" % (prevline, prevcol),
"%d.0" % lineno)
prevcol = 0
prevline = lineno
sourceline = sourcetext[lineno]
prevcol = prevcol + ws_width(sourceline, prevcol)
colorize(ntype, nstr, lineno, prevcol)
# point prevline/prevcol to 1st char after token:
if endpos:
prevline, prevcol, sourceline = endpos
else:
prevcol = prevcol + len(nstr)
else:
nodes = node[1:] + nodes
# end of last token to EOF is a comment...
start = "%d.%d" % (prevline or 1, prevcol)
tag_add('python:comment', start, tkinter.END)
self.__viewer.context.message_clear()
self.tag_add = None
def close(self):
self.show("Colorizing Python source text - parsing...")
import parser
try:
nodes = parser.ast2list(parser.suite(self.__source), 1)
except parser.ParserError, err:
self.__viewer.context.message(
"Syntax error in Python source: %s" % err)
return
def parseSource(sourcecode, indexbuff):
"""Parses python source code and puts the resulting index into the buffer.
"""
# Parse the source to an Abstract Syntax Tree
ast = parser.suite(sourcecode)
astlist = parser.ast2list(ast, True)
# Set these globals before each file's AST is walked
global sourcelinehassymbol
sourcelinehassymbol = False
global currentlinenum
currentlinenum = 0
# Walk the AST to index the rest of the file
walkAst(astlist, indexbuff)
def parseSource(sourcecode, indexbuff):
"""Parses python source code and puts the resulting index into the buffer.
"""
# Parse the source to an Abstract Syntax Tree
ast = parser.suite(sourcecode)
astlist = parser.ast2list(ast, True)
# Set these globals before each file's AST is walked
global sourcelinehassymbol
sourcelinehassymbol = False
global currentlinenum
currentlinenum = 0
# Walk the AST to index the rest of the file
walkAst(astlist, indexbuff)
def get_const_list( rhs_eqnStr= "A*sin(B*x)*x**2"):
errorStrL = [] # assume no error for now
tokenD = {}
functionD = {}
errorStr = ''
def search(st):
if not isinstance(st, list):
return
if st[0] in [token.NAME]:
#print st[1]
if st[1] in legalFuncL:
functionD[st[1]] = st[1]
else:
tokenD[st[1]] = st[1]
else:
for s in st[1:]:
search(s)
try:
ast = parser.expr( rhs_eqnStr )
eqnL = parser.ast2list( ast )
search( eqnL )
#print tokenD
except:
errorStrL = ['ERROR in Equation String']
tbStr = traceback.format_exc() # get error message
tbL = tbStr.split('\n')
active = 0
for line in tbL:
if active:
errorStrL.append( line )
if line.find('''File "<string>"''')>=0:
active = 1
if not active:
errorStrL = tbL[-4:] # next best guess at interesting part of message
errorStrL.append('Please Correct Error and Try Again')
if errorStrL:
errorStr = '\n'.join(errorStrL)
return tokenD, functionD, errorStr
def _parseconf(confstr):
"""
Parse the configuration *confstr* string and remove anything else
than the supported constructs, which are:
Assignments, bool, dict list, string, float, bool, and, or, xor,
arithmetics, string expressions and if..then..else.
The *entire* statement containing the unsupported statement is removed
from the parser; the effect is that the whole expression is ignored
from the 'root' down.
The modified AST object is returned to the Python parser for evaluation.
"""
# Initialise the parse tree, convert to list format and get a list of
# the symbol ID's for the unwanted statements. Might raise SyntaxError.
ast = parser.suite(confstr)
#ast = parser.expr(confstr)
stmts = parser.ast2list(ast)
rmsym = _get_forbidden_symbols()
result = list()
# copy 256: 'single_input', 257: 'file_input' or 258: 'eval_input'. The
# parse tree must begin with one of these to compile back to an AST obj.
result.append(stmts[0])
# NOTE: This might be improved with reduce(...) builtin!? How can we get
# line number for better warnings?
for i in range(1, len(stmts)):
# censor the parse tree produced from parsing the configuration.
if _check_ast(stmts[i], rmsym):
result.append(stmts[i])
else:
pass
return parser.sequence2ast(result)
def astl(s): return parser.ast2list(parser.expr(s)) def pret(ast, level=0):
def process_fn(fn_ast_list, call_graph):
dummy, dummy, func_name = fn_ast_list[:3]
dummy, func_name = func_name
calls = set()
find_fn_call(fn_ast_list, calls)
call_graph[func_name] = list(calls)
def construct_call_graph(ast_list, call_graph):
code = ast_list[0]
if code == symbol.funcdef:
process_fn(ast_list, call_graph)
def generate_dot_code(python_code):
ast = parser.suite(python_code)
ast_list = parser.ast2list(ast)
call_graph = {}
construct_call_graph(ast_list, call_graph)
dot = []
dot.append('digraph G {')
dot.append('rankdir=LR')
]:
i: 0()
o: 197(t), 333(f)
nname: 0
n 0(None)[__doc__ = '\ngenerates call graph of given python code file\nin dot format input for graphviz.\n\nlimitations:\n* statically tried to figure out functions calls\n* does not understand classes\n* algorithm is naive and may not statically find\n all cases\n'
import sys
import parser
import symbol
def pretty(s):
l = ast2list(parser.expr(s))
print l
pret(l)
def pretty(s):
l=ast2list(parser.expr(s))
print l
pret(l)
def astl(s):
return parser.ast2list(parser.expr(s))
