def pretty_print(self, node: Optional[ast.AST]) -> str:
"""
Given some AST type expression,
pretty print the type so that it is readable
"""
if node is None:
return ""
return code_gen.to_source(node, indent_with="").rstrip()
def __call__(self, selection):
if not self.is_initialized:
self._initialize()
try:
parse_result = self.expression.parseString(selection,
parseAll=True)
except ParseException as e:
msg = str(e)
lines = [
"%s: %s" % (msg, selection),
" " * (12 + len("%s: " % msg) + e.loc) + "^^^"
]
raise ValueError('\n'.join(lines))
# Change __ATOM__ in function bodies. It must bind to the arg
# name specified below (i.e. 'atom')
astnode = self.transformer.visit(deepcopy(parse_result[0].ast()))
# Special check for a single literal
if isinstance(astnode, ast.Num) or isinstance(astnode, ast.Str):
raise ValueError("Cannot use a single literal as a boolean.")
if PY2:
args = [ast.Name(id='atom', ctx=ast.Param())]
signature = ast.arguments(args=args,
vararg=None,
kwarg=None,
defaults=[])
else:
args = [ast.arg(arg='atom', annotation=None)]
signature = ast.arguments(args=args,
vararg=None,
kwarg=None,
posonlyargs=[],
kwonlyargs=[],
defaults=[],
kw_defaults=[])
func = ast.Expression(body=ast.Lambda(signature, astnode))
source = code_gen.to_source(
astnode, pretty_source=lambda src: ''.join(src[:-1]))
expr = eval(
compile(ast.fix_missing_locations(func), '<string>', mode='eval'),
SELECTION_GLOBALS)
return _ParsedSelection(expr, source, astnode)
def to_py_str(t: ast.AST) -> str:
"""Return a string of the Python code which would generate the input
AST node."""
return codegen.to_source(t)
def convert(srctree,
dsttree=dsttree,
readonly=False,
dumpall=False,
ignore_exceptions=False,
fullcomp=False):
"""Walk the srctree, and convert/copy all python files
into the dsttree
"""
if fullcomp:
allow_ast_comparison()
parse_file = code_to_ast.parse_file
find_py_files = code_to_ast.find_py_files
srctree = os.path.normpath(srctree)
if not readonly:
dsttree = os.path.normpath(dsttree)
logging.info('')
logging.info('Trashing ' + dsttree)
shutil.rmtree(dsttree, True)
unknown_src_nodes = set()
unknown_dst_nodes = set()
badfiles = set()
broken = []
oldpath = None
allfiles = find_py_files(srctree, None if readonly else dsttree)
for srcpath, fname in allfiles:
# Create destination directory
if not readonly and srcpath != oldpath:
oldpath = srcpath
if srcpath >= srctree:
dstpath = srcpath.replace(srctree, dsttree, 1)
if not dstpath.startswith(dsttree):
raise ValueError("%s not a subdirectory of %s" %
(dstpath, dsttree))
else:
assert srctree.startswith(srcpath)
dstpath = dsttree
os.makedirs(dstpath)
srcfname = os.path.join(srcpath, fname)
logging.info('Converting %s' % srcfname)
try:
srcast = parse_file(srcfname)
except SyntaxError:
badfiles.add(srcfname)
continue
try:
dsttxt = to_source(srcast)
except:
if not ignore_exceptions:
raise
dsttxt = ''
if not readonly:
dstfname = os.path.join(dstpath, fname)
try:
with open(dstfname, 'wb') as f:
f.write(out_prep(dsttxt))
except UnicodeEncodeError:
badfiles.add(dstfname)
# As a sanity check, make sure that ASTs themselves
# round-trip OK
try:
dstast = ast.parse(dsttxt) if readonly else parse_file(dstfname)
except SyntaxError:
dstast = []
if fullcomp:
unknown_src_nodes.update(strip_tree(srcast))
unknown_dst_nodes.update(strip_tree(dstast))
bad = srcast != dstast
else:
bad = not fast_compare(srcast, dstast)
if dumpall or bad:
srcdump = dump_tree(srcast)
dstdump = dump_tree(dstast)
logging.warning(' calculating dump -- %s' %
('bad' if bad else 'OK'))
if bad:
broken.append(srcfname)
if dumpall or bad:
if not readonly:
try:
with open(dstfname[:-3] + '.srcdmp', 'wb') as f:
f.write(out_prep(srcdump))
except UnicodeEncodeError:
badfiles.add(dstfname[:-3] + '.srcdmp')
try:
with open(dstfname[:-3] + '.dstdmp', 'wb') as f:
f.write(out_prep(dstdump))
except UnicodeEncodeError:
badfiles.add(dstfname[:-3] + '.dstdmp')
elif dumpall:
sys.stdout.write('\n\nAST:\n\n ')
sys.stdout.write(srcdump.replace('\n', '\n '))
sys.stdout.write('\n\nDecompile:\n\n ')
sys.stdout.write(dsttxt.replace('\n', '\n '))
sys.stdout.write('\n\nNew AST:\n\n ')
sys.stdout.write('(same as old)' if dstdump == srcdump else
dstdump.replace('\n', '\n '))
sys.stdout.write('\n')
if badfiles:
logging.warning('\nFiles not processed due to syntax errors:')
for fname in sorted(badfiles):
logging.warning(' %s' % fname)
if broken:
logging.warning('\nFiles failed to round-trip to AST:')
for srcfname in broken:
logging.warning(' %s' % srcfname)
ok_to_strip = 'col_offset _precedence _use_parens lineno _p_op _pp'
ok_to_strip = set(ok_to_strip.split())
bad_nodes = (unknown_dst_nodes | unknown_src_nodes) - ok_to_strip
if bad_nodes:
logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes)
logging.info('\n')
return broken
def convert(srctree, dsttree=dsttree, readonly=False, dumpall=False):
"""Walk the srctree, and convert/copy all python files
into the dsttree
"""
allow_ast_comparison()
parse_file = code_to_ast.parse_file
find_py_files = code_to_ast.find_py_files
srctree = os.path.normpath(srctree)
if not readonly:
dsttree = os.path.normpath(dsttree)
logging.info('')
logging.info('Trashing ' + dsttree)
shutil.rmtree(dsttree, True)
unknown_src_nodes = set()
unknown_dst_nodes = set()
badfiles = set()
broken = []
# TODO: When issue #26 resolved, remove UnicodeDecodeError
handled_exceptions = SyntaxError, UnicodeDecodeError
oldpath = None
allfiles = find_py_files(srctree, None if readonly else dsttree)
for srcpath, fname in allfiles:
# Create destination directory
if not readonly and srcpath != oldpath:
oldpath = srcpath
if srcpath >= srctree:
dstpath = srcpath.replace(srctree, dsttree, 1)
if not dstpath.startswith(dsttree):
raise ValueError("%s not a subdirectory of %s" %
(dstpath, dsttree))
else:
assert srctree.startswith(srcpath)
dstpath = dsttree
os.makedirs(dstpath)
srcfname = os.path.join(srcpath, fname)
logging.info('Converting %s' % srcfname)
try:
srcast = parse_file(srcfname)
except handled_exceptions:
badfiles.add(srcfname)
continue
dsttxt = to_source(srcast)
if not readonly:
dstfname = os.path.join(dstpath, fname)
try:
with open(dstfname, 'w') as f:
f.write(dsttxt)
except UnicodeEncodeError:
badfiles.add(dstfname)
# As a sanity check, make sure that ASTs themselves
# round-trip OK
try:
dstast = ast.parse(dsttxt) if readonly else parse_file(dstfname)
except SyntaxError:
dstast = []
unknown_src_nodes.update(strip_tree(srcast))
unknown_dst_nodes.update(strip_tree(dstast))
if dumpall or srcast != dstast:
srcdump = dump_tree(srcast)
dstdump = dump_tree(dstast)
bad = srcdump != dstdump
logging.warning(' calculating dump -- %s' %
('bad' if bad else 'OK'))
if bad:
broken.append(srcfname)
if dumpall or bad:
if not readonly:
try:
with open(dstfname[:-3] + '.srcdmp', 'w') as f:
f.write(srcdump)
except UnicodeEncodeError:
badfiles.add(dstfname[:-3] + '.srcdmp')
try:
with open(dstfname[:-3] + '.dstdmp', 'w') as f:
f.write(dstdump)
except UnicodeEncodeError:
badfiles.add(dstfname[:-3] + '.dstdmp')
elif dumpall:
sys.stdout.write('\n\nAST:\n\n ')
sys.stdout.write(srcdump.replace('\n', '\n '))
sys.stdout.write('\n\nDecompile:\n\n ')
sys.stdout.write(dsttxt.replace('\n', '\n '))
sys.stdout.write('\n\nNew AST:\n\n ')
sys.stdout.write('(same as old)' if dstdump == srcdump
else dstdump.replace('\n', '\n '))
sys.stdout.write('\n')
if badfiles:
logging.warning('\nFiles not processed due to syntax errors:')
for fname in sorted(badfiles):
logging.warning(' %s' % fname)
if broken:
logging.warning('\nFiles failed to round-trip to AST:')
for srcfname in broken:
logging.warning(' %s' % srcfname)
ok_to_strip = 'col_offset _precedence _use_parens lineno _p_op _pp'
ok_to_strip = set(ok_to_strip.split())
bad_nodes = (unknown_dst_nodes | unknown_src_nodes) - ok_to_strip
if bad_nodes:
logging.error('\nERROR -- UNKNOWN NODES STRIPPED: %s' % bad_nodes)
logging.info('\n')
def main(fname):
p = Parser.fromfile(fname)
m = p.parse()
print(to_source(m))