def main(): if len(sys.argv[1:]) is not 2: return ast1 = pythonparser.parse(sys.argv[1] + '\n') ast2 = pythonparser.parse(sys.argv[2] + '\n') print pythonparser.algorithm.compare(ast1, ast2)
def parse(flag, codeArray, indent_level):
try:
if flag == 1:
# only one item
p = pythonparser.parse(codeArray[0])
v = Visitor()
res = v.visit(p)
print json.dumps(res, indent=indent_level)
elif flag == 2:
# multi-element
res = [[] for x in range(len(codeArray))]
for i, ary in enumerate(codeArray):
res[i] = [[] for y in range(len(ary))]
for j, code in enumerate(ary):
p = pythonparser.parse(code)
v = Visitor()
res[i][j] = v.visit(p)
print json.dumps(res, indent=indent_level)
except pythonparser.diagnostic.Error as e:
error_obj = {'type': 'parse_error'}
diag = e.diagnostic
loc = diag.location
error_obj['loc'] = {
'start': {'line': loc.begin().line(), 'column': loc.begin().column()},
'end': {'line': loc.end().line(), 'column': loc.end().column()}
}
error_obj['message'] = diag.message()
print json.dumps(error_obj, indent=indent_level)
sys.exit(1)
def _parse_file(arg, cmd_options):
if cmd_options['verbose']:
print "Parsing file %(file)s..." % {'file': arg}
if java_pattern.search(arg) != None:
javaparser.parse(arg, cmd_options)
elif xml_pattern.search(arg) != None:
xmlparser.parse(arg, cmd_options)
elif python_pattern.search(arg) != None:
pythonparser.parse(arg, cmd_options)
elif php_pattern.search(arg) != None:
phpparser.parse(arg, cmd_options)
def _ParseAndVisit(source):
mod = pythonparser.parse(source)
future_features = stmt.visit_future(mod)
b = block.ModuleBlock('__main__', '<test>', source, future_features)
visitor = stmt.StatementVisitor(b)
visitor.visit(mod)
return visitor
def testFutureFeatures(self):
testcases = [
('from __future__ import print_function',
imputil.FutureFeatures(print_function=True)),
("""\
"module docstring"
from __future__ import print_function
""", imputil.FutureFeatures(print_function=True)),
("""\
"module docstring"
from __future__ import print_function, with_statement
from __future__ import nested_scopes
""", imputil.FutureFeatures(print_function=True)),
('from __future__ import absolute_import',
imputil.FutureFeatures(absolute_import=True)),
('from __future__ import absolute_import, print_function',
imputil.FutureFeatures(absolute_import=True,
print_function=True)),
('foo = 123\nfrom __future__ import print_function',
imputil.FutureFeatures()),
('import os\nfrom __future__ import print_function',
imputil.FutureFeatures()),
]
for tc in testcases:
source, want = tc
mod = pythonparser.parse(textwrap.dedent(source))
_, got = imputil.parse_future_features(mod)
self.assertEqual(want, got)
def testRelativeModuleMemberMixed(self):
imp1 = copy.deepcopy(self.fred_import)
imp1.add_binding(imputil.Import.MEMBER, 'qux', 'qux')
imp2 = copy.deepcopy(self.quux_import)
imp2.add_binding(imputil.Import.MODULE, 'quux', 2)
node = pythonparser.parse('from .fred import qux, quux').body[0]
self._assert_imports_equal([imp1, imp2], self.bar_importer.visit(node))
def testRelativeModuleFromSubModule(self):
imp = copy.deepcopy(self.foo2_import)
imp.add_binding(imputil.Import.MODULE, 'foo', 1)
baz_script = os.path.join(self.pydir, 'bar', 'baz.py')
importer = imputil.Importer(self.rootdir, 'bar.baz', baz_script, False)
node = pythonparser.parse('from . import foo').body[0]
self._assert_imports_equal([imp], importer.visit(node))
def testImportPackageModuleAbsoluteImport(self):
imp = copy.deepcopy(self.baz_import)
imp.add_binding(imputil.Import.MODULE, 'baz', 0)
bar_script = os.path.join(self.pydir, 'bar', '__init__.py')
importer = imputil.Importer(self.rootdir, 'bar', bar_script, True)
got = importer.visit(pythonparser.parse('import baz').body[0])
self._assert_imports_equal([imp], got)
def testImportPackageModuleRelativeFromSubModule(self):
imp = copy.deepcopy(self.baz2_import)
imp.add_binding(imputil.Import.MODULE, 'baz', 1)
foo_script = os.path.join(self.pydir, 'bar', 'foo.py')
importer = imputil.Importer(self.rootdir, 'bar.foo', foo_script, False)
got = importer.visit(pythonparser.parse('import baz').body[0])
self._assert_imports_equal([imp], got)
def _ParseAndVisit(source):
mod = pythonparser.parse(source)
_, future_features = imputil.parse_future_features(mod)
importer = imputil.Importer(None, 'foo', 'foo.py', False)
b = block.ModuleBlock(importer, '__main__', '<test>', source,
future_features)
visitor = stmt.StatementVisitor(b)
visitor.visit(mod)
return visitor
def collect_imports(modname, script, gopath, package_dir=''):
parser.patch_pythonparser()
with open(script) as py_file:
py_contents = py_file.read()
mod = pythonparser.parse(py_contents)
future_node, future_features = parse_future_features(mod)
importer = Importer(gopath,
modname,
script,
future_features.absolute_import,
package_dir=package_dir)
collector = _ImportCollector(importer, future_node)
collector.visit(mod)
return collector.imports
def testImportFromFuture(self):
testcases = [
('from __future__ import print_function',
imputil.FutureFeatures(print_function=True)),
('from __future__ import generators', imputil.FutureFeatures()),
('from __future__ import generators, print_function',
imputil.FutureFeatures(print_function=True)),
]
for tc in testcases:
source, want = tc
mod = pythonparser.parse(textwrap.dedent(source))
node = mod.body[0]
got = imputil._make_future_features(node) # pylint: disable=protected-access
self.assertEqual(want.__dict__, got.__dict__)
def testImportFromFuture(self):
testcases = [
('from __future__ import print_function',
stmt.FUTURE_PRINT_FUNCTION),
('from __future__ import generators', 0),
('from __future__ import generators, print_function',
stmt.FUTURE_PRINT_FUNCTION),
]
for i, tc in enumerate(testcases):
source, want_flags = tc
mod = pythonparser.parse(textwrap.dedent(source))
node = mod.body[0]
got = stmt.import_from_future(node)
msg = '#{}: want {}, got {}'.format(i, want_flags, got)
self.assertEqual(want_flags, got, msg=msg)
def testImportFromFutureParseError(self):
testcases = [
# NOTE: move this group to testImportFromFuture as they are implemented
# by grumpy
('from __future__ import division',
r'future feature \w+ not yet implemented'),
('from __future__ import braces', 'not a chance'),
('from __future__ import nonexistant_feature',
r'future feature \w+ is not defined'),
]
for tc in testcases:
source, want_regexp = tc
mod = pythonparser.parse(source)
node = mod.body[0]
self.assertRaisesRegexp(util.ParseError, want_regexp,
imputil._make_future_features, node) # pylint: disable=protected-access
def testVisitFutureLate(self):
testcases = [
# future after normal imports
"""\
import os
from __future__ import print_function
""",
# future after non-docstring expression
"""
asd = 123
from __future__ import print_function
"""
]
for source in testcases:
mod = pythonparser.parse(textwrap.dedent(source))
self.assertRaises(util.LateFutureError, imputil.visit_future, mod)
def parse_out(self,path, gen_grammar_rule=False):
outputs=[]
bug_cnt=0
with open(path) as f:
for line in f:
src=line.strip()
src=src.replace('''§''','\n')
for key in bug_fix:
if key in src:
bug_cnt+=1
src=src.replace(key,bug_fix[key])
try:
outputs.append(pythonparser.parse(src, gen_grammar_rule))
except:
print(src)
print(line)
input()
if gen_grammar_rule:
self.grammar=pythonparser.grammar
return outputs
def testImportFromFutureParseError(self):
testcases = [
# NOTE: move this group to testImportFromFuture as they are implemented
# by grumpy
('from __future__ import absolute_import',
r'future feature \w+ not yet implemented'),
('from __future__ import division',
r'future feature \w+ not yet implemented'),
('from __future__ import unicode_literals',
r'future feature \w+ not yet implemented'),
('from __future__ import braces', 'not a chance'),
('from __future__ import nonexistant_feature',
r'future feature \w+ is not defined'),
]
for tc in testcases:
source, want_regexp = tc
mod = pythonparser.parse(source)
node = mod.body[0]
self.assertRaisesRegexp(util.ParseError, want_regexp,
stmt.import_from_future, node)
def _parse_and_visit(stream, script, modname):
patch_pythonparser()
gopath = os.environ['GOPATH']
stream.seek(0)
py_contents = stream.read()
mod = pythonparser.parse(py_contents)
# Do a pass for compiler directives from `from __future__ import *` statements
future_node, future_features = imputil.parse_future_features(mod)
importer = imputil.Importer(gopath, modname, script,
future_features.absolute_import)
full_package_name = modname.replace('.', '/')
mod_block = block.ModuleBlock(importer, full_package_name, script,
py_contents, future_features)
visitor = stmt.StatementVisitor(mod_block, future_node)
# Indent so that the module body is aligned with the goto labels.
with visitor.writer.indent_block():
visitor.visit(mod)
return visitor, mod_block
def testVisitFuture(self):
testcases = [
('from __future__ import print_function',
imputil.FUTURE_PRINT_FUNCTION, 1),
("""\
"module docstring"
from __future__ import print_function
""", imputil.FUTURE_PRINT_FUNCTION, 3),
("""\
"module docstring"
from __future__ import print_function, with_statement
from __future__ import nested_scopes
""", imputil.FUTURE_PRINT_FUNCTION, 4),
]
for tc in testcases:
source, flags, lineno = tc
mod = pythonparser.parse(textwrap.dedent(source))
future_features = imputil.visit_future(mod)
self.assertEqual(future_features.parser_flags, flags)
self.assertEqual(future_features.future_lineno, lineno)
def count_total_data(source_code):
global func_count_map
global func_node_map
global list_map
global dict_map
global tuple_map
global set_map
global map_map
source_code = source_code.strip()
new_src_code = ""
for a_src in source_code.split("\n"):
if a_src.find('class ') > -1 and a_src.find("()") > -1:
a_src = a_src.replace("()", "(self)")
new_src_code = new_src_code + a_src + "\n"
p = pythonparser.parse(new_src_code)
count_object_type(p.body)
total_array = [
total_list_data(list_map),
total_dict_data(dict_map),
total_tuple_data(tuple_map),
total_set_data(set_map),
total_map_data(map_map)
]
list_map = {}
dict_map = {}
tuple_map = {}
set_map = {}
map_map = {}
func_count_map = {}
func_node_map = {}
return total_array
def _visit_import(self, source, path=None):
if not path:
path = MockPath()
visitor = imputil.ImportVisitor(path)
visitor.visit(pythonparser.parse(source).body[0])
return visitor.imports
def _ParseExpr(expr):
return pythonparser.parse(expr).body[0].value
def main():
if len(sys.argv[1:]) is not 1:
return
ast = pythonparser.parse(sys.argv[1] + '\n')
print ast
def testUndefinedFutureRaises(self):
mod = pythonparser.parse('from __future__ import foo')
self.assertRaisesRegexp(util.ParseError,
'future feature foo is not defined',
imputil.parse_future_features, mod)
def testUnimplementedFutureRaises(self):
mod = pythonparser.parse('from __future__ import division')
msg = 'future feature division not yet implemented by grumpy'
self.assertRaisesRegexp(util.ParseError, msg,
imputil.parse_future_features, mod)
(options, args) = parser.parse_args()
if options.pyfile:
code = open(options.pyfile).read()
else:
code = args[0]
# make sure it ends with a newline to get parse() to work:
if code[-1] != '\n':
code += '\n'
indent_level = None
if options.pp:
indent_level = 2
try:
p = pythonparser.parse(code)
v = Visitor()
res = v.visit(p)
print json.dumps(res, indent=indent_level)
except pythonparser.diagnostic.Error as e:
error_obj = {'type': 'parse_error'}
diag = e.diagnostic
loc = diag.location
error_obj['loc'] = {
'start': {
'line': loc.begin().line(),
'column': loc.begin().column()
},
'end': {
def testGlobalUsedPriorToDeclaration(self):
node = pythonparser.parse('foo = 42\nglobal foo')
visitor = block.BlockVisitor()
self.assertRaisesRegexp(util.ParseError,
'used prior to global declaration',
visitor.generic_visit, node)
def testImportLateFuture(self):
mod = pythonparser.parse(
'import os\nfrom __future__ import print_function')
visitor = imputil.ImportVisitor(MockPath())
self.assertRaises(util.LateFutureError, visitor.visit, mod)
def _ParseStmt(stmt_str):
return pythonparser.parse(stmt_str).body[0]
def init_module(module):
func_def = parse(inspect.getsource(_gcd)).body[0]
function, _ = module.compile_function(func_def,
{"a": VInt(64), "b": VInt(64)})
function.linkage = "internal"
def testImportFromFuture(self):
mod = pythonparser.parse('from __future__ import print_function')
visitor = imputil.ImportVisitor(MockPath(), mod.body[0])
visitor.visit(mod)
self.assertEqual([], visitor.imports)