def checkfilename(brokencode):
try:
symtable.symtable(brokencode, "spam", "exec")
except SyntaxError as e:
self.assertEqual(e.filename, "spam")
else:
self.fail("no SyntaxError for %r" % (brokencode,))
def checkfilename(brokencode):
try:
symtable.symtable(brokencode, "spam", "exec")
except SyntaxError as e:
self.assertEqual(e.filename, "spam")
else:
self.fail("no SyntaxError for %r" % (brokencode,))
def test_bytes(self):
top = symtable.symtable(TEST_CODE.encode('utf8'), "?", "exec")
self.assertIsNotNone(find_block(top, "Mine"))
code = b'# -*- coding: iso8859-15 -*-\nclass \xb4: pass\n'
top = symtable.symtable(code, "?", "exec")
self.assertIsNotNone(find_block(top, "\u017d"))
def checkfilename(brokencode, offset):
try:
symtable.symtable(brokencode, 'spam', 'exec')
except SyntaxError as e:
self.assertEqual(e.filename, 'spam')
self.assertEqual(e.lineno, 1)
self.assertEqual(e.offset, offset)
else:
self.fail('no SyntaxError for %r' % (brokencode, ))
def test_annotated(self):
st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
st2 = st1.get_children()[0]
self.assertTrue(st2.lookup('x').is_local())
self.assertTrue(st2.lookup('x').is_annotated())
self.assertFalse(st2.lookup('x').is_global())
st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
st4 = st3.get_children()[0]
self.assertTrue(st4.lookup('x').is_local())
self.assertFalse(st4.lookup('x').is_annotated())
def test_annotated(self):
st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
st2 = st1.get_children()[0]
self.assertTrue(st2.lookup('x').is_local())
self.assertTrue(st2.lookup('x').is_annotated())
self.assertFalse(st2.lookup('x').is_global())
st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
st4 = st3.get_children()[0]
self.assertTrue(st4.lookup('x').is_local())
self.assertFalse(st4.lookup('x').is_annotated())
def test_walk_rules_over_simple_assignment_examples_with_funcs(self):
# Equivalent examples
examples = [
"""
import pandas as pd
df = pd.DataFrame(data={"Price": [100.]})
def mod(d):
d["Price2"] = d["Price"]
mod(df)"""
]
for code_str in examples:
root_node = ast.parse(code_str, '<unknown>', 'exec')
table = symtable.symtable(code_str, '<unknown>', 'exec')
# Get the sub-exprs corresponding to the column assignment
assignment_stmts = [
n for n in ast.walk(root_node) if isinstance(n, ast.Assign)
]
assert len(assignment_stmts) == 2
df_symbol_ref = SymbolTypeReferant(table.lookup("df"))
df_price_ref = ColumnTypeReferant(df_symbol_ref, ("Price", ))
df_price2_ref = ColumnTypeReferant(df_symbol_ref, ("Price2", ))
# Make sure the two columns on `df` are unified despite their
# unification happening as the result of the side effects in
# `mod`'s type signature.
visitor = WalkRulesVisitor(table)
visitor.visit(root_node)
type_env = visitor.type_environment
self.assertTrue(
self.types_connected(type_env[df_price_ref],
type_env[df_price2_ref], visitor))
def test_overwriting_subscript_doesnt_connect_types(self):
# Equivalent examples
examples = [
"""
import pandas as pd
d = pd.DataFrame(data={"Price": [100.]})
d["Price2"] = d["Price"]"""
]
for code_str in examples:
root_node = ast.parse(code_str, '<unknown>', 'exec')
table = symtable.symtable(code_str, '<unknown>', 'exec')
# Get the sub-exprs corresponding to the column assignment
assignment_stmts = [
n for n in ast.walk(root_node) if isinstance(n, ast.Assign)
]
assert len(assignment_stmts) == 2
p2_expr: ast.Subscript = assignment_stmts[-1].targets[0]
p1_expr: ast.Subscript = assignment_stmts[-1].value
assert isinstance(p2_expr, ast.Subscript)
assert isinstance(p1_expr, ast.Subscript)
# Assert that an equality constraint is baked for these
# expressions. Remember types are not unified yet, so
# we need the type judgement from the `WalkRulesVisitor`
# specifically for the expressions `p1_expr`, `p2_expr`
# above.
visitor = WalkRulesVisitor(table)
visitor.visit(root_node)
# Make sure that `d["Price"]` and `d["Price2"]` are equal
self.assertTrue(self.types_connected(p1_expr, p2_expr, visitor))
def parse(self):
'''
Parse the input expression to get list of identifiers.
'''
try:
symTable = symtable.symtable(self.string, 'string', 'eval')
except:
raise IOError('Not a valid python math expression \n' +
self.string)
idents = symTable.get_identifiers()
known = []
unknown = []
for ident in idents:
if ident not in self._restricted:
unknown.append(ident)
else:
known.append(ident)
for val in unknown:
band = val.split('_')[0]
if len(band)!=1:
raise IOError('Multi character variables in input expressions represent functions or constants. Unknown function or constant : %s'%(val))
elif (band.lower() != band):
raise IOError('Single character upper case letters are used for constant. No available constant named %s'%(val))
return unknown, known
def __tests():
import symtable
class ClassFinder(ast.NodeVisitor):
def __init__(self, name):
ast.NodeVisitor.__init__(self)
self.name = name
self.classes = []
def visit_ClassDef(self, node):
if node.name == self.name:
self.classes.append(node)
with open('tests/game2.py') as source_file:
code = source_file.read()
ast_node = ast.parse(code)
symbol_table = symtable.symtable(code, 'game2.py', 'exec')
class_finder = ClassFinder('Game')
class_finder.visit(ast_node)
subclass = class_finder.classes[0]
subclass_symbol_table = symbol_table.lookup('Game').get_namespace()
superclasses, subclass = SuperClassExtractor.extract_into_ideal_cohesion(subclass, subclass_symbol_table)
for superclass_node in superclasses:
print to_source(superclass_node)
# print
# print ast.dump(superclass_node)
print
print '#' * 200
print
print to_source(subclass)
def __init__(self, code, ns=None):
global __default_namespace__
if ns is None:
ns = __default_namespace__
var = [v for v in symtable.symtable(code, '<string>', 'exec').get_identifiers() if v in ns]
super(PythonCode, self).__init__(var=var, namespace=ns)
self.code = compile(code, '<string>', 'exec')
def parse_module(module_name):
filename = module_name + ".py"
with open(os.path.join(SOLUTIONS_DIR, filename), 'r') as f:
source = f.read()
srclines = source.splitlines()
module = ast.parse(source, filename)
table = symtable.symtable(source, filename, "exec")
stmts = list(get_stmts(module))
last_fun = stmts[0][0]
lines = {name:"\n".join(srclines[s:e]).strip() for (name,(s,e)) in stmts}
imports = dict(get_imports(module))
def parse_dependencies(name):
import builtins
for tab in get_child_tables(table.lookup(name).get_namespace()):
for g in tab.get_globals():
if g in dir(builtins):
continue
if table.lookup(g).is_imported():
imported = imports[g]
if imported[0] != "leetcode":
yield imported
else:
yield (module_name, g)
return last_fun, lines, {name:tuple(parse_dependencies(name)) for name in lines}
def test_assign_1():
file = 'files/class1.py'
logger = logging.getLogger('test_assign_1')
logger.setLevel(logging.DEBUG)
m = Module()
v = ValueCollector("test", logger=logger, module=m)
with open(file, 'r') as f:
code = f.read()
sym_table = symtable.symtable(code, file, 'exec')
tree = ast.parse(code)
tree = (Transform1(module=m, logger=logger,
sym_table=sym_table)).visit(tree)
code = astor.to_source(tree)
collector = Collector(module=m, logger=logger, sym_table=sym_table)
print(tree)
collector.visit(tree)
tree = (Transform2(module=m,
logger=logger,
sym_table=sym_table,
collector=collector)).visit(tree)
analyzer = ValueCollector("main",
True,
top_level=True,
module=m,
logger=logger,
sym_table=sym_table)
analyzer.do_visit(tree)
program = analyzer.output_nodes[-1][0]
json.dump(program, fp=sys.stderr)
def test_type_eq_after_augmented_assignment(self):
# Equivalent examples
examples = [
"import pandas as pd\n"
"df = pd.DataFrame(data={'Price': [10.], 'Price2': [20.]})\n"
f"df['Price2'] {op_str} df['Price']"
"" for op_str in ("+=", "-=")
]
for code_str in examples:
root_node = ast.parse(code_str, '<unknown>', 'exec')
table = symtable.symtable(code_str, '<unknown>', 'exec')
# Get the sub-exprs corresponding to the augmented assignment
assignment_stmts = [
n for n in ast.walk(root_node) if isinstance(n, ast.AugAssign)
]
assert len(assignment_stmts) == 1
df_symbol_ref = SymbolTypeReferant(table.lookup("df"))
df_price_ref = ColumnTypeReferant(df_symbol_ref, ("Price", ))
df_price2_ref = ColumnTypeReferant(df_symbol_ref, ("Price2", ))
# Make sure the two columns on `df` are unified
visitor = WalkRulesVisitor(table)
visitor.visit(root_node)
type_env = visitor.type_environment
self.assertTrue(
self.types_connected(type_env[df_price_ref],
type_env[df_price2_ref], visitor))
def emit_java_symtable_test(prog, name):
#print("\n# {}\n{}".format(name, prog))
tree = ast.parse(prog, "m")
try:
c = compile(tree, name, 'exec')
except SyntaxError as se:
#print("INVALID:", name, se)
return False
# Get the symbol tables
mst = symtable.symtable(prog, "<module>", 'exec')
stlist = list_symbol_tables(mst)
# Generate test for Java
print(" @Test public void {}() {{".format(name))
print(" // @formatter:off")
emit_java_ast(prog, tree)
print(" // @formatter:on")
print()
print("checkExample(module, new RefSymbol[][]{ //", name)
for st in stlist:
emit_java_symbol_table(st)
print("});")
print("}\n")
return True
def test_true_false_and_null_dont_raise(self):
for c in ["True", "False", "None"]:
code_str = "x = " + c
root_node = ast.parse(code_str, '<unknown>', 'exec')
table = symtable.symtable(code_str, '<unknown>', 'exec')
visitor = WalkRulesVisitor(table)
visitor.visit(root_node)
def __init__(self, source: str, type_graph: TypeLatticeGenerator):
self.__type_graph = type_graph
self.__node_to_id: Dict[Any, int] = {}
self.__id_to_node: List[Any] = []
self.__symbol_to_supernode_id: Dict[Symbol, int] = {}
self.__edges: Dict[EdgeType, Dict[int, Set[int]]] = {
e: defaultdict(set)
for e in EdgeType
}
self.__ast = parse(source)
self.__scope_symtable = [symtable(source, 'file.py', 'exec')]
self.__imported_symbols = {
} # type: Dict[TypeAnnotationNode, TypeAnnotationNode]
# For the CHILD edges
self.__current_parent_node: Optional[AST] = None
# For the NEXT_TOKEN edges
self.__backbone_sequence: List[TokenNode] = []
self.__prev_token_node: Optional[TokenNode] = None
# For the RETURNS_TO edge
self.__return_scope: Optional[AST] = None
# For the OCCURRENCE_OF and Supernodes
self.__variable_like_symbols: Dict[Any, SymbolInformation] = {}
# Last Lexical Use
self.__last_lexical_use: Dict[Any, Any] = {}
def main():
"""Write symtable_data_test.go"""
path = "symtable_data_test.go"
out = [
"""// Test data generated by make_symtable_test.py - do not edit
package symtable
import (
"github.com/ncw/gpython/py"
)
var symtableTestData = []struct {
in string
mode string // exec, eval or single
out *SymTable
exceptionType *py.Type
errString string
}{"""
]
for x in inp:
source, mode = x[:2]
if len(x) > 2:
exc = x[2]
try:
table = symtable(source, "<string>", mode)
except exc as e:
error = e.msg
else:
raise ValueError("Expecting exception %s" % exc)
dumped_symtable = "nil"
gostring = "nil"
exc_name = "py.%s" % exc.__name__
else:
table = symtable(source, "<string>", mode)
exc_name = "nil"
error = ""
dumped_symtable = dump_symtable(table)
out.append(
'{"%s", "%s", %s, %s, "%s"},' %
(escape(source), mode, dumped_symtable, exc_name, escape(error)))
out.append("}\n")
print("Writing %s" % path)
with open(path, "w") as f:
f.write("\n".join(out))
f.write("\n")
subprocess.check_call(["gofmt", "-w", path])
def __init__(self, source_no_encoding):
self.pubapi = []
self.modnames = []
self.symtab = symtable.symtable(source_no_encoding, "-", "exec")
cst = parser.suite(source_no_encoding)
elements = parser.ast2tuple(cst, line_info=1)
self.names = {}
self.walk(elements, [self.symtab])
def __init__(self, expr, range, ns=None):
# find the variables in the expression
global __default_namespace__
if ns is None:
ns = __default_namespace__
var = [v for v in symtable.symtable(expr, '<string>', 'exec').get_identifiers() if v in ns]
super(ExpressionSlider, self).__init__(var=var, values=range, ns=ns, enabled=False)
self.code = compile(expr, '<string>', 'eval')
def _inspect(self,string): symtab = symtable.symtable(string,'rule',self.mode) symbols = symtab.get_symbols() defs = [sym.get_name() for sym in symbols if sym.is_assigned()] refs = [sym.get_name() for sym in symbols if sym.is_referenced()] self.all_symbols = frozenset(symtab.get_identifiers()) self.sym_assigned = defset = frozenset(defs) self.sym_internal = frozenset(defset.intersection(refs))
def get_imports(s):
imported = []
table = symtable.symtable(s, "string", "exec")
for i in table.get_symbols():
if i.is_imported():
imported.append(i.get_name())
return imported
def _inspect(self, string):
symtab = symtable.symtable(string, 'rule', self.mode)
symbols = symtab.get_symbols()
defs = [sym.get_name() for sym in symbols if sym.is_assigned()]
refs = [sym.get_name() for sym in symbols if sym.is_referenced()]
self.all_symbols = frozenset(symtab.get_identifiers())
self.sym_assigned = defset = frozenset(defs)
self.sym_internal = frozenset(defset.intersection(refs))
def _get_from_symtab(self):
"""
Get the variables used in the 'f' function.
"""
variables = set()
table = symtable.symtable(self.f_code, "<string>", "exec")
if table.has_children():
variables.update(self._walk_children(table))
return variables
def get_symtable(path):
import symtable
try:
st = symtables[path]
except KeyError:
with open(path, 'r') as f:
st = symtable.symtable(f.read(), path, 'exec')
symtables[path] = st
return st
def get_globals():
module = inspect.getmodule(get_globals)
source = inspect.getsource(module)
top = symtable.symtable(source, "<string>", "exec")
return top.get_identifiers()
def get_locals(func):
source = inspect.getsource(func)
top = symtable.symtable(source, "<string>", "exec")
func = top.get_children()[0] #since we are passing only the func code.
return func.get_locals()
def setUp(self) -> None:
""" Setup the symbol table for Dafny Visitor
Notice that no transition are provided
"""
sym_tab = symtable.symtable(_test_ioa, "<unknown>", 'exec')
ns = _IOANamespace(sym_tab)
self._dfy_visitor = _ToDafnyVisitor(ns, 1)
self._dfy_visitor._current_namespace.enter_automaton("TestAut")
self._dfy_visitor._current_namespace.enter_transition("trans")
def from_ast_node(cls, ast_node, filename):
# ast_node -> {name}
raise NotImplementedError
table = symtable.symtable(ast_node, filename, 'exec')
return symtable2forgots(table, ())
c = CollectAllUnboundNamesVisitor()
c.visit(ast_node)
return c.unbound_names
def get_symtable(path): import symtable try: st = symtables[path] except KeyError: with open(path, 'r') as f: st = symtable.symtable(f.read(), path, 'exec') symtables[path] = st return st
def __find_class_symbol_tables(self):
class_names = self.__get_class_names()
source_code = self.get_source_code()
module_symbol_table = symtable.symtable(source_code, self.file_path, 'exec')
symbol_tables = []
for name in class_names:
class_symbol_table = module_symbol_table.lookup(name).get_namespace()
symbol_tables.append(class_symbol_table)
return symbol_tables
def test_import_stats_from_scipy(self):
code_str = "from scipy import stats; import scipy"
root_node = ast.parse(code_str, '<unknown>', 'exec')
table = symtable.symtable(code_str, '<unknown>', 'exec')
visitor = WalkRulesVisitor(table)
visitor.visit(root_node)
self.assertIsInstance(
visitor.type_environment.get(
SymbolTypeReferant(table.lookup("stats"))),
ScipyStatsModuleType)
def __init__(self, source_no_encoding, pubapi):
# Our public API (__all__)
self.pubapi = pubapi
# Names of imported modules
self.modnames = []
self.symtab = symtable.symtable(source_no_encoding, "-", "exec")
cst = parser.suite(source_no_encoding)
elements = parser.ast2tuple(cst, line_info=1)
self.names = {}
self.walk(elements, [self.symtab])
def __init__(self, code, ns=None):
global __default_namespace__
if ns is None:
ns = __default_namespace__
var = [
v for v in symtable.symtable(code, '<string>',
'exec').get_identifiers() if v in ns
]
super(PythonCode, self).__init__(var=var, namespace=ns)
self.code = compile(code, '<string>', 'exec')
def build_symbol_table_for_text(text, path, namespace, types_table, methods_table):
try:
tree = ast.parse(text)
except SyntaxError:
return None, []
root_table = symtable.symtable(text, "<string>", "exec")
visitor = Visitor(path, namespace, root_table, types_table, methods_table)
visitor.visit(tree)
result = visitor.symtable[-1] if visitor.symtable else None
return result, visitor.errors
def __init__(self, source_no_encoding, pubapi):
# Our public API (__all__)
self.pubapi = pubapi
# Names of imported modules
self.modnames = []
self.symtab = symtable.symtable(source_no_encoding, "-", "exec")
cst = parser.suite(source_no_encoding)
elements = parser.ast2tuple(cst, line_info=1)
self.names = {}
self.walk(elements, [self.symtab])
def __init__(self, source, path='<string>'):
self.source = source
self.path = path
self.ast = ast.parse(source, path)
self.symtable = symtable.symtable(source, path, 'exec')
self.tokens = tokenize_string(source)
cw = ChainWalker(self.ast, self.symtable)
self.nodes = cw.nodes
TokenAssociator(self.nodes, self.tokens)
self.ast_map = {node.ast_node: node for node in self.nodes}
def compile_to_strict(
self,
code: str,
builtins: Dict[str, Any] = __builtins__,
modules: Optional[Dict[str, Dict[str, Any]]] = None,
globals: Optional[Dict[str, Any]] = None,
track_import_call: bool = False,
import_call_tracker: Optional[Set[str]] = None,
) -> StrictModule:
code = dedent(code)
root = ast.parse(code)
name = "foo"
filename = "foo.py"
symbols = symtable.symtable(code, filename, "exec")
RewriterTestPreprocessor().visit(root)
root = rewrite(
root,
symbols,
filename,
name,
builtins=builtins,
track_import_call=track_import_call,
)
c = strict_compile(name, filename, root)
def freeze_type(freeze: Type[object]) -> None:
pass
def loose_slots(freeze: Type[object]) -> None:
pass
def strict_slots(typ: Type[object]) -> Type[object]:
return typ
def track_import_call(mod: str) -> None:
if import_call_tracker is not None:
import_call_tracker.add(mod)
fixed_modules = modules or dict(FIXED_MODULES)
fixed_modules.update(
__strict__={
"freeze_type": freeze_type,
"loose_slots": loose_slots,
"track_import_call": track_import_call,
"strict_slots": strict_slots,
})
additional_dicts = globals or {}
additional_dicts.update({
"<fixed-modules>": fixed_modules,
"<builtins>": builtins
})
d, m = self._exec_strict_code(c,
name,
additional_dicts=additional_dicts)
return m
def show_module(prog, name="<module>"):
""" Compile a module and dump the symbol tables
"""
if isinstance(prog, symtable.SymbolTable):
# Already compiled
mst = prog
else:
mst = symtable.symtable(prog, name, 'exec')
stlist = list_symbol_tables(mst)
for st in stlist:
show_symbol_table(st)
def format_table(code_string):
s_table = symtable(code_string, "string", "exec")
table = []
for child in s_table.get_children():
row = {
"name": child.get_name(),
"scope": "global",
"children": [subchild.get_name() for subchild in child.get_children()],
}
table.append(row)
return s_table, table
def format_table(code_string):
s_table = symtable(code_string, 'string', 'exec')
table = []
for child in s_table.get_children():
row = {
'name': child.get_name(),
'scope': 'global',
'children':
[subchild.get_name() for subchild in child.get_children()]
}
table.append(row)
return s_table, table
def getDefinitions(self, doc, identifier):
if doc.get_language().get_name() != "Python":
return
doc_location = doc.get_location()
with open(doc_location.get_path()) as f:
table = symtable.symtable(
f.read(),
doc_location.get_basename(),
"exec",
)
for line in self.generateDefLines(table, identifier):
yield doc_location, line, "", doc_location.get_path()
def load_file(filename):
filename = os.path.abspath(filename)
with open(filename) as fp:
syminfo = symtable.symtable(fp.read() + '\n', filename, 'exec')
if(os.path.splitext(filename)[1] == '.py'):
try:
py_compile.compile(filename, filename+'c', doraise=True)
except py_compile.PyCompileError, msg:
print str(msg)
print 'Couldn\'t compile %s, stopping.' % filename
os._exit(0)
filename += 'c'
def test_annotated(self):
st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec')
st2 = st1.get_children()[0]
self.assertTrue(st2.lookup('x').is_local())
self.assertTrue(st2.lookup('x').is_annotated())
self.assertFalse(st2.lookup('x').is_global())
st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec')
st4 = st3.get_children()[0]
self.assertTrue(st4.lookup('x').is_local())
self.assertFalse(st4.lookup('x').is_annotated())
# Test that annotations in the global scope are valid after the
# variable is declared as nonlocal.
st5 = symtable.symtable('global x\nx: int', 'test', 'exec')
self.assertTrue(st5.lookup("x").is_global())
# Test that annotations for nonlocals are valid after the
# variable is declared as nonlocal.
st6 = symtable.symtable('def g():\n'
' x = 2\n'
' def f():\n'
' nonlocal x\n'
' x: int',
'test', 'exec')
def symtabdump(fn):
if not os.path.exists(fn):
print "%s doesn't exist" % fn
raise SystemExit()
text = open(fn).read()
mod = symtable.symtable(text, os.path.split(fn)[1], "exec")
def getidents(obj, indent=""):
ret = ""
ret += """%sSym_type: %s
%sSym_name: %s
%sSym_lineno: %s
%sSym_nested: %s
%sSym_haschildren: %s
""" % (
indent, obj.get_type(),
indent, obj.get_name(),
indent, obj.get_lineno(),
indent, obj.is_nested(),
indent, obj.has_children())
if obj.get_type() == "function":
ret += "%sFunc_params: %s\n%sFunc_locals: %s\n%sFunc_globals: %s\n%sFunc_frees: %s\n" % (
indent, sorted(obj.get_parameters()),
indent, sorted(obj.get_locals()),
indent, sorted(obj.get_globals()),
indent, sorted(obj.get_frees()))
elif obj.get_type() == "class":
ret += "%sClass_methods: %s\n" % (
indent, sorted(obj.get_methods()))
ret += "%s-- Identifiers --\n" % indent
for ident in sorted(obj.get_identifiers()):
info = obj.lookup(ident)
ret += "%sname: %s\n %sis_referenced: %s\n %sis_imported: %s\n %sis_parameter: %s\n %sis_global: %s\n %sis_declared_global: %s\n %sis_local: %s\n %sis_free: %s\n %sis_assigned: %s\n %sis_namespace: %s\n %snamespaces: [\n%s %s]\n" % (
indent, info.get_name(),
indent, info.is_referenced(),
indent, info.is_imported(),
indent, info.is_parameter(),
indent, info.is_global(),
indent, info.is_declared_global(),
indent, info.is_local(),
indent, info.is_free(),
indent, info.is_assigned(),
indent, info.is_namespace(),
indent, '\n'.join([getidents(x, indent + " ") for x in info.get_namespaces()]),
indent
)
return ret
return getidents(mod)
def test_filename_correct(self):
### Bug tickler: SyntaxError file name correct whether error raised
### while parsing or building symbol table.
def checkfilename(brokencode):
try:
symtable.symtable(brokencode, "spam", "exec")
except SyntaxError as e:
self.assertEqual(e.filename, "spam")
else:
self.fail("no SyntaxError for %r" % (brokencode,))
checkfilename("def f(x): foo)(") # parse-time
checkfilename("def f(x): global x") # symtable-build-time
symtable.symtable("pass", b"spam", "exec")
with self.assertRaises(TypeError):
symtable.symtable("pass", bytearray(b"spam"), "exec")
symtable.symtable("pass", memoryview(b"spam"), "exec")
with self.assertRaises(TypeError):
symtable.symtable("pass", list(b"spam"), "exec")
def __init__(self, source_no_encoding):
ast = compiler.parse(source_no_encoding)
self.pubapi = None
self.matches = 0
compiler.walk(ast, self)
if self.pubapi == None:
# Didn't find __all__.
if conf.allpublic:
symtab = symtable.symtable(source_no_encoding, "-", "exec")
self.pubapi = filter(lambda s: s[0] != "_",
symtab.get_identifiers())
else:
self.pubapi = []
if self.matches > 1:
print >>sys.stderr, "Warning: Found multiple __all__ definitions"
print >>sys.stderr, "Using last definition"
def to_one_line(original):
# original :: string
# :: string
t = ast.parse(original)
table = symtable.symtable(original, '<string>', 'exec')
original = original.strip()
# If there's only one line anyways, be lazy
if len(original.splitlines()) == 1 and \
len(t.body) == 1 and \
type(t.body[0]) in (ast.Delete, ast.Assign, ast.AugAssign, ast.Print,
ast.Raise, ast.Assert, ast.Import, ast.ImportFrom,
ast.Exec, ast.Global, ast.Expr, ast.Pass):
return original
return get_init_code(t, table)
def __init__(self, name="", path=None, file_name="<unknown>", code=None):
CKClass.__init__(self, name, None, None, None)
self.path = path
self.descriptor = pyclbr.readmodule_ex(self.name, path)
self.file_name = file_name
self.code = code
if self.code:
self.symbol_table = symtable.symtable(self.code, self.file_name, 'exec')
self.ast_node = ast.parse(self.code, self.file_name)
self.classes = {}
class_node_finder = ClassNodeFinder(self.ast_node)
for class_name in self.descriptor.keys():
class_descriptor = self.descriptor[class_name]
if isinstance(class_descriptor, pyclbr.Class):
if class_descriptor.module == self.name:
class_table = self.symbol_table.lookup(class_name).get_namespace()
class_node = class_node_finder.find(class_name)
ck_class = CKClass(class_name, class_descriptor, class_table, class_node)
ck_class.extract_references()
self.classes[self.name + "." + class_name] = ck_class
def get_locals(func):
source = inspect.getsource(func)
top = symtable.symtable(source, "<string>", "exec")
func = top.get_children()[0] #since we are passing only the func code.
return func.get_locals()
def test_exec(self):
symbols = symtable.symtable("def f(x): return x", "?", "exec")
"""Module symbol-table generator"""
from compiler import ast
from compiler.consts import SC_LOCAL, SC_GLOBAL, SC_FREE, SC_CELL, SC_UNKNOWN
from compiler.misc import mangle
import types
import sys
MANGLE_LEN = 256
class Scope:
# XXX how much information do I need about each name?
def __init__(self, name, module, klass=None):
self.name = name
self.module = module
self.defs = {}
self.uses = {}
self.globals = {}
self.params = {}
self.frees = {}
self.cells = {}
self.children = []
# nested is true if the class could contain free variables,
# i.e. if it is nested within another function.
self.nested = None
self.generator = None
self.klass = None
if klass is not None:
for i in range(len(klass)):
if klass[i] != '_':
self.klass = klass[i:]
break
def parse_code(source_code, filename='app.py'):
# type: (str, str) -> ParsedCode
parsed = ast.parse(source_code, filename)
table = symtable.symtable(source_code, filename, 'exec')
return ParsedCode(parsed, ChainedSymbolTable(table, table))
__author__ = 'anthonymcclay'
__project__ = 'LearnSmart1'
__date__ = '10/23/16'
__revision__ = '$'
__revision_date__ = '$'
#!/usr/bin/python
code = compile('a + 5', 'file.py', 'eval')
a = 5
print("code : ", eval(code))
# import parser module
import parser
st = parser.expr('b + 5')
code = st.compile('file.py')
b = 15
print("Parser code : ", eval(code))
# import systable module - access to the compiler’s symbol tables
import symtable
table = symtable.symtable("def func(): pass", "string", "exec")
print("func :", table.lookup("func").is_namespace())
def get_globals():
module = inspect.getmodule(get_globals)
source = inspect.getsource(module)
top = symtable.symtable(source, "<string>", "exec")
return top.get_identifiers()
if __name__ == "__main__":
import sys
from compiler import parseFile, walk
import symtable
def get_names(syms):
return [s for s in [s.get_name() for s in syms.get_symbols()]
if not (s.startswith('_[') or s.startswith('.'))]
for file in sys.argv[1:]:
print file
f = open(file)
buf = f.read()
f.close()
syms = symtable.symtable(buf, file, "exec")
mod_names = get_names(syms)
tree = parseFile(file)
s = SymbolVisitor()
walk(tree, s)
# compare module-level symbols
names2 = s.scopes[tree].get_names()
if not list_eq(mod_names, names2):
print
print "oops", file
print sorted(mod_names)
print sorted(names2)
sys.exit(-1)
def test_eval(self):
symbols = symtable.symtable("42", "?", "eval")
def test_single(self):
symbols = symtable.symtable("42", "?", "single")
if __name__ == '__main__':
import sys
from compiler import parseFile, walk
import symtable
def get_names(syms):
return [ s for s in [ s.get_name() for s in syms.get_symbols() ] if not (s.startswith('_[') or s.startswith('.')) ]
for file in sys.argv[1:]:
print file
f = open(file)
buf = f.read()
f.close()
syms = symtable.symtable(buf, file, 'exec')
mod_names = get_names(syms)
tree = parseFile(file)
s = SymbolVisitor()
walk(tree, s)
names2 = s.scopes[tree].get_names()
if not list_eq(mod_names, names2):
print
print 'oops', file
print sorted(mod_names)
print sorted(names2)
sys.exit(-1)
d = {}
d.update(s.scopes)
del d[tree]
scopes = d.values()
