def ast_repr(x):
"""Similar to repr(), but returns an AST instead of a String, which when
evaluated will return the given value."""
tx = type(x)
if tx in (int, float):
return ast.Num(n=x)
elif tx is bytes:
return ast.Bytes(s=x)
elif isinstance(x, str):
return ast.Str(s=x)
elif tx is list:
return ast.List(elts=list(map(ast_repr, x)))
elif tx is dict:
return ast.Dict(keys=list(map(ast_repr, x.keys())),
values=list(map(ast_repr, x.values())))
elif tx is set:
return ast.Set(elts=list(map(ast_repr, x)))
elif tx is Literal:
return x.body
elif tx is Captured:
return compat.Call(ast.Name(id="Captured"),
[x.val, ast_repr(x.name)], [])
elif tx in (bool, type(None)):
return ast.NameConstant(value=x)
elif isinstance(x, ast.AST):
fields = [ast.keyword(a, ast_repr(b)) for a, b in ast.iter_fields(x)]
# This hard-codes an expectation that ast classes will be
# bound to the name `ast`. There must be a better way.
return compat.Call(
ast.Attribute(value=ast.Name(id='ast', ctx=ast.Load()),
attr=x.__class__.__name__,
ctx=ast.Load()), [], fields)
raise Exception("Don't know how to ast_repr this: ", x)
def obj_to_ast(obj):
if isinstance(obj, tuple):
return ast.Tuple(elts=tuple(map(obj_to_ast, obj)))
elif isinstance(obj, dict):
k, v = unzip([(obj_to_ast(k), obj_to_ast(v)) for k, v in obj.items()])
return ast.Dict(k, v)
elif isinstance(obj, list):
return ast.List(list(map(obj_to_ast, obj)))
elif isinstance(obj, type):
return ast.Name(id=obj.__name__)
elif isinstance(obj, int):
return ast.Num(obj)
elif isinstance(obj, str):
return ast.Str(obj)
elif obj is None:
return ast.NameConstant(None)
elif isinstance(obj, (typing._GenericAlias, typing._SpecialForm)):
# TODO: types
# issue was in pandas, where importing pandas._typing.Axis would
# resolve module to typing, attempt to do "from typing import Axis"
return ast.NameConstant(None)
elif isinstance(obj, float) and math.isinf(obj):
return parse_expr('float("inf")')
elif isinstance(obj, bytes):
return ast.Bytes(s=obj)
else:
raise ObjConversionException(f"No converter for {obj}")
def value2pyliteral(val): # noqa: C901
"""
Transforms a python value into a literal in AST form.
"""
if val is None:
return ast.NameConstant(value=None)
elif val is ...:
return ast.Ellipsis()
elif val is True:
return ast.NameConstant(value=True)
elif val is False:
return ast.NameConstant(value=False)
elif isinstance(val, int):
return ast.Num(n=val)
elif isinstance(val, float):
return ast.Num(n=val)
elif isinstance(val, str):
return ast.Str(s=val)
elif isinstance(val, bytes):
return ast.Bytes(s=val)
elif isinstance(val, tuple):
return ast.Tuple(elts=[value2pyliteral(item) for item in val])
elif isinstance(val, list):
return ast.List(elts=[value2pyliteral(item) for item in val])
elif isinstance(val, dict):
return ast.Dict(
# .keys() and .values() have been documented to return things in the
# same order since Py2
keys=[value2pyliteral(item) for item in val.keys()],
values=[value2pyliteral(item) for item in val.values()],
)
elif isinstance(val, set):
return ast.Set(elts=[value2pyliteral(item) for item in val])
else:
raise ValueError(f"Can't translate {val!r} into a literal")
def test_deprecated_constant_nodes(self):
self.assertAstEqualsSource(
ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(3)),
"spam = 3")
self.assertAstEqualsSource(
ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(-93)),
"spam = -93")
self.assertAstEqualsSource(
ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(837.3888)),
"spam = 837.3888")
self.assertAstEqualsSource(
ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(-0.9877)),
"spam = -0.9877")
self.assertAstEqualsSource(ast.Ellipsis(), "...")
if sys.version_info >= (3, 0):
self.assertAstEqualsSource(
ast.Assign(targets=[ast.Name(id='spam')],
value=ast.Bytes(b"Bytes")), "spam = b'Bytes'")
self.assertAstEqualsSource(
ast.Assign(targets=[ast.Name(id='spam')], value=ast.Str("String")),
"spam = 'String'")
def test_unused_constants(self):
number = ast.Num(1)
string = ast.Str('A')
bytes_ = ast.Bytes(b'A')
module = ast.Module([ast.Expr(expr)
for expr in (number, string, bytes_)])
new_ast = self.transform.visit(module)
self.assertFalse(new_ast.body)
def _store_code(self, code_name, mcode):
# store marshal data in the dict
name = ast.Name(id='__lazy_data', ctx=ast.Load())
index = ast.Index(ast.Str(code_name))
target = ast.Subscript(value=name, slice=index, ctx=ast.Store())
assign = ast.Assign(targets=[target], value=ast.Bytes(mcode))
ast.fix_missing_locations(assign)
return assign
def bytes_word_visitor(node: concat.parse.BytesWordNode):
"""Converts a BytesWordNode to the Python expression `push(b'...')`."""
load = ast.Load()
bytes = ast.Bytes(s=node.value)
push_func = ast.Name(id='push', ctx=load)
py_node = ast.Call(func=push_func, args=[bytes], keywords=[])
py_node.lineno, py_node.col_offset = node.location
return py_node
def string_action(s, loc, tokens):
value = ast.literal_eval(tokens[0])
if PY3:
if isinstance(value, text_type):
return ast.Str(s=value)
else:
return ast.Bytes(s=value)
else:
return ast.Str(s=value)
def compile_terminal(sexp: List, closure: Dict) -> ast.AST:
"""
Compiles primitive type variable (language built-in) to AST node.
`closure` contains name bindings for target function. Variables of
non-primitive (built-in) types (without corresponding AST node)
are given a unique random name, converted to ast.Name and stored
in closure for the target function.
"""
if type(sexp) is bool:
return ast.NameConstant(value=sexp), closure
elif type(sexp) in (int, float, complex):
return ast.Num(n=sexp), closure
elif type(sexp) is str:
return ast.Str(s=sexp), closure
elif type(sexp) is bytes:
return ast.Bytes(s=sexp), closure
elif type(sexp) is list:
elts = []
for e in sexp:
e, closure = compile_terminal(e, closure)
elts.append(e)
return ast.List(elts=elts, ctx=ast.Load()), closure
elif type(sexp) is tuple:
elts = []
for e in sexp:
e, closure = compile_terminal(e, closure)
elts.append(e)
return ast.Tuple(elts=elts, ctx=ast.Load()), closure
elif type(sexp) is dict:
keys, values = [], []
for k, v in sexp.items():
k, closure = compile_terminal(k, closure)
v, closure = compile_terminal(v, closure)
keys.append(k)
values.append(v)
return ast.Dict(keys=keys, values=values), closure
elif type(sexp) is set:
elts = []
for e in sexp:
e, closure = compile_terminal(e, closure)
elts.append(e)
return ast.Set(elts=elts), closure
else:
# Generate random name and store variable in closure.
name = '_%s' % uuid.uuid4().hex
closure[name] = sexp
return ast.Name(id=name, ctx=ast.Load()), closure
def test_simple_types(self):
# string, integer, float, force casting
xml = Parser(self.get_xml("basic_types.xml"))
module = xml.parse()
code = compile(module, "<ast>", "exec")
mymodule = ast.Module([
ast.Expr(ast.Str("batuhan")),
ast.Expr(ast.Num(15)),
ast.Expr(ast.Num(15.5)),
ast.Expr(ast.Str("13")),
ast.Expr(ast.Ellipsis()),
ast.Expr(ast.Bytes(bytes("a", "ASCII"))),
ast.Expr(ast.Bytes(bytes("ş", "utf-8"))),
])
ast.fix_missing_locations(mymodule)
mycode = compile(mymodule, "<ast>", "exec")
pprint(mymodule)
self.assertEqual(code, mycode)
def visit_Constant(self, node):
if node.value is None:
new_node = ast.NameConstant(node.value)
elif isinstance(node.value, bool):
new_node = ast.NameConstant(node.value)
elif isinstance(node.value, (int, float, complex)):
new_node = ast.Num(node.value)
elif isinstance(node.value, str):
new_node = ast.Str(node.value)
else:
new_node = ast.Bytes(node.value)
ast.copy_location(new_node, node)
return new_node
def wrapper(*args, **kwargs):
_args = []
assert args[2]
for arg in args[2]:
if (type(arg) == int):
_args.append(ast.Num(arg))
elif (type(arg) == str):
_args.append(ast.Str(arg))
elif (type(arg) == bytes):
_args.append(ast.Bytes(arg))
args[2].clear()
args[2].append(_args)
return w_func(*args, **kwargs)
def test_tree_node(self):
self.assertEqual("(|py-Bytes| #(97 98))",
self.function(ast.Bytes(b'\x61\x62')))
self.assertEqual("(|py-Str| #(0 0 0 97))", self.function(ast.Str('a')))
self.assertEqual("(|py-Num| 97)", self.function(ast.Num(97)))
self.assertEqual("(|py-NameConstant| t)",
self.function(ast.NameConstant(True)))
self.assertEqual("(|py-NameConstant| nil)",
self.function(ast.NameConstant(False)))
self.assertEqual("(|py-NameConstant| |None|)",
self.function(ast.NameConstant(None)))
self.assertEqual(
"(|py-Module| " + "((|py-Expr| (|py-NameConstant| |None|))" +
" (|py-Expr| (|py-NameConstant| |None|))" +
" (|py-Expr| (|py-NameConstant| |None|))))",
self.function(ast.Module([ast.Expr(ast.NameConstant(None))] * 3)))
def test__get_literal_value(self):
new_context = context.Context()
value = ast.Num(42)
expected = value.n
self.assertEqual(expected, new_context._get_literal_value(value))
value = ast.Str('spam')
expected = value.s
self.assertEqual(expected, new_context._get_literal_value(value))
value = ast.List([ast.Str('spam'), ast.Num(42)], ast.Load())
expected = [ast.Str('spam').s, ast.Num(42).n]
self.assertListEqual(expected, new_context._get_literal_value(value))
value = ast.Tuple([ast.Str('spam'), ast.Num(42)], ast.Load())
expected = (ast.Str('spam').s, ast.Num(42).n)
self.assertTupleEqual(expected, new_context._get_literal_value(value))
value = ast.Set([ast.Str('spam'), ast.Num(42)])
expected = set([ast.Str('spam').s, ast.Num(42).n])
self.assertSetEqual(expected, new_context._get_literal_value(value))
value = ast.Dict(['spam', 'eggs'], [42, 'foo'])
expected = dict(spam=42, eggs='foo')
self.assertDictEqual(expected, new_context._get_literal_value(value))
value = ast.Ellipsis()
self.assertIsNone(new_context._get_literal_value(value))
value = ast.Name('spam', ast.Load())
expected = value.id
self.assertEqual(expected, new_context._get_literal_value(value))
if six.PY3:
value = ast.NameConstant(True)
expected = str(value.value)
self.assertEqual(expected, new_context._get_literal_value(value))
if six.PY3:
value = ast.Bytes(b'spam')
expected = value.s
self.assertEqual(expected, new_context._get_literal_value(value))
self.assertIsNone(new_context._get_literal_value(None))
def _to_entry(param):
if isinstance(param, int):
return ast.Num(param)
if isinstance(param, float):
return ast.Num(param)
if isinstance(param, bool):
return ast.NameConstant(value=param)
if isinstance(param, str):
return ast.Str(s=param)
if isinstance(param, bytes):
return ast.Bytes(s=param)
if isinstance(param, list):
return ast.parse(str(param)).body[0].value
if isinstance(param, dict):
return ast.parse(str(param)).body[0].value
if isinstance(param, tuple):
return ast.parse(str(param)).body[0].value
if isinstance(param, set):
return ast.parse(str(param)).body[0].value
def test__get_literal_value(self):
new_context = context.Context()
value = ast.Num(42)
expected = value.n
self.assertEqual(expected, new_context._get_literal_value(value))
value = ast.Str("spam")
expected = value.s
self.assertEqual(expected, new_context._get_literal_value(value))
value = ast.List([ast.Str("spam"), ast.Num(42)], ast.Load())
expected = [ast.Str("spam").s, ast.Num(42).n]
self.assertListEqual(expected, new_context._get_literal_value(value))
value = ast.Tuple([ast.Str("spam"), ast.Num(42)], ast.Load())
expected = (ast.Str("spam").s, ast.Num(42).n)
self.assertTupleEqual(expected, new_context._get_literal_value(value))
value = ast.Set([ast.Str("spam"), ast.Num(42)])
expected = {ast.Str("spam").s, ast.Num(42).n}
self.assertSetEqual(expected, new_context._get_literal_value(value))
value = ast.Dict(["spam", "eggs"], [42, "foo"])
expected = dict(spam=42, eggs="foo")
self.assertDictEqual(expected, new_context._get_literal_value(value))
value = ast.Ellipsis()
self.assertIsNone(new_context._get_literal_value(value))
value = ast.Name("spam", ast.Load())
expected = value.id
self.assertEqual(expected, new_context._get_literal_value(value))
value = ast.Bytes(b"spam")
expected = value.s
self.assertEqual(expected, new_context._get_literal_value(value))
self.assertIsNone(new_context._get_literal_value(None))
def basicTypeSpecialFunction(cv):
"""If you're in a number or string (which has no metadata), move up to the AST to make the special functions work."""
if isinstance(cv, SwapVector) or isinstance(cv, MoveVector):
return cv
if (cv.path[0] in [('n', 'Number'), ('s', 'String'), ('id', 'Name'), ('arg', 'Argument'),
('value', 'Name Constant'), ('s', 'Bytes'), ('name', 'Alias')]):
cvCopy = cv.deepcopy()
cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start))
if cv.path[0] == ('n', 'Number'):
cv.newSubtree = ast.Num(cv.newSubtree)
elif cv.path[0] == ('s', 'String'):
cv.newSubtree = ast.Str(cv.newSubtree)
elif cv.path[0] == ('id', 'Name'):
cv.newSubtree = ast.Name(cv.newSubtree, cv.oldSubtree.ctx)
elif cv.path[0] == ('arg', 'Argument'):
cv.newSubtree = ast.arg(cv.newSubtree, cv.oldSubtree.annotation)
elif cv.path[0] == ('value', 'Name Constant'):
cv.newSubtree = ast.NameConstant(cv.newSubtree)
elif cv.path[0] == ('s', 'Bytes'):
cv.newSubtree = ast.Bytes(cv.newSubtree)
elif cv.path[0] == ('name', 'Alias'):
cv.newSubtree = ast.alias(cv.newSubtree, cv.oldSubtree.asname)
cv.path = cv.path[1:]
return cv
def test_Bytes(self):
for thing in (b'1', b'123'):
self.verify(ast.Bytes(thing), repr(thing))
def p_atom12(self, p):
''' atom : atom_bytes_list '''
s = ast.Bytes(s=p[1])
p[0] = s
def visit_Call(self, node):
if self.just_imports:
return self.generic_visit(node)
if isinstance(node.func, ast.Name) and node.func.id == "__define":
if len(node.args) != 2:
raise CWrongSyntax(node.lineno, node.col_offset, "",
"__define needs 2 arguments")
if not isinstance(node.args[0], ast.Name):
raise CWrongSyntax(
node.lineno, node.col_offset, self.filename,
"__define needs a name and an integer as arguments")
if isinstance(node.args[1], ast.UnaryOp) and isinstance(
node.args[1].op, ast.USub):
val = -node.args[1].operand.n
elif isinstance(node.args[1], ast.Num) and isinstance(
node.args[1].n, int):
val = node.args[1].n
else:
raise CWrongSyntax(
node.lineno, node.col_offset, self.filename,
"__define needs an integer as second argument")
#print("===>>>> ADDING",node.args[0].id,"AS",node.args[1].n)
self.allnames[node.args[0].id] = val
return None
elif isinstance(node.func, ast.Name) and node.func.id == "__lookup":
if len(node.args) != 1:
raise CWrongSyntax(node.lineno, node.col_offset, "",
"__lookup need 1 argument")
if not isinstance(node.args[0], ast.Name):
raise CWrongSyntax(node.lineno, node.col_offset, self.filename,
"__lookup needs a name as first argument")
try:
val = lookup_table[node.args[0].id]
except:
raise CWrongSyntax(node.lineno, node.col_offset, self.filename,
"__lookup can't find " + node.args[0].id)
debug("__lookup value assigned to", node.args[0].id, "is",
str(val))
if isinstance(val, str):
return ast.Str(val)
elif isinstance(val, bytes):
return ast.Bytes(val)
elif isinstance(val, int) or isinstance(val, float):
return ast.Num(val)
return None
elif isinstance(node.func, ast.Name) and node.func.id == "__cdefine":
if len(node.args) > 2:
raise CWrongSyntax(node.lineno, node.col_offset, "",
"__cdefine needs 2 arguments at most")
if not isinstance(node.args[0], ast.Name):
raise CWrongSyntax(node.lineno, node.col_offset, self.filename,
"__cdefine needs a name as first argument")
val = None
if len(node.args) == 2:
if isinstance(node.args[1], ast.UnaryOp) and isinstance(
node.args[1].op, ast.USub):
val = -node.args[1].operand.n
elif isinstance(node.args[1], ast.Num) and isinstance(
node.args[1].n, int):
val = node.args[1].n
else:
raise CWrongSyntax(
node.lineno, node.col_offset, self.filename,
"__cdefine needs an integer as second argument")
#print("===>>>> ADDING",node.args[0].id,"AS",node.args[1].n)
if "CDEFS" not in self.defines:
self.defines["CDEFS"] = []
if val is None:
self.defines["CDEFS"].append(node.args[0].id)
else:
self.defines["CDEFS"].append(node.args[0].id + "=" + str(val))
return None
elif isinstance(node.func, ast.Name) and node.func.id == "__cfile":
if len(node.args) > 1:
raise CWrongSyntax(node.lineno, node.col_offset, "",
"__cfile needs 1 argument at most")
if not isinstance(node.args[0], ast.Str):
raise CWrongSyntax(node.lineno, node.col_offset, self.filename,
"__cfile needs a string as first argument")
#print("__cfile: ", os.path.join(self.curpath,node.args[0].s))
self.cfiles.add(fs.path(self.curpath, node.args[0].s))
return None
else:
return self.generic_visit(node)
def visitBytes(self, n, *args):
return ast.Bytes(s=n.s)
def visitBlameCast(self, n, *args):
def get_type(ty):
if isinstance(ty, retic_ast.OutputAlias) or isinstance(
ty, retic_ast.ClassOutputAlias):
return ty.underlying
else:
return ty
val = self.dispatch(n.value)
args = [n.responsible, n.tag.to_ast()]
src = get_type(n.src)
trg = get_type(n.trg)
if isinstance(trg, retic_ast.Int) or isinstance(
trg, retic_ast.SingletonInt):
fn = '__retic_check_int__'
args = []
elif isinstance(trg, retic_ast.Float):
fn = '__retic_check_float__'
args = []
elif isinstance(trg, retic_ast.Void):
fn = '__retic_check_none__'
args = []
elif isinstance(trg, retic_ast.Str):
fn = '__retic_check_str__'
args = []
elif isinstance(trg, retic_ast.Bool):
fn = '__retic_check_bool__'
args = []
elif isinstance(trg, retic_ast.Function):
fn = '__retic_cast_callable__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset
]
elif isinstance(get_type(n.type), retic_ast.Set):
fn = '__retic_cast_set__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset
]
elif isinstance(get_type(n.type), retic_ast.List):
fn = '__retic_cast_list__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset
]
elif isinstance(get_type(n.type), retic_ast.Dict):
fn = '__retic_cast_dict__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset
]
elif isinstance(get_type(n.type), retic_ast.Tuple):
fn = '__retic_cast_tuple__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset,
ast.Num(n=len(trg.elts),
lineno=val.lineno,
col_offset=val.col_offset)
]
elif isinstance(get_type(n.type), retic_ast.HTuple):
fn = '__retic_cast_htuple__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset
]
elif isinstance(get_type(n.type), retic_ast.Module):
fn = '__retic_cast_module__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset
]
elif isinstance(get_type(n.type), retic_ast.Instance):
fn = '__retic_cast_instance__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset,
n.trg.to_ast(lineno=val.lineno, col_offset=val.col_offset)
]
elif isinstance(get_type(n.type), retic_ast.Class):
fn = '__retic_cast_class__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset,
n.trg.to_ast(lineno=val.lineno,
col_offset=val.col_offset).args[0]
]
elif isinstance(get_type(n.type), retic_ast.Union):
fn = '__retic_cast_union__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset,
n.trg.to_ast(lineno=val.lineno,
col_offset=val.col_offset).args[0]
]
elif isinstance(get_type(n.type), retic_ast.Structural):
fn = '__retic_cast_structural__'
args = [
ast.Bytes(s=pickle.dumps(src), lineno=n.lineno),
ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno,
n.col_offset,
ast.List(elts=[
ast.Str(s=k, lineno=val.lineno, col_offset=val.col_offset)
for k in get_type(n.type).members
],
ctx=ast.Load(),
lineno=val.lineno,
col_offset=val.col_offset)
]
else:
raise exc.InternalReticulatedError(n.type)
return ast_trans.Call(func=ast.Name(id=fn,
ctx=ast.Load(),
lineno=n.lineno,
col_offset=n.col_offset),
args=[val] + args,
keywords=[],
starargs=None,
kwargs=None,
lineno=val.lineno,
col_offset=val.col_offset)
def Bytes(draw) -> ast.Bytes:
return ast.Bytes(draw(binary(max_size=3)))
def visit_str(node):
# Converts text-strings to NumExpr-supported byte-strings
return ast.Bytes(node.s.encode())
def visit_Str(self, node):
return ast.Bytes(node.s.encode('ascii'))
def generate_bytes(max_depth=None):
seq = generate_word().encode('utf-8')
return ast.Bytes(seq)
