def ast_repr(x): """Similar to repr(), but returns an AST instead of a String, which when evaluated will return the given value.""" tx = type(x) if tx in (int, float): return ast.Num(n=x) elif tx is bytes: return ast.Bytes(s=x) elif isinstance(x, str): return ast.Str(s=x) elif tx is list: return ast.List(elts=list(map(ast_repr, x))) elif tx is dict: return ast.Dict(keys=list(map(ast_repr, x.keys())), values=list(map(ast_repr, x.values()))) elif tx is set: return ast.Set(elts=list(map(ast_repr, x))) elif tx is Literal: return x.body elif tx is Captured: return compat.Call(ast.Name(id="Captured"), [x.val, ast_repr(x.name)], []) elif tx in (bool, type(None)): return ast.NameConstant(value=x) elif isinstance(x, ast.AST): fields = [ast.keyword(a, ast_repr(b)) for a, b in ast.iter_fields(x)] # This hard-codes an expectation that ast classes will be # bound to the name `ast`. There must be a better way. return compat.Call( ast.Attribute(value=ast.Name(id='ast', ctx=ast.Load()), attr=x.__class__.__name__, ctx=ast.Load()), [], fields) raise Exception("Don't know how to ast_repr this: ", x)
def obj_to_ast(obj): if isinstance(obj, tuple): return ast.Tuple(elts=tuple(map(obj_to_ast, obj))) elif isinstance(obj, dict): k, v = unzip([(obj_to_ast(k), obj_to_ast(v)) for k, v in obj.items()]) return ast.Dict(k, v) elif isinstance(obj, list): return ast.List(list(map(obj_to_ast, obj))) elif isinstance(obj, type): return ast.Name(id=obj.__name__) elif isinstance(obj, int): return ast.Num(obj) elif isinstance(obj, str): return ast.Str(obj) elif obj is None: return ast.NameConstant(None) elif isinstance(obj, (typing._GenericAlias, typing._SpecialForm)): # TODO: types # issue was in pandas, where importing pandas._typing.Axis would # resolve module to typing, attempt to do "from typing import Axis" return ast.NameConstant(None) elif isinstance(obj, float) and math.isinf(obj): return parse_expr('float("inf")') elif isinstance(obj, bytes): return ast.Bytes(s=obj) else: raise ObjConversionException(f"No converter for {obj}")
def value2pyliteral(val): # noqa: C901 """ Transforms a python value into a literal in AST form. """ if val is None: return ast.NameConstant(value=None) elif val is ...: return ast.Ellipsis() elif val is True: return ast.NameConstant(value=True) elif val is False: return ast.NameConstant(value=False) elif isinstance(val, int): return ast.Num(n=val) elif isinstance(val, float): return ast.Num(n=val) elif isinstance(val, str): return ast.Str(s=val) elif isinstance(val, bytes): return ast.Bytes(s=val) elif isinstance(val, tuple): return ast.Tuple(elts=[value2pyliteral(item) for item in val]) elif isinstance(val, list): return ast.List(elts=[value2pyliteral(item) for item in val]) elif isinstance(val, dict): return ast.Dict( # .keys() and .values() have been documented to return things in the # same order since Py2 keys=[value2pyliteral(item) for item in val.keys()], values=[value2pyliteral(item) for item in val.values()], ) elif isinstance(val, set): return ast.Set(elts=[value2pyliteral(item) for item in val]) else: raise ValueError(f"Can't translate {val!r} into a literal")
def test_deprecated_constant_nodes(self): self.assertAstEqualsSource( ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(3)), "spam = 3") self.assertAstEqualsSource( ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(-93)), "spam = -93") self.assertAstEqualsSource( ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(837.3888)), "spam = 837.3888") self.assertAstEqualsSource( ast.Assign(targets=[ast.Name(id='spam')], value=ast.Num(-0.9877)), "spam = -0.9877") self.assertAstEqualsSource(ast.Ellipsis(), "...") if sys.version_info >= (3, 0): self.assertAstEqualsSource( ast.Assign(targets=[ast.Name(id='spam')], value=ast.Bytes(b"Bytes")), "spam = b'Bytes'") self.assertAstEqualsSource( ast.Assign(targets=[ast.Name(id='spam')], value=ast.Str("String")), "spam = 'String'")
def test_unused_constants(self): number = ast.Num(1) string = ast.Str('A') bytes_ = ast.Bytes(b'A') module = ast.Module([ast.Expr(expr) for expr in (number, string, bytes_)]) new_ast = self.transform.visit(module) self.assertFalse(new_ast.body)
def _store_code(self, code_name, mcode): # store marshal data in the dict name = ast.Name(id='__lazy_data', ctx=ast.Load()) index = ast.Index(ast.Str(code_name)) target = ast.Subscript(value=name, slice=index, ctx=ast.Store()) assign = ast.Assign(targets=[target], value=ast.Bytes(mcode)) ast.fix_missing_locations(assign) return assign
def bytes_word_visitor(node: concat.parse.BytesWordNode): """Converts a BytesWordNode to the Python expression `push(b'...')`.""" load = ast.Load() bytes = ast.Bytes(s=node.value) push_func = ast.Name(id='push', ctx=load) py_node = ast.Call(func=push_func, args=[bytes], keywords=[]) py_node.lineno, py_node.col_offset = node.location return py_node
def string_action(s, loc, tokens): value = ast.literal_eval(tokens[0]) if PY3: if isinstance(value, text_type): return ast.Str(s=value) else: return ast.Bytes(s=value) else: return ast.Str(s=value)
def compile_terminal(sexp: List, closure: Dict) -> ast.AST: """ Compiles primitive type variable (language built-in) to AST node. `closure` contains name bindings for target function. Variables of non-primitive (built-in) types (without corresponding AST node) are given a unique random name, converted to ast.Name and stored in closure for the target function. """ if type(sexp) is bool: return ast.NameConstant(value=sexp), closure elif type(sexp) in (int, float, complex): return ast.Num(n=sexp), closure elif type(sexp) is str: return ast.Str(s=sexp), closure elif type(sexp) is bytes: return ast.Bytes(s=sexp), closure elif type(sexp) is list: elts = [] for e in sexp: e, closure = compile_terminal(e, closure) elts.append(e) return ast.List(elts=elts, ctx=ast.Load()), closure elif type(sexp) is tuple: elts = [] for e in sexp: e, closure = compile_terminal(e, closure) elts.append(e) return ast.Tuple(elts=elts, ctx=ast.Load()), closure elif type(sexp) is dict: keys, values = [], [] for k, v in sexp.items(): k, closure = compile_terminal(k, closure) v, closure = compile_terminal(v, closure) keys.append(k) values.append(v) return ast.Dict(keys=keys, values=values), closure elif type(sexp) is set: elts = [] for e in sexp: e, closure = compile_terminal(e, closure) elts.append(e) return ast.Set(elts=elts), closure else: # Generate random name and store variable in closure. name = '_%s' % uuid.uuid4().hex closure[name] = sexp return ast.Name(id=name, ctx=ast.Load()), closure
def test_simple_types(self): # string, integer, float, force casting xml = Parser(self.get_xml("basic_types.xml")) module = xml.parse() code = compile(module, "<ast>", "exec") mymodule = ast.Module([ ast.Expr(ast.Str("batuhan")), ast.Expr(ast.Num(15)), ast.Expr(ast.Num(15.5)), ast.Expr(ast.Str("13")), ast.Expr(ast.Ellipsis()), ast.Expr(ast.Bytes(bytes("a", "ASCII"))), ast.Expr(ast.Bytes(bytes("ş", "utf-8"))), ]) ast.fix_missing_locations(mymodule) mycode = compile(mymodule, "<ast>", "exec") pprint(mymodule) self.assertEqual(code, mycode)
def visit_Constant(self, node): if node.value is None: new_node = ast.NameConstant(node.value) elif isinstance(node.value, bool): new_node = ast.NameConstant(node.value) elif isinstance(node.value, (int, float, complex)): new_node = ast.Num(node.value) elif isinstance(node.value, str): new_node = ast.Str(node.value) else: new_node = ast.Bytes(node.value) ast.copy_location(new_node, node) return new_node
def wrapper(*args, **kwargs): _args = [] assert args[2] for arg in args[2]: if (type(arg) == int): _args.append(ast.Num(arg)) elif (type(arg) == str): _args.append(ast.Str(arg)) elif (type(arg) == bytes): _args.append(ast.Bytes(arg)) args[2].clear() args[2].append(_args) return w_func(*args, **kwargs)
def test_tree_node(self): self.assertEqual("(|py-Bytes| #(97 98))", self.function(ast.Bytes(b'\x61\x62'))) self.assertEqual("(|py-Str| #(0 0 0 97))", self.function(ast.Str('a'))) self.assertEqual("(|py-Num| 97)", self.function(ast.Num(97))) self.assertEqual("(|py-NameConstant| t)", self.function(ast.NameConstant(True))) self.assertEqual("(|py-NameConstant| nil)", self.function(ast.NameConstant(False))) self.assertEqual("(|py-NameConstant| |None|)", self.function(ast.NameConstant(None))) self.assertEqual( "(|py-Module| " + "((|py-Expr| (|py-NameConstant| |None|))" + " (|py-Expr| (|py-NameConstant| |None|))" + " (|py-Expr| (|py-NameConstant| |None|))))", self.function(ast.Module([ast.Expr(ast.NameConstant(None))] * 3)))
def test__get_literal_value(self): new_context = context.Context() value = ast.Num(42) expected = value.n self.assertEqual(expected, new_context._get_literal_value(value)) value = ast.Str('spam') expected = value.s self.assertEqual(expected, new_context._get_literal_value(value)) value = ast.List([ast.Str('spam'), ast.Num(42)], ast.Load()) expected = [ast.Str('spam').s, ast.Num(42).n] self.assertListEqual(expected, new_context._get_literal_value(value)) value = ast.Tuple([ast.Str('spam'), ast.Num(42)], ast.Load()) expected = (ast.Str('spam').s, ast.Num(42).n) self.assertTupleEqual(expected, new_context._get_literal_value(value)) value = ast.Set([ast.Str('spam'), ast.Num(42)]) expected = set([ast.Str('spam').s, ast.Num(42).n]) self.assertSetEqual(expected, new_context._get_literal_value(value)) value = ast.Dict(['spam', 'eggs'], [42, 'foo']) expected = dict(spam=42, eggs='foo') self.assertDictEqual(expected, new_context._get_literal_value(value)) value = ast.Ellipsis() self.assertIsNone(new_context._get_literal_value(value)) value = ast.Name('spam', ast.Load()) expected = value.id self.assertEqual(expected, new_context._get_literal_value(value)) if six.PY3: value = ast.NameConstant(True) expected = str(value.value) self.assertEqual(expected, new_context._get_literal_value(value)) if six.PY3: value = ast.Bytes(b'spam') expected = value.s self.assertEqual(expected, new_context._get_literal_value(value)) self.assertIsNone(new_context._get_literal_value(None))
def _to_entry(param): if isinstance(param, int): return ast.Num(param) if isinstance(param, float): return ast.Num(param) if isinstance(param, bool): return ast.NameConstant(value=param) if isinstance(param, str): return ast.Str(s=param) if isinstance(param, bytes): return ast.Bytes(s=param) if isinstance(param, list): return ast.parse(str(param)).body[0].value if isinstance(param, dict): return ast.parse(str(param)).body[0].value if isinstance(param, tuple): return ast.parse(str(param)).body[0].value if isinstance(param, set): return ast.parse(str(param)).body[0].value
def test__get_literal_value(self): new_context = context.Context() value = ast.Num(42) expected = value.n self.assertEqual(expected, new_context._get_literal_value(value)) value = ast.Str("spam") expected = value.s self.assertEqual(expected, new_context._get_literal_value(value)) value = ast.List([ast.Str("spam"), ast.Num(42)], ast.Load()) expected = [ast.Str("spam").s, ast.Num(42).n] self.assertListEqual(expected, new_context._get_literal_value(value)) value = ast.Tuple([ast.Str("spam"), ast.Num(42)], ast.Load()) expected = (ast.Str("spam").s, ast.Num(42).n) self.assertTupleEqual(expected, new_context._get_literal_value(value)) value = ast.Set([ast.Str("spam"), ast.Num(42)]) expected = {ast.Str("spam").s, ast.Num(42).n} self.assertSetEqual(expected, new_context._get_literal_value(value)) value = ast.Dict(["spam", "eggs"], [42, "foo"]) expected = dict(spam=42, eggs="foo") self.assertDictEqual(expected, new_context._get_literal_value(value)) value = ast.Ellipsis() self.assertIsNone(new_context._get_literal_value(value)) value = ast.Name("spam", ast.Load()) expected = value.id self.assertEqual(expected, new_context._get_literal_value(value)) value = ast.Bytes(b"spam") expected = value.s self.assertEqual(expected, new_context._get_literal_value(value)) self.assertIsNone(new_context._get_literal_value(None))
def basicTypeSpecialFunction(cv): """If you're in a number or string (which has no metadata), move up to the AST to make the special functions work.""" if isinstance(cv, SwapVector) or isinstance(cv, MoveVector): return cv if (cv.path[0] in [('n', 'Number'), ('s', 'String'), ('id', 'Name'), ('arg', 'Argument'), ('value', 'Name Constant'), ('s', 'Bytes'), ('name', 'Alias')]): cvCopy = cv.deepcopy() cv.oldSubtree = deepcopy(cvCopy.traverseTree(cv.start)) if cv.path[0] == ('n', 'Number'): cv.newSubtree = ast.Num(cv.newSubtree) elif cv.path[0] == ('s', 'String'): cv.newSubtree = ast.Str(cv.newSubtree) elif cv.path[0] == ('id', 'Name'): cv.newSubtree = ast.Name(cv.newSubtree, cv.oldSubtree.ctx) elif cv.path[0] == ('arg', 'Argument'): cv.newSubtree = ast.arg(cv.newSubtree, cv.oldSubtree.annotation) elif cv.path[0] == ('value', 'Name Constant'): cv.newSubtree = ast.NameConstant(cv.newSubtree) elif cv.path[0] == ('s', 'Bytes'): cv.newSubtree = ast.Bytes(cv.newSubtree) elif cv.path[0] == ('name', 'Alias'): cv.newSubtree = ast.alias(cv.newSubtree, cv.oldSubtree.asname) cv.path = cv.path[1:] return cv
def test_Bytes(self): for thing in (b'1', b'123'): self.verify(ast.Bytes(thing), repr(thing))
def p_atom12(self, p): ''' atom : atom_bytes_list ''' s = ast.Bytes(s=p[1]) p[0] = s
def visit_Call(self, node): if self.just_imports: return self.generic_visit(node) if isinstance(node.func, ast.Name) and node.func.id == "__define": if len(node.args) != 2: raise CWrongSyntax(node.lineno, node.col_offset, "", "__define needs 2 arguments") if not isinstance(node.args[0], ast.Name): raise CWrongSyntax( node.lineno, node.col_offset, self.filename, "__define needs a name and an integer as arguments") if isinstance(node.args[1], ast.UnaryOp) and isinstance( node.args[1].op, ast.USub): val = -node.args[1].operand.n elif isinstance(node.args[1], ast.Num) and isinstance( node.args[1].n, int): val = node.args[1].n else: raise CWrongSyntax( node.lineno, node.col_offset, self.filename, "__define needs an integer as second argument") #print("===>>>> ADDING",node.args[0].id,"AS",node.args[1].n) self.allnames[node.args[0].id] = val return None elif isinstance(node.func, ast.Name) and node.func.id == "__lookup": if len(node.args) != 1: raise CWrongSyntax(node.lineno, node.col_offset, "", "__lookup need 1 argument") if not isinstance(node.args[0], ast.Name): raise CWrongSyntax(node.lineno, node.col_offset, self.filename, "__lookup needs a name as first argument") try: val = lookup_table[node.args[0].id] except: raise CWrongSyntax(node.lineno, node.col_offset, self.filename, "__lookup can't find " + node.args[0].id) debug("__lookup value assigned to", node.args[0].id, "is", str(val)) if isinstance(val, str): return ast.Str(val) elif isinstance(val, bytes): return ast.Bytes(val) elif isinstance(val, int) or isinstance(val, float): return ast.Num(val) return None elif isinstance(node.func, ast.Name) and node.func.id == "__cdefine": if len(node.args) > 2: raise CWrongSyntax(node.lineno, node.col_offset, "", "__cdefine needs 2 arguments at most") if not isinstance(node.args[0], ast.Name): raise CWrongSyntax(node.lineno, node.col_offset, self.filename, "__cdefine needs a name as first argument") val = None if len(node.args) == 2: if isinstance(node.args[1], ast.UnaryOp) and isinstance( node.args[1].op, ast.USub): val = -node.args[1].operand.n elif isinstance(node.args[1], ast.Num) and isinstance( node.args[1].n, int): val = node.args[1].n else: raise CWrongSyntax( node.lineno, node.col_offset, self.filename, "__cdefine needs an integer as second argument") #print("===>>>> ADDING",node.args[0].id,"AS",node.args[1].n) if "CDEFS" not in self.defines: self.defines["CDEFS"] = [] if val is None: self.defines["CDEFS"].append(node.args[0].id) else: self.defines["CDEFS"].append(node.args[0].id + "=" + str(val)) return None elif isinstance(node.func, ast.Name) and node.func.id == "__cfile": if len(node.args) > 1: raise CWrongSyntax(node.lineno, node.col_offset, "", "__cfile needs 1 argument at most") if not isinstance(node.args[0], ast.Str): raise CWrongSyntax(node.lineno, node.col_offset, self.filename, "__cfile needs a string as first argument") #print("__cfile: ", os.path.join(self.curpath,node.args[0].s)) self.cfiles.add(fs.path(self.curpath, node.args[0].s)) return None else: return self.generic_visit(node)
def visitBytes(self, n, *args): return ast.Bytes(s=n.s)
def visitBlameCast(self, n, *args): def get_type(ty): if isinstance(ty, retic_ast.OutputAlias) or isinstance( ty, retic_ast.ClassOutputAlias): return ty.underlying else: return ty val = self.dispatch(n.value) args = [n.responsible, n.tag.to_ast()] src = get_type(n.src) trg = get_type(n.trg) if isinstance(trg, retic_ast.Int) or isinstance( trg, retic_ast.SingletonInt): fn = '__retic_check_int__' args = [] elif isinstance(trg, retic_ast.Float): fn = '__retic_check_float__' args = [] elif isinstance(trg, retic_ast.Void): fn = '__retic_check_none__' args = [] elif isinstance(trg, retic_ast.Str): fn = '__retic_check_str__' args = [] elif isinstance(trg, retic_ast.Bool): fn = '__retic_check_bool__' args = [] elif isinstance(trg, retic_ast.Function): fn = '__retic_cast_callable__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset ] elif isinstance(get_type(n.type), retic_ast.Set): fn = '__retic_cast_set__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset ] elif isinstance(get_type(n.type), retic_ast.List): fn = '__retic_cast_list__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset ] elif isinstance(get_type(n.type), retic_ast.Dict): fn = '__retic_cast_dict__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset ] elif isinstance(get_type(n.type), retic_ast.Tuple): fn = '__retic_cast_tuple__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset, ast.Num(n=len(trg.elts), lineno=val.lineno, col_offset=val.col_offset) ] elif isinstance(get_type(n.type), retic_ast.HTuple): fn = '__retic_cast_htuple__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset ] elif isinstance(get_type(n.type), retic_ast.Module): fn = '__retic_cast_module__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset ] elif isinstance(get_type(n.type), retic_ast.Instance): fn = '__retic_cast_instance__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset, n.trg.to_ast(lineno=val.lineno, col_offset=val.col_offset) ] elif isinstance(get_type(n.type), retic_ast.Class): fn = '__retic_cast_class__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset, n.trg.to_ast(lineno=val.lineno, col_offset=val.col_offset).args[0] ] elif isinstance(get_type(n.type), retic_ast.Union): fn = '__retic_cast_union__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset, n.trg.to_ast(lineno=val.lineno, col_offset=val.col_offset).args[0] ] elif isinstance(get_type(n.type), retic_ast.Structural): fn = '__retic_cast_structural__' args = [ ast.Bytes(s=pickle.dumps(src), lineno=n.lineno), ast.Bytes(s=pickle.dumps(trg), lineno=n.lineno), n.lineno, n.col_offset, ast.List(elts=[ ast.Str(s=k, lineno=val.lineno, col_offset=val.col_offset) for k in get_type(n.type).members ], ctx=ast.Load(), lineno=val.lineno, col_offset=val.col_offset) ] else: raise exc.InternalReticulatedError(n.type) return ast_trans.Call(func=ast.Name(id=fn, ctx=ast.Load(), lineno=n.lineno, col_offset=n.col_offset), args=[val] + args, keywords=[], starargs=None, kwargs=None, lineno=val.lineno, col_offset=val.col_offset)
def Bytes(draw) -> ast.Bytes: return ast.Bytes(draw(binary(max_size=3)))
def visit_str(node): # Converts text-strings to NumExpr-supported byte-strings return ast.Bytes(node.s.encode())
def visit_Str(self, node): return ast.Bytes(node.s.encode('ascii'))
def generate_bytes(max_depth=None): seq = generate_word().encode('utf-8') return ast.Bytes(seq)