def __call__(self, selection):
if not self.is_initialized:
self._initialize()
try:
parse_result = self.expression.parseString(selection, parseAll=True)
except ParseException as e:
msg = str(e)
lines = ["%s: %s" % (msg, selection),
" " * (12 + len("%s: " % msg) + (e.loc)) + "^^^"]
raise ValueError('\n'.join(lines))
# Change __ATOM__ in function bodies. It must bind to the arg
# name specified below (i.e. 'atom')
astnode = self.transformer.visit(deepcopy(parse_result[0].ast()))
if PY2:
args = [ast.Name(id='atom', ctx=ast.Param())]
signature = ast.arguments(args=args, vararg=None, kwarg=None,
defaults=[])
else:
args = [ast.arg(arg='atom', annotation=None)]
signature = ast.arguments(args=args, vararg=None, kwarg=None,
kwonlyargs=[], defaults=[],
kw_defaults=[])
func = ast.Expression(body=ast.Lambda(signature, astnode))
source = codegen.to_source(astnode)
expr = eval(
compile(ast.fix_missing_locations(func), '<string>', mode='eval'),
SELECTION_GLOBALS)
return _ParsedSelection(expr, source, astnode)
def _make_fn(name, chain_fn, args, defaults):
args_with_self = ['_self'] + list(args)
arguments = [_ast.Name(id=arg, ctx=_ast.Load()) for arg in args_with_self]
defs = [_ast.Name(id='_def{0}'.format(idx), ctx=_ast.Load()) for idx, _ in enumerate(defaults)]
if _PY2:
parameters = _ast.arguments(args=[_ast.Name(id=arg, ctx=_ast.Param()) for arg in args_with_self],
defaults=defs)
else:
parameters = _ast.arguments(args=[_ast.arg(arg=arg) for arg in args_with_self],
kwonlyargs=[],
defaults=defs,
kw_defaults=[])
module_node = _ast.Module(body=[_ast.FunctionDef(name=name,
args=parameters,
body=[_ast.Return(value=_ast.Call(func=_ast.Name(id='_chain', ctx=_ast.Load()),
args=arguments,
keywords=[]))],
decorator_list=[])])
module_node = _ast.fix_missing_locations(module_node)
# compile the ast
code = compile(module_node, '<string>', 'exec')
# and eval it in the right context
globals_ = {'_chain': chain_fn}
locals_ = dict(('_def{0}'.format(idx), value) for idx, value in enumerate(defaults))
eval(code, globals_, locals_)
# extract our function from the newly created module
return locals_[name]
def visit_ListComp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iterList = []
varList = []
for gen in node.generators:
iterList.append(self.make_Iterator(gen))
varList.append(ast.Name(gen.target.id, ast.Param()))
# If dim = 1, product is useless
if len(iterList) == 1:
iterAST = iterList[0]
varAST = ast.arguments([varList[0]], None, None, [])
else:
prodName = ast.Attribute(value=ast.Name(id="itertools", ctx=ast.Load()), attr="product", ctx=ast.Load())
iterAST = ast.Call(prodName, iterList, [], None, None)
varAST = ast.arguments([ast.Tuple(varList, ast.Store())], None, None, [])
mapName = ast.Attribute(value=ast.Name(id="__builtin__", ctx=ast.Load()), attr="map", ctx=ast.Load())
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return ast.Call(mapName, [ldmap, iterAST], [], None, None)
else:
return self.generic_visit(node)
def visit_GeneratorExp(self, node):
if node in self.optimizable_comprehension:
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [ast.Name(gen.target.id, ast.Param())
for gen in node.generators]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], None, None, [])
else:
prodName = ast.Attribute(
value=ast.Name(id='itertools', ctx=ast.Load()),
attr='product', ctx=ast.Load())
iterAST = ast.Call(prodName, iters, [], None, None)
varAST = ast.arguments([ast.Tuple(variables, ast.Store())],
None, None, [])
imapName = ast.Attribute(
value=ast.Name(id='itertools', ctx=ast.Load()),
attr='imap', ctx=ast.Load())
ldBodyimap = node.elt
ldimap = ast.Lambda(varAST, ldBodyimap)
return ast.Call(imapName, [ldimap, iterAST], [], None, None)
else:
return self.generic_visit(node)
def p_top_func(p):
"""topfunc : FUNC '(' params ')' '{' opt_newline body '}'"""
if p[3]:
args = ast.arguments([ast.Name('self', ast.Param())], None, None, [])
else:
args = ast.arguments([], None, None, [])
p[7] = [RewriteInjected([param[0].id for param in p[3]]).visit(node) for node in p[7]]
p[0] = [ast.FunctionDef("top", args, p[7], [])]
def p_func(p):
"""func : FUNC ID '(' params ')' '{' opt_newline body '}'"""
if p[4]:
arg_names, defaults = tuple([filter(lambda x: x is not None, item) for item in zip(*p[4])])
args = ast.arguments(list(arg_names), None, None, list(defaults))
else:
args = ast.arguments([], None, None, [])
p[0] = ast.FunctionDef(p[2], args, p[8], [])
def p_lambda(p):
"""lambda : '@' '(' params ')' expr"""
if p[3]:
arg_names, defaults = tuple([filter(lambda x: x is not None, item) for item in zip(*p[3])])
args = ast.arguments(list(arg_names), None, None, list(defaults))
else:
args = ast.arguments([], None, None, [])
p[0] = ast.Lambda(args, p[5])
def test_arguments(self):
x = ast.arguments()
self.assertEqual(x._fields, ('args', 'vararg', 'kwarg', 'defaults'))
with self.assertRaises(AttributeError):
x.vararg
x = ast.arguments(1, 2, 3, 4)
self.assertEqual(x.vararg, 2)
def test_arguments(self):
x = ast.arguments()
self.assertEqual(x._fields, ("args", "vararg", "kwarg", "defaults"))
with self.assertRaises(AttributeError):
x.vararg
x = ast.arguments(1, 2, 3, 4)
self.assertEqual(x.vararg, 2)
def make_lambda(expression, args, env=None):
# type: (ast.Expression, List[str], Dict[str, Any]) -> types.FunctionType
"""
Create an lambda function from a expression AST.
Parameters
----------
expression : ast.Expression
The body of the lambda.
args : List[str]
A list of positional argument names
env : Optional[Dict[str, Any]]
Extra environment to capture in the lambda's closure.
Returns
-------
func : types.FunctionType
"""
# lambda *{args}* : EXPRESSION
lambda_ = ast.Lambda(
args=ast.arguments(
args=[ast.arg(arg=arg, annotation=None) for arg in args],
varargs=None,
varargannotation=None,
kwonlyargs=[],
kwarg=None,
kwargannotation=None,
defaults=[],
kw_defaults=[]),
body=expression.body,
)
lambda_ = ast.copy_location(lambda_, expression.body)
# lambda **{env}** : lambda *{args}*: EXPRESSION
outer = ast.Lambda(
args=ast.arguments(
args=[ast.arg(arg=name, annotation=None) for name in (env or {})],
varargs=None,
varargannotation=None,
kwonlyargs=[],
kwarg=None,
kwargannotation=None,
defaults=[],
kw_defaults=[],
),
body=lambda_,
)
exp = ast.Expression(body=outer, lineno=1, col_offset=0)
ast.fix_missing_locations(exp)
GLOBALS = __GLOBALS.copy()
GLOBALS["__builtins__"] = {}
# pylint: disable=eval-used
fouter = eval(compile(exp, "<lambda>", "eval"), GLOBALS)
assert isinstance(fouter, types.FunctionType)
finner = fouter(**env)
assert isinstance(finner, types.FunctionType)
return finner
def p_function_def(p):
"""function_def : type NAME new_scope LPAREN params_def_list RPAREN LBRACE func_statement_list RBRACE
| void NAME new_scope LPAREN params_def_list RPAREN LBRACE func_statement_list RBRACE
"""
if len(p[8]) > 0:
p[0] = ast.FunctionDef(name=p[2], args=ast.arguments(args=p[5], vararg=None, kwarg=None, defaults=[]), body=p[8], decorator_list=[], type=p[1], level=0, globals=global_names())
else:
p[0] = ast.FunctionDef(name=p[2], args=ast.arguments(args=p[5], vararg=None, kwarg=None, defaults=[]), body=[ast.Pass()], decorator_list=[], type=p[1], level=0, globals=global_names())
pop_scope()
symbol_stack[0].append(p[2])
def func_super(self, type_, inst):
self.visit(
ast_call(
ast.FunctionDef(
'',
ast.arguments([ast_store('p'), ast_store('i')], None, None, []),
[
ast.FunctionDef(
'tmp',
ast.arguments([], None, None, []),
[
ast.Expr(
ast_call(
ast_load('this.__bind__'),
ast_load('i')
)
),
],
[]
),
ast.Assign(
[ast_load('tmp.prototype')],
ast_load('p')
),
ast.Return(
ast_call(
ast_load('new'),
ast_load('tmp'),
)
)
],
[]
),
ast.Attribute(
ast.Subscript(
ast_load('self.__mro__'),
ast.Index(
ast.BinOp(
ast_call(
ast_load('self.__mro__.indexOf'),
type_,
),
ast.Sub(),
ast.Num(1)
)
),
ast.Load()
),
'prototype',
ast.Load()
),
inst,
)
)
def test_arguments(self):
x = ast.arguments()
self.assertEqual(x._fields, ('args', 'vararg', 'varargannotation',
'kwonlyargs', 'kwarg', 'kwargannotation',
'defaults', 'kw_defaults'))
with self.assertRaises(AttributeError):
x.vararg
x = ast.arguments(*range(1, 9))
self.assertEqual(x.vararg, 2)
def map_args(self, arg_items, body):
if len(arg_items) == 1:
arg = ast.arguments(args=[arg_items[0]], vararg=None, kwarg=None, defaults=[])
return arg, body
arg = ast.Name(id=self.make_name("_a"), ctx=ast.Load())
maps = self.get_slice(arg, arg_items)
if isinstance(body, list):
body = [NamesMapper(maps).visit(i) for i in body]
else:
body = NamesMapper(maps).visit(body)
arg = ast.arguments(args=[arg], vararg=None, kwarg=None, defaults=[])
return arg, body
def test_arguments(self):
x = ast.arguments()
self.assertEqual(
x._fields,
("args", "vararg", "varargannotation", "kwonlyargs", "kwarg", "kwargannotation", "defaults", "kw_defaults"),
)
with self.assertRaises(AttributeError):
x.vararg
x = ast.arguments(*range(1, 9))
self.assertEqual(x.vararg, 2)
def ast_lambda(name, body):
if PYTHON_VERSION is 2:
return ast.Lambda(args=ast.arguments(args=[name],
defaults=[]), body=body)
elif PYTHON_VERSION is 3:
return ast.Lambda(args=ast.arguments(args=[ast.arg(arg=name.id)],
defaults=[],
kwonlyargs=[],
kw_defaults=[]),
body=body)
else:
invalid_python_version()
def create_bind(gs):
l = len(gs)
g = gs[0]
iter_call = g.iter
if l == 1:
la = ast.Lambda(args=ast.arguments(args=[g.target], defaults=[]),
body=func_call(name('unit'), args=[node.elt])) # ast.Str(s="TODO")) # TODO
return func_call(name('bind'), [iter_call, la])
if l > 1:
la = ast.Lambda(args=ast.arguments(args=[g.target], defaults=[]),
body=create_bind(gs[1:]))
return func_call(name('bind'), [iter_call, la])
raise Exception('Empty generators for list comprehension')
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
if six.PY2:
func_def = ast.FunctionDef(
name=name.encode('utf-8'),
args=ast.arguments(
args=[ast.Name(n, ast.Param()) for n in arg_names],
vararg=None,
kwarg=None,
defaults=[ex_literal(None) for _ in arg_names],
),
body=statements,
decorator_list=[],
)
else:
func_def = ast.FunctionDef(
name=name,
args=ast.arguments(
args=[ast.arg(arg=n, annotation=None) for n in arg_names],
kwonlyargs=[],
kw_defaults=[],
defaults=[ex_literal(None) for _ in arg_names],
),
body=statements,
decorator_list=[],
)
# The ast.Module signature changed in 3.8 to accept a list of types to
# ignore.
if sys.version_info >= (3, 8):
mod = ast.Module([func_def], [])
else:
mod = ast.Module([func_def])
ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec')
# Debug: show bytecode.
if debug:
dis.dis(prog)
for const in prog.co_consts:
if isinstance(const, types.CodeType):
dis.dis(const)
the_locals = {}
exec(prog, {}, the_locals)
return the_locals[name]
def decorator(wrapped):
spec = inspect.getargspec(wrapped)
name = wrapped.__name__
assert spec.varargs is not None
# Example was generated with print ast.dump(ast.parse("def f(a, b, *args, **kwds): return call_wrapped((a, b), args, kwds)"), include_attributes=True)
# http://code.activestate.com/recipes/578353-code-to-source-and-back/ helped a lot
# http://stackoverflow.com/questions/10303248#29927459
if sys.hexversion < 0x03000000:
wrapper_ast_args = ast.arguments(
args=[ast.Name(id=a, ctx=ast.Param(), lineno=1, col_offset=0) for a in spec.args],
vararg=spec.varargs,
kwarg=spec.keywords,
defaults=[]
)
else:
wrapper_ast_args = ast.arguments(
args=[ast.arg(arg=a, annotation=None, lineno=1, col_offset=0) for a in spec.args],
vararg=None if spec.varargs is None else ast.arg(arg=spec.varargs, annotation=None, lineno=1, col_offset=0),
kwonlyargs=[],
kw_defaults=[],
kwarg=None if spec.keywords is None else ast.arg(arg=spec.keywords, annotation=None, lineno=1, col_offset=0),
defaults=[]
)
wrapper_ast = ast.Module(body=[ast.FunctionDef(
name=name,
args=wrapper_ast_args,
body=[ast.Return(value=ast.Call(
func=ast.Name(id="wrapped", ctx=ast.Load(), lineno=1, col_offset=0),
args=[ast.Name(id=a, ctx=ast.Load(), lineno=1, col_offset=0) for a in spec.args],
keywords=[],
starargs=ast.Call(
func=ast.Name(id="flatten", ctx=ast.Load(), lineno=1, col_offset=0),
args=[ast.Name(id=spec.varargs, ctx=ast.Load(), lineno=1, col_offset=0)],
keywords=[], starargs=None, kwargs=None, lineno=1, col_offset=0
),
kwargs=None if spec.keywords is None else ast.Name(id=spec.keywords, ctx=ast.Load(), lineno=1, col_offset=0),
lineno=1, col_offset=0
), lineno=1, col_offset=0)],
decorator_list=[],
lineno=1,
col_offset=0
)])
wrapper_code = [c for c in compile(wrapper_ast, "<ast_in_variadic_py>", "exec").co_consts if isinstance(c, types.CodeType)][0]
wrapper = types.FunctionType(wrapper_code, {"wrapped": wrapped, "flatten": flatten}, argdefs=spec.defaults)
functools.update_wrapper(wrapper, wrapped)
if wrapper.__doc__ is not None:
wrapper.__doc__ = "Note that this function is variadic. See :ref:`variadic-functions`.\n\n" + wrapper.__doc__
return wrapper
def compile_expr(self, target_type):
if target_type is None:
exprtype = None
else:
assert hasattr(target_type, 'paramtypes')
assert hasattr(target_type, 'exprtype')
exprtype = target_type.exprtype
if _chill_util.python_version_major == 2:
return _pyast.Lambda(
_pyast.arguments([_pyast.Name(p, _pyast.Param()) for p in self.params], None, None, []),
self.expr.compile_expr(exprtype))
else:
return _pyast.Lambda(
_pyast.arguments([_pyast.arg(p, None) for p in self.params], None, None, [], None, None, [], []),
self.expr.compile_expr(exprtype))
def CmpOp_In(self, left, comparator):
self.visit(ast_call(
ast.FunctionDef(
'',
ast.arguments([ast_load('l'), ast_load('c')], None, None, []),
[
ast.Return(
ast.IfExp(
ast_call(
ast_load('Array.isArray'),
ast_load('c'),
),
ast.Compare(
ast_call(
ast_load('Array.prototype.indexOf.call'),
ast_load('c'),
ast_load('l'),
),
[ast.Gt()],
[ast.Num(-1)]
),
ast_call(
ast_load('JS'),
ast.Str("l in c"),
)
)
)
],
[]
),
left,
comparator
))
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
func_def = ast.FunctionDef(
name,
ast.arguments(
[ast.Name(n, ast.Param()) for n in arg_names],
None, None,
[ex_literal(None) for _ in arg_names],
),
statements,
[],
)
mod = ast.Module([func_def])
ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec')
# Debug: show bytecode.
if debug:
dis.dis(prog)
for const in prog.co_consts:
if isinstance(const, types.CodeType):
dis.dis(const)
the_locals = {}
exec prog in {}, the_locals
return the_locals[name]
def build_Assign_Destructuring(self, targets, value):
scope = self.stack[-1].scope
global_scope = scope.get_global_scope()
assignments = []
for target, i in zip(targets.elts, range(len(targets.elts))):
if isinstance(target, _ast.Name):
if scope.is_global(target.id):
global_scope.declare(target.id)
else:
scope.declare(target.id)
target = ast.Name(target.id, ast.Load())
assignments.append(
ast.Assign(
[target],
ast.Subscript(
ast_load('v'),
ast.Index(ast.Num(i)),
ast.Load()
)
)
)
return ast_call(
ast.FunctionDef(
'',
ast.arguments([ast_store('v')], None, None, []),
assignments + [
ast.Return(ast_load('v'))
],
[]
),
value
)
def test_lambda(self):
a = ast.arguments([], None, None, [], None, None, [], [])
self.expr(ast.Lambda(a, ast.Name("x", ast.Store())),
"must have Load context")
def fac(args):
return ast.Lambda(args, ast.Name("x", ast.Load()))
self._check_arguments(fac, self.expr)
def make_rtstate_func(self, name, lineno=None):
body = [ast.Assign([ast.Name('config', ast.Store())],
ast.Attribute(ast.Name('rtstate', ast.Load()),
'config', ast.Load()))]
funcargs = ast.arguments([ast.Name('rtstate', ast.Param())], None,
None, [])
return ast.FunctionDef(name, funcargs, body, [], lineno=lineno)
def visit_Function(self, node, fstate):
name = self.visit(node.name, fstate)
defaults = ast.List([self.visit(x, fstate) for x in node.defaults],
ast.Load())
arg_names = ast.Tuple([ast.Str(x.name) for x in node.args], ast.Load())
buffer_name = self.ident_manager.temporary()
func_fstate = fstate.derive()
func_fstate.analyze_identfiers(node.args)
func_fstate.analyze_identfiers(node.body)
func_fstate.buffer = buffer_name
internal_name = fstate.ident_manager.temporary()
body = [ast.Assign([ast.Name(buffer_name, ast.Store())],
ast.List([], ast.Load()))]
body.extend(self.visit_block(node.body, func_fstate))
funcargs = ast.arguments([self.visit(x, func_fstate)
for x in node.args], None, None, [])
self.inject_scope_code(func_fstate, body)
body.append(ast.Return(self.make_call('rtstate.info.concat_template_data',
[ast.Name(buffer_name, ast.Load())])))
# XXX: because inner_functions are prepended, the config alias is not
# yet set up so we have to use rtstate.config. Is that bad? I mean,
# it's certainly not beautiful, but not sure what a beautiful solution
# would look like.
fstate.inner_functions.append((
ast.FunctionDef(internal_name, funcargs, body, [],
lineno=node.lineno),
ast.Assign([ast.Name(internal_name, ast.Store())],
self.make_call('rtstate.config.wrap_function',
[name, ast.Name(internal_name, ast.Load()),
arg_names, defaults], lineno=node.lineno))
))
return ast.Name(internal_name, ast.Load())
def make_lambda(expression, args, values):
def make_arg(name):
if sys.version_info >= (3, 0):
return ast.arg(arg=name, annotation=None)
else:
return ast.Name(id=name, ctx=ast.Param(), lineno=1, col_offset=0)
lambda_ = ast.Lambda(
args=ast.arguments(
args=[make_arg(arg) for arg in args + values],
varargs=None,
varargannotation=None,
kwonlyargs=[],
kwarg=None,
kwargannotation=None,
defaults=[ast.Num(i) for i in range(len(values))],
kw_defaults=[]),
body=expression.body,
)
lambda_ = ast.copy_location(lambda_, expression.body)
exp = ast.Expression(body=lambda_, lineno=1, col_offset=0)
ast.dump(exp)
ast.fix_missing_locations(exp)
GLOBALS = __GLOBALS.copy()
GLOBALS["__builtins__"] = {}
return eval(compile(exp, "<lambda>", "eval"), GLOBALS)
def compile_func(arg_names, statements, name='_the_func', debug=False):
"""Compile a list of statements as the body of a function and return
the resulting Python function. If `debug`, then print out the
bytecode of the compiled function.
"""
arguments = ast.arguments(
args=[_to_arg(n) for n in arg_names],
defaults=[ex_literal(None) for _ in arg_names],
kwonlyargs=[],
kw_defaults=[]
)
func_def = ast.FunctionDef(
name=name,
args=arguments,
body=statements,
decorator_list=[]
)
mod = ast.Module([func_def])
ast.fix_missing_locations(mod)
prog = compile(mod, '<generated>', 'exec')
# Debug: show bytecode.
if debug:
dis.dis(prog)
for const in prog.co_consts:
if isinstance(const, types.CodeType):
dis.dis(const)
the_locals = {}
exec(prog, {}, the_locals)
return the_locals[name]
def visit_GeneratorExp(self, node):
# first build the function that yields the elements.
# we don't to any depth traversal here to attempt to
# simply expressions. We do this later on the
# function.
yieldstmt = ast.Expr()
yieldstmt.value = ast.Yield()
yieldstmt.value.value = node.elt
body = [yieldstmt]
for generator in reversed(node.generators):
loop = self.build_comprehension(generator, body)
body = [loop]
func = ast.FunctionDef()
funcname = self.id_factory("generator")
func.name = funcname
args = ast.arguments()
args.args = []
args.vararg = None
args.kwarg = None
args.defaults = []
func.args = args
func.decorator_list = []
func.body = body
# Now simplify all expressions inside the generator.
# We can do this now as we have a proper function
statements = self.visit(func)
self.execute(statements)
return self.visit(mk_call(funcname))
def getRTL():
from ast import FunctionDef, For, Print, Name, Load, Store, arguments
functions = []
# imprimir(...)
# Recibe cero o mas expresiones separadas por coma y las
# imprme en pantalla separadas por un espacio
#
# Codigo Python
# def imprimir(*args):
# for arg in args:
# print arg,
# print
#
functions.append(
FunctionDef(
name="imprimir",
args=arguments(args=[], vararg="args", kwarg=None, defaults=[]),
body=[
For(
target=Name(id="arg", ctx=Store()),
iter=Name(id="args", ctx=Load()),
body=[Print(dest=None, values=[Name(id="arg", ctx=Load())], nl=False)],
orelse=[],
),
Print(dest=None, values=[], nl=True),
],
decorator_list=[],
)
)
return functions
def test_one_arg_lambda(self):
"""
This method tests the squaring lambda function, a one argument lambda function.
"""
square_lambda_node = ast.Lambda(args=ast.arguments([SymbolRef("x")],
None, None, None),
body=Mul(SymbolRef("x"),
SymbolRef("x")))
# simulating __call__()
type_inferred_node = self.mini__call__(square_lambda_node)
self.assertEqual(str(type_inferred_node), "float apply(float x) {\n" + \
" return x * x;\n}")
def p_parameters(p):
'''parameters : "(" ")"
| "(" varargslist ")"'''
if len(p) == 3:
p[0] = ast.arguments([],
None,
None, [],
lineno=p.get_item(1).lineno,
col_offset=p.get_item(1).lexpos)
else:
p[0] = p[2]
return
def get_funcdef(value):
return astor.to_source(
ast.Module(body=[
ast.FunctionDef(name='get_value',
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[ast.Return(value=value)],
decorator_list=[],
returns=None)
]))
def create_dict(self, resource):
"""Create a subclass from dict
class FieldContainer(typing.Dict[K, V]):
pass
"""
value_type = resource.properties[0].type
if value_type.name in BUILTIN_TYPES:
value_node = BUILTIN_TYPES[value_type.name]
else:
value_node = ast.Str(s=value_type.name, kind=None)
bases = [
ast.Subscript(
value=ast.Name(id="typing.Dict"),
slice=ast.Index(value=ast.Tuple(elts=[ast.Name(id="str"), value_node])),
)
]
class_node = ast.ClassDef(
name=resource.name, bases=bases, keywords=[], decorator_list=[], body=[]
)
node = ast.FunctionDef(
name="__repr__",
args=ast.arguments(
args=[ast.arg(arg="self", annotation=None)],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
),
body=[],
decorator_list=[],
returns=ast.Name(id="str"),
)
# Cheating a bit here, but using ast nodes for this results in a lot of
# code.
repr_statement = ast.Return(
value=ast.Name(
id="'"
+ resource.name
+ "(%s)' % (', '.join(f'{k}={v!r}' for k, v in self.items()))"
)
)
node.body.append(repr_statement)
class_node.body.append(node)
return class_node
def visit_ListComp(self, t):
t = self.generic_visit(t)
add_element = ast.Attribute(ast.Name('.elements', load), 'append',
load)
body = ast.Expr(Call(add_element, [t.elt]))
for loop in reversed(t.generators):
for test in reversed(loop.ifs):
body = ast.If(test, [body], [])
body = ast.For(loop.target, loop.iter, [body], [])
fn = [body, ast.Return(ast.Name('.elements', load))]
args = ast.arguments([ast.arg('.elements', None)], None, [], None, [],
[])
result = Call(Function('<listcomp>', args, fn), [ast.List([], load)])
return ast.copy_location(result, t)
def __call__(self, *arguments, **kwargs):
varargs = [a for a in arguments if isinstance(a, Args)]
kwargs = list(kwargs.values())
arguments = [a for a in arguments if not isinstance(a, (Args, dict))]
return FunctionSignature(
self.name,
ast.arguments(
args=list(arguments),
vararg=varargs[0] if varargs else None,
kwonlyargs=[],
kw_defaults=[],
kwarg=kwargs[0] if kwargs else None,
defaults=[],
))
def compile_predicate(
sexp: List,
funcname: str = None,
force_func: bool = False,
force_lambda: bool = False) -> Union[FunctionType, LambdaType]:
"""
Compiles s-expression into predicate function.
S-expression is validated by the grammar (`predicate.yml`) and,
if ill-formed, `None` is returned.
By default, s-expressions with one or zero arguments are compiled
to lambdas and everything else to proper Python function. This is
in order to allow passing arguments via kwargs instead of regular
args. Although argument names are always read in a DFS manner,
with complex s-expressions, keeping track of the argument order is
very cumbersome. Compilation to either lambda or function can be
forced with `force_lambda` and `force_function`, respectively.
If compiling to function, `compile_predicate` can be provided with
the name. If `funcname` is null, random UUID is generated.
"""
if not grammar.matches(sexp):
return None
argnames = read_argnames(sexp)
exp, env = compile_sexpr(sexp, closure={})
funcname = funcname or '_%s' % uuid.uuid4().hex
arglist = [ast.arg(arg=name, annotation=None) for name in argnames]
args = ast.arguments(
args=arglist,
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
)
if force_func or not force_lambda and len(argnames) > 1:
define = define_func
else:
define = define_lambda
func_def = define(funcname, args, exp)
func_def = ast.fix_missing_locations(func_def)
module = ast.Module(body=[func_def])
exec(compile(module, '<string>', mode='exec'), env)
return env[funcname]
def visit_arguments(self, node):
ret = []
for arg_node in node.args:
if isinstance(arg_node, ast.arg):
new_node = ast.Name(ctx=ast.Param(), id=arg_node.arg)
ret.append(new_node)
elif isinstance(arg_node, ast.Name):
ret.append(arg_node)
else:
raise TypeError('Cannot transform node %r' % arg_node)
return ast.arguments(args=ret,
defaults=node.defaults,
kwarg=node.kwarg,
vararg=node.vararg)
def arguments(args=None, vararg=None,
kwonlyargs=None, kwarg=None,
defaults=None, kw_defaults=None):
if args is None:
args = []
if kwonlyargs is None:
kwonlyargs = []
if defaults is None:
defaults = []
if kw_defaults is None:
kw_defaults = []
args = ast.arguments(args, vararg, kwonlyargs, kw_defaults,
kwarg, defaults)
return fac(args)
def visit_arguments(self, node):
extra_args = [self._make_arg(node.vararg),
[self._make_arg(n) for n in node.kwonlyargs],
self._visit(node.kw_defaults),
self._make_arg(node.kwarg),
self._visit(node.defaults), ]
if sys.version_info.minor >= 8:
extra_args.insert(0, [self._make_arg(arg) for arg in
node.posonlyargs])
new_node = ast.arguments(
[self._make_arg(n) for n in node.args],
*extra_args
)
return new_node
def _get_ast_arguments_for_pdb_function(pdb_function):
args = []
defaults = []
pdb_params, has_run_mode_param = get_pdb_params_with_fixed_run_mode(
pdb_function.params)
if has_run_mode_param:
defaults.append(ast.Name(id=_DEFAULT_RUN_MODE_NAME))
for pdb_param in pdb_params:
args.append(pdb_param.name)
return ast.arguments(args=args, vararg=None, kwarg=None, defaults=defaults)
def visit_Module(self, node):
self.generic_visit(node)
var_alias = ast.alias("Var", None)
runtime_alias = ast.alias("Runtime", None)
importStmt = ast.ImportFrom("terpret_run_runtime",
[var_alias, runtime_alias], 0)
fun_name = "__generated_%s" % str(uuid.uuid4()).replace('-', '')
arguments = ast.arguments(
[ast.Name(self.__runtime_arg, ast.Param())], None, None, [])
fun_def = ast.FunctionDef(name=fun_name,
args=arguments,
body=[importStmt] + node.body,
decorator_list=[])
return (fun_name, ast.Module([fun_def]))
def _gf(body):
# args: m, message, c, client, _ueri
func = ast.AsyncFunctionDef(
'ex',
ast.arguments([], [
ast.arg(i, None, None)
for i in ['m', 'message', 'c', 'client', '_ueri']
], None, [], [], None, []), body, [], None, None)
ast.fix_missing_locations(func)
mod = ast.parse('')
mod.body = [func]
fl = locals().copy()
exec(compile(mod, '<ast>', 'exec'), globals(), fl)
return fl['ex']
def __init__(self, **kwargs):
self.argument_effects = kwargs.get('argument_effects',
(UpdateEffect(), ) * 11)
self.global_effects = kwargs.get('global_effects', False)
self.return_alias = kwargs.get('return_alias',
lambda x: {UnboundValue})
self.return_type = kwargs.get('return_type', None)
self.args = ast.arguments(
[ast.Name(n, ast.Param()) for n in kwargs.get('args', [])], None,
None, map(to_ast, kwargs.get('defaults', [])))
self.return_range = kwargs.get("return_range",
lambda call: UNKNOWN_RANGE)
self.return_range_content = kwargs.get("return_ange_content",
lambda c: UNKNOWN_RANGE)
def __init__(self, function):
expression = function(None)
if getattr(expression, "copy", None):
expression = expression.copy()
func_code = getattr(function, "__code__", None) or function.func_code
self.file = {
"lineno": func_code.co_firstlineno,
"filename": func_code.co_filename,
"col_offset": 0,
"ctx": Load(),
}
self.file_store = dict(self.file, ctx=Store())
self.expression = expression
body = self.build_expression(expression)
if isinstance(body, list):
pass
elif not isinstance(body, stmt):
body = [Return(value=body, **self.file)]
else:
body = [body]
self.ast = Module(
body=[
FunctionDef(
name=func_code.co_name,
args=arguments(
args=[arg(arg="self", annotation=None)
], # function.func_code.co_varnames
defaults=[],
vararg=None,
kwarg=None,
kwonlyargs=[],
kw_defaults=[],
posonlyargs=[],
),
kwarg=[],
kw_defaults=[],
vararg=[],
kwonlyargs=[],
body=body,
decorator_list=[],
),
],
type_ignores=[],
**self.file,
)
fix_missing_locations(self.ast)
self.code = compile(self.ast,
mode="exec",
filename=self.file["filename"])
def functionalize(node):
# changes node in-place, but returns it anyway
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
for i, x in enumerate(node.args):
if (node.func.id, i) in expectFunction:
numargs = highestDollar(x)
if numargs > 0:
# the parameter name "$args" can't conflict with any valid Python names
out = ast.Lambda(ast.arguments([ast.Name("$args", ast.Param())], None, None, []), dollarToArg(x))
out.lineno, out.col_offset = x.lineno, x.col_offset
out.args.lineno, out.args.col_offset = x.lineno, x.col_offset
out.args.args[0].lineno, out.args.args[0].col_offset = x.lineno, x.col_offset
out.args.args[0].ctx.lineno, out.args.args[0].ctx.col_offset = x.lineno, x.col_offset
node.args[i] = out
for keyword in node.keywords:
if (node.func.id, keyword.arg) in expectFunction:
x = keyword.value
numargs = highestDollar(x)
if numargs > 0:
out = ast.Lambda(ast.arguments([ast.Name("$args", ast.Param())], None, None, []), dollarToArg(x))
out.lineno, out.col_offset = x.lineno, x.col_offset
out.args.lineno, out.args.col_offset = x.lineno, x.col_offset
out.args.args[0].lineno, out.args.args[0].col_offset = x.lineno, x.col_offset
out.args.args[0].ctx.lineno, out.args.args[0].ctx.col_offset = x.lineno, x.col_offset
keyword.value = out
for field in node._fields:
subnode = getattr(node, field)
if isinstance(subnode, ast.AST):
functionalize(subnode)
elif isinstance(subnode, list):
for x in subnode:
if isinstance(x, ast.AST):
functionalize(x)
return node
def __call__(self, selection):
if not self.is_initialized:
self._initialize()
try:
parse_result = self.expression.parseString(selection, parseAll=True)
except ParseException as e:
msg = str(e)
lines = ["%s: %s" % (msg, selection),
" " * (12 + len("%s: " % msg) + e.loc) + "^^^"]
raise ValueError('\n'.join(lines))
# Change __ATOM__ in function bodies. It must bind to the arg
# name specified below (i.e. 'atom')
astnode = self.transformer.visit(deepcopy(parse_result[0].ast()))
# Special check for a single literal
if isinstance(astnode, ast.Num) or isinstance(astnode, ast.Str):
raise ValueError("Cannot use a single literal as a boolean.")
if PY2:
args = [ast.Name(id='atom', ctx=ast.Param())]
signature = ast.arguments(args=args, vararg=None, kwarg=None,
defaults=[])
else:
args = [ast.arg(arg='atom', annotation=None)]
signature = ast.arguments(args=args, vararg=None, kwarg=None,
kwonlyargs=[], defaults=[],
kw_defaults=[])
func = ast.Expression(body=ast.Lambda(signature, astnode))
source = codegen.to_source(astnode)
expr = eval(
compile(ast.fix_missing_locations(func), '<string>', mode='eval'),
SELECTION_GLOBALS)
return _ParsedSelection(expr, source, astnode)
def visit_ListComp(self, node):
if node in self.optimizable_comprehension:
self.generic_visit(node)
iterList = []
varList = []
for gen in node.generators:
iterList.append(self.make_Iterator(gen))
varList.append(ast.Name(gen.target.id, ast.Param()))
# If dim = 1, product is useless
if len(iterList) == 1:
iterAST = iterList[0]
varAST = ast.arguments([varList[0]], None, None, [])
else:
prodName = ast.Attribute(
value=ast.Name(id='itertools', ctx=ast.Load()),
attr='product', ctx=ast.Load())
iterAST = ast.Call(prodName, iterList, [], None, None)
varAST = ast.arguments([ast.Tuple(varList, ast.Store())],
None, None, [])
mapName = ast.Attribute(
value=ast.Name(id='__builtin__', ctx=ast.Load()),
attr='map', ctx=ast.Load())
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return ast.Call(mapName, [ldmap, iterAST], [], None, None)
else:
return self.generic_visit(node)
def visit_While(self, node):
self.generic_visit(node)
tFun_name = self.freshName("cond")
bFun_name = self.freshName("body")
tFun = ast.FunctionDef(name=tFun_name,
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[]),
body=[ast.Return(node.test)],
decorator_list=[])
bFun = ast.FunctionDef(name=bFun_name,
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[]),
body=node.body,
decorator_list=[])
ast.fix_missing_locations(tFun)
ast.fix_missing_locations(bFun)
new_node = ast.Expr(
ast.Call(func=ast.Name(id='_while', ctx=ast.Load()),
args=[
ast.Name(id=tFun_name, ctx=ast.Load()),
ast.Name(id=bFun_name, ctx=ast.Load()),
],
keywords=[]))
ast.fix_missing_locations(new_node)
mod = [tFun, bFun, new_node]
return mod
def visit_GeneratorExp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [
ast.Name(gen.target.id, ast.Param()) for gen in node.generators
]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], None, None, [])
else:
prodName = ast.Attribute(value=ast.Name(id='itertools',
ctx=ast.Load()),
attr='product',
ctx=ast.Load())
iterAST = ast.Call(prodName, iters, [], None, None)
varAST = ast.arguments([ast.Tuple(variables, ast.Store())],
None, None, [])
imapName = ast.Attribute(value=ast.Name(id='itertools',
ctx=ast.Load()),
attr='imap',
ctx=ast.Load())
ldBodyimap = node.elt
ldimap = ast.Lambda(varAST, ldBodyimap)
return ast.Call(imapName, [ldimap, iterAST], [], None, None)
else:
return self.generic_visit(node)
def generateFermiSplitFunction(funcName, tanhModuleName="math"):
marginArgName = "margin"
lesArgName = "les"
lesArg = ast.Name(id=lesArgName)
greatArgName = "gret"
valueArgName = "val"
yield ast.ImportFrom(module=tanhModuleName,
names=[ast.alias(name=tanhName, asname=None)],
level=0)
yield ast.Assign(targets=[globalInvTemp], value=astNum(n=100))
yield ast.FunctionDef(
name=funcName,
args=ast.arguments(args=[
ast.arg(arg=valueArgName, annotation=floatType),
ast.arg(arg=marginArgName, annotation=floatType),
ast.arg(arg=lesArgName, annotation=floatType),
ast.arg(arg=greatArgName, annotation=floatType),
ast.arg(arg=inverseTemperatureArgName, annotation=floatType)
],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[
ast.Return(value=ast.BinOp(
left=lesArg,
op=ast.Add(),
right=ast.BinOp(left=ast.BinOp(
left=astNum(n=0.5),
op=ast.Mult(),
right=ast.BinOp(left=tanhCall(
ast.BinOp(left=ast.BinOp(left=ast.Name(
id=valueArgName),
op=ast.Sub(),
right=ast.Name(
id=marginArgName)),
op=ast.Mult(),
right=globalInvTemp)),
op=ast.Add(),
right=astNum(n=1))),
op=ast.Mult(),
right=ast.BinOp(left=ast.Name(id=greatArgName),
op=ast.Sub(),
right=lesArg))))
],
decorator_list=[],
returns=floatType)
def visit_ListComp(self, node):
"""Generate a curried lambda function
[x + y for x, y in [[1, 4], [2, 5], [3, 6]]]
becomes
[[1, 4], [2, 5], [3, 6]]].map(([x, y]) => x + y)
"""
try:
generator, = node.generators
except ValueError:
raise NotImplementedError(
'Only single loop comprehensions are allowed'
)
names = find_names(generator.target)
argslist = [ast.arg(arg=name.id, annotation=None) for name in names]
if len(names) <= 1:
signature = ast.arguments(
args=argslist,
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
)
else:
signature = ast.List(elts=argslist, ctx=ast.Load())
array = generator.iter
lam_sig = functools.partial(ast.Lambda, args=signature)
filters = generator.ifs
if filters:
filt = ast.BoolOp(op=ast.And(), values=filters)
# array.filter
method = ast.Attribute(value=array, attr='filter', ctx=ast.Load())
# array.filter(func)
array = ast.Call(
func=method, args=[lam_sig(body=filt)], keywords=[]
)
method = ast.Attribute(value=array, attr='map', ctx=ast.Load())
mapped = ast.Call(
func=method, args=[lam_sig(body=node.elt)], keywords=[]
)
result = self.visit(mapped)
return result
def visit_AnyComp(self, node, comp_type, comp_module, comp_method):
node.elt = self.visit(node.elt)
name = "{0}_comprehension{1}".format(comp_type, self.count)
self.count += 1
args = self.passmanager.gather(ImportedIds, node, self.ctx)
self.count_iter = 0
starget = "__target"
body = reduce(self.nest_reducer,
reversed(node.generators),
ast.Expr(
ast.Call(
ast.Attribute(
ast.Name(comp_module, ast.Load()),
comp_method,
ast.Load()),
[ast.Name(starget, ast.Load()), node.elt],
[],
None,
None
)
)
)
# add extra metadata to this node
metadata.add(body, metadata.Comprehension(starget))
init = ast.Assign(
[ast.Name(starget, ast.Store())],
ast.Call(
ast.Attribute(
ast.Name('__builtin__', ast.Load()),
comp_type,
ast.Load()
),
[], [], None, None)
)
result = ast.Return(ast.Name(starget, ast.Load()))
sargs = sorted(ast.Name(arg, ast.Param()) for arg in args)
fd = ast.FunctionDef(name,
ast.arguments(sargs, None, None, []),
[init, body, result],
[])
self.ctx.module.body.append(fd)
return ast.Call(
ast.Name(name, ast.Load()),
[ast.Name(arg.id, ast.Load()) for arg in sargs],
[],
None,
None
) # no sharing !
def create_function_def(name, localization, decorators, context, line=0, column=0):
"""
Creates a FunctionDef node, that represent a function declaration. This is used in type inference code, so every
created function has the following parameters (type_of_self, localization, *varargs, **kwargs) for methods and
(localization, *varargs, **kwargs) for functions.
:param name: Name of the function
:param localization: Localization parameter
:param decorators: Decorators of the function, mainly the norecursion one
:param context: Context passed to this method
:param line: Line
:param column: Column
:return: An AST FunctionDef node
"""
function_def_arguments = ast.arguments()
function_def_arguments.args = [localization]
isconstructor = is_constructor(name)
ismethod = is_method(context)
function_def = ast.FunctionDef()
function_def.lineno = line
function_def.col_offset = column
function_def.name = name
function_def.args = function_def_arguments
function_def_arguments.args = []
if isconstructor:
function_def_arguments.args.append(core_language_copy.create_Name('type_of_self'))
if ismethod and not isconstructor:
function_def_arguments.args.append(core_language_copy.create_Name('type_of_self'))
function_def_arguments.args.append(localization)
function_def_arguments.kwarg = "kwargs"
function_def_arguments.vararg = "varargs"
function_def_arguments.defaults = []
if data_structures_copy.is_iterable(decorators):
function_def.decorator_list = decorators
else:
function_def.decorator_list = [decorators]
function_def.body = []
return function_def
def gen_test_method(self):
return ast.FunctionDef(
name='test_requests',
args=ast.arguments(
args=[ast.Name(id='self', ctx=ast.Param())],
defaults=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
returns=None,
),
body=self.gen_test_method_body(),
decorator_list=[],
)
def visit_GeneratorExp(self, node):
body = ast.Expr(value=ast.Yield(node.elt), lineno=None)
for comp_node in reversed(node.generators):
for if_node in reversed(comp_node.ifs):
body = ast.If(test=if_node, body=[body], orelse=[], lineno=None) # pylint: disable=redefined-variable-type
body = ast.For(target=comp_node.target, iter=comp_node.iter,
body=[body], orelse=[], lineno=None)
args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[])
node = ast.FunctionDef(name='<generator>', args=args, body=[body])
gen_func = self.visit_function_inline(node)
result = self.block.alloc_temp()
self.writer.write_checked_call2(
result, '{}.Call(πF, nil, nil)', gen_func.expr)
return result
def create_function_nodes(statement_group, function_meta):
args = ast.arguments(args=[ast.Name(id=variable.name) for variable in function_meta["args"]], vararg=None, kwarg=None,
defaults=[])
function_nodes = {}
function_body = [statement.get_ast() for statement in statement_group]
if function_meta["has_return_statement"]:
name = helper.generate_function_name()
function_nodes[name] = ast.FunctionDef(name=name, args=args, body=function_body, decorator_list=[])
else:
for ret_variable in function_meta["returns"]:
name = helper.generate_function_name()
return_statement = ast.Return(value=ast.Name(id=ret_variable.name))
function_nodes[name] = ast.FunctionDef(name=name, args=args, body=function_body + [return_statement],
decorator_list=[])
return function_nodes
def visit_Module(self, node):
module_body = list()
init_body = list()
for stmt in node.body:
if type(stmt) in ExtractTopLevelStmts.TYPEDEFS:
module_body.append(stmt)
else:
init_body.append(stmt)
init = ast.FunctionDef('__init__',
ast.arguments([], None, None, []),
init_body,
[])
module_body.append(init)
node.body = module_body
return node
def isl2py_fn(node, fn_name):
""" Transform an ISL AST node to a Python AST function"""
return pyast.FunctionDef(
name=fn_name,
args=pyast.arguments(
args=[],
defaults=[],
vararg=None,
kwonlyargs=[],
kwarg=None,
kw_defaults=[],
),
body=isl2py_ast(node),
decorator_list=[],
)
def visit_Subscript(self, node):
from ast import Expr, Call, Name, Load, Lambda, arguments, arg
m = match('E[ _expr | _x in _set]', node)
if m:
x_s = m['_x'].id # name of dummy vaqriable
expr = m['_expr']
sset = m['_set']
res = Call(func=Name(id='Ex', ctx=Load()),
args=[Lambda(args=arguments(args=[arg(arg=x_s, annotation=None)],
vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=expr), sset],
keywords=[], starargs=None, kwargs=None)
return res
m = match('Sum[ _expr | _x in _set]', node)
if m:
x_s = m['_x'].id # name of dummy vaqriable
expr = m['_expr']
sset = m['_set']
res = Call(func=Name(id='Sum', ctx=Load()),
args=[Lambda(args=arguments(args=[arg(arg=x_s, annotation=None)],
vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=expr), sset],
keywords=[], starargs=None, kwargs=None)
return res
return node
