def visit_FunctionDef(self, node):
writer.write('def %s(args, kwargs):' % node.name)
writer.push()
# new pythonjs' python function arguments handling
# create the structure representing the functions arguments
# first create the defaultkwargs JSObject
writer.write('var(signature, arguments)')
L = len(node.args.defaults)
kwargsdefault = map(lambda x: keyword(self.visit(x[0]), x[1]), zip(node.args.args[-L:], node.args.defaults))
kwargsdefault = Call(
Name('JSObject', None),
[],
kwargsdefault,
None,
None
)
args = Call(
Name('JSArray', None),
map(lambda x: Str(x.id), node.args.args),
[],
None,
None
)
keywords = list([
keyword(Name('kwargs', None), kwargsdefault),
keyword(Name('args', None), args),
])
if node.args.vararg:
keywords.append(keyword(Name('vararg', None), Str(node.args.vararg)))
if node.args.kwarg:
keywords.append(keyword(Name('varkwarg', None), Str(node.args.kwarg)))
prebody = list()
# create a JS Object to store the value of each parameter
signature = ', '.join(map(lambda x: '%s=%s' % (self.visit(x.arg), self.visit(x.value)), keywords))
writer.write('signature = JSObject(%s)' % signature)
writer.write('arguments = get_arguments(signature, args, kwargs)')
# # then for each argument assign its value
for arg in node.args.args:
writer.write("""JS("var %s = arguments['%s']")""" % (arg.id, arg.id))
if node.args.vararg:
writer.write("""JS("var %s arguments['%s']")""" % (node.args.vararg, node.args.vararg))
# turn it into a list
expr = '%s = get_attribute(list, "__call__")(create_array(%s), {});'
expr = expr % (node.args.vararg, node.args.vararg)
writer.write(expr)
if node.args.kwarg:
writer.write("""JS('var %s = arguments["%s"]')""" % (node.args.kwarg, node.args.kwarg))
expr = '%s = get_attribute(dict, "__call__")(create_array(%s), {});'
expr = expr % (node.args.kwarg, node.args.kwarg)
writer.write(expr)
map(self.visit, node.body)
writer.pull()
# apply decorators
for decorator in reversed(node.decorator_list):
writer.write('%s = %s(create_array(%s))' % (node.name, self.visit(decorator), node.name))
def visit_Compare (self, node):
self.generic_visit (node)
ans= node
# ls () < "bar.txt" > "foo.txt"
# Compare(left=Call(func=Name(id='ls', ctx=Load()), args=[], keywords=[],
# starargs=None, kwargs=None),
# ops=[Lt(), Gt()],
# comparators=[Str(s='bar.txt'), Str(s='foo.txt')]))
if is_executable (node.left):
# yes, they're reversed, but it makes more sense to me like this
for comp, op in zip (node.ops, node.comparators):
if type (comp)==Gt:
# > means _out
update_keyword (node.left, keyword (arg='_out', value=op))
ans= node.left
elif type (comp)==GtE:
# >= means _out+_err
update_keyword (node.left, keyword (arg='_out', value=op))
update_keyword (node.left, keyword (arg='_err_to_out', value=op))
ans= node.left
elif type (comp)==Lt:
# < means _in
update_keyword (node.left, keyword (arg='_in', value=op))
ans= node.left
return ans
def test_keywords_wildcard(self):
apf = ASTPatternFinder(prepare_pattern("f(e=4, ??=??)"))
it = apf.scan_ast(self.ast)
assert_ast_like(next(it), ast.Call(keywords=[ast.keyword(arg='d'),
ast.keyword(arg='e'),])
)
self.assert_no_more(it)
def test_Call(self):
name = ast.Name('spam', ast.Load())
args = ([ast.Num(42)], '42'), ([], None)
keywords = ([ast.keyword('X', ast.Num(42))], 'X=42'), ([], None)
starargs = (ast.Name('args', ast.Load()), '*args'), (None, None)
kwargs = (ast.Name('kwargs', ast.Load()), '**kwargs'), (None, None)
for arg in args:
for keyword in keywords:
for stararg in starargs:
for kwarg in kwargs:
node = ast.Call(name, arg[0], keyword[0], stararg[0],
kwarg[0])
expect = 'spam({})'.format(','.join(x for x in
(arg[1], keyword[1], stararg[1], kwarg[1])
if x))
self.verify(node, expect)
self.verify(ast.Call(name, [ast.Num(2), ast.Num(3)], [], None, None),
'spam(2,3)')
self.verify(ast.Call(name, [],
[ast.keyword('X', ast.Num(0)),
ast.keyword('Y', ast.Num(1))],
None, None),
'spam(X=0,Y=1)')
# A single genexp doesn't need parentheses.
genexp = self.seq_comp_test(ast.GeneratorExp, '()')
self.verify(ast.Call(name, [genexp], [], None, None),
'spam(w for x in y if 2 for a in b)')
self.verify(ast.Call(name, [genexp, genexp], [], None, None),
'spam((w for x in y if 2 for a in b),'
'(w for x in y if 2 for a in b))')
def visit_Call(self, node, parent):
if node in self.seen:
return node
self.seen.add(node)
callName = dict(ast.iter_fields(dict(ast.iter_fields(node))['func'])).get('id', None)
callType = dict(ast.iter_fields(dict(ast.iter_fields(node))['func'])).get('attr',None)
#print ast.dump(dict(ast.iter_fields(node))['func']), callType, node.lineno, node.col_offset
#print callName, self.localFuncs
if callName in self.localFuncs:
#print ast.dump(node)
#print callName, node.lineno, node.col_offset
dict(ast.iter_fields(node))['keywords'].append(ast.keyword(arg=self.calleeInfo, value=ast.Str(s=str(node.lineno) + "-" + str(node.col_offset))))
if callType in erps.keys() or callType == "stocPrim":
if callType not in self.primsNumArgs:
self.primsNumArgs[callType] = len(inspect.getargspec(globals()[callType]).args)
namedArgs = map(lambda x: dict(ast.iter_fields(x))['arg'], dict(ast.iter_fields(node))['keywords'])
numArgs = len(namedArgs) + len(dict(ast.iter_fields(node))['args'])
#print callType, node.lineno, node.col_offset
#print ast.dump(parent)
if not ('name' in namedArgs or numArgs == self.primsNumArgs[callType]): #check if name already supplied
dict(ast.iter_fields(node))['keywords'].append(ast.keyword(arg='name', value=ast.BinOp(left=ast.BinOp(left=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.funcStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None), op=ast.Add(), right=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.locStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)), op=ast.Add(), right=ast.BinOp(left=ast.Str(s=str(node.lineno) + "-" + str(node.col_offset)), op=ast.Add(), right=ast.Call(func=ast.Name(id='str', ctx=ast.Load()), args=[ast.Name(id=self.loopStack, ctx=ast.Load())], keywords=[], starargs=None, kwargs=None)))))
ast.fix_missing_locations(node)
#print map(ast.dump, dict(ast.iter_fields(node))['keywords'])
self.generic_visit(node)
return node
def visit_BinOp(self, node):
if node.op.__class__ in self.operators:
sympy_class = self.operators[node.op.__class__]
right = self.visit(node.right)
if isinstance(node.op, ast.Sub):
right = ast.UnaryOp(op=ast.USub(), operand=right)
elif isinstance(node.op, ast.Div):
right = ast.Call(
func=ast.Name(id='Pow', ctx=ast.Load()),
args=[right, ast.UnaryOp(op=ast.USub(), operand=ast.Num(1))],
keywords=[ast.keyword(arg='evaluate', value=ast.Name(id='False', ctx=ast.Load()))],
starargs=None,
kwargs=None
)
new_node = ast.Call(
func=ast.Name(id=sympy_class, ctx=ast.Load()),
args=[self.visit(node.left), right],
keywords=[ast.keyword(arg='evaluate', value=ast.Name(id='False', ctx=ast.Load()))],
starargs=None,
kwargs=None
)
if sympy_class in ('Add', 'Mul'):
# Denest Add or Mul as appropriate
new_node.args = self.flatten(new_node.args, sympy_class)
return new_node
return node
def _compile_directive_call_assets(self, el, options):
""" This special 't-call' tag can be used in order to aggregate/minify javascript and css assets"""
if len(el):
raise SyntaxError("t-call-assets cannot contain children nodes")
# self._get_asset(xmlid, options, css=css, js=js, debug=values.get('debug'), async=async, values=values)
return [
self._append(ast.Call(
func=ast.Attribute(
value=ast.Name(id='self', ctx=ast.Load()),
attr='_get_asset',
ctx=ast.Load()
),
args=[
ast.Str(el.get('t-call-assets')),
ast.Name(id='options', ctx=ast.Load()),
],
keywords=[
ast.keyword('css', self._get_attr_bool(el.get('t-css', True))),
ast.keyword('js', self._get_attr_bool(el.get('t-js', True))),
ast.keyword('debug', ast.Call(
func=ast.Attribute(
value=ast.Name(id='values', ctx=ast.Load()),
attr='get',
ctx=ast.Load()
),
args=[ast.Str('debug')],
keywords=[], starargs=None, kwargs=None
)),
ast.keyword('async', self._get_attr_bool(el.get('async', False))),
ast.keyword('values', ast.Name(id='values', ctx=ast.Load())),
],
starargs=None, kwargs=None
))
]
def visit_Compare (self, node):
self.generic_visit (node)
ans= node
# ls () < "bar.txt" > "foo.txt"
# Compare(left=Call(func=Name(id='ls', ctx=Load()), args=[], keywords=[],
# starargs=None, kwargs=None),
# ops=[Lt(), Gt()],
# comparators=[Str(s='bar.txt'), Str(s='foo.txt')]))
if self.is_executable (node.left):
# yes, they're reversed, but it makes more sense to me like this
for comp, op in zip (node.ops, node.comparators):
if type (comp)==Gt:
# > means _out
node.left.keywords.append (keyword (arg='_out', value=op))
ans= node.left
elif type (comp)==Lt:
# < means _in
# now, _in works differently
# a string is written directly to the stdin,
# instead of creating a file() with that name
# so, we do it ourseleves.
if type (op)==Str:
op= Call (func=Name (id='open', ctx=Load ()), args=[op],
keywords=[], starargs=None, kwargs=None)
node.left.keywords.append (keyword (arg='_in', value=op))
ast.fix_missing_locations (node.left)
ans= node.left
return ans
def visit_Call(self, node):
if isinstance(node.func, ast.Name) and isinstance(node.func.ctx, ast.Load):
if node.func.id in map(lambda x: x.value, list(TradingCommands)):
parser = LocationPatcher(node)
node.keywords.append(parser.visit(ast.keyword(arg="lineno", value=ast.Num(n=node.lineno))))
node.keywords.append(parser.visit(ast.keyword(arg="col_offset", value=ast.Num(n=node.col_offset))))
return node
def make_argparse_arguments(self):
for arg, name, help, action in self.macro_handler.args:
expr = ast.parse('parser.add_argument()').body[0]
expr.value.args = [ast.Str(arg)]
expr.value.keywords = [
ast.keyword('dest', ast.Str(name)),
ast.keyword('action', ast.Str(action)),
ast.keyword('help', ast.Str(help)),
]
yield expr
def nameAndArgsToKeyword(name, args):
if name in groupbySet:
arg = 'groupby'
keyword = ast.keyword(arg=arg, value = ast.Dict(keys=args, values=args))
elif name in aggregationSet:
arg = 'aggregate'
keyword = ast.keyword(arg=arg, value = ast.Dict(keys=[ast.Str(name),], values=[ast.List(elts=args)]))
else:
raise NotImplementedError
return keyword
def test_call(self):
func = ast.Name("x", ast.Load())
args = [ast.Name("y", ast.Load())]
keywords = [ast.keyword("w", ast.Name("z", ast.Load()))]
call = ast.Call(ast.Name("x", ast.Store()), args, keywords)
self.expr(call, "must have Load context")
call = ast.Call(func, [None], keywords)
self.expr(call, "None disallowed")
bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))]
call = ast.Call(func, args, bad_keywords)
self.expr(call, "must have Load context")
def test_wildcard_call_mixed_args(self):
pat = prepare_pattern("f(1, ??, a=2, **{'b':3})")
assert isinstance(pat.args, listmiddle)
assert_ast_like(pat.args.front[0], ast.Num(n=1))
assert not hasattr(pat, 'starargs')
assert isinstance(pat.keywords, types.FunctionType)
kwargs_dict = ast.Dict(keys=[ast.Str(s='b')], values=[ast.Num(n=3)])
if sys.version_info < (3, 5):
assert_ast_like(pat.kwargs, kwargs_dict)
else:
pat.keywords([ast.keyword(arg=None, value=kwargs_dict),
ast.keyword(arg='a', value=ast.Num(n=2))], [])
def create_super_call(self, node):
super_call = utils.create_ast('super().{}()'.format(node.name)).body[0]
for arg in node.args.args[1:-len(node.args.defaults) or None]:
super_call.value.args.append(ast.Name(id=arg.arg, ctx=ast.Load()))
for arg, default in zip(node.args.args[-len(node.args.defaults):], node.args.defaults):
super_call.value.keywords.append(ast.keyword(arg=arg.arg, value=default))
for arg, default in zip(node.args.kwonlyargs, node.args.kw_defaults):
super_call.value.keywords.append(ast.keyword(arg=arg.arg, value=default))
if node.args.vararg:
super_call.value.starargs = ast.Name(id=node.args.vararg, ctx=ast.Load())
if node.args.kwarg:
super_call.value.kwargs = ast.Name(id=node.args.kwarg, ctx=ast.Load())
return super_call
def test_pos_final_wildcard(self):
apf = ASTPatternFinder(prepare_pattern("f(1, ??)"))
it = apf.scan_ast(self.ast)
assert_ast_like(next(it), ast.Call(args=[ast.Num(n=1)]))
assert_ast_like(next(it), ast.Call(args=[ast.Num(n=1), ast.Num(n=2)]))
assert_ast_like(next(it), ast.Call(starargs=ast.Name(id='c')))
assert_ast_like(next(it), ast.Call(args=[ast.Num(n=1)],
keywords=[ast.keyword(arg='d'),
ast.keyword(arg='e'),
])
)
assert_ast_like(next(it), ast.Call(kwargs=ast.Name(id='k')))
self.assert_no_more(it)
def visit_BinOp (self, node):
self.generic_visit (node)
# BinOp( left=Call(...), op=BitOr(), right=Call(...))
if type (node.op)==BitOr:
# pipe
# BinOp (left, BitOr, right) -> right (left, ...)
# check the left and right; if they're calls to CommandWrapper._create
# then do the magic
both= True
for child in (node.left, node.right):
both= both and self.is_executable (child)
if both:
# right (left, ...)
# I can't believe it's this easy
node.left.keywords.append (keyword (arg='_out',
value=Name (id='Capture', ctx=Load ())))
ast.fix_missing_locations (node.left)
node.right.args.insert (0, node.left)
node= node.right
# Call(func=Call(func=Attribute(value=Name(id='CommandWrapper', ctx=Load()),
# attr='_create', ctx=Load()),
# args=[Str(s='grep')], keywords=[], starargs=None, kwargs=None),
# args=[Call(func=Call(func=Attribute(value=Name(id='CommandWrapper', ctx=Load()),
# attr='_create', ctx=Load()),
# args=[Str(s='ls')], keywords=[], starargs=None, kwargs=None),
# args=[], keywords=[keyword(arg='_out',
# value=Name(id='Capture', ctx=Load()))],
# starargs=None, kwargs=None),
# Str(s='foo')],
# keywords=[], starargs=None, kwargs=None)
elif type (node.op)==RShift:
# BinOp(left=Call(func=Name(id='ls', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None),
# op=RShift(),
# right=Str(s='foo.txt'))
if self.is_executable (node.left):
node.left.keywords.append (keyword (arg='_out',
value=Call (func=Name (id='open', ctx=Load ()),
args=[node.right, Str (s='ab')], keywords=[], starargs=None, kwargs=None)))
ast.fix_missing_locations (node.left)
node= node.left
return node
def ClassDef(*, name, bases, keywords, starargs, kwargs, body, decorator_list, lineno=None, col_offset=None):
if flags.PY_VERSION == 2:
# Ignore keywords and starargs, they shouldnt be there in the first place if Py2
return ast.ClassDef(name=name, bases=bases, body=body,
decorator_list=decorator_list, lineno=lineno, col_offset=col_offset)
elif flags.PY_VERSION == 3:
if flags.PY3_VERSION >= 5:
if starargs is not None:
bases += [ast.Starred(starargs, ast.Load())]
if kwargs is not None:
keywords += [ast.keyword(arg=None, value=kwargs)]
if lineno is not None and col_offset is not None:
return ast.ClassDef(name=name, bases=bases, keywords=keywords, body=body,
decorator_list=decorator_list, lineno=lineno, col_offset=col_offset)
else:
return ast.ClassDef(name=name, bases=bases, keywords=keywords, body=body,
decorator_list=decorator_list)
else:
if lineno is not None and col_offset is not None:
return ast.ClassDef(name=name, bases=bases, keywords=keywords,
starargs=starargs, kwargs=kwargs, body=body,
decorator_list=decorator_list, lineno=lineno, col_offset=col_offset)
else:
return ast.ClassDef(name=name, bases=bases, keywords=keywords,
starargs=starargs, kwargs=kwargs, body=body,
decorator_list=decorator_list)
def test_classdef(self):
def cls(bases=None, keywords=None, starargs=None, kwargs=None,
body=None, decorator_list=None):
if bases is None:
bases = []
if keywords is None:
keywords = []
if body is None:
body = [ast.Pass()]
if decorator_list is None:
decorator_list = []
return ast.ClassDef("myclass", bases, keywords, starargs,
kwargs, body, decorator_list)
self.stmt(cls(bases=[ast.Name("x", ast.Store())]),
"must have Load context")
self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]),
"must have Load context")
self.stmt(cls(starargs=ast.Name("x", ast.Store())),
"must have Load context")
self.stmt(cls(kwargs=ast.Name("x", ast.Store())),
"must have Load context")
self.stmt(cls(body=[]), "empty body on ClassDef")
self.stmt(cls(body=[None]), "None disallowed")
self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]),
"must have Load context")
def get_call_ast(func_name, args=None, kwargs=None, return_type=None):
""" Return an AST representing the call to a function with the name
func_name, passing it the arguments args (given as a list) and the
keyword arguments kwargs (given as a dictionary).
func_name -- either the name of a callable as a string, or an AST
representing a callable expression
return_type -- if this is not None, return a TypedCall object with this
return type instead """
if args is None:
args = []
# convert keyword argument dict to a list of (key, value) pairs
keywords = []
if kwargs is not None:
for (key, value) in kwargs.iteritems():
keywords.append(ast.keyword(arg=key, value=value))
# get or generate the AST representing the callable
if isinstance(func_name, ast.AST):
func_ast = func_name
else:
func_ast = ast.Name(id=func_name, ctx=ast.Load)
# if no return type is given, return a simple Call AST
if return_type is None:
return ast.Call(func=func_ast, args=args, keywords=keywords,
starargs=None, kwargs=None)
# if a return type is given, return a TypedCall AST
else:
return TypedCall(func=func_ast, args=args, keywords=keywords,
return_type=return_type)
def visit_Call(self, call):
new_func, func_expl = self.visit(call.func)
arg_expls = []
new_args = []
new_kwargs = []
new_star = new_kwarg = None
for arg in call.args:
res, expl = self.visit(arg)
new_args.append(res)
arg_expls.append(expl)
for keyword in call.keywords:
res, expl = self.visit(keyword.value)
new_kwargs.append(ast.keyword(keyword.arg, res))
arg_expls.append(keyword.arg + "=" + expl)
if call.starargs:
new_star, expl = self.visit(call.starargs)
arg_expls.append("*" + expl)
if call.kwargs:
new_kwarg, expl = self.visit(call.kwargs)
arg_expls.append("**" + expl)
expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg)
res = self.assign(new_call)
res_expl = self.explanation_param(self.display(res))
outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
return res, outer_expl
def visit_Call_35(self, call):
"""
visit `ast.Call` nodes on Python3.5 and after
"""
new_func, func_expl = self.visit(call.func)
arg_expls = []
new_args = []
new_kwargs = []
for arg in call.args:
res, expl = self.visit(arg)
arg_expls.append(expl)
new_args.append(res)
for keyword in call.keywords:
res, expl = self.visit(keyword.value)
new_kwargs.append(ast.keyword(keyword.arg, res))
if keyword.arg:
arg_expls.append(keyword.arg + "=" + expl)
else: ## **args have `arg` keywords with an .arg of None
arg_expls.append("**" + expl)
expl = "%s(%s)" % (func_expl, ', '.join(arg_expls))
new_call = ast.Call(new_func, new_args, new_kwargs)
res = self.assign(new_call)
res_expl = self.explanation_param(self.display(res))
outer_expl = "%s\n{%s = %s\n}" % (res_expl, res_expl, expl)
return res, outer_expl
def do_for(parser, token):
'''
{% for a, b, c in iterable %}
{% endfor %}
We create the structure:
for a, b, c in iterable:
with helpers['forwrapper'](context, a=a, b=b, c=c):
...
'''
code = ast.parse('for %s: pass' % token, mode='exec')
loop = code.body[0]
loop.iter = wrap_name_in_context(loop.iter)
body, _ = parser.parse_nodes_until('endfor')
if isinstance(loop.target, ast.Tuple):
targets = [elt.id for elt in loop.target.elts]
else:
targets = [loop.target.id]
kwargs = [
ast.keyword(arg=elt, value=ast.Name(id=elt, ctx=ast.Load()))
for elt in targets
]
loop.body = [_create_with_scope(body, kwargs)]
return loop
def generate(self, element:Element, GC:GenerationContext):
acode = element.code
sig = acode[1].code
mac_name = sig[0].code.full_name
param_names = [p.code.full_name for p in sig[1:]]
body = [s.code for s in acode[2:]]
params_code = ast.List(elts=[ast.Str(s) for s in param_names],
ctx=ast.Load())
# rcode_code =
# __aky__.Macro( ... )
mac_code = ast.Call(func=ast.Attribute(value=ast.Name(id="__aky__", ctx=ast.Load()),
attr="Macro", ctx=ast.Load()),
args=[],
keywords=[ast.keyword(arg="specs",
value=ast.Tuple(elts=[params_code,
rcode_code],
ctx=ast.Load()))])
target_code = ast.Subscript(value=ast.Name(id="__macros__", ctx=ast.Load()),
slice=ast.Index(ast.Str(mac_name)),
ctx=ast.Store())
#
assign_code = ast.Assign(targets=[target_code],
value=mac_code)
return assign_code
def visit_Call(self, node):
if hasattr(node.func, 'id') and node.func.id in ('JS', 'toString', 'JSObject', 'JSArray', 'var'):
return self.generic_visit(node)
return Call(
Call(
Name('get_attribute', None),
[self.visit(node.func), Str('__call__')],
None,
None,
None,
),
[
Call(
Name('JSArray', None),
map(self.visit, node.args),
None,
None,
None
),
Call(
Name('JSObject', None),
None,
map(lambda x: keyword(Name(x.arg, None), self.visit(x.value)), node.keywords),
None,
None
),
],
None,
None,
None,
)
def peval_call(state, ctx, func, args=[], keywords=[]):
assert all(type(arg) != ast.Starred for arg in args)
assert all(kw.arg is not None for kw in keywords)
keyword_expressions = [kw.value for kw in keywords]
state, results = map_peval_expression(
state, dict(func=func, args=args, keywords=keyword_expressions), ctx)
if all_known_values_or_none(results):
values = map_get_value(results)
kwds = {kw.arg: value for kw, value in zip(keywords, values['keywords'])}
success, value = try_eval_call(
values['func'], args=values['args'], keywords=kwds)
if success:
return state, KnownValue(value=value)
state, nodes = map_reify(state, results)
# restoring the keyword list
nodes['keywords'] = [
ast.keyword(arg=kw.arg, value=expr)
for kw, expr in zip(keywords, nodes['keywords'])]
return state, ast.Call(**nodes)
def visit_argument(self, values, ctx):
"""argument: test [comp_for] | test '=' test"""
if len(values) == 1:
node = self.visit(values[0], ctx)
elif len(values) == 2:
test = self.visit(values[0], ctx)
comp_for = self.visit(values[1], ctx)
# TODO
node = ast.GeneratorExp()
node.elt = test
node.generators = [comp_for]
elif len(values) == 3:
node = ast.keyword()
arg = self.visit(values[0], ctx)
node.arg = arg.id
node.value = self.visit(values[2], ctx)
else:
raise ValueError
return node
def visitCall(self, n, *args):
return ast_trans.Call(func=self.dispatch(n.func, *args),
args=self.reduce(n.args, *args),
keywords=[ast.keyword(arg=k.arg, value=self.dispatch(k.value, *args))\
for k in n.keywords],
starargs=self.dispatch(n.starargs, *args) if getattr(n, 'starargs', None) else None,
kwargs=self.dispatch(n.kwargs, *args) if getattr(n, 'kwargs', None) else None)
def test_ClassDef(self):
# class X: pass
cls = ast.ClassDef('X', [], [], None, None, [ast.Pass()], [])
self.verify(cls, 'class X:pass')
# class X(Y): pass
cls = ast.ClassDef('X', [ast.Name('Y', ast.Load())], [], None, None,
[ast.Pass()], [])
self.verify(cls, 'class X(Y):pass')
# class X(Y=42): pass
cls = ast.ClassDef('X', [], [ast.keyword('Y', ast.Num(42))], None,
None, [ast.Pass()], [])
self.verify(cls, 'class X(Y=42):pass')
# class X(Z, Y=42): pass
cls.bases.append(ast.Name('Z', ast.Load()))
self.verify(cls, 'class X(Z,Y=42):pass')
# class X(*args): pass
cls = ast.ClassDef('X', [], [], ast.Name('args', ast.Load()), None,
[ast.Pass()], [])
self.verify(cls, 'class X(*args):pass')
# class X(Y, *args): pass
cls.bases.append(ast.Name('Y', ast.Load()))
self.verify(cls, 'class X(Y,*args):pass')
# class X(**kwargs): pass
cls = ast.ClassDef('X', [], [], None, ast.Name('kwargs', ast.Load()),
[ast.Pass()], [])
self.verify(cls, 'class X(**kwargs):pass')
# class X(Y, **kwargs): pass
cls.bases.append(ast.Name('Y', ast.Load()))
self.verify(cls, 'class X(Y,**kwargs):pass')
# Decorators
cls = ast.ClassDef('X', [], [], None, None, [ast.Pass()],
[ast.Name('dec1', ast.Load()),
ast.Name('dec2', ast.Load())])
self.verify(cls, '@dec1\n@dec2\nclass X:pass')
def visitCall(self, n, *args):
self.dispatch(n.func, *args)
self.dispatch(n.args, *args)
[ast.keyword(arg=k.arg, value=self.dispatch(k.value, *args)) for k in n.keywords]
self.dispatch(n.starargs, *args) if getattr(n, 'starargs', None) else None
self.dispatch(n.kwargs, *args) if getattr(n, 'kwargs', None) else None
if flags.PY3_VERSION <= 4:
args, starargs, keywords, kwargs = n.args, n.starargs, n.keywords, n.kwargs
else:
args = [a for a in n.args if not isinstance(a, ast.Starred)]
starargs = [a.value for a in n.args if isinstance(a, ast.Starred)]
keywords = [k for k in n.keywords if k.arg is not None]
kwargs = [k.value for k in n.keywords if k.arg is None]
if len(starargs) > 1 or len(kwargs) > 1:
raise Exception()
else:
starargs = starargs[0] if len(starargs) else None
kwargs = kwargs[0] if len(kwargs) else None
ty, tyerr = consistency.apply(n.func, n.func.retic_type, args, keywords,
starargs, kwargs)
if not ty:
raise tyerr
else:
n.retic_type = ty
def ast_repr(x):
"""Similar to repr(), but returns an AST instead of a String, which when
evaluated will return the given value."""
if type(x) in (int, float):
return ast.Num(n=x)
elif type(x) in (str, unicode):
return ast.Str(s=x)
elif type(x) is list:
return ast.List(elts=map(ast_repr, x))
elif type(x) is dict:
return ast.Dict(keys=map(ast_repr, x.keys()), values=map(ast_repr, x.values()))
elif type(x) is set:
return ast.Set(elts=map(ast_repr, x))
elif type(x) is Literal:
return x.body
elif type(x) is Captured:
return ast.Call(ast.Name(id="Captured"), [x.val, ast_repr(x.name)], [], None, None)
elif x is None:
return ast.Name(id="None")
elif x is True:
return ast.Name(id="True")
elif x is False:
return ast.Name(id="False")
elif isinstance(x, ast.AST):
fields = [ast.keyword(a, ast_repr(b)) for a, b in ast.iter_fields(x)]
return ast.Call(ast.Name(id=x.__class__.__name__), [], fields, None, None)
raise Exception("Don't know how to ast_repr this: ", x)
def build(self) -> typing.Optional[ast.ClassDef]:
base_class: typing.Union[ast.Attribute, ast.Name]
# Create the base class
if not self.resource.base or self.resource.base.name == "object":
base_class = ast.Attribute(value=ast.Name(id="marshmallow"),
attr="Schema")
else:
if self.resource.base.name in FIELD_TYPES:
return None
base_class = ast.Name(id=self.resource.base.name + "Schema")
# Define the base class
class_node = ast.ClassDef(
name=self.resource.name + "Schema",
bases=[base_class],
keywords=[],
decorator_list=[],
body=[],
)
doc_string = f"Marshmallow schema for :class:`commercetools.types.{self.resource.name}`."
class_node.body.append(ast.Expr(value=ast.Str(s=doc_string)))
# Add the field definitions
for prop in self.resource.properties:
node = self._create_schema_property(prop)
if node:
class_node.body.append(node)
# Add the Meta class
class_node.body.append(
ast.ClassDef(
name="Meta",
bases=[],
keywords=[],
body=[
ast.Assign(
targets=[ast.Name(id="unknown")],
value=ast.Name(id="marshmallow.EXCLUDE"),
)
],
decorator_list=[],
))
# Create the post_load() method
post_load_node = self._create_marshmallow_hook("post_load")
if self.contains_regex_field:
post_load_node.body.append(
self._create_regex_call("_regex", "postprocess"))
post_load_node.body.append(
ast.Return(value=ast.Call(
func=ast.Name(id=f"types.{self.resource.name}"),
args=[],
keywords=[ast.keyword(arg=None, value=ast.Name(id="data"))],
)))
class_node.body.append(post_load_node)
# Create the pre_load() method
if self.contains_regex_field:
node = self._create_marshmallow_hook("pre_load")
node.body.append(self._create_regex_call("_regex", "preprocess"))
node.body.append(ast.Return(value=ast.Name(id="data")))
class_node.body.append(node)
node = self._create_marshmallow_hook("pre_dump")
node.body.append(self._create_regex_call("_regex", "preprocess"))
node.body.append(ast.Return(value=ast.Name(id="data")))
class_node.body.append(node)
node = self._create_marshmallow_hook("post_dump")
node.body.append(self._create_regex_call("_regex", "postprocess"))
node.body.append(ast.Return(value=ast.Name(id="data")))
class_node.body.append(node)
d_field = self.resource.get_discriminator_field()
if d_field:
post_load_node.body.insert(
0,
ast.Delete(targets=[
ast.Subscript(
value=ast.Name(id="data"),
slice=ast.Index(value=ast.Str(
s=d_field.attribute_name)),
)
]),
)
return class_node
def visit_For(self, node: ast.For):
# making sure that we don't have for/else case
if len(node.orelse) > 0:
# this is illegal syntax
lines = [n.lineno for n in node.orelse]
print_src(self.fn_src, lines)
raise SyntaxError("Illegal Syntax: you are not allowed to use "
"for/else in code block")
# get the target
iter_ = node.iter
iter_src = astor.to_source(iter_)
try:
iter_obj = eval(iter_src, self.global_, self.local)
iter_ = list(iter_obj)
except RuntimeError:
print_src(self.fn_src, node.iter.lineno)
raise SyntaxError("Unable to statically evaluate loop iter")
for v in iter_:
if not isinstance(v, (int, str)):
print_src(self.fn_src, node.iter.lineno)
raise SyntaxError("Loop iter has to be either integer or "
"string, got " + str(type(v)))
target = node.target
if not isinstance(target, ast.Name):
print_src(self.fn_src, node.iter.lineno)
raise SyntaxError("Unable to parse loop "
"target " + astor.to_source(target))
# if not in debug mode and we are allowed to do so
if not self.generator.debug and not self.unroll_for and isinstance(iter_obj, range) and \
self.__loop_self_var(target, node.body):
# return it as a function call
_vars = iter_obj.start, iter_obj.stop, iter_obj.step, node.lineno
_vars = [ast.Num(n=n) for n in _vars]
keywords = []
if self.generator.debug:
keywords = [
ast.keyword(arg="f_ln",
value=ast.Constant(value=node.lineno))
]
# we redirect the var to one of the scope vars. the index is based
# on the line number
index_num = node.lineno
# create the for statement in the scope
for_stmt = _kratos.ForStmt(target.id, iter_obj.start,
iter_obj.stop, iter_obj.step)
self.scope.for_stmt[index_num] = for_stmt
# set the for iter var
# redirect the variable to env
self.scope.iter_var[index_num] = for_stmt.get_iter_var()
index = ast.Subscript(slice=ast.Index(value=ast.Num(n=index_num)),
value=ast.Attribute(value=ast.Name(
id="scope", ctx=ast.Load()),
attr="iter_var",
ctx=ast.Load()))
for_node = ast.Call(func=ast.Attribute(value=ast.Name(
id="scope", ctx=ast.Load()),
attr="for_",
ctx=ast.Load()),
args=[ast.Str(s=target.id)] + _vars,
keywords=keywords,
ctx=ast.Load())
body_visitor = StaticElaborationNodeVisitor.NameVisitor(
target, index)
body = []
for i in range(len(node.body)):
n = self.visit(body_visitor.visit(node.body[i]))
if isinstance(n, list):
for nn in n:
body.append(nn)
else:
body.append(n)
body_node = ast.Call(func=ast.Attribute(attr="loop",
value=for_node,
ctx=ast.Load()),
args=body,
keywords=[])
# create an entry for the target
self.local[str(target.id)] = 0
return self.visit(ast.Expr(body_node))
else:
new_node = []
for value in iter_:
loop_body = copy.deepcopy(node.body)
for n in loop_body:
# need to replace all the reference to
visitor = StaticElaborationNodeVisitor.NameVisitor(
target, value)
n = visitor.visit(n)
self.key_pair.append((target.id, value))
n = self.visit(n)
self.key_pair.pop(len(self.key_pair) - 1)
if isinstance(n, list):
for n_ in n:
new_node.append(n_)
self.target_node[n_] = (target.id, value)
else:
new_node.append(n)
self.target_node[n] = (target.id, value)
return new_node
def visit_If(self, node: ast.If):
predicate = node.test
# if it's a var comparison, we change it to eq functional call
predicate = self.__change_if_predicate(predicate)
# we only replace stuff if the predicate has something to do with the
# verilog variable
predicate_src = astor.to_source(predicate)
has_var = False
try:
predicate_value = eval(predicate_src, self.global_, self.local)
except (_kratos.exception.InvalidConversionException,
_kratos.exception.UserException,
_kratos.exception.VarException) as _:
has_var = True
predicate_value = None
# if's a kratos var, we continue
if not has_var and not isinstance(predicate_value, _kratos.Var):
if not isinstance(predicate_value, bool):
print_src(self.fn_src, predicate.lineno)
raise SyntaxError("Cannot statically evaluate if predicate")
if predicate_value:
for i, n in enumerate(node.body):
if_exp = StaticElaborationNodeVisitor(
self.generator, self.fn_src, self.scope,
self.unroll_for, self.local, self.global_,
self.filename, self.scope_ln)
node.body[i] = if_exp.visit(n)
return node.body
else:
for i, n in enumerate(node.orelse):
if_exp = StaticElaborationNodeVisitor(
self.generator, self.fn_src, self.scope,
self.unroll_for, self.local, self.global_,
self.filename, self.scope_ln)
node.orelse[i] = if_exp.visit(n)
return node.orelse
else:
# need to convert the logical operators to either reduced function calls, or
# expression or
if_test = LogicOperatorVisitor(self.local, self.global_,
self.filename, self.scope_ln)
predicate = if_test.visit(node.test)
expression = node.body
else_expression = node.orelse
if self.generator.debug:
keywords_if = [
ast.keyword(arg="f_ln", value=ast.Constant(value=node.lineno))
]
# do our best guess
if len(else_expression) > 0:
if else_expression[0].lineno != expression[0].lineno:
ln = else_expression[0].lineno + 1
else:
ln = else_expression[0].lineno
keywords_else = [
ast.keyword(arg="f_ln", value=ast.Constant(value=ln))
]
else:
keywords_else = []
else:
keywords_if = []
keywords_else = []
for key, value in self.key_pair:
keywords_if.append(ast.keyword(arg=key, value=ast.Str(s=value)))
if_node = ast.Call(func=ast.Attribute(value=ast.Name(id="scope",
ctx=ast.Load()),
attr="if_",
ctx=ast.Load()),
args=[predicate] + expression,
keywords=keywords_if,
ctx=ast.Load)
else_node = ast.Call(func=ast.Attribute(attr="else_",
value=if_node,
ctx=ast.Load()),
args=else_expression,
keywords=keywords_else)
return self.visit(ast.Expr(value=else_node))
def name(tree):
"Splices a string value into the quoted code snippet as a Name."
# TODO: another hard-coded call now assuming `ast.Name`
return Literal(compat.Call(ast.Attribute(
value=ast.Name(id='ast', ctx=ast.Load()),
attr='Name', ctx=ast.Load()), [], [ast.keyword("id", tree)]))
def compile_func(gen: 'Generator', func: Callable, strategy: Strategy, with_hooks: bool = False) -> Callable:
"""
The compilation basically assigns functionality to each of the operator calls as
governed by the semantics (strategy). Memoization is done with the keys as the `func`,
the class of the `strategy` and the `with_hooks` argument.
Args:
gen (Generator): The generator object containing the function to compile
func (Callable): The function to compile
strategy (Strategy): The strategy governing the behavior of the operators
with_hooks (bool): Whether support for hooks is required
Returns:
The compiled function
"""
if isinstance(strategy, PartialReplayStrategy):
strategy = strategy.backup_strategy
if with_hooks:
cache = CompilationCache.WITH_HOOKS[strategy.__class__]
else:
cache = CompilationCache.WITHOUT_HOOKS[strategy.__class__]
if func in cache:
return cache[func]
cache[func] = None
source_code, start_lineno = inspect.getsourcelines(func)
source_code = ''.join(source_code)
f_ast = astutils.parse(textwrap.dedent(source_code))
# This matches up line numbers with original file and is thus super useful for debugging
ast.increment_lineno(f_ast, start_lineno - 1)
# Remove the ``@generator`` decorator to avoid recursive compilation
f_ast.decorator_list = [d for d in f_ast.decorator_list
if (not isinstance(d, ast.Name) or d.id != 'generator') and
(not isinstance(d, ast.Attribute) or d.attr != 'generator') and
(not (isinstance(d, ast.Call) and isinstance(d.func,
ast.Name)) or d.func.id != 'generator')]
# Get all the external dependencies of this function.
# We rely on a modified closure function adopted from the ``inspect`` library.
closure_vars = getclosurevars_recursive(func, f_ast)
g = {**closure_vars.nonlocals.copy(), **closure_vars.globals.copy()}
known_ops: Set[str] = strategy.get_known_ops()
known_methods: Set[str] = strategy.get_known_methods()
op_info_constructor = OpInfoConstructor()
delayed_compilations: List[Tuple[Generator, str]] = []
ops = {}
handlers = {}
op_infos = {}
op_idx: int = 0
composition_cnt: int = 0
for n in astutils.preorder_traversal(f_ast):
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id in known_ops:
# Rename the function call, and assign a new function to be called during execution.
# This new function is determined by the semantics (strategy) being used for compilation.
# Also determine if there any eligible hooks for this operator call.
op_idx += 1
handler_idx = len(handlers)
op_info: OpInfo = op_info_constructor.get(n, gen.name, gen.group)
n.keywords.append(ast.keyword(arg='model', value=ast.Name(_GEN_MODEL_VAR, ctx=ast.Load())))
n.keywords.append(ast.keyword(arg='op_info', value=ast.Name(f"_op_info_{op_idx}", ctx=ast.Load())))
op_infos[f"_op_info_{op_idx}"] = op_info
n.keywords.append(ast.keyword(arg='handler', value=ast.Name(f"_handler_{handler_idx}", ctx=ast.Load())))
handler = strategy.get_op_handler(op_info)
handlers[f"_handler_{handler_idx}"] = handler
if not with_hooks:
n.func = astutils.parse(f"{_GEN_STRATEGY_VAR}.generic_op").value
else:
n.keywords.append(ast.keyword(arg=_GEN_HOOK_VAR, value=ast.Name(_GEN_HOOK_VAR, ctx=ast.Load())))
n.keywords.append(ast.keyword(arg=_GEN_STRATEGY_VAR, value=ast.Name(_GEN_STRATEGY_VAR, ctx=ast.Load())))
n.func.id = _GEN_HOOK_WRAPPER
ops[_GEN_HOOK_WRAPPER] = hook_wrapper
if returns_lambda(handler):
n.func = ast.Call(func=n.func, args=n.args[:], keywords=n.keywords[:])
n.keywords = []
n.args = [n.args[0]]
ast.fix_missing_locations(n)
elif isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id in known_methods:
# Similar in spirit to the known_ops case, just much less fancy stuff to do.
# Only need to get the right handler which we will achieve by simply making this
# a method call instead of a regular call.
n.func = ast.Attribute(value=ast.Name(_GEN_STRATEGY_VAR, ctx=ast.Load()), attr=n.func.id, ctx=ast.Load())
ast.fix_missing_locations(n)
elif isinstance(n, ast.Call):
# Try to check if it is a call to a Generator
# TODO : Can we be more sophisticated in our static analysis here
try:
function = eval(astunparse.unparse(n.func), g)
if isinstance(function, Generator):
call_id = f"{_GEN_COMPOSITION_ID}_{composition_cnt}"
composition_cnt += 1
n.func.id = call_id
n.keywords.append(ast.keyword(arg=_GEN_EXEC_ENV_VAR,
value=ast.Name(_GEN_EXEC_ENV_VAR, ctx=ast.Load())))
n.keywords.append(ast.keyword(arg=_GEN_STRATEGY_VAR,
value=ast.Name(_GEN_STRATEGY_VAR, ctx=ast.Load())))
n.keywords.append(ast.keyword(arg=_GEN_MODEL_VAR,
value=ast.Name(_GEN_MODEL_VAR, ctx=ast.Load())))
n.keywords.append(ast.keyword(arg=_GEN_HOOK_VAR,
value=ast.Name(_GEN_HOOK_VAR, ctx=ast.Load())))
ast.fix_missing_locations(n)
# We delay compilation to handle mutually recursive generators
delayed_compilations.append((function, call_id))
except:
pass
# Add the execution environment argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_EXEC_ENV_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the strategy argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_STRATEGY_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the strategy argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_MODEL_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
# Add the hook argument to the function
f_ast.args.kwonlyargs.append(ast.arg(arg=_GEN_HOOK_VAR, annotation=None))
f_ast.args.kw_defaults.append(ast.NameConstant(value=None))
ast.fix_missing_locations(f_ast)
# New name so it doesn't clash with original
func_name = f"{_GEN_COMPILED_TARGET_ID}_{len(cache)}"
g.update({k: v for k, v in ops.items()})
g.update({k: v for k, v in handlers.items()})
g.update({k: v for k, v in op_infos.items()})
module = ast.Module()
module.body = [f_ast]
# Passing ``g`` to exec allows us to execute all the new functions
# we assigned to every operator call in the previous AST walk
exec(compile(module, filename=inspect.getabsfile(func), mode="exec"), g)
result = g[func.__name__]
if inspect.ismethod(func):
result = result.__get__(func.__self__, func.__self__.__class__)
# Restore the correct namespace so that tracebacks contain actual function names
g[gen.name] = gen
g[func_name] = result
cache[func] = result
# Handle the delayed compilations now that we have populated the cache
for gen, call_id in delayed_compilations:
compiled_func = compile_func(gen, gen.func, strategy, with_hooks)
if gen.caching and isinstance(strategy, DfsStrategy):
# Add instructions for using cached result if any
g[call_id] = cache_wrapper(compiled_func)
else:
g[call_id] = compiled_func
return result
""".format(comment=docstring_header_and_return_str)
config_tbl_ast = Assign(
targets=[Name("config_tbl", Store())],
value=Call(
func=Name("Table", Load()),
args=[
set_value("config_tbl"),
Name("metadata", Load()),
Call(
func=Name("Column", Load()),
args=[set_value("dataset_name"),
Name("String", Load())],
keywords=[
keyword(arg="doc",
value=set_value("name of dataset"),
identifier=None),
keyword(arg="default",
value=set_value("mnist"),
identifier=None),
keyword(arg="primary_key",
value=set_value(True),
identifier=None),
],
expr=None,
expr_func=None,
),
Call(
func=Name("Column", Load()),
args=[set_value("tfds_dir"),
Name("String", Load())],
def _create_schema_field(param: ServiceParameter):
"""Generate a field assignment for a marshmallow schema"""
keywords = []
imports = []
methods = []
code_name = snakeit(param.name)
if code_name != param.name:
keywords.append(
ast.keyword(arg="data_key", value=ast.Str(s=param.name,
kind=None)))
if not param.required:
keywords.append(
ast.keyword(arg="required",
value=ast.Constant(value=False, kind=None)))
if param.name.startswith("/"):
placeholder = param.extra_data["(placeholderParam)"]
code_name = snakeit(placeholder["paramName"])
imports.append(("marshmallow", "fields"))
imports.append(("marshmallow", ))
serialize_func = ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"), attr="Dict"),
args=[],
keywords=[],
)
# TODO: can break if there is a suffix
key_name = placeholder["template"].replace(
"<%s>" % placeholder["placeholder"], "")
code = ast.parse(
textwrap.dedent(
"""
@marshmallow.post_dump
def _%(target_name)s_post_dump(self, data, **kwrags):
values = data.pop('%(target_name)s')
if not values:
return data
for key, val in values.items():
data[f"%(key_name)s{key}"] = val
return data
@marshmallow.pre_load
def _%(target_name)s_post_load(self, data, **kwrags):
items = {}
for key in list(data.keys()):
if key.startswith("%(key_name)s"):
items[key[%(key_len)d:]] = data[key]
del data[key]
data["%(target_name)s"] = items
return data
""" % {
"target_name": code_name,
"key_name": key_name,
"key_len": len(key_name),
}))
methods.extend(code.body)
elif param.type == "string":
imports.append(("commercetools.helpers", "OptionalList"))
imports.append(("marshmallow", "fields"))
serialize_func = ast.Call(
func=ast.Name(id="OptionalList"),
args=[
ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"),
attr="String"),
args=[],
keywords=[],
)
],
keywords=keywords,
)
elif param.type == "number":
imports.append(("marshmallow", "fields"))
serialize_func = ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"), attr="Int"),
args=[],
keywords=keywords,
)
elif param.type == "boolean":
keywords.append(
ast.keyword(arg="missing",
value=ast.Constant(value=False, kind=None)))
imports.append(("marshmallow", "fields"))
serialize_func = ast.Call(
func=ast.Attribute(value=ast.Name(id="fields"), attr="Bool"),
args=[],
keywords=keywords,
)
elif param.type == "file":
return None, []
else:
raise NotImplementedError(param)
node = ast.Assign(targets=[ast.Name(id=code_name)],
value=serialize_func,
simple=1)
return node, methods, imports
def _get_property_field(self, prop):
if prop.type is None:
return ast.Call(
func=ast.Name(id=FIELD_TYPES["object"]),
args=[],
keywords=[
ast.keyword(arg="allow_none", value=ast.NameConstant(True))
],
)
elif prop.type.enum:
return ast.Call(
func=ast.Name(id="marshmallow_enum.EnumField"),
args=[
ast.Attribute(value=ast.Name(id="types"),
attr=prop.type.name)
],
keywords=[
ast.keyword(arg="by_value", value=ast.NameConstant(True))
],
)
elif prop.type.discriminator:
return self._create_discriminator_field(prop.type)
elif prop.type.name.startswith("/"):
return ast.Call(func=ast.Name(id=prop.type.name + "Field"),
args=[],
keywords=[])
elif prop.type.name in FIELD_TYPES:
node = ast.Call(
func=ast.Name(id=FIELD_TYPES[prop.type.name]),
args=[],
keywords=[
ast.keyword(arg="allow_none", value=ast.NameConstant(True))
],
)
if prop.type.name == "array":
assert prop.items, f"The array property {prop.name} has no items"
assert prop.items_types
# TODO: We for now assume that the items are all subclasses of
# the first item. We shouldn't do that :-)
if prop.items_types[0].discriminator:
node.args.append(
self._create_discriminator_field(prop.items_types[0]))
else:
node.args.append(
self._create_nested_field(prop.items_types[0]))
return node
# Dict Field
elif "asMap" in prop.type.annotations:
return ast.Call(
func=ast.Name(id=prop.type.name + "Field"),
args=[],
keywords=[
ast.keyword(arg="allow_none", value=ast.NameConstant(True))
],
)
elif prop.type.base and prop.type.base.name == "string":
return ast.Call(func=ast.Name(id="marshmallow.fields.String"),
args=[],
keywords=[])
else:
return self._create_nested_field(prop.type)
def py_emit(node: ast.Call, ctx: Context):
py_emit(node.func, ctx)
has_star = False
has_key = False
has_star_star = False
for each in node.args:
if isinstance(each, ast.Starred):
has_star = True
break
for each in node.keywords:
if each.arg:
has_key = True
break
for each in node.keywords:
if each.arg is None:
has_star_star = True
break
# positional arguments
if has_star or has_star_star:
arg_count = 0
arg_tuple_count = 0
for each in node.args:
if not isinstance(each, ast.Starred):
py_emit(each, ctx)
arg_count += 1
else:
if arg_count:
ctx.bc.append(
Instr("BUILD_TUPLE", arg_count, lineno=node.lineno))
arg_tuple_count += 1
arg_count = 0
py_emit(each.value, ctx)
arg_tuple_count += 1
if arg_count:
ctx.bc.append(Instr("BUILD_TUPLE", arg_count, lineno=node.lineno))
arg_tuple_count += 1
if arg_tuple_count > 1:
ctx.bc.append(
Instr("BUILD_TUPLE_UNPACK_WITH_CALL",
arg_tuple_count,
lineno=node.lineno))
elif arg_tuple_count == 1:
pass
elif arg_tuple_count == 0:
ctx.bc.append(Instr("BUILD_TUPLE", 0, lineno=node.lineno))
else:
for each in node.args:
py_emit(each, ctx)
# keyword arguments
if has_star or has_star_star:
karg_pack_count = 0
keys = []
values = []
karg_count = 0
# use dummy node handle trailing keyword arguments
dummy_node = ast.keyword(arg=None)
node.keywords.append(dummy_node)
for each in node.keywords:
if each.arg:
keys.append(each.arg)
values.append(each.value)
karg_count += 1
else:
if karg_count:
karg_pack_count += 1
if karg_count > 1:
for value in values:
py_emit(value, ctx)
ctx.bc.append(
Instr("LOAD_CONST",
tuple(keys),
lineno=node.lineno))
ctx.bc.append(
Instr("BUILD_CONST_KEY_MAP",
karg_count,
lineno=node.lineno))
elif karg_count == 1:
ctx.bc.append(
Instr("LOAD_CONST", keys[0], lineno=node.lineno))
py_emit(values[0], ctx)
ctx.bc.append(Instr("BUILD_MAP", 1,
lineno=node.lineno))
keys = []
values = []
karg_count = 0
if each is dummy_node:
break
py_emit(each.value, ctx)
karg_pack_count += 1
node.keywords.pop(-1) # pop dummy node
if karg_pack_count > 1:
ctx.bc.append(
Instr("BUILD_MAP_UNPACK_WITH_CALL",
karg_pack_count,
lineno=node.lineno))
else:
keys = []
for each in node.keywords:
py_emit(each.value, ctx)
keys.append(each.arg)
if keys:
ctx.bc.append(Instr("LOAD_CONST", tuple(keys), lineno=node.lineno))
if has_star or has_star_star:
ctx.bc.append(
Instr("CALL_FUNCTION_EX",
has_star_star | has_key,
lineno=node.lineno))
elif has_key:
ctx.bc.append(
Instr("CALL_FUNCTION_KW",
len(node.args) + len(node.keywords),
lineno=node.lineno))
else:
ctx.bc.append(
Instr('CALL_FUNCTION', len(node.args), lineno=node.lineno))
def _gen_extractors(self, request):
stmts = []
jextractors = request.config.get("extract-jsonpath", BetterDict())
for varname in jextractors:
cfg = ensure_is_dict(jextractors, varname, "jsonpath")
stmts.append(ast.Assign(
targets=[ast.Name(id=varname, ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="response", ctx=ast.Load()),
attr="extract_jsonpath",
ctx=ast.Load()
),
args=[self.gen_expr(cfg['jsonpath']), self.gen_expr(cfg.get('default', 'NOT_FOUND'))],
keywords=[],
starargs=None,
kwargs=None
)
))
extractors = request.config.get("extract-regexp", BetterDict())
for varname in extractors:
cfg = ensure_is_dict(extractors, varname, "regexp")
# TODO: support non-'body' value of 'subject'
stmts.append(ast.Assign(
targets=[ast.Name(id=varname, ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="response", ctx=ast.Load()),
attr="extract_regex",
ctx=ast.Load()
),
args=[self.gen_expr(cfg['regexp']), self.gen_expr(cfg.get('default', 'NOT_FOUND'))],
keywords=[],
starargs=None,
kwargs=None
)
))
# TODO: css/jquery extractor?
xpath_extractors = request.config.get("extract-xpath", BetterDict())
for varname in xpath_extractors:
cfg = ensure_is_dict(xpath_extractors, varname, "xpath")
parser_type = 'html' if cfg.get('use-tolerant-parser', True) else 'xml'
validate = cfg.get('validate-xml', False)
stmts.append(ast.Assign(
targets=[ast.Name(id=varname, ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="response", ctx=ast.Load()),
attr="extract_xpath",
ctx=ast.Load()
),
args=[self.gen_expr(cfg['xpath'])],
keywords=[ast.keyword(arg="default", value=cfg.get('default', 'NOT_FOUND')),
ast.keyword(arg="parser_type", value=parser_type),
ast.keyword(arg="validate", value=validate)],
starargs=None,
kwargs=None,
)
))
return stmts
def gen_request_lines(self, req):
apiritif_http = ast.Attribute(value=ast.Name(id='apiritif', ctx=ast.Load()),
attr='http', ctx=ast.Load())
target = ast.Name(id='target', ctx=ast.Load())
requestor = target if self.__access_method == "target" else apiritif_http
method = req.method.lower()
think_time = dehumanize_time(req.priority_option('think-time', default=None))
named_args = self._extract_named_args(req)
if req.label:
label = req.label
else:
label = req.url
lines = []
tran = ast.Attribute(value=ast.Name(id='apiritif', ctx=ast.Load()),
attr="transaction", ctx=ast.Load())
transaction = ast.With(
context_expr=ast.Call(
func=tran,
args=[self.gen_expr(label)],
keywords=[],
starargs=None,
kwargs=None
),
optional_vars=None,
body=[
ast.Assign(
targets=[
ast.Name(id="response", ctx=ast.Store())
],
value=ast.Call(
func=ast.Attribute(value=requestor, attr=method, ctx=ast.Load()),
args=[self.gen_expr(req.url)],
keywords=[ast.keyword(arg=name, value=self.gen_expr(value))
for name, value in iteritems(named_args)],
starargs=None,
kwargs=None
)
)
],
)
lines.append(transaction)
lines.extend(self._gen_assertions(req))
lines.extend(self._gen_jsonpath_assertions(req))
lines.extend(self._gen_xpath_assertions(req))
lines.extend(self._gen_extractors(req))
if think_time:
lines.append(ast.Expr(ast.Call(func=ast.Attribute(value=ast.Name(id="time", ctx=ast.Load()),
attr="sleep",
ctx=ast.Load()),
args=[self.gen_expr(think_time)],
keywords=[],
starargs=None,
kwargs=None)))
return lines
def test_empty_init(self):
# Jython 2.5.0 did not allow empty constructors for many ast node types
# but CPython ast nodes do allow this. For the moment, I don't see a
# reason to allow construction of the super types (like ast.AST and
# ast.stmt) as well as the op types that are implemented as enums in
# Jython (like boolop), but I've left them in but commented out for
# now. We may need them in the future since CPython allows this, but
# it may fall under implementation detail.
#ast.AST()
ast.Add()
ast.And()
ast.Assert()
ast.Assign()
ast.Attribute()
ast.AugAssign()
ast.AugLoad()
ast.AugStore()
ast.BinOp()
ast.BitAnd()
ast.BitOr()
ast.BitXor()
ast.BoolOp()
ast.Break()
ast.Call()
ast.ClassDef()
ast.Compare()
ast.Continue()
ast.Del()
ast.Delete()
ast.Dict()
ast.Div()
ast.Ellipsis()
ast.Eq()
ast.Exec()
ast.Expr()
ast.Expression()
ast.ExtSlice()
ast.FloorDiv()
ast.For()
ast.FunctionDef()
ast.GeneratorExp()
ast.Global()
ast.Gt()
ast.GtE()
ast.If()
ast.IfExp()
ast.Import()
ast.ImportFrom()
ast.In()
ast.Index()
ast.Interactive()
ast.Invert()
ast.Is()
ast.IsNot()
ast.LShift()
ast.Lambda()
ast.List()
ast.ListComp()
ast.Load()
ast.Lt()
ast.LtE()
ast.Mod()
ast.Module()
ast.Mult()
ast.Name()
ast.Not()
ast.NotEq()
ast.NotIn()
ast.Num()
ast.Or()
ast.Param()
ast.Pass()
ast.Pow()
ast.Print()
ast.RShift()
ast.Raise()
ast.Repr()
ast.Return()
ast.Slice()
ast.Store()
ast.Str()
ast.Sub()
ast.Subscript()
ast.Suite()
ast.TryExcept()
ast.TryFinally()
ast.Tuple()
ast.UAdd()
ast.USub()
ast.UnaryOp()
ast.While()
ast.With()
ast.Yield()
ast.alias()
ast.arguments()
#ast.boolop()
#ast.cmpop()
ast.comprehension()
#ast.excepthandler()
#ast.expr()
#ast.expr_context()
ast.keyword()
async def async_eval(self, code, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globals().copy()
# This code saves __name__ and __package into a kwarg passed to the function.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there's no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, 'exec')
code = root.body
# If we can use it as a lambda return (but multiline)
if isinstance(code[-1], ast.Expr):
# Change it to a return statement
code[-1] = ast.copy_location(ast.Return(code[-1].value), code[-1])
# globals().update(**<global_args>)
glob_copy = ast.Expr(ast.Call(func=ast.Attribute(value=ast.Call(func=ast.Name(id='globals', ctx=ast.Load()),
args=[], keywords=[]),
attr='update', ctx=ast.Load()),
args=[], keywords=[ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))]))
glob_copy.lineno = 0
glob_copy.col_offset = 0
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
a.lineno = 0
a.col_offset = 0
args += [a]
fun = ast.AsyncFunctionDef('tmp', ast.arguments(
args=[],
vararg=None,
kwonlyargs=args,
posonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[None for i in range(len(args))]), code, [], None
)
fun.lineno = 0
fun.col_offset = 0
mod = ast.Module([fun], type_ignores=[])
comp = compile(mod, '<string>', 'exec')
exec(comp, {}, locs)
with self.temp_stdio() as out:
result = await locs["tmp"](**kwargs)
try:
globals().clear()
# Inconsistent state
finally:
globals().update(**globs)
return result, out.getvalue()
def visit_Call(self, node):
new_node = self.generic_visit(node)
if isinstance(node.func, ast.Name) and node.func.id in self.functions:
new_node.keywords.append(ast.keyword(arg='evaluate', value=ast.NameConstant(value=False, ctx=ast.Load())))
return new_node
async def meval(code, globs, **kwargs):
# Note to self: please don't set globals here as they will be lost.
# Don't clutter locals
locs = {}
# Restore globals later
globs = globs.copy()
# This code saves __name__ and __package into a kwarg passed to the func.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there's no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, "exec")
code = root.body
ret_name = "_ret"
ok = False
while True:
if ret_name in globs.keys():
ret_name = "_" + ret_name
continue
for node in ast.walk(root):
if isinstance(node, ast.Name) and node.id == ret_name:
ret_name = "_" + ret_name
break
ok = True
if ok:
break
if not code:
return None
if not any(isinstance(node, ast.Return) for node in code):
for i in range(len(code)):
if isinstance(code[i], ast.Expr):
if (i == len(code) - 1
or not isinstance(code[i].value, ast.Call)):
code[i] = ast.copy_location(
ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(
id=ret_name, ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[code[i].value],
keywords=[])), code[-1])
else:
for node in code:
if isinstance(node, ast.Return):
node.value = ast.List(elts=[node.value], ctx=ast.Load())
code.append(
ast.copy_location(
ast.Return(value=ast.Name(id=ret_name, ctx=ast.Load())), code[-1]))
# globals().update(**<global_args>)
glob_copy = ast.Expr(
ast.Call(
func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals",
ctx=ast.Load()),
args=[],
keywords=[]),
attr="update",
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))
]))
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
ret_decl = ast.Assign(targets=[ast.Name(id=ret_name, ctx=ast.Store())],
value=ast.List(elts=[], ctx=ast.Load()))
ast.fix_missing_locations(ret_decl)
code.insert(1, ret_decl)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
ast.fix_missing_locations(a)
args += [a]
args = ast.arguments(args=[],
vararg=None,
kwonlyargs=args,
kwarg=None,
defaults=[],
kw_defaults=[None for i in range(len(args))])
args.posonlyargs = []
fun = ast.AsyncFunctionDef(name="tmp",
args=args,
body=code,
decorator_list=[],
returns=None)
ast.fix_missing_locations(fun)
mod = ast.parse("")
mod.body = [fun]
comp = compile(mod, "<string>", "exec")
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
for i in range(len(r)):
if hasattr(r[i], "__await__"):
r[i] = await r[i] # workaround for 3.5
i = 0
while i < len(r) - 1:
if r[i] is None:
del r[i]
else:
i += 1
if len(r) == 1:
[r] = r
elif not r:
r = None
return r
def make_update_method(self, method, node, return_obj, module_name):
"""Create the `.update_by_*(self, id, actions, ...)` method"""
response_schema_name = return_obj.name + "Schema"
self.add_import_statement(
module_name,
f"commercetools._schemas.{return_obj.package_name}",
response_schema_name,
)
input_obj = self._types.get(method.input_type)
if not input_obj:
return
action_obj = self._types.get(method.input_type + "Action")
input_schema_name = input_obj.name + "Schema"
self.add_import_statement(
module_name, f"commercetools.types.{input_obj.package_name}",
input_obj.name)
self.add_import_statement(
module_name,
f"commercetools.types.{action_obj.package_name}",
action_obj.name,
)
self.add_import_statement(
module_name,
f"commercetools._schemas.{input_obj.package_name}",
input_schema_name,
)
node.args.args.append(
ast.arg(
arg="actions",
annotation=ast.Subscript(
value=ast.Name(id="typing.List"),
slice=ast.Index(value=ast.Name(id=action_obj.name)),
),
))
node.args.kwonlyargs.append(
ast.arg(arg="force_update", annotation=ast.Name(id="bool")))
node.args.kw_defaults.append(ast.Constant(value=False, kind=None))
# update_action = types.ProductUpdate(version=version, actions=actions)
node.body.append(
ast.Assign(
targets=[ast.Name(id="update_action")],
value=ast.Call(
func=ast.Name(id=input_obj.name),
args=[],
keywords=[
ast.keyword(arg="version",
value=ast.Name(id="version")),
ast.keyword(arg="actions",
value=ast.Name(id="actions")),
],
),
))
node.body.append(
ast.Return(
self._call_client(
method=f"_post",
endpoint=_create_endpoint_fstring(method.path),
params=ast.Name("params"),
data_object=ast.Name("update_action"),
request_schema_cls=ast.Name(id=input_schema_name),
response_schema_cls=ast.Name(id=response_schema_name),
force_update=ast.Name(id="force_update"),
)))
return node
def __build_function(self, dom_name, full_name, func_params):
assert 'name' in func_params
func_name = func_params['name']
docstr = self.__build_desc_string(dom_name, func_name, func_params)
args = [ast.arg('self', None)]
message_params = []
func_body = []
if docstr:
func_body.append(ast.Expr(ast.Str("\n"+docstr+"\n\t\t")))
for param in func_params.get("parameters", []):
argname = param['name']
param_optional = param.get("optional", False)
if param_optional is False:
message_params.append(ast.keyword(argname, ast.Name(id=argname, ctx=ast.Load())))
args.append(ast.arg(argname, None))
if self.do_debug_prints:
func_body.append(self.__build_debug_print(argname, argname))
param_type = param.get("type", None)
if param_type in CHECKS:
if param_optional:
check = self.__build_conditional_arg_check(argname, CHECKS[param_type])
else:
check = self.__build_unconditional_arg_check(argname, CHECKS[param_type])
if check:
func_body.append(check)
optional_params = [param.get("name") for param in func_params.get("parameters", []) if param.get("optional", False)]
func_kwargs = None
if len(optional_params):
value = ast.List(elts=[ast.Str(s=param, ctx=ast.Store()) for param in optional_params], ctx=ast.Load())
create_list = ast.Assign(targets=[ast.Name(id='expected', ctx=ast.Store())], value=value)
func_body.append(create_list)
passed_arg_list = ast.Assign(targets=[ast.Name(id='passed_keys', ctx=ast.Store())],
value=ast.Call(func=ast.Name(id='list', ctx=ast.Load()),
args=[ast.Call(func=ast.Attribute(value=ast.Name(id='kwargs', ctx=ast.Load()), attr='keys', ctx=ast.Load()), args=[], keywords=[])],
keywords=[]))
func_body.append(passed_arg_list)
comprehension = ast.comprehension(target=ast.Name(id='key', ctx=ast.Store()), iter=ast.Name(id='passed_keys', ctx=ast.Load()), ifs=[], is_async=False)
comparator = ast.Name(id='expected', ctx=ast.Load())
listcomp = ast.ListComp(elt=ast.Compare(left=ast.Name(id='key', ctx=ast.Load()), ops=[ast.In()], comparators=[comparator]), generators=[comprehension])
check_message = ast.BinOp(
left = ast.Str(s="Allowed kwargs are {}. Passed kwargs: %s".format(optional_params)),
op = ast.Mod(),
right = ast.Name(id='passed_keys', ctx=ast.Load()),
lineno = self.__get_line())
kwarg_check = ast.Assert(test=ast.Call(func=ast.Name(id='all', ctx=ast.Load()), args=[listcomp], keywords=[]), msg=check_message)
func_body.append(kwarg_check)
func_kwargs = ast.Name(id='kwargs', ctx=ast.Load())
fname = "{}.{}".format(dom_name, func_name)
fname = ast.Str(s=fname, ctx=ast.Load())
if (sys.version_info[0], sys.version_info[1]) == (3, 5) or \
(sys.version_info[0], sys.version_info[1]) == (3, 6) or \
(sys.version_info[0], sys.version_info[1]) == (3, 7) or \
(sys.version_info[0], sys.version_info[1]) == (3, 8):
# More irritating minor semantic differences in the AST between 3.4 and 3.5
if func_kwargs:
message_params.append(ast.keyword(arg=None, value=ast.Name(id='kwargs', ctx=ast.Load())))
communicate_call = ast.Call(
func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), ctx=ast.Load(), attr='synchronous_command'),
args=[fname],
keywords=message_params)
elif (sys.version_info[0], sys.version_info[1]) == (3,4):
communicate_call = ast.Call(
func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), ctx=ast.Load(), attr='synchronous_command'),
args=[fname],
kwargs=func_kwargs,
keywords=message_params)
else:
print("Version:", sys.version_info)
raise RuntimeError("This script only functions on python 3.4, 3.5, 3.6, or 3.7. Active python version {}.{}".format(*sys.version_info))
do_communicate = ast.Assign(targets=[ast.Name(id='subdom_funcs', ctx=ast.Store())], value=communicate_call)
func_ret = ast.Return(value=ast.Name(id='subdom_funcs', ctx=ast.Load()))
if len(optional_params) and self.do_debug_prints:
func_body.append(self.__build_debug_print('kwargs', 'kwargs'))
func_body.append(do_communicate)
func_body.append(func_ret)
if len(optional_params):
kwarg = ast.arg(arg='kwargs', annotation=None)
else:
kwarg = None
sig = ast.arguments(
args=args,
vararg=None,
varargannotation=None,
posonlyargs=[],
kwonlyargs=[],
kwarg=kwarg,
kwargannotation=None,
defaults=[],
kw_defaults=[])
func = ast.FunctionDef(
name = "{}_{}".format(full_name, func_name),
args = sig,
body = func_body,
decorator_list = [],
lineno = self.__get_line(),
col_offset = 0,
)
return func
def _compile_directive_call_assets(self, el, options):
""" This special 't-call' tag can be used in order to aggregate/minify javascript and css assets"""
if len(el):
raise SyntaxError("t-call-assets cannot contain children nodes")
# nodes = self._get_asset_nodes(xmlid, options, css=css, js=js, debug=values.get('debug'), async=async, values=values)
#
# for index, (tagName, t_attrs, content) in enumerate(nodes):
# if index:
# append('\n ')
# append('<')
# append(tagName)
#
# self._post_processing_att(tagName, t_attrs, options)
# for name, value in t_attrs.items():
# if value or isinstance(value, string_types)):
# append(u' ')
# append(name)
# append(u'="')
# append(escape(pycompat.to_text((value)))
# append(u'"')
#
# if not content and tagName in self._void_elements:
# append('/>')
# else:
# append('>')
# if content:
# append(content)
# append('</')
# append(tagName)
# append('>')
#
space = el.getprevious() is not None and el.getprevious().tail or el.getparent().text
sep = u'\n' + space.rsplit('\n').pop()
return [
ast.Assign(
targets=[ast.Name(id='nodes', ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id='self', ctx=ast.Load()),
attr='_get_asset_nodes',
ctx=ast.Load()
),
args=[
ast.Str(el.get('t-call-assets')),
ast.Name(id='options', ctx=ast.Load()),
],
keywords=[
ast.keyword('css', self._get_attr_bool(el.get('t-css', True))),
ast.keyword('js', self._get_attr_bool(el.get('t-js', True))),
ast.keyword('debug', ast.Call(
func=ast.Attribute(
value=ast.Name(id='values', ctx=ast.Load()),
attr='get',
ctx=ast.Load()
),
args=[ast.Str('debug')],
keywords=[], starargs=None, kwargs=None
)),
ast.keyword('async_load', self._get_attr_bool(el.get('async_load', False))),
ast.keyword('defer_load', self._get_attr_bool(el.get('defer_load', False))),
ast.keyword('lazy_load', self._get_attr_bool(el.get('lazy_load', False))),
ast.keyword('values', ast.Name(id='values', ctx=ast.Load())),
],
starargs=None, kwargs=None
)
),
ast.For(
target=ast.Tuple(elts=[
ast.Name(id='index', ctx=ast.Store()),
ast.Tuple(elts=[
ast.Name(id='tagName', ctx=ast.Store()),
ast.Name(id='t_attrs', ctx=ast.Store()),
ast.Name(id='content', ctx=ast.Store())
], ctx=ast.Store())
], ctx=ast.Store()),
iter=ast.Call(
func=ast.Name(id='enumerate', ctx=ast.Load()),
args=[ast.Name(id='nodes', ctx=ast.Load())],
keywords=[],
starargs=None, kwargs=None
),
body=[
ast.If(
test=ast.Name(id='index', ctx=ast.Load()),
body=[self._append(ast.Str(sep))],
orelse=[]
),
self._append(ast.Str(u'<')),
self._append(ast.Name(id='tagName', ctx=ast.Load())),
] + self._append_attributes() + [
ast.If(
test=ast.BoolOp(
op=ast.And(),
values=[
ast.UnaryOp(ast.Not(), ast.Name(id='content', ctx=ast.Load()), lineno=0, col_offset=0),
ast.Compare(
left=ast.Name(id='tagName', ctx=ast.Load()),
ops=[ast.In()],
comparators=[ast.Attribute(
value=ast.Name(id='self', ctx=ast.Load()),
attr='_void_elements',
ctx=ast.Load()
)]
),
]
),
body=[self._append(ast.Str(u'/>'))],
orelse=[
self._append(ast.Str(u'>')),
ast.If(
test=ast.Name(id='content', ctx=ast.Load()),
body=[self._append(ast.Name(id='content', ctx=ast.Load()))],
orelse=[]
),
self._append(ast.Str(u'</')),
self._append(ast.Name(id='tagName', ctx=ast.Load())),
self._append(ast.Str(u'>')),
]
)
],
orelse=[]
)
]
def compile_ui_files(module, import_instructions=None):
"""
Reads recursively all *.py files in root directory looking for
"generate_pyfile_from_uifile" calls.
If this call is found, execute it in order to compile ui file.
import_instructions : python code containing required imports.
example :
>>> import_instructions = 'from openalea.vpltk.qt.designer import generate_pyfile_from_uifile\n'
if None, uses default imports : generate_pyfile_from_uifile, Path, hardbook and get_data
"""
import ast
from openalea.core import codegen
if import_instructions is None:
import_instructions = "from openalea.vpltk.qt.designer import generate_pyfile_from_uifile\n"
module = __import__(module)
paths = []
for root in module.__path__:
root = Path(root)
for py in root.walkfiles('*.py'):
paths.append((root, py))
for root, py in paths:
f = open(py)
lines = f.readlines()
f.close()
code = ''.join(lines)
try:
r = ast.parse(code)
except SyntaxError:
print 'SYNTAX ERROR: cannot read ...', py
else:
for instr in r.body:
if isinstance(instr, ast.Expr):
value = instr.value
if isinstance(value, ast.Call):
try:
func_name = value.func.id
except AttributeError:
pass
else:
if func_name == 'generate_pyfile_from_uifile':
true = ast.parse('True').body[0]
for keyword in value.keywords:
if keyword.arg == 'force':
keyword.value = true
break
else:
value.keywords.append(ast.keyword('force', true))
src = codegen.to_source(instr)
if py.startswith('./') or py.startswith('.\\'):
py = Path(py[2:])
name = replaceext(root.parent.relpathto(py), '').replace(os.sep, '.')
src = src.replace('__name__', repr(name))
try:
code = compile(import_instructions + src, "<string>", "exec")
exec code
except Exception as e:
print repr(e)
print 'COMPILATION ERROR: cannot compile', py
print
def keyword(arg, value):
return _ast.keyword(str(arg), value)
def ast_default_field(value):
return Call(
func=Name(id='field', ctx=Load()),
args=[],
keywords=[keyword(arg='default', value=Constant(value=value, kind=None))]
)
def visit_Call(self, node): # pylint: disable=invalid-name
self.generic_visit(node)
# ignore if the function is not 'nni.*'
if type(node.func) is not ast.Attribute:
return node
if type(node.func.value) is not ast.Name:
return node
if node.func.value.id != 'nni':
return node
# ignore if its not a search space function (e.g. `report_final_result`)
func = node.func.attr
if func not in _ss_funcs:
return node
self.last_line = node.lineno
if node.keywords:
# there is a `name` argument
assert len(
node.keywords
) == 1, 'Smart parameter has keyword argument other than "name"'
assert node.keywords[
0].arg == 'name', 'Smart paramater\'s keyword argument is not "name"'
assert type(
node.keywords[0].value
) is ast.Str, 'Smart parameter\'s name must be string literal'
name = node.keywords[0].value.s
specified_name = True
else:
# generate the missing name automatically
name = '__line' + str(str(node.args[-1].lineno))
specified_name = False
node.keywords = list()
if func in ('choice', 'function_choice'):
# we will use keys in the dict as the choices, which is generated by code_generator according to the args given by user
assert len(node.args
) == 1, 'Smart parameter has arguments other than dict'
# check if it is a number or a string and get its value accordingly
args = [
key.n if type(key) is ast.Num else key.s
for key in node.args[0].keys
]
else:
# arguments of other functions must be literal number
assert all(
type(arg) is ast.Num for arg in node.args
), 'Smart parameter\'s arguments must be number literals'
args = [arg.n for arg in node.args]
key = self.module_name + '/' + name + '/' + func
# store key in ast.Call
node.keywords.append(ast.keyword(arg='key', value=ast.Str(s=key)))
if func == 'function_choice':
func = 'choice'
value = {'_type': func, '_value': args}
if specified_name:
# multiple functions with same name must have identical arguments
old = self.search_space.get(key)
assert old is None or old == value, 'Different smart parameters have same name'
else:
# generated name must not duplicate
assert key not in self.search_space, 'Only one smart parameter is allowed in a line'
self.search_space[key] = value
return node
def visit_BinOp(self, node):
if node.op.__class__ in self.operators:
sympy_class = self.operators[node.op.__class__]
right = self.visit(node.right)
left = self.visit(node.left)
if isinstance(node.left, ast.UnaryOp) and (isinstance(
node.right,
ast.UnaryOp) == 0) and sympy_class in ('Mul', ):
left, right = right, left
if isinstance(node.op, ast.Sub):
right = ast.Call(func=ast.Name(id='Mul', ctx=ast.Load()),
args=[
ast.UnaryOp(op=ast.USub(),
operand=ast.Num(1)), right
],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
if isinstance(node.op, ast.Div):
if isinstance(node.left, ast.UnaryOp):
if isinstance(node.right, ast.UnaryOp):
left, right = right, left
left = ast.Call(func=ast.Name(id='Pow', ctx=ast.Load()),
args=[
left,
ast.UnaryOp(op=ast.USub(),
operand=ast.Num(1))
],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
else:
right = ast.Call(func=ast.Name(id='Pow', ctx=ast.Load()),
args=[
right,
ast.UnaryOp(op=ast.USub(),
operand=ast.Num(1))
],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
new_node = ast.Call(func=ast.Name(id=sympy_class, ctx=ast.Load()),
args=[left, right],
keywords=[
ast.keyword(arg='evaluate',
value=ast.NameConstant(
value=False,
ctx=ast.Load()))
],
starargs=None,
kwargs=None)
if sympy_class in ('Add', 'Mul'):
# Denest Add or Mul as appropriate
new_node.args = self.flatten(new_node.args, sympy_class)
return new_node
return node
async def meval(code, **kwargs):
# Don't clutter locals
locs = {}
# Restore globals later
globs = globals().copy()
# This code saves __name__ and __package into a kwarg passed to the function.
# It is set before the users code runs to make sure relative imports work
global_args = "_globs"
while global_args in globs.keys():
# Make sure there's no name collision, just keep prepending _s
global_args = "_" + global_args
kwargs[global_args] = {}
for glob in ["__name__", "__package__"]:
# Copy data to args we are sending
kwargs[global_args][glob] = globs[glob]
root = ast.parse(code, "exec")
code = root.body
if isinstance(
code[-1],
ast.Expr): # If we can use it as a lambda return (but multiline)
code[-1] = ast.copy_location(ast.Return(
code[-1].value), code[-1]) # Change it to a return statement
# globals().update(**<global_args>)
glob_copy = ast.Expr(
ast.Call(
func=ast.Attribute(value=ast.Call(func=ast.Name(id="globals",
ctx=ast.Load()),
args=[],
keywords=[]),
attr="update",
ctx=ast.Load()),
args=[],
keywords=[
ast.keyword(arg=None,
value=ast.Name(id=global_args, ctx=ast.Load()))
]))
ast.fix_missing_locations(glob_copy)
code.insert(0, glob_copy)
args = []
for a in list(map(lambda x: ast.arg(x, None), kwargs.keys())):
ast.fix_missing_locations(a)
args += [a]
args = ast.arguments(args=[],
vararg=None,
kwonlyargs=args,
kwarg=None,
defaults=[],
kw_defaults=[None for i in range(len(args))])
if int.from_bytes(importlib.util.MAGIC_NUMBER[:-2], 'little') >= 3410:
args.posonlyargs = []
fun = ast.AsyncFunctionDef(name="tmp",
args=args,
body=code,
decorator_list=[],
returns=None)
ast.fix_missing_locations(fun)
mod = ast.parse("")
mod.body = [fun]
comp = compile(mod, "<string>", "exec")
exec(comp, {}, locs)
r = await locs["tmp"](**kwargs)
if isinstance(r, types.CoroutineType) or isinstance(r, _asyncio.Future):
r = await r # workaround for 3.5
try:
globals().clear()
# Inconsistent state
finally:
globals().update(**globs)
return r
def transform(tree, *, forcing_mode, stop, **kw):
def rec(tree,
forcing_mode=forcing_mode
): # shorthand that defaults to current mode
return transform.recurse(tree, forcing_mode=forcing_mode)
# Forcing references (Name, Attribute, Subscript):
# x -> f(x)
# a.x -> f(force1(a).x)
# a.b.x -> f(force1(force1(a).b).x)
# a[j] -> f((force1(a))[force(j)])
# a[j][k] -> f(force1(force1(a)[force(j)])[force(k)])
#
# where f is force, force1 or identity (optimized away) depending on
# where the term appears; j and k may be indices or slices.
#
# Whenever not in Load context, f is identity.
#
# The idea is to apply just the right level of forcing to be able to
# resolve the reference, and then decide what to do with the resolved
# reference based on where it appears.
#
# For example, when subscripting a list, force1 it to unwrap it from
# a promise if it happens to be inside one, but don't force its elements
# just for the sake of resolving the reference. Then, apply f to the
# whole subscript term (forcing the accessed slice of the list, if necessary).
def f(tree):
if type(tree.ctx) is Load:
if forcing_mode == "full":
return hq[force(ast_literal[tree])]
elif forcing_mode == "flat":
return hq[force1(ast_literal[tree])]
# else forcing_mode == "off"
return tree
if type(tree) in (FunctionDef, AsyncFunctionDef, Lambda):
if type(tree) is Lambda and id(tree) not in userlambdas:
pass # ignore macro-introduced lambdas
else:
stop()
# mark this definition as lazy, and insert the interface wrapper
# to allow also strict code to call this function
if type(tree) is Lambda:
lam = tree
tree = hq[mark_lazy(ast_literal[tree])]
tree = sort_lambda_decorators(tree)
lam.body = rec(lam.body)
else:
tree.decorator_list = rec(tree.decorator_list)
k = suggest_decorator_index("mark_lazy",
tree.decorator_list)
if k is not None:
tree.decorator_list.insert(k, hq[mark_lazy])
else:
# mark_lazy should generally be as innermost as possible
# (so that e.g. the curry decorator will see the function as lazy)
tree.decorator_list.append(hq[mark_lazy])
tree.body = rec(tree.body)
elif type(tree) is Call:
def transform_arg(tree):
# add any needed force() invocations inside the tree,
# but leave the top level of simple references untouched.
isref = type(tree) in (Name, Attribute, Subscript)
tree = rec(tree, forcing_mode=("off" if isref else "full"))
if not isref: # (re-)thunkify expr; a reference can be passed as-is.
tree = lazyrec(tree)
return tree
def transform_starred(tree, dstarred=False):
isref = type(tree) in (Name, Attribute, Subscript)
tree = rec(tree, forcing_mode=("off" if isref else "full"))
# lazify items if we have a literal container
# we must avoid lazifying any other exprs, since a Lazy cannot be unpacked.
if is_literal_container(tree, maps_only=dstarred):
tree = lazyrec(tree)
return tree
# let bindings have a role similar to function arguments, so auto-lazify there
# (LHSs are always new names, so no infinite loop trap for the unwary)
if islet(tree):
stop()
view = ExpandedLetView(tree)
if view.mode == "let":
for b in view.bindings.elts: # b = (name, value)
b.elts[1] = transform_arg(b.elts[1])
else: # view.mode == "letrec":
for b in view.bindings.elts: # b = (name, (lambda e: ...))
thelambda = b.elts[1]
thelambda.body = transform_arg(thelambda.body)
if view.body: # let decorators have no body inside the Call node
thelambda = view.body
thelambda.body = rec(thelambda.body)
# namelambda() is used by let[] and do[]
# Lazy() is a strict function, takes a lambda, constructs a Lazy object
# _autoref_resolve doesn't need any special handling
elif isdo(tree) or is_decorator(tree.func, "namelambda") or \
any(isx(tree.func, s) for s in _ctorcalls_all) or isx(tree.func, isLazy) or \
any(isx(tree.func, s) for s in ("_autoref_resolve", "AutorefMarker")):
# here we know the operator (.func) to be one of specific names;
# don't transform it to avoid confusing lazyrec[] (important if this
# is an inner call in the arglist of an outer, lazy call, since it
# must see any container constructor calls that appear in the args)
stop()
# TODO: correct forcing mode? We shouldn't need to forcibly use "full",
# since lazycall() already fully forces any remaining promises
# in the args when calling a strict function.
tree.args = rec(tree.args)
tree.keywords = rec(tree.keywords)
# Python 3.4
if hasattr(tree, "starargs"):
tree.starargs = rec(tree.starargs)
if hasattr(tree, "kwargs"): tree.kwargs = rec(tree.kwargs)
else:
stop()
ln, co = tree.lineno, tree.col_offset
thefunc = rec(tree.func)
# TODO: test *args support in Python 3.5+ (this **should** work according to the AST specs)
adata = []
for x in tree.args:
if type(x) is Starred: # *args in Python 3.5+
v = transform_starred(x.value)
v = Starred(value=q[ast_literal[v]],
lineno=ln,
col_offset=co)
else:
v = transform_arg(x)
adata.append(v)
# TODO: test **kwargs support in Python 3.5+ (this **should** work according to the AST specs)
kwdata = []
for x in tree.keywords:
if x.arg is None: # **kwargs in Python 3.5+
v = transform_starred(x.value, dstarred=True)
else:
v = transform_arg(x.value)
kwdata.append((x.arg, v))
# Construct the call
mycall = Call(func=hq[lazycall],
args=[q[ast_literal[thefunc]]] +
[q[ast_literal[x]] for x in adata],
keywords=[
keyword(arg=k, value=q[ast_literal[x]])
for k, x in kwdata
],
lineno=ln,
col_offset=co)
if hasattr(tree, "starargs"): # *args in Python 3.4
if tree.starargs is not None:
mycall.starargs = transform_starred(tree.starargs)
else:
mycall.starargs = None
if hasattr(tree, "kwargs"): # **kwargs in Python 3.4
if tree.kwargs is not None:
mycall.kwargs = transform_starred(tree.kwargs,
dstarred=True)
else:
mycall.kwargs = None
tree = mycall
elif type(tree) is Subscript: # force only accessed part of obj[...]
stop()
tree.slice = rec(tree.slice, forcing_mode="full")
# resolve reference to the actual container without forcing its items.
tree.value = rec(tree.value, forcing_mode="flat")
tree = f(tree)
elif type(tree) is Attribute:
# a.b.c --> f(force1(force1(a).b).c) (Load)
# --> force1(force1(a).b).c (Store)
# attr="c", value=a.b
# attr="b", value=a
# Note in case of assignment to a compound, only the outermost
# Attribute is in Store context.
#
# Recurse in flat mode. Consider lst = [[1, 2], 3]
# lst[0] --> f(force1(lst)[0]), but
# lst[0].append --> force1(force1(force1(lst)[0]).append)
# Hence, looking up an attribute should only force **the object**
# so that we can perform the attribute lookup on it, whereas
# looking up values should finally f() the whole slice.
# (In the above examples, we have omitted f() when it is identity;
# in reality there is always an f() around the whole expr.)
stop()
tree.value = rec(tree.value, forcing_mode="flat")
tree = f(tree)
elif type(tree) is Name and type(tree.ctx) is Load:
stop() # must not recurse when a Name changes into a Call.
tree = f(tree)
return tree
def handleNode(self, node, parent, scope):
# Scope variables
if isinstance(node, ast.Name) and not isinstance(node.ctx, ast.Param):
return ast.Subscript(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=node.id)),
ctx=node.ctx
)
# Global
if isinstance(node, ast.Global):
res = []
for name in node.names:
res.append(ast.Expr(value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
attr="inheritVariable",
ctx=ast.Load()
),
args=[
ast.Name(id="scope0", ctx=ast.Load()),
ast.Str(s=name)
],
keywords=[], starargs=None, kwargs=None
)))
return res
# Import
if isinstance(node, ast.Import) or isinstance(node, ast.ImportFrom):
none = ast.Name(id="None", ctx=ast.Load())
names = ast.List(elts=[
ast.Tuple(elts=[
ast.Str(s=name.name),
ast.Str(s=name.asname) if name.asname else none
], ctx=ast.Load())
for name in node.names
], ctx=ast.Load())
if isinstance(node, ast.ImportFrom):
from_ = ast.Str(node.module) if node.module else none
level = ast.Num(node.level or 0)
else:
from_ = none
level = none
return ast.Expr(value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
attr="import_",
ctx=ast.Load()
),
args=[names, from_, level],
keywords=[], starargs=None, kwargs=None
))
if isinstance(node, ast.FunctionDef):
scope += 1
if isinstance(node, ast.Lambda):
# Handle arguments (default values) in parent scope
args = self.handleNode(node.args, node, scope)
if args is not None:
node.args = args
# Now increment scope and handle body
scope += 1
body = self.handleNode(node.body, node, scope)
if body is not None:
node.body = body
else:
for fieldname, value in ast.iter_fields(node):
if isinstance(value, ast.AST):
res = self.handleNode(value, node, scope)
if res is not None:
setattr(node, fieldname, res)
elif isinstance(value, list):
result = []
for child in value:
val = self.handleNode(child, node, scope)
if val is None:
result.append(child)
elif isinstance(val, list):
result += val
else:
result.append(val)
setattr(node, fieldname, result)
# Add scope to functions
if isinstance(node, ast.FunctionDef):
node.body.insert(0, ast.Assign(
targets=[ast.Name(id="scope%s" % scope, ctx=ast.Store())],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="scope%s" % (scope-1), ctx=ast.Load()),
attr="inherit",
ctx=ast.Load()
),
args=[], keywords=[],
starargs=None, kwargs=None
)
))
# Arguments
for arg in node.args.args:
node.body.insert(1, ast.Assign(
targets=[ast.Subscript(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=arg.id)),
ctx=ast.Store()
)],
value=ast.Name(id=arg.id, ctx=ast.Load())
))
# Vararg
if node.args.vararg is not None:
node.body.insert(1, ast.Assign(
targets=[ast.Subscript(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=node.args.vararg)),
ctx=ast.Store()
)],
value=ast.Name(id=node.args.vararg, ctx=ast.Load())
))
# Kwarg
if node.args.kwarg is not None:
node.body.insert(1, ast.Assign(
targets=[ast.Subscript(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=node.args.kwarg)),
ctx=ast.Store()
)],
value=ast.Name(id=node.args.kwarg, ctx=ast.Load())
))
# Save functions (not methods) to scope
if isinstance(node, ast.FunctionDef) and not isinstance(parent, ast.ClassDef):
oldname = node.name
node.name = "_user_%s_%s_" % (oldname, scope)
return [
node,
ast.Assign(
targets=[ast.Subscript(
value=ast.Name(id="scope%s" % (scope-1), ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=oldname)),
ctx=ast.Store()
)],
value=ast.Name(id=node.name, ctx=ast.Load())
)
]
# Save classes to scope
if isinstance(node, ast.ClassDef) and not isinstance(parent, ast.ClassDef):
oldname = node.name
node.name = "_user_%s_%s_" % (oldname, scope)
return [
node,
ast.Assign(
targets=[ast.Subscript(
value=ast.Name(id="scope%s" % scope, ctx=ast.Load()),
slice=ast.Index(value=ast.Str(s=oldname)),
ctx=ast.Store()
)],
value=ast.Name(id=node.name, ctx=ast.Load())
)
]
# Print
if isinstance(node, ast.Print):
true = ast.Name(id="True", ctx=ast.Load())
false = ast.Name(id="False", ctx=ast.Load())
none = ast.Name(id="None", ctx=ast.Load())
return ast.Expr(value=ast.Call(
func=ast.Subscript(
value=ast.Attribute(
value=ast.Name(id="scope0", ctx=ast.Load()),
attr="inherits",
ctx=ast.Load()
),
slice=ast.Index(value=ast.Str(s="print")),
ctx=ast.Load()
),
args=node.values, keywords=[
ast.keyword(arg="nl", value=true if node.nl else false),
ast.keyword(arg="dest", value=node.dest or none)
],
starargs=None, kwargs=None
))
# Add scope to lambdas
if isinstance(node, ast.Lambda):
# lambda a: a
# ->
# lambda a: (lambda scope1: scope1["a"])(scope0.extend({"a": a}))
# We save everything to dict, don't assign automatically
dct = ast.Dict(keys=[], values=[])
# Arguments
for arg in node.args.args:
dct.keys.append(ast.Str(s=arg.id))
dct.values.append(ast.Name(id=arg.id, ctx=ast.Load()))
# Vararg
if node.args.vararg is not None:
dct.keys.append(ast.Str(s=node.args.vararg))
dct.values.append(ast.Name(id=node.args.vararg, ctx=ast.Load()))
# Kwarg
if node.args.kwarg is not None:
dct.keys.append(ast.Str(s=node.args.kwarg))
dct.values.append(ast.Name(id=node.args.kwarg, ctx=ast.Load()))
node.body = ast.Call(
func=ast.Lambda(
args=ast.arguments(
args=[ast.Name(id="scope%s" % scope, ctx=ast.Load())],
vararg=None, kwarg=None, defaults=[]
),
body=node.body
),
args=[
ast.Call(
func=ast.Attribute(
value=ast.Name(id="scope%s" % (scope - 1), ctx=ast.Load()),
attr="extend",
ctx=ast.Load()
),
args=[dct], keywords=[],
starargs=None, kwargs=None
)
],
keywords=[], starargs=None, kwargs=None
)
# Now do something to prevent object.__subclasses__() hacks and others
if (
isinstance(node, ast.Attribute) and
((
node.attr.startswith("__") and
node.attr.endswith("__")
) or (
node.attr.startswith("func_")
))
):
return ast.Subscript(
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="scope0", ctx=ast.Load()),
attr="safeAttr",
ctx=ast.Load()
),
args=[node.value],
keywords=[], starargs=None, kwargs=None
),
slice=ast.Index(value=ast.Str(node.attr)),
ctx=node.ctx
)
def visit_FunctionDef(self, node):
""" Instrument a function definition by creating a new report builder
for this stack frame and putting it in a local variable. The local
variable has the same name as the global variable so all calls can
use the same CONTEXT_NAME symbol, but it means that I had to use this:
x = globals()['x'].start_frame()
Kind of ugly, but I think it was worth it to handle recursive calls.
"""
new_node = self.generic_visit(node)
line_numbers = set()
find_line_numbers(new_node, line_numbers)
first_line_number = min(line_numbers)
last_line_number = max(line_numbers)
args = [Num(n=first_line_number),
Num(n=last_line_number)]
start_frame_keywords = []
for decorator in new_node.decorator_list:
if getattr(decorator, 'id', None) == 'traced':
start_frame_keywords.append(
keyword(arg='is_decorated', value=Name(id='True',
ctx=Load())))
try_body = new_node.body
globals_call = Call(func=Name(id='globals', ctx=Load()),
args=[],
keywords=[],
starargs=None,
kwargs=None)
global_context = Subscript(value=globals_call,
slice=Index(value=Str(s=CONTEXT_NAME)),
ctx=Load())
start_frame_call = Call(func=Attribute(value=global_context,
attr='start_frame',
ctx=Load()),
args=args,
keywords=start_frame_keywords,
starargs=None,
kwargs=None)
context_assign = Assign(targets=[Name(id=CONTEXT_NAME, ctx=Store())],
value=start_frame_call)
new_node.body = [context_assign]
if isinstance(try_body[0], Expr) and isinstance(try_body[0].value, Str):
# Move docstring back to top of function.
# noinspection PyUnresolvedReferences
new_node.body.insert(0, try_body.pop(0))
# trace function parameter values
for target in new_node.args.args:
if isinstance(target, Name) and target.id == 'self':
continue
if arg and isinstance(target, arg) and target.arg == 'self':
continue
new_node.body.append(self._trace_assignment(target, node.lineno))
if new_node.args.vararg is not None:
new_node.body.append(
self._trace_assignment(new_node.args.vararg, node.lineno))
if new_node.args.kwarg is not None:
new_node.body.append(
self._trace_assignment(new_node.args.kwarg, node.lineno))
if try_body:
handler_body = [self._create_context_call('exception'),
Raise()]
new_node.body.append(
TryExcept(body=try_body,
handlers=[ExceptHandler(body=handler_body)],
orelse=[],
finalbody=[]))
self._set_statement_line_numbers(try_body, first_line_number)
self._set_statement_line_numbers(handler_body, last_line_number)
return new_node
def add_kwarg_to_super_call(super_call, kwarg):
super_call.value.keywords.append(
ast.keyword(arg=None, value=ast.Name(id=kwarg.arg,
ctx=ast.Load())))
def ast_list(tree):
"""Splices a list of ASTs into the quoted code snippet as a List node."""
return Literal(compat.Call(ast.Attribute(
value=ast.Name(id='ast', ctx=ast.Load()),
attr='List', ctx=ast.Load()), [], [ast.keyword("elts", tree)]))
],
func=Attribute(
Name("math_ops", Load()),
"square",
Load(),
),
keywords=[],
expr=None,
expr_func=None,
)
],
func=Attribute(Name("K", Load()), "mean", Load()),
keywords=[
keyword(
arg="axis",
value=UnaryOp(USub(), set_value(1)),
identifier=None,
)
],
expr=None,
expr_func=None,
),
expr=None,
),
],
decorator_list=[],
name="__call__",
returns=None,
arguments_args=None,
identifier_name=None,
stmt=None,
