def setUp(self):
self.source = """
def test_fn(x, y):
z = 1
if x > y:
z = x * x
z = z + y
return z
"""
self.all_name_ids = {
'x': [
gast.Param(),
gast.Load(),
gast.Load(),
gast.Load(),
],
'y': [
gast.Param(),
gast.Load(),
gast.Load(),
],
'z': [
gast.Store(),
gast.Store(),
gast.Load(),
gast.Store(),
gast.Load(),
]
}
def setUp(self):
self.source = """
def test_fn(x, y):
a = 1
x = y + a
if x > y:
z = x * x
z = z + a
else:
z = y * y
return z
"""
self.all_name_ids = {
'x': [
gast.Param(), gast.Store(), gast.Load(), gast.Load(),
gast.Load()
],
'a': [gast.Store(), gast.Load(), gast.Load()],
'y': [
gast.Param(),
gast.Load(),
gast.Load(),
gast.Load(),
gast.Load(),
],
'z': [
gast.Store(),
gast.Load(),
gast.Store(),
gast.Store(),
gast.Load(),
]
}
def create_while_node(condition_name, body_name, loop_var_names):
while_args = []
while_args.append(
gast.Name(id=condition_name,
ctx=gast.Param(),
annotation=None,
type_comment=None))
while_args.append(
gast.Name(id=body_name,
ctx=gast.Param(),
annotation=None,
type_comment=None))
assign_targets = [
gast.Name(id=var_name,
ctx=gast.Param(),
annotation=None,
type_comment=None) for var_name in loop_var_names
]
while_args.append(gast.List(elts=assign_targets, ctx=gast.Param()))
while_func_id = gast.parse('fluid.layers.while_loop').body[0].value
while_node = gast.Call(func=while_func_id, args=while_args, keywords=[])
assign_node = gast.Assign(
targets=[gast.Tuple(elts=assign_targets, ctx=gast.Store())],
value=while_node)
return assign_node
def visit_Lambda(self, node):
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', mangle('functools'))])
self.imports.append(import_)
functools_module = MODULES['functools']
self.global_declarations[mangle('functools')] = functools_module
self.generic_visit(node)
forged_name = "{0}_lambda{1}".format(self.prefix,
len(self.lambda_functions))
ii = self.passmanager.gather(ImportedIds, node, self.ctx)
ii.difference_update(self.lambda_functions) # remove current lambdas
binded_args = [ast.Name(iin, ast.Load(), None) for iin in sorted(ii)]
node.args.args = (
[ast.Name(iin, ast.Param(), None)
for iin in sorted(ii)] + node.args.args)
forged_fdef = ast.FunctionDef(forged_name, copy(node.args),
[ast.Return(node.body)], [], None)
self.lambda_functions.append(forged_fdef)
self.global_declarations[forged_name] = forged_fdef
proxy_call = ast.Name(forged_name, ast.Load(), None)
if binded_args:
return ast.Call(
ast.Attribute(ast.Name(mangle('functools'), ast.Load(), None),
"partial", ast.Load()),
[proxy_call] + binded_args, [])
else:
return proxy_call
def test_ast_to_object(self):
node = gast.FunctionDef(
name='f',
args=gast.arguments(
args=[gast.Name('a', gast.Param(), None)],
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[]),
body=[
gast.Return(
gast.BinOp(
op=gast.Add(),
left=gast.Name('a', gast.Load(), None),
right=gast.Num(1)))
],
decorator_list=[],
returns=None)
module, source, _ = compiler.ast_to_object(node)
expected_source = """
# coding=utf-8
def f(a):
return a + 1
"""
self.assertEqual(
textwrap.dedent(expected_source).strip(),
source.strip())
self.assertEqual(2, module.f(1))
with open(module.__file__, 'r') as temp_output:
self.assertEqual(
textwrap.dedent(expected_source).strip(),
temp_output.read().strip())
def visit_FunctionDef(self, node):
self.update = True
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import([ast.alias('functools', mangle('functools'))])
self.ctx.module.body.insert(0, import_)
functools_module = MODULES['functools']
self.global_declarations[mangle('functools')] = functools_module
self.ctx.module.body.append(node)
former_name = node.name
seed = 0
new_name = "pythran_{}{}"
while new_name.format(former_name, seed) in self.identifiers:
seed += 1
new_name = new_name.format(former_name, seed)
self.identifiers.add(new_name)
ii = self.gather(ImportedIds, node)
binded_args = [
ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii)
]
node.args.args = (
[ast.Name(iin, ast.Param(), None, None)
for iin in sorted(ii)] + node.args.args)
metadata.add(node, metadata.Local())
class Renamer(ast.NodeTransformer):
def visit_Call(self, node):
self.generic_visit(node)
if (isinstance(node.func, ast.Name)
and node.func.id == former_name):
node.func.id = new_name
node.args = ([
ast.Name(iin, ast.Load(), None, None)
for iin in sorted(ii)
] + node.args)
return node
Renamer().visit(node)
node.name = new_name
self.global_declarations[node.name] = node
proxy_call = ast.Name(new_name, ast.Load(), None, None)
new_node = ast.Assign([ast.Name(former_name, ast.Store(), None, None)],
ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(),
None, None), "partial",
ast.Load()),
[proxy_call] + binded_args,
[],
))
self.generic_visit(node)
return new_node
def outline(name, formal_parameters, out_parameters, stmts, has_return):
args = ast.arguments(
[ast.Name(fp, ast.Param(), None) for fp in formal_parameters], None,
[], [], None, [])
if isinstance(stmts, ast.expr):
assert not out_parameters, "no out parameters with expr"
fdef = ast.FunctionDef(name, args, [ast.Return(stmts)], [], None)
else:
fdef = ast.FunctionDef(name, args, stmts, [], None)
# this is part of a huge trick that plays with delayed type inference
# it basically computes the return type based on out parameters, and
# the return statement is unconditionally added so if we have other
# returns, there will be a computation of the output type based on the
# __combined of the regular return types and this one The original
# returns have been patched above to have a different type that
# cunningly combines with this output tuple
#
# This is the only trick I found to let pythran compute both the output
# variable type and the early return type. But hey, a dirty one :-/
stmts.append(
ast.Return(
ast.Tuple(
[ast.Name(fp, ast.Load(), None) for fp in out_parameters],
ast.Load())))
if has_return:
pr = PatchReturn(stmts[-1])
pr.visit(fdef)
return fdef
def visit_arg(self, node):
new_node = gast.Name(
self._visit(node.arg),
gast.Param(),
self._visit(node.annotation),
)
return ast.copy_location(new_node, node)
def visit_Lambda(self, node):
op = issimpleoperator(node)
if op is not None:
if mangle('operator') not in self.global_declarations:
import_ = ast.Import(
[ast.alias('operator', mangle('operator'))])
self.imports.append(import_)
operator_module = MODULES['operator']
self.global_declarations[mangle('operator')] = operator_module
return ast.Attribute(
ast.Name(mangle('operator'), ast.Load(), None, None), op,
ast.Load())
self.generic_visit(node)
forged_name = "{0}_lambda{1}".format(self.prefix,
len(self.lambda_functions))
ii = self.gather(ImportedIds, node)
ii.difference_update(self.lambda_functions) # remove current lambdas
binded_args = [
ast.Name(iin, ast.Load(), None, None) for iin in sorted(ii)
]
node.args.args = (
[ast.Name(iin, ast.Param(), None, None)
for iin in sorted(ii)] + node.args.args)
for patternname, pattern in self.patterns.items():
if issamelambda(pattern, node):
proxy_call = ast.Name(patternname, ast.Load(), None, None)
break
else:
duc = ExtendedDefUseChains()
nodepattern = deepcopy(node)
duc.visit(ast.Module([ast.Expr(nodepattern)], []))
self.patterns[forged_name] = nodepattern, duc
forged_fdef = ast.FunctionDef(forged_name, copy(node.args),
[ast.Return(node.body)], [], None,
None)
metadata.add(forged_fdef, metadata.Local())
self.lambda_functions.append(forged_fdef)
self.global_declarations[forged_name] = forged_fdef
proxy_call = ast.Name(forged_name, ast.Load(), None, None)
if binded_args:
if MODULES['functools'] not in self.global_declarations.values():
import_ = ast.Import(
[ast.alias('functools', mangle('functools'))])
self.imports.append(import_)
functools_module = MODULES['functools']
self.global_declarations[mangle(
'functools')] = functools_module
return ast.Call(
ast.Attribute(
ast.Name(mangle('functools'), ast.Load(), None, None),
"partial", ast.Load()), [proxy_call] + binded_args, [])
else:
return proxy_call
def __init__(self, **kwargs):
self.argument_effects = kwargs.get('argument_effects',
(UpdateEffect(), ) * DefaultArgNum)
self.global_effects = kwargs.get('global_effects', False)
self.return_alias = kwargs.get('return_alias',
lambda x: {UnboundValue})
self.args = ast.arguments([
ast.Name(n, ast.Param(), None, None)
for n in kwargs.get('args', [])
], [], None, [
ast.Name(n, ast.Param(), None, None)
for n in kwargs.get('kwonlyargs', [])
], [], None, [to_ast(d) for d in kwargs.get('defaults', [])])
self.return_range = kwargs.get("return_range",
lambda call: UNKNOWN_RANGE)
self.return_range_content = kwargs.get("return_range_content",
lambda c: UNKNOWN_RANGE)
def visit_ListComp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iterList = []
varList = []
for gen in node.generators:
iterList.append(self.make_Iterator(gen))
varList.append(ast.Name(gen.target.id, ast.Param(), None))
# If dim = 1, product is useless
if len(iterList) == 1:
iterAST = iterList[0]
varAST = ast.arguments([varList[0]], None, [], [], None, [])
else:
self.use_itertools = True
prodName = ast.Attribute(value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None),
attr='product',
ctx=ast.Load())
varid = varList[0].id # retarget this id, it's free
renamings = {v.id: (i, ) for i, v in enumerate(varList)}
node.elt = ConvertToTuple(varid, renamings).visit(node.elt)
iterAST = ast.Call(prodName, iterList, [])
varAST = ast.arguments([ast.Name(varid, ast.Param(), None)],
None, [], [], None, [])
mapName = ast.Attribute(value=ast.Name(id='__builtin__',
ctx=ast.Load(),
annotation=None),
attr='map',
ctx=ast.Load())
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return ast.Call(mapName, [ldmap, iterAST], [])
else:
return self.generic_visit(node)
def _make_annotated_arg(self, parent, identifier, annotation):
if identifier is None:
return None
new_node = gast.Name(
self._visit(identifier),
gast.Param(),
self._visit(annotation),
)
return ast.copy_location(new_node, parent)
def visit_Lambda(self, node):
self.generic_visit(node)
for i, arg in enumerate(node.args.args):
renamings = OrderedDict()
self.traverse_tuples(arg, (), renamings)
if renamings:
nname = self.get_new_id()
node.args.args[i] = ast.Name(nname, ast.Param(), None)
node.body = ConvertToTuple(nname, renamings).visit(node.body)
return node
def visit_Compare(self, node):
node = self.generic_visit(node)
if len(node.ops) > 1:
# in case we have more than one compare operator
# we generate an auxiliary function
# that lazily evaluates the needed parameters
imported_ids = self.passmanager.gather(ImportedIds, node, self.ctx)
imported_ids = sorted(imported_ids)
binded_args = [ast.Name(i, ast.Load(), None) for i in imported_ids]
# name of the new function
forged_name = "{0}_compare{1}".format(self.prefix,
len(self.compare_functions))
# call site
call = ast.Call(ast.Name(forged_name, ast.Load(), None),
binded_args, [])
# new function
arg_names = [ast.Name(i, ast.Param(), None) for i in imported_ids]
args = ast.arguments(arg_names, None, [], [], None, [])
body = [] # iteratively fill the body (yeah, feel your body!)
if is_trivially_copied(node.left):
prev_holder = node.left
else:
body.append(
ast.Assign([ast.Name('$0', ast.Store(), None)], node.left))
prev_holder = ast.Name('$0', ast.Load(), None)
for i, exp in enumerate(node.comparators):
if is_trivially_copied(exp):
holder = exp
else:
body.append(
ast.Assign(
[ast.Name('${}'.format(i + 1), ast.Store(), None)],
exp))
holder = ast.Name('${}'.format(i + 1), ast.Load(), None)
cond = ast.Compare(prev_holder, [node.ops[i]], [holder])
body.append(
ast.If(
cond, [ast.Pass()],
[ast.Return(path_to_attr(('__builtin__', 'False')))]))
prev_holder = holder
body.append(ast.Return(path_to_attr(('__builtin__', 'True'))))
forged_fdef = ast.FunctionDef(forged_name, args, body, [], None)
self.compare_functions.append(forged_fdef)
return call
else:
return node
def visitComp(self, node, make_attr):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [
ast.Name(gen.target.id, ast.Param(), None, None)
for gen in node.generators
]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], [], None, [], [], None,
[])
else:
self.use_itertools = True
prodName = ast.Attribute(value=ast.Name(id=mangle('itertools'),
ctx=ast.Load(),
annotation=None,
type_comment=None),
attr='product',
ctx=ast.Load())
varid = variables[0].id # retarget this id, it's free
renamings = {v.id: (i, ) for i, v in enumerate(variables)}
node.elt = ConvertToTuple(varid, renamings).visit(node.elt)
iterAST = ast.Call(prodName, iters, [])
varAST = ast.arguments(
[ast.Name(varid, ast.Param(), None, None)], [], None, [],
[], None, [])
ldBodymap = node.elt
ldmap = ast.Lambda(varAST, ldBodymap)
return make_attr(ldmap, iterAST)
else:
return self.generic_visit(node)
def visit_Print(self, node):
self.generic_visit(node)
for n in node.values:
n.ctx = gast.Param()
call_node = gast.Call(func=gast.Name('print', gast.Load(), None),
args=node.values,
keywords=[])
anno.setanno(call_node.func, 'live_val', print)
anno.setanno(call_node.func, 'fqn', 'print')
anno.setanno(call_node, 'args_scope', anno.getanno(node, 'args_scope'))
node = gast.Expr(call_node)
return node
def visit_arg(self, node):
if sys.version_info.minor < 8:
extra_arg = None
else:
extra_arg = self._visit(node.type_comment)
new_node = gast.Name(
node.arg, # micro-optimization here, don't call self._visit
gast.Param(),
self._visit(node.annotation),
extra_arg # type_comment
)
return ast.copy_location(new_node, node)
def visit_arg(self, node):
if sys.version_info.minor < 8:
extra_args = [None]
else:
extra_args = [self._visit(node.type_comment)]
new_node = gast.Name(
self._visit(node.arg),
gast.Param(),
self._visit(node.annotation),
*extra_args # type_comment
)
ast.copy_location(new_node, node)
return new_node
def visit_AnyComp(self, node, comp_type, *path):
self.update = True
node.elt = self.visit(node.elt)
name = "{0}_comprehension{1}".format(comp_type, self.count)
self.count += 1
args = self.gather(ImportedIds, node)
self.count_iter = 0
starget = "__target"
body = reduce(self.nest_reducer,
reversed(node.generators),
ast.Expr(
ast.Call(
reduce(lambda x, y: ast.Attribute(x, y,
ast.Load()),
path[1:],
ast.Name(path[0], ast.Load(),
None, None)),
[ast.Name(starget, ast.Load(), None, None),
node.elt],
[],
)
)
)
# add extra metadata to this node
metadata.add(body, metadata.Comprehension(starget))
init = ast.Assign(
[ast.Name(starget, ast.Store(), None, None)],
ast.Call(
ast.Attribute(
ast.Name('builtins', ast.Load(), None, None),
comp_type,
ast.Load()
),
[], [],)
)
result = ast.Return(ast.Name(starget, ast.Load(), None, None))
sargs = [ast.Name(arg, ast.Param(), None, None) for arg in args]
fd = ast.FunctionDef(name,
ast.arguments(sargs, [], None, [], [], None, []),
[init, body, result],
[], None, None)
metadata.add(fd, metadata.Local())
self.ctx.module.body.append(fd)
return ast.Call(
ast.Name(name, ast.Load(), None, None),
[ast.Name(arg.id, ast.Load(), None, None) for arg in sargs],
[],
) # no sharing !
def make_Iterator(self, gen):
if gen.ifs:
ldFilter = ast.Lambda(
ast.arguments([ast.Name(gen.target.id, ast.Param(), None)],
None, [], [], None, []),
ast.BoolOp(ast.And(), gen.ifs)
if len(gen.ifs) > 1 else gen.ifs[0])
ifilterName = ast.Attribute(value=ast.Name(id=MODULE,
ctx=ast.Load(),
annotation=None),
attr=IFILTER,
ctx=ast.Load())
return ast.Call(ifilterName, [ldFilter, gen.iter], [])
else:
return gen.iter
def generate_FunctionDef(self):
"""Generate a FunctionDef node."""
# Generate the arguments, register them as available
arg_vars = self.sample_node_list(
low=2, high=10, generator=lambda: self.generate_Name(gast.Param()))
args = gast.arguments(arg_vars, None, [], [], None, [])
# Generate the function body
body = self.sample_node_list(
low=1, high=N_FUNCTIONDEF_STATEMENTS, generator=self.generate_statement)
body.append(self.generate_Return())
fn_name = self.generate_Name().id
node = gast.FunctionDef(fn_name, args, body, (), None)
return node
def visit_BinOp(self, node):
self.generic_visit(node)
wl, wr = [self.result[x].isweak() for x in (node.left, node.right)]
if(isinstance(node.op, ast.Add) and any([wl, wr]) and
not all([wl, wr])):
# assumes the + operator always has the same operand type
# on left and right side
F = operator.add
else:
def F(x, y):
return ExpressionType(operator_to_lambda[type(node.op)],
[x, y])
fake_node = ast.Name("#", ast.Param(), None)
self.combine(fake_node, node.left, F)
self.combine(fake_node, node.right, F)
self.combine(node, fake_node)
del self.result[fake_node]
def test_load_ast(self):
node = gast.FunctionDef(
name='f',
args=gast.arguments(
args=[
gast.Name(
'a', ctx=gast.Param(), annotation=None, type_comment=None)
],
posonlyargs=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[
gast.Return(
gast.BinOp(
op=gast.Add(),
left=gast.Name(
'a',
ctx=gast.Load(),
annotation=None,
type_comment=None),
right=gast.Constant(1, kind=None)))
],
decorator_list=[],
returns=None,
type_comment=None)
module, source, _ = loader.load_ast(node)
expected_source = """
# coding=utf-8
def f(a):
return (a + 1)
"""
self.assertEqual(
textwrap.dedent(expected_source).strip(),
source.strip())
self.assertEqual(2, module.f(1))
with open(module.__file__, 'r') as temp_output:
self.assertEqual(
textwrap.dedent(expected_source).strip(),
temp_output.read().strip())
def visit_While(self, node):
self.generic_visit(node)
# Scrape out the data flow analysis
body_scope = anno.getanno(node, 'body_scope')
parent_scope_values = anno.getanno(node, 'parent_scope_values')
body_closure = tuple(body_scope.modified - body_scope.created)
def template(
state_args, # pylint:disable=unused-argument
state_locals,
state_results, # pylint:disable=unused-argument
test_name,
test, # pylint:disable=unused-argument
body_name,
body,
state_init):
def test_name(state_args): # pylint:disable=function-redefined,unused-argument
return test
def body_name(state_args): # pylint:disable=function-redefined,unused-argument
body # pylint:disable=pointless-statement
return state_locals
state_results = tf.while_loop(test_name, body_name, [state_init]) # pylint:disable=undefined-variable
test_name = self.namer.new_symbol('loop_test', body_scope.used)
body_name = self.namer.new_symbol('loop_body', body_scope.used)
node = templates.replace(
template,
state_args=self._tuple_or_item(
gast.Name(n, gast.Param(), None) for n in body_closure),
state_locals=self._ast_tuple_or_item(
(gast.Name(n, gast.Load(), None) for n in body_closure),
gast.Load()),
state_results=self._ast_tuple_or_item(
(gast.Name(n, gast.Store(), None) for n in body_closure),
gast.Store()),
test_name=gast.Name(test_name, gast.Load(), None),
test=node.test,
body_name=gast.Name(body_name, gast.Load(), None),
body=node.body,
state_init=[parent_scope_values.getval(n) for n in body_closure])
return node
def visit_Assign(self, node):
self.generic_visit(node)
val = node.value
if isinstance(val, gast.Call):
if isinstance(val.func, gast.Call):
if isinstance(val.func.func, gast.Attribute):
if isinstance(val.func.func.value, gast.Name):
if (val.func.func.value.id == 'tfe'
and val.func.func.attr
== 'value_and_gradients_function'):
# pylint:disable=unused-argument,undefined-variable
def template(loss_var, loss_fn, args, d_vars,
wrt_vars):
loss_var = loss_fn(args)
d_vars = tf.gradients(loss_var, [wrt_vars])
# pylint:enable=unused-argument,undefined-variable
# How to get these values? Print out the node.
loss_var = gast.Name(node.targets[0].elts[0].id,
gast.Store(), None)
loss_fn = gast.Name(val.func.args[0].id,
gast.Load(), None)
args = tuple(
gast.Name(a.id, gast.Param(), None)
for a in val.args)
d_vars = node.targets[0].elts[1]
wrt_vars = [
val.args[e.n] for e in val.func.args[1].elts
]
node = templates.replace(template,
loss_var=loss_var,
loss_fn=loss_fn,
args=args,
d_vars=d_vars,
wrt_vars=wrt_vars)
return node
def visit_GeneratorExp(self, node):
self.update = True
node.elt = self.visit(node.elt)
name = "generator_expression{0}".format(self.count)
self.count += 1
args = self.passmanager.gather(ImportedIds, node, self.ctx)
self.count_iter = 0
body = reduce(self.nest_reducer, reversed(node.generators),
ast.Expr(ast.Yield(node.elt)))
sargs = [ast.Name(arg, ast.Param(), None) for arg in args]
fd = ast.FunctionDef(name, ast.arguments(sargs, None, [], [], None,
[]), [body], [], None)
self.ctx.module.body.append(fd)
return ast.Call(
ast.Name(name, ast.Load(), None),
[ast.Name(arg.id, ast.Load(), None) for arg in sargs],
[],
) # no sharing !
def save_arguments(module_name, elements):
""" Recursively save arguments name and default value. """
for elem, signature in elements.items():
if isinstance(signature, dict): # Submodule case
save_arguments(module_name + (elem, ), signature)
else:
# use introspection to get the Python obj
try:
themodule = __import__(".".join(module_name))
obj = getattr(themodule, elem)
spec = inspect.getargspec(obj)
assert not signature.args.args
signature.args.args = [
ast.Name(arg, ast.Param(), None) for arg in spec.args
]
if spec.defaults:
signature.args.defaults = [
to_ast(default) for default in spec.defaults
]
except (AttributeError, ImportError, TypeError, ToNotEval):
pass
def visit_GeneratorExp(self, node):
if node in self.optimizable_comprehension:
self.update = True
self.generic_visit(node)
iters = [self.make_Iterator(gen) for gen in node.generators]
variables = [
ast.Name(gen.target.id, ast.Param(), None)
for gen in node.generators
]
# If dim = 1, product is useless
if len(iters) == 1:
iterAST = iters[0]
varAST = ast.arguments([variables[0]], None, [], [], None, [])
else:
prodName = ast.Attribute(value=ast.Name(id='itertools',
ctx=ast.Load(),
annotation=None),
attr='product',
ctx=ast.Load())
iterAST = ast.Call(prodName, iters, [])
varAST = ast.arguments([ast.Tuple(variables, ast.Store())],
None, [], [], None, [])
imapName = ast.Attribute(value=ast.Name(id=MODULE,
ctx=ast.Load(),
annotation=None),
attr=IMAP,
ctx=ast.Load())
ldBodyimap = node.elt
ldimap = ast.Lambda(varAST, ldBodyimap)
return ast.Call(imapName, [ldimap, iterAST], [])
else:
return self.generic_visit(node)
def _wrap_into_factory(nodes, entity_name, inner_factory_name,
outer_factory_name, closure_vars, factory_args,
future_features):
"""Wraps an AST into the body of a factory with consistent lexical context.
The AST is expected to define some symbol with a name given by `entity_name`.
This mechanism ensures that the resulting transformed entity has lexical
scoping identical to that of the source entity, while allowing extra
parametrization.
Two nested factories achieve the following:
1. The inner factory dynamically creates the entity represented by `nodes`.
2. The inner factory is parametrized by a custom set of arguments.
3. The inner factory has a closure identical to that of the transformed
entity.
4. The inner factory has local variables named like `args`, which `nodes` may
use as additional parameters.
5. The inner factory returns the variables given by `entity_name`.
6. The outer factory is niladic.
7. The outer factory has no closure.
8. The outer factory creates the necessary lexical scope for the inner
factory, so that the loaded code has the given configuration for
closure/globals.
9. The outer factory returns the inner factory.
Roughly speaking, the following code is generated:
from __future__ import future_feature_1
from __future__ import future_feature_2
...
def outer_factory():
closure_var_1 = None
closure_var_2 = None
...
def inner_factory(arg_1, arg_2, ...):
<<nodes>>
return entity
return inner_factory
The lexical scoping is created using dummy symbol declarations which create
local fariables in the body of the outer factory, so that the Python parser
correctly marks them as free non-global variables upon load (that is, it
creates cell slots for each symbol. Thes symbols are initialized with None,
but their values are not expected to be used; instead, the caller is expected
to replace them with the cells of the source entity. For more details, see:
https://docs.python.org/3/reference/executionmodel.html#binding-of-names
Args:
nodes: Tuple[ast.AST], the source code to wrap.
entity_name: Union[Text, ast.AST], the name of the principal entity that
`nodes` define.
inner_factory_name: Text, the name of the inner factory.
outer_factory_name: Text, the name of the outer factory.
closure_vars: Iterable[Text], names of the closure variables for the inner
factory.
factory_args: Iterable[Text], names of additional arguments for the
inner factory. Useful to configure variables that the converted code can
use. Typically, these are modules.
future_features: Iterable[Text], names of future statements to associate the
code with.
Returns:
ast.AST
"""
dummy_closure_defs = []
for var_name in closure_vars:
template = """
var_name = None
"""
dummy_closure_defs.extend(templates.replace(template, var_name=var_name))
if future_features:
future_imports = gast.ImportFrom(
module='__future__',
names=[gast.alias(name=name, asname=None) for name in future_features],
level=0)
else:
future_imports = []
factory_args = [
gast.Name(name, ctx=gast.Param(), annotation=None, type_comment=None)
for name in factory_args
]
template = """
future_imports
def outer_factory_name():
dummy_closure_defs
def inner_factory_name(factory_args):
entity_defs
return entity_name
return inner_factory_name
"""
return templates.replace(
template,
dummy_closure_defs=dummy_closure_defs,
entity_defs=nodes,
entity_name=entity_name,
factory_args=factory_args,
future_imports=future_imports,
inner_factory_name=inner_factory_name,
outer_factory_name=outer_factory_name)
def get_while_stmt_nodes(self, node):
# TODO: consider while - else in python
if not self.name_visitor.is_control_flow_loop(node):
return [node]
loop_var_names, create_var_names = self.name_visitor.get_loop_var_names(
node)
new_stmts = []
# Python can create variable in loop and use it out of loop, E.g.
#
# while x < 10:
# x += 1
# y = x
# z = y
#
# We need to create static variable for those variables
for name in create_var_names:
if "." not in name:
new_stmts.append(create_static_variable_gast_node(name))
# while x < 10 in dygraph should be convert into static tensor < 10
for name in loop_var_names:
new_stmts.append(to_static_variable_gast_node(name))
logical_op_transformer = LogicalOpTransformer(node.test)
cond_value_node = logical_op_transformer.transform()
condition_func_node = gast.FunctionDef(
name=unique_name.generate(WHILE_CONDITION_PREFIX),
args=gast.arguments(args=[
gast.Name(id=name,
ctx=gast.Param(),
annotation=None,
type_comment=None) for name in loop_var_names
],
posonlyargs=[],
vararg=None,
kwonlyargs=[],
kw_defaults=None,
kwarg=None,
defaults=[]),
body=[gast.Return(value=cond_value_node)],
decorator_list=[],
returns=None,
type_comment=None)
for name in loop_var_names:
if "." in name:
rename_transformer = RenameTransformer(condition_func_node)
rename_transformer.rename(
name, unique_name.generate(GENERATE_VARIABLE_PREFIX))
new_stmts.append(condition_func_node)
new_body = node.body
new_body.append(
gast.Return(
value=generate_name_node(loop_var_names, ctx=gast.Load())))
body_func_node = gast.FunctionDef(
name=unique_name.generate(WHILE_BODY_PREFIX),
args=gast.arguments(args=[
gast.Name(id=name,
ctx=gast.Param(),
annotation=None,
type_comment=None) for name in loop_var_names
],
posonlyargs=[],
vararg=None,
kwonlyargs=[],
kw_defaults=None,
kwarg=None,
defaults=[]),
body=new_body,
decorator_list=[],
returns=None,
type_comment=None)
for name in loop_var_names:
if "." in name:
rename_transformer = RenameTransformer(body_func_node)
rename_transformer.rename(
name, unique_name.generate(GENERATE_VARIABLE_PREFIX))
new_stmts.append(body_func_node)
while_loop_node = create_while_node(condition_func_node.name,
body_func_node.name,
loop_var_names)
new_stmts.append(while_loop_node)
return new_stmts
