def transform(tree, *, forcing_mode, stop, **kw):
def rec(tree,
forcing_mode=forcing_mode
): # shorthand that defaults to current mode
return transform.recurse(tree, forcing_mode=forcing_mode)
# Forcing references (Name, Attribute, Subscript):
# x -> f(x)
# a.x -> f(force1(a).x)
# a.b.x -> f(force1(force1(a).b).x)
# a[j] -> f((force1(a))[force(j)])
# a[j][k] -> f(force1(force1(a)[force(j)])[force(k)])
#
# where f is force, force1 or identity (optimized away) depending on
# where the term appears; j and k may be indices or slices.
#
# Whenever not in Load context, f is identity.
#
# The idea is to apply just the right level of forcing to be able to
# resolve the reference, and then decide what to do with the resolved
# reference based on where it appears.
#
# For example, when subscripting a list, force1 it to unwrap it from
# a promise if it happens to be inside one, but don't force its elements
# just for the sake of resolving the reference. Then, apply f to the
# whole subscript term (forcing the accessed slice of the list, if necessary).
def f(tree):
if type(tree.ctx) is Load:
if forcing_mode == "full":
return hq[force(ast_literal[tree])]
elif forcing_mode == "flat":
return hq[force1(ast_literal[tree])]
# else forcing_mode == "off"
return tree
if type(tree) in (FunctionDef, AsyncFunctionDef, Lambda):
if type(tree) is Lambda and id(tree) not in userlambdas:
pass # ignore macro-introduced lambdas
else:
stop()
# mark this definition as lazy, and insert the interface wrapper
# to allow also strict code to call this function
if type(tree) is Lambda:
lam = tree
tree = hq[passthrough_lazy_args(ast_literal[tree])]
# TODO: This doesn't really do anything; we don't here see the chain
# TODO: of Call nodes (decorators) that surround the Lambda node.
tree = sort_lambda_decorators(tree)
lam.body = rec(lam.body)
else:
k = suggest_decorator_index("passthrough_lazy_args",
tree.decorator_list)
# Force the decorators only after `suggest_decorator_index`
# has suggested us where to put ours.
# TODO: could make `suggest_decorator_index` ignore a `force()` wrapper.
tree.decorator_list = rec(tree.decorator_list)
if k is not None:
tree.decorator_list.insert(k,
hq[passthrough_lazy_args])
else:
# passthrough_lazy_args should generally be as innermost as possible
# (so that e.g. the curry decorator will see the function as lazy)
tree.decorator_list.append(hq[passthrough_lazy_args])
tree.body = rec(tree.body)
elif type(tree) is Call:
def transform_arg(tree):
# add any needed force() invocations inside the tree,
# but leave the top level of simple references untouched.
isref = type(tree) in (Name, Attribute, Subscript)
tree = rec(tree, forcing_mode=("off" if isref else "full"))
if not isref: # (re-)thunkify expr; a reference can be passed as-is.
tree = lazyrec(tree)
return tree
def transform_starred(tree, dstarred=False):
isref = type(tree) in (Name, Attribute, Subscript)
tree = rec(tree, forcing_mode=("off" if isref else "full"))
# lazify items if we have a literal container
# we must avoid lazifying any other exprs, since a Lazy cannot be unpacked.
if is_literal_container(tree, maps_only=dstarred):
tree = lazyrec(tree)
return tree
# let bindings have a role similar to function arguments, so auto-lazify there
# (LHSs are always new names, so no infinite loop trap for the unwary)
if islet(tree):
stop()
view = ExpandedLetView(tree)
if view.mode == "let":
for b in view.bindings.elts: # b = (name, value)
b.elts[1] = transform_arg(b.elts[1])
else: # view.mode == "letrec":
for b in view.bindings.elts: # b = (name, (lambda e: ...))
thelambda = b.elts[1]
thelambda.body = transform_arg(thelambda.body)
if view.body: # let decorators have no body inside the Call node
thelambda = view.body
thelambda.body = rec(thelambda.body)
# namelambda() is used by let[] and do[]
# Lazy() is a strict function, takes a lambda, constructs a Lazy object
# _autoref_resolve doesn't need any special handling
elif (isdo(tree) or is_decorator(tree.func, "namelambda")
or any(isx(tree.func, s) for s in _ctorcalls_all)
or isx(tree.func, isLazy) or any(
isx(tree.func, s)
for s in ("_autoref_resolve", "AutorefMarker"))):
# here we know the operator (.func) to be one of specific names;
# don't transform it to avoid confusing lazyrec[] (important if this
# is an inner call in the arglist of an outer, lazy call, since it
# must see any container constructor calls that appear in the args)
stop()
# TODO: correct forcing mode for `rec`? We shouldn't need to forcibly use "full",
# since maybe_force_args() already fully forces any remaining promises
# in the args when calling a strict function.
tree.args = rec(tree.args)
tree.keywords = rec(tree.keywords)
# Python 3.4
if hasattr(tree,
"starargs"): # pragma: no cover, Python 3.4 only.
tree.starargs = rec(tree.starargs)
if hasattr(tree,
"kwargs"): # pragma: no cover, Python 3.4 only.
tree.kwargs = rec(tree.kwargs)
else:
stop()
ln, co = tree.lineno, tree.col_offset
thefunc = rec(tree.func)
adata = []
for x in tree.args:
if type(x) is Starred: # *args in Python 3.5+
v = transform_starred(x.value)
v = Starred(value=q[ast_literal[v]],
lineno=ln,
col_offset=co)
else:
v = transform_arg(x)
adata.append(v)
kwdata = []
for x in tree.keywords:
if x.arg is None: # **kwargs in Python 3.5+
v = transform_starred(x.value, dstarred=True)
else:
v = transform_arg(x.value)
kwdata.append((x.arg, v))
# Construct the call
mycall = Call(func=hq[maybe_force_args],
args=[q[ast_literal[thefunc]]] +
[q[ast_literal[x]] for x in adata],
keywords=[
keyword(arg=k, value=q[ast_literal[x]])
for k, x in kwdata
],
lineno=ln,
col_offset=co)
if hasattr(
tree,
"starargs"): # *args in Python 3.4 # pragma: no cover
if tree.starargs is not None:
mycall.starargs = transform_starred(tree.starargs)
else:
mycall.starargs = None
if hasattr(tree, "kwargs"
): # **kwargs in Python 3.4 # pragma: no cover
if tree.kwargs is not None:
mycall.kwargs = transform_starred(tree.kwargs,
dstarred=True)
else:
mycall.kwargs = None
tree = mycall
elif type(tree) is Subscript: # force only accessed part of obj[...]
stop()
tree.slice = rec(tree.slice, forcing_mode="full")
# resolve reference to the actual container without forcing its items.
tree.value = rec(tree.value, forcing_mode="flat")
tree = f(tree)
elif type(tree) is Attribute:
# a.b.c --> f(force1(force1(a).b).c) (Load)
# --> force1(force1(a).b).c (Store)
# attr="c", value=a.b
# attr="b", value=a
# Note in case of assignment to a compound, only the outermost
# Attribute is in Store context.
#
# Recurse in flat mode. Consider lst = [[1, 2], 3]
# lst[0] --> f(force1(lst)[0]), but
# lst[0].append --> force1(force1(force1(lst)[0]).append)
# Hence, looking up an attribute should only force **the object**
# so that we can perform the attribute lookup on it, whereas
# looking up values should finally f() the whole slice.
# (In the above examples, we have omitted f() when it is identity;
# in reality there is always an f() around the whole expr.)
stop()
tree.value = rec(tree.value, forcing_mode="flat")
tree = f(tree)
elif type(tree) is Name and type(tree.ctx) is Load:
stop() # must not recurse when a Name changes into a Call.
tree = f(tree)
return tree
def transform(tree, *, quotelevel, set_ctx, stop, **kw):
# Not tuples but syntax: leave alone the:
# - binding pair "tuples" of let, letseq, letrec, their d*, b* variants,
# and let_syntax, abbrev
# - subscript part of an explicit do[], do0[]
# but recurse inside them.
#
# let and do have not expanded yet when prefix runs (better that way!).
if islet(tree, expanded=False):
stop()
view = UnexpandedLetView(tree)
for binding in view.bindings:
if type(binding) is not Tuple:
assert False, "prefix: expected a tuple in let binding position" # pragma: no cover
_, value = binding.elts # leave name alone, recurse into value
binding.elts[1] = transform.recurse(value, quotelevel=quotelevel)
if view.body:
view.body = transform.recurse(view.body, quotelevel=quotelevel)
return tree
elif isdo(tree, expanded=False):
stop()
view = UnexpandedDoView(tree)
view.body = [transform.recurse(expr, quotelevel=quotelevel) for expr in view.body]
return tree
# integration with testing framework
if isunexpandedtestmacro(tree):
stop()
if type(tree.slice) is not Index:
assert False, "prefix: Slice and ExtSlice not implemented in analysis of testing macro arguments" # pragma: no cover
body = tree.slice.value
if type(body) is Tuple:
# skip the transformation of the argument tuple itself, but transform its elements
body.elts = [transform.recurse(expr, quotelevel=quotelevel) for expr in body.elts]
else:
tree.slice.value = transform.recurse(tree.slice.value, quotelevel=quotelevel)
return tree
# general case
# macro-created nodes might not have a ctx, but we run in the first pass.
if not (type(tree) is Tuple and type(tree.ctx) is Load):
return tree
op, *data = tree.elts
while True:
if isunquote(op):
if quotelevel < 1:
assert False, "unquote while not in quote" # pragma: no cover
quotelevel -= 1
elif isquote(op):
quotelevel += 1
else:
break
set_ctx(quotelevel=quotelevel)
if not len(data):
assert False, "a prefix tuple cannot contain only quote/unquote operators" # pragma: no cover
op, *data = data
if quotelevel > 0:
quoted = [op] + data
if any(iskwargs(x) for x in quoted):
assert False, "kw(...) may only appear in a prefix tuple representing a function call" # pragma: no cover
return q[(ast_literal[quoted],)]
# (f, a1, ..., an) --> f(a1, ..., an)
posargs = [x for x in data if not iskwargs(x)]
# TODO: tag *args and **kwargs in a kw() as invalid, too (currently just ignored)
invalids = list(flatmap(lambda x: x.args, filter(iskwargs, data)))
if invalids:
assert False, "kw(...) may only specify named args" # pragma: no cover
kwargs = flatmap(lambda x: x.keywords, filter(iskwargs, data))
kwargs = list(rev(uniqify(rev(kwargs), key=lambda x: x.arg))) # latest wins, but keep original ordering
thecall = Call(func=op, args=posargs, keywords=list(kwargs))
if PYTHON34: # pragma: no cover, Python 3.4 only
thecall.starargs = None
thecall.kwargs = None
return thecall