def args(args=None):
return _ast.arguments(
args=args,
posonlyargs=[],
kwonlyargs=[],
kw_defaults=[],
defaults=[])
def arguments(args: List[_ast.arg], vararg: _ast.arg) -> _ast.arguments:
return _ast.arguments(
args=args,
posonlyargs=[],
vararg=vararg,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
)
def make_function(code, defaults=None, lineno=0):
from meta.decompiler.disassemble import disassemble
instructions = Instructions(disassemble(code))
stmnts = instructions.stmnt()
if code.co_flags & 2:
vararg = None
kwarg = None
varnames = list(code.co_varnames[:code.co_argcount])
co_locals = list(code.co_varnames[code.co_argcount:])
#have var args
if code.co_flags & 4:
vararg = co_locals.pop(0)
#have kw args
if code.co_flags & 8:
kwarg = co_locals.pop()
args = [_ast.Name(id=argname, ctx=_ast.Param(), lineno=lineno, col_offset=0) for argname in varnames]
args = _ast.arguments(args=args,
defaults=defaults if defaults else [],
kwarg=kwarg,
vararg=vararg,
lineno=lineno, col_offset=0
)
if code.co_name == '<lambda>':
if len(stmnts) == 2:
if isinstance(stmnts[0], _ast.If) and isinstance(stmnts[1], _ast.Return):
assert len(stmnts[0].body) == 1
assert isinstance(stmnts[0].body[0], _ast.Return)
stmnts = [_ast.Return(_ast.IfExp(stmnts[0].test, stmnts[0].body[0].value, stmnts[1].value))]
assert len(stmnts) == 1, stmnts
assert isinstance(stmnts[0], _ast.Return)
stmnt = stmnts[0].value
ast_obj = _ast.Lambda(args=args, body=stmnt, lineno=lineno, col_offset=0)
else:
if instructions.seen_yield:
return_ = stmnts[-1]
assert isinstance(return_, _ast.Return)
assert isinstance(return_.value, _ast.Name)
assert return_.value.id == 'None'
return_.value = None
ast_obj = _ast.FunctionDef(name=code.co_name, args=args, body=stmnts, decorator_list=[], lineno=lineno, col_offset=0)
return ast_obj
def make_function(code, defaults=None, lineno=0):
from graphlab.meta.decompiler.disassemble import disassemble
instructions = Instructions(disassemble(code))
stmnts = instructions.stmnt()
if code.co_flags & 2:
vararg = None
kwarg = None
varnames = list(code.co_varnames[:code.co_argcount])
co_locals = list(code.co_varnames[code.co_argcount:])
#have var args
if code.co_flags & 4:
vararg = co_locals.pop(0)
#have kw args
if code.co_flags & 8:
kwarg = co_locals.pop()
args = [_ast.Name(id=argname, ctx=_ast.Param(), lineno=lineno, col_offset=0) for argname in varnames]
args = _ast.arguments(args=args,
defaults=defaults if defaults else [],
kwarg=kwarg,
vararg=vararg,
lineno=lineno, col_offset=0
)
if code.co_name == '<lambda>':
if len(stmnts) == 2:
if isinstance(stmnts[0], _ast.If) and isinstance(stmnts[1], _ast.Return):
assert len(stmnts[0].body) == 1
assert isinstance(stmnts[0].body[0], _ast.Return)
stmnts = [_ast.Return(_ast.IfExp(stmnts[0].test, stmnts[0].body[0].value, stmnts[1].value))]
assert len(stmnts) == 1, stmnts
assert isinstance(stmnts[0], _ast.Return)
stmnt = stmnts[0].value
ast_obj = _ast.Lambda(args=args, body=stmnt, lineno=lineno, col_offset=0)
else:
if instructions.seen_yield:
return_ = stmnts[-1]
assert isinstance(return_, _ast.Return)
assert isinstance(return_.value, _ast.Name)
assert return_.value.id == 'None'
return_.value = None
ast_obj = _ast.FunctionDef(name=code.co_name, args=args, body=stmnts, decorator_list=[], lineno=lineno, col_offset=0)
return ast_obj
def _replace_where_block_with_function(self, with_node):
return _ast.FunctionDef(name=self.feature_name + '_where',
args=_ast.arguments(
args=[_ast.arg(arg='self'), _ast.arg(arg='injectable_values')],
kwonlyargs=[],
kw_defaults=[],
defaults=[]
),
body=copy.deepcopy(with_node.body),
decorator_list=[],
returns=None)
def async_function(self, items):
if len(items) == 2:
(name, body) = items
args = []
else:
(name, args, body) = items
return _ast.AsyncFunctionDef(name=name,
args=_ast.arguments(args=args,
defaults=[],
vararg=None,
kwarg=None),
body=body,
decorator_list=[])
def constructor_method(self, items):
(args, body) = items
method_args = [_ast.arg(arg='self', annotation=None)]
method_args.extend(args)
return _ast.FunctionDef(name='__init__',
args=_ast.arguments(args=method_args,
defaults=[],
vararg=None,
kwarg=None),
body=body,
decorator_list=[])
def new_func(func_name, args, body, decorators=None, defaults=None):
"""AST function."""
return FunctionDef(name=func_name,
returns=None,
args=arguments(
args=args,
vararg=None,
kwarg=None,
kw_defaults=[],
kwonlyargs=[],
defaults=[] if defaults is None else defaults),
body=body,
decorator_list=[] if decorators is None else decorators)
def test_functiondef(self):
import _ast as ast
fAst = ast.FunctionDef(
name="foo",
args=ast.arguments(
args=[], vararg=None, kwarg=None, defaults=[],
kwonlyargs=[], kw_defaults=[]),
body=[], decorator_list=[], lineno=5, col_offset=0)
exprAst = ast.Interactive(body=[fAst])
compiled = compile(exprAst, "<foo>", "single")
#
d = {}
eval(compiled, d, d)
assert type(d['foo']) is type(lambda: 42)
assert d['foo']() is None
def test_functiondef(self):
import _ast as ast
fAst = ast.FunctionDef(
name="foo",
args=ast.arguments(
args=[], vararg=None, kwarg=None, defaults=[],
kwonlyargs=[], kw_defaults=[]),
body=[], decorator_list=[], lineno=5, col_offset=0)
exprAst = ast.Interactive(body=[fAst])
compiled = compile(exprAst, "<foo>", "single")
#
d = {}
eval(compiled, d, d)
assert type(d['foo']) is type(lambda: 42)
assert d['foo']() is None
def _createrespond(self):
"""Create the `respond()` method on the final class."""
args = arguments(
args=[Name(id='self', ctx=Param())],
vararg=None,
kwarg=None,
defaults=[])
# the actual respond function
respond = FunctionDef(
name='respond',
args=args,
body=self.ast_.body, # whatever is left at the top level
decorator_list=[])
ast.copy_location(self.ast_, respond)
ast.fix_missing_locations(respond)
return respond
def arguments(args=(), keys=(), values=(), vararg_name=None, kwarg_name=None):
"""Creates an _ast.FunctionDef node.
Args:
args: A list of args.
keys: A list of keys, must be the same length as values.
values: A list of values, correspond to keys.
vararg_name: The name of the vararg variable, or None.
kwarg_name: The name of the kwargs variable, or None.
Raises:
ValueError: If len(keys) != len(values).
Returns:
An _ast.FunctionDef node.
"""
args, defaults = _ToArgsDefaults(args=args, keys=keys, values=values)
return _ast.arguments(args=args,
defaults=defaults,
vararg=vararg_name,
kwarg=kwarg_name)
def visit_GeneratorExp(self, node):
"""Rewrite the GeneratorExp visitor function to turn the
generator expression into a iterator function. This is
necessary to be able to correctly label any random functions
that get called from within the generator expression.
Basically, this function creates a function, and transforms
the generator into a for loop that yields values from the.
The function name is then returned, so that the parent node
can handle the assignment properly.
"""
# make an identifier for the list
self.newline(node)
iden = self._gen_iden(node)
argids = []
for gen in node.generators:
argval = gen.iter
argid = self._gen_iden(gen.iter)
self.visit(
_ast.Assign(targets=[_ast.Name(id=argid, ctx=_ast.Store())],
value=argval))
argids.append(argid)
elt = node.elt
def parse_generator(nodes, ids):
node = nodes[0]
tempnode = _ast.For()
tempnode.target = node.target
tempnode.iter = _ast.Name(id=ids[0], ctx=_ast.Load())
if len(nodes) == 1:
yield_node = _ast.Expr(value=_ast.Yield(value=elt))
body = [yield_node]
else:
body = [parse_generator(nodes[1:], ids[1:])]
if len(node.ifs) == 1:
ifnode = _ast.If(test=node.ifs[0], body=body, orelse=[])
tempnode.body = [ifnode]
elif len(node.ifs) > 1:
ifnode = _ast.If(test=_ast.BoolOp(op=_ast.And(),
values=node.ifs),
body=body,
orelse=[])
tempnode.body = [ifnode]
else:
tempnode.body = body
tempnode.orelse = None
return tempnode
node = _ast.FunctionDef(
name=iden,
args=_ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]),
body=[parse_generator(node.generators, argids)],
decorator_list=[])
self.visit(node)
return iden
def make_function(code,
defaults=None,
annotations=(),
kw_defaults=(),
lineno=0):
from ..decompiler.disassemble import disassemble
instructions = Instructions(disassemble(code))
stmnts = instructions.stmnt()
if code.co_flags & 2:
vararg = None
kwarg = None
varnames = list(code.co_varnames[:code.co_argcount])
kwonly_varnames = list(code.co_varnames[code.co_argcount:code.co_argcount +
code.co_kwonlyargcount])
co_locals = list(code.co_varnames[code.co_argcount +
code.co_kwonlyargcount:])
assert (len(kw_defaults) % 2) == 0
kw_defaults = list(kw_defaults)
kw_default_dict = {}
while kw_defaults:
name = kw_defaults.pop(0)
value = kw_defaults.pop(0)
kw_default_dict[name.s] = value
kw_defaults = []
for argname in kwonly_varnames:
kw_defaults.append(kw_default_dict.pop(argname))
#have var args
if code.co_flags & 4:
vararg = co_locals.pop(0)
#have kw args
if code.co_flags & 8:
kwarg = co_locals.pop()
args = []
annotation_names = [annotation.arg for annotation in annotations]
for argname in varnames:
if argname in annotation_names:
arg = [
annotation for annotation in annotations
if annotation.arg == argname
][0]
else:
arg = _ast.arg(annotation=None,
arg=argname,
lineno=lineno,
col_offset=0) #@UndefinedVariable
args.append(arg)
kwonlyargs = []
for argname in kwonly_varnames:
if argname in annotation_names:
arg = [
annotation for annotation in annotations
if annotation.arg == argname
][0]
else:
arg = _ast.arg(annotation=None,
arg=argname,
lineno=lineno,
col_offset=0) #@UndefinedVariable
kwonlyargs.append(arg)
if 'return' in annotation_names:
arg = [
annotation for annotation in annotations
if annotation.arg == 'return'
][0]
returns = arg.annotation
else:
returns = None
if vararg in annotation_names:
arg = [
annotation for annotation in annotations
if annotation.arg == vararg
][0]
varargannotation = arg.annotation
else:
varargannotation = None
if kwarg in annotation_names:
arg = [
annotation for annotation in annotations if annotation.arg == kwarg
][0]
kwargannotation = arg.annotation
else:
kwargannotation = None
args = _ast.arguments(args=args,
defaults=defaults if defaults else [],
kwarg=kwarg,
vararg=vararg,
kw_defaults=kw_defaults,
kwonlyargs=kwonlyargs,
kwargannotation=kwargannotation,
varargannotation=varargannotation,
lineno=lineno,
col_offset=0)
if code.co_name == '<lambda>':
if len(stmnts) == 2:
if isinstance(stmnts[0], _ast.If) and isinstance(
stmnts[1], _ast.Return):
assert len(stmnts[0].body) == 1
assert isinstance(stmnts[0].body[0], _ast.Return)
stmnts = [
_ast.Return(
_ast.IfExp(stmnts[0].test, stmnts[0].body[0].value,
stmnts[1].value))
]
assert isinstance(stmnts[0], _ast.Return)
stmnt = stmnts[0].value
ast_obj = _ast.Lambda(args=args,
body=stmnt,
lineno=lineno,
col_offset=0)
else:
if instructions.seen_yield:
return_ = stmnts[-1]
assert isinstance(return_, _ast.Return)
assert isinstance(return_.value, _ast.Name)
assert return_.value.id == 'None'
return_.value = None
ast_obj = _ast.FunctionDef(name=code.co_name,
args=args,
body=stmnts,
decorator_list=[],
returns=returns,
lineno=lineno,
col_offset=0)
return ast_obj
def make_function(code, defaults=None, annotations=(), kw_defaults=(), lineno=0):
from graphlab.meta.decompiler.disassemble import disassemble
instructions = Instructions(disassemble(code))
stmnts = instructions.stmnt()
if code.co_flags & 2:
vararg = None
kwarg = None
varnames = list(code.co_varnames[:code.co_argcount])
kwonly_varnames = list(code.co_varnames[code.co_argcount:code.co_argcount + code.co_kwonlyargcount])
co_locals = list(code.co_varnames[code.co_argcount + code.co_kwonlyargcount:])
assert (len(kw_defaults) % 2) == 0
kw_defaults = list(kw_defaults)
kw_default_dict = {}
while kw_defaults:
name = kw_defaults.pop(0)
value = kw_defaults.pop(0)
kw_default_dict[name.s] = value
kw_defaults = []
for argname in kwonly_varnames:
kw_defaults.append(kw_default_dict.pop(argname))
#have var args
if code.co_flags & 4:
vararg = co_locals.pop(0)
#have kw args
if code.co_flags & 8:
kwarg = co_locals.pop()
args = []
annotation_names = [annotation.arg for annotation in annotations]
for argname in varnames:
if argname in annotation_names:
arg = [annotation for annotation in annotations if annotation.arg == argname][0]
else:
arg = _ast.arg(annotation=None, arg=argname, lineno=lineno, col_offset=0) #@UndefinedVariable
args.append(arg)
kwonlyargs = []
for argname in kwonly_varnames:
if argname in annotation_names:
arg = [annotation for annotation in annotations if annotation.arg == argname][0]
else:
arg = _ast.arg(annotation=None, arg=argname, lineno=lineno, col_offset=0) #@UndefinedVariable
kwonlyargs.append(arg)
if 'return' in annotation_names:
arg = [annotation for annotation in annotations if annotation.arg == 'return'][0]
returns = arg.annotation
else:
returns = None
if vararg in annotation_names:
arg = [annotation for annotation in annotations if annotation.arg == vararg][0]
varargannotation = arg.annotation
else:
varargannotation = None
if kwarg in annotation_names:
arg = [annotation for annotation in annotations if annotation.arg == kwarg][0]
kwargannotation = arg.annotation
else:
kwargannotation = None
args = _ast.arguments(args=args,
defaults=defaults if defaults else [],
kwarg=kwarg,
vararg=vararg,
kw_defaults=kw_defaults,
kwonlyargs=kwonlyargs,
kwargannotation=kwargannotation,
varargannotation=varargannotation,
lineno=lineno, col_offset=0
)
if code.co_name == '<lambda>':
if len(stmnts) == 2:
if isinstance(stmnts[0], _ast.If) and isinstance(stmnts[1], _ast.Return):
assert len(stmnts[0].body) == 1
assert isinstance(stmnts[0].body[0], _ast.Return)
stmnts = [_ast.Return(_ast.IfExp(stmnts[0].test, stmnts[0].body[0].value, stmnts[1].value))]
assert isinstance(stmnts[0], _ast.Return)
stmnt = stmnts[0].value
ast_obj = _ast.Lambda(args=args, body=stmnt, lineno=lineno, col_offset=0)
else:
if instructions.seen_yield:
return_ = stmnts[-1]
assert isinstance(return_, _ast.Return)
assert isinstance(return_.value, _ast.Name)
assert return_.value.id == 'None'
return_.value = None
ast_obj = _ast.FunctionDef(name=code.co_name, args=args,
body=stmnts, decorator_list=[],
returns=returns,
lineno=lineno, col_offset=0)
return ast_obj
def wrap_func(self, func):
if self.func:
raise RuntimeError("func is already set.")
if not inspect.isfunction(func):
raise TypeError("func:%r is not a function or method." % func)
# Get function source code.
#
# HACK: Patch issue 1218234_. If getsource() is called on a
# function, it's module is modified and reloaded afterward, and
# getsource() is called again, then the returned source is the
# cached source that was returned from the first call to
# getsource(). This is due to the caching in the linecache module
# not being tightly coupled with reload().
#
# .. 1218234: http://bugs.python.org/issue1218234
func_file = inspect.getsourcefile(func)
linecache.checkcache(func_file) # HACK: issue 1218234
func_src, lineno = inspect.getsourcelines(func)
# Dedent decorators and function def.
dedent_func_lines(func_src)
func_src = ''.join(func_src)
# Parse function source code to AST.
mod_ast = ast.parse(func_src)
func_ast = mod_ast.body[0]
# Wrap function in an enclosing function to create closure. This is
# so that we can pass along our variables to the template function.
if inspect.ismethod(func):
enc_name = '__pdt_enc_%s_%s_%s' % (func.__module__, (func.im_self or func.im_class).__name__, func.__name__)
else:
enc_name = '__pdt_enc_%s_%s' % (func.__module__, func.__name__)
enc_vars = {
'__pdt_io_factory': self.io_factory,
'__pdt_io_args': self.io_args,
'__pdt_io_kw': self.io_kw
}
# def __pdt_enc_func(...):
mod_ast.body[0] = _ast.FunctionDef(enc_name, _ast.arguments([
_ast.Name('__pdt_io_factory', _ast_param),
_ast.Name('__pdt_io_args', _ast_param),
_ast.Name('__pdt_io_kw', _ast_param)
], None, None, []), [
# def func(...):
# ...
func_ast,
# return func
_ast.Return(_ast.Name(func.__name__, _ast_load))
], [])
# Get template global namespace.
# .. NOTE: This has to be the actual function globals (module dict)
# reference and NOT A COPY.
func_globals = func.__globals__
# Remove our decorator to prevent recursive wrapping. It is safe to
# clear the whole list because decorators before ours have not yet
# been called (but will be), and any decorators after ours would
# have already caused an error with the call to get the function
# source.
func_ast.decorator_list = []
# Add doc string and create template buffer at beginning of
# function.
func_body = []
if self.doc:
func_body.append(
# """..."""
_ast.Expr(_ast.Str(self.doc))
)
func_body += [
# __pdt_buff = __pdt_io_factory(*__pdt_io_args, **__pdt_io_kw)
_ast.Assign([_ast.Name('__pdt_buff', _ast_store)], _ast.Call(_ast.Name('__pdt_io_factory', _ast_load), [], [], _ast.Name('__pdt_io_args', _ast_load), _ast.Name('__pdt_io_kw', _ast_load))),
# __pdt_write = __pdt_buff.write
_ast.Assign([_ast.Name('__pdt_write', _ast_store)], _ast.Attribute(_ast.Name('__pdt_buff', _ast_load), 'write', _ast_load)),
# __pdt_getvalue = __pdt_buff.getvalue
_ast.Assign([_ast.Name('__pdt_getvalue', _ast_store)], _ast.Attribute(_ast.Name('__pdt_buff', _ast_load), 'getvalue', _ast_load))
]
# Wrap expressions to write to buffer.
node_lists = [func_ast.body]
while node_lists:
nodes = node_lists.pop()
for i, node in enumerate(nodes):
if isinstance(node, _ast.Expr):
# expr -> __pdt_write(expr)
nodes[i] = _ast.Expr(_ast.Call(_ast.Name('__pdt_write', _ast_load), [node.value], [], None, None))
elif isinstance(node, _ast.Return):
if not node.value:
# return -> return __pdt_getvalue()
node.value = _ast.Call(_ast.Name('__pdt_getvalue', _ast_load), [], [], None, None)
elif isinstance(node, (_ast.If, _ast.While, _ast.For)):
node_lists += [node.body, node.orelse]
elif isinstance(node, _ast.TryExcept):
node_lists += [node.body, node.orelse, node.handlers]
elif isinstance(node, _ast.TryFinally):
node_lists += [node.body, node.finalbody]
elif isinstance(node, (_ast.ExceptHandler, _ast.With)):
node_lists.append(node.body)
elif isinstance(node, _ast.Yield):
raise TypeError("Generator functions are not supported.")
func_body += func_ast.body
# Return buffer at end of function.
func_body.append(
# return __pdt_getvalue()
_ast.Return(_ast.Call(_ast.Name('__pdt_getvalue', _ast_load), [], [], None, None))
)
func_ast.body = func_body
# XXX
'''
import sys
sys.path.append("../dev")
from astpp import dump
print "LINENO", lineno
'''
# Generate line and column information for modified AST.
ast.fix_missing_locations(mod_ast)
# Fix line numbers.
ast.increment_lineno(mod_ast, lineno - 1)
# XXX
'''
print "%s()" % func.__name__
print dump(mod_ast, True, True, ' ')
'''
# Compile template function.
exec compile(mod_ast, func_file, 'exec') in func_globals
# Store compiled template function.
self.func = func_globals[enc_name](**enc_vars)
del func_globals[enc_name]
def visit_GeneratorExp(self, node):
"""Rewrite the GeneratorExp visitor function to turn the
generator expression into a iterator function. This is
necessary to be able to correctly label any random functions
that get called from within the generator expression.
Basically, this function creates a function, and transforms
the generator into a for loop that yields values from the.
The function name is then returned, so that the parent node
can handle the assignment properly.
"""
# make an identifier for the list
self.newline(node)
iden = self._gen_iden(node)
argids = []
for gen in node.generators:
argval = gen.iter
argid = self._gen_iden(gen.iter)
self.visit(_ast.Assign(
targets=[_ast.Name(id=argid, ctx=_ast.Store())],
value=argval))
argids.append(argid)
elt = node.elt
def parse_generator(nodes, ids):
node = nodes[0]
tempnode = _ast.For()
tempnode.target = node.target
tempnode.iter = _ast.Name(id=ids[0], ctx=_ast.Load())
if len(nodes) == 1:
yield_node = _ast.Expr(value=_ast.Yield(value=elt))
body = [yield_node]
else:
body = [parse_generator(nodes[1:], ids[1:])]
if len(node.ifs) == 1:
ifnode = _ast.If(
test=node.ifs[0],
body=body,
orelse=[])
tempnode.body = [ifnode]
elif len(node.ifs) > 1:
ifnode = _ast.If(
test=_ast.BoolOp(
op=_ast.And(),
values=node.ifs),
body=body,
orelse=[])
tempnode.body = [ifnode]
else:
tempnode.body = body
tempnode.orelse = None
return tempnode
node = _ast.FunctionDef(
name=iden,
args=_ast.arguments(args=[], vararg=None, kwarg=None, defaults=[]),
body=[parse_generator(node.generators, argids)],
decorator_list=[])
self.visit(node)
return iden