def __init__(self, func):
self.ast = decompile_func(func)
if isinstance(self.ast, ast.Lambda):
self.ast = ast.FunctionDef(
name="f",
args=self.ast.args,
body=[ast.Return(value=self.ast.body, lineno=1, col_offset=0)],
lineno=self.ast.lineno,
col_offset=self.ast.col_offset,
decorator_list=[])
def _get_ast(func):
if int(os.environ.get('NUMBA_FORCE_META_AST', 0)):
func_def = decompile_func(func)
assert isinstance(func_def, ast.FunctionDef)
return func_def
try:
source = inspect.getsource(func)
except IOError:
return decompile_func(func)
else:
source = textwrap.dedent(source)
# Split off decorators
decorators = 0
while not source.startswith('def'): # decorator can have multiple lines
decorator, sep, source = source.partition('\n')
decorators += 1
module_ast = ast.parse(source)
# fix line numbering
lineoffset = func.func_code.co_firstlineno + decorators
ast.increment_lineno(module_ast, lineoffset)
assert len(module_ast.body) == 1
func_def = module_ast.body[0]
_fix_ast(func_def)
assert isinstance(func_def, ast.FunctionDef)
return func_def
def _get_ast(func):
if os.environ.get('NUMBA_FORCE_META_AST'):
func_def = decompile_func(func)
assert isinstance(func_def, ast.FunctionDef)
return func_def
try:
source = inspect.getsource(func)
except IOError:
return decompile_func(func)
else:
source = textwrap.dedent(source)
if source.startswith('@'):
decorator, sep, source = source.partition('\n')
while not source.startswith(
'def'): # decorator can have multiple lines
decorator, sep, source = source.partition('\n')
module_ast = ast.parse(source)
# fix line numbering
lineoffset = func.func_code.co_firstlineno
ast.increment_lineno(module_ast, lineoffset)
assert len(module_ast.body) == 1
func_def = module_ast.body[0]
_fix_ast(func_def)
assert isinstance(func_def, ast.FunctionDef)
return func_def
def _get_ast(func):
if os.environ.get('NUMBA_FORCE_META_AST'):
func_def = decompile_func(func)
assert isinstance(func_def, ast.FunctionDef)
return func_def
try:
source = inspect.getsource(func)
except IOError:
return decompile_func(func)
else:
if source.lstrip().startswith('@'):
decorator, sep, source = source.partition('\n')
source = textwrap.dedent(source)
module_ast = ast.parse(source)
# fix line numbering
lineoffset = func.func_code.co_firstlineno + 1
ast.increment_lineno(module_ast, lineoffset)
assert len(module_ast.body) == 1
func_def = module_ast.body[0]
_fix_ast(func_def)
assert isinstance(func_def, ast.FunctionDef)
return func_def
def hash_obj(obj):
"""
return a hash of any python object
"""
from meta.decompiler import decompile_func
import ast
def obj_digest(to_digest):
return hashlib.sha1(dill.dumps(to_digest)).hexdigest()
def iter_digest(to_digest):
return obj_digest(reduce(operator.add, list(map(hash_obj, to_digest))))
if (not isinstance(obj, np.ndarray)) and hasattr(
obj, '__iter__') and (len(obj) > 1):
if isinstance(obj, dict):
return iter_digest(iter(list(obj.items())))
else:
return iter_digest(obj)
else:
# Functions receive special treatment, such that code changes alter
# the hash value
if hasattr(obj, '__call__'):
try:
return obj_digest(ast.dump(decompile_func(obj)))
# This covers an exception that happens in meta.decompiler under
# certain situations. TODO: replace this workaround with something
# better.
except IndexError:
return obj_digest(dill.dumps(obj))
else:
return obj_digest(obj)
def get_ast(fn):
# type: (Callable) -> ast.FunctionDef
"""Return FunctionDef AST node for given function."""
try:
return ast.parse(get_source(fn)).body[0]
except IOError:
return decompile_func(fn)
def hash_obj(obj):
"""
return a hash of any python object
"""
from meta.decompiler import decompile_func
import ast
def obj_digest(to_digest):
return hashlib.sha1(dill.dumps(to_digest)).hexdigest()
def iter_digest(to_digest):
return obj_digest(reduce(operator.add, list(map(hash_obj, to_digest))))
if (not isinstance(obj, np.ndarray)) and hasattr(obj, '__iter__') and (len(obj) > 1):
if isinstance(obj, dict):
return iter_digest(iter(list(obj.items())))
else:
return iter_digest(obj)
else:
# Functions receive special treatment, such that code changes alter
# the hash value
if hasattr(obj, '__call__'):
try:
return obj_digest(ast.dump(decompile_func(obj)))
# This covers an exception that happens in meta.decompiler under
# certain situations. TODO: replace this workaround with something
# better.
except IndexError:
return obj_digest(dill.dumps(obj))
else:
return obj_digest(obj)
def wrap(func, types=types, name=name, prefix=_prefix):
if name is None:
name = func.__name__
decompiled = decompile_func(func)
self.functions[name] = dict(func=func,
prefix=prefix,
ast=decompiled,
types=types,
code=None)
return func
def wrap(func, types=types, name=name, prefix=prefix):
if name is None:
name = func.__name__
decompiled = decompile_func(func)
self.functions[name] = dict(func=func,
prefix=prefix,
ast=decompiled,
types=types,
code=None)
return func
def eval_where(closure):
"""
Given a closure, parse and evaluate it.
Return a WHERE expression.
"""
parser = ParseFilter()
parser.env = make_env(closure)
ast_top = decompile_func(closure)
result = parser.visit(ast_top)
return result
def eval_where(closure):
"""
Given a closure, parse and evaluate it.
Return a WHERE expression.
"""
parser = ParseFilter()
parser.env = make_env(closure)
ast_top = decompile_func(closure)
result = parser.visit(ast_top)
return result
def _get_ast(func, flags=0):
if (int(os.environ.get('NUMBA_FORCE_META_AST', 0))
or func.__name__ == '<lambda>'):
func_def = decompile_func(func)
if isinstance(func_def, ast.Lambda):
func_def = ast.FunctionDef(name='<lambda>',
args=func_def.args,
body=[ast.Return(func_def.body)],
decorator_list=[])
assert isinstance(func_def, ast.FunctionDef)
return func_def
try:
linecache.checkcache(inspect.getsourcefile(func))
source = inspect.getsource(func)
source_module = inspect.getmodule(func)
except IOError:
return decompile_func(func)
else:
# Split off decorators
# TODO: This is not quite correct, we can have comments or strings
# starting at column 0 and an indented function !
source = textwrap.dedent(source)
decorators = 0
while not source.lstrip().startswith(
'def'): # decorator can have multiple lines
assert source
decorator, sep, source = source.partition('\n')
decorators += 1
if (hasattr(source_module, "print_function")
and hasattr(source_module.print_function, "compiler_flag")):
flags |= source_module.print_function.compiler_flag
source_file = getattr(source_module, '__file__', '<unknown file>')
module_ast = compile(source, source_file, "exec",
ast.PyCF_ONLY_AST | flags, True)
lineoffset = func.__code__.co_firstlineno + decorators - 1
ast.increment_lineno(module_ast, lineoffset)
assert len(module_ast.body) == 1
func_def = module_ast.body[0]
_fix_ast(func_def)
assert isinstance(func_def, ast.FunctionDef)
return func_def
def get_ast(func):
if func.__name__ == '<lambda>':
func_def = decompile_func(func)
if isinstance(func_def, Lambda):
func_def = FunctionDef(name='<lambda>',
args=func_def.args,
body=[Return(func_def.body)],
decorator_list=[])
assert isinstance(func_def, FunctionDef)
return func_def
try:
linecache.checkcache(inspect.getsourcefile(func))
source = inspect.getsource(func)
source_module = inspect.getmodule(func)
except IOError:
return decompile_func(func)
else:
# Split off decorators
# TODO: This is not quite correct, we can have comments or strings
# starting at column 0 and an indented function !
source = textwrap.dedent(source)
decorators = 0
# decorator can have multiple lines
while not source.lstrip().startswith('def'):
assert source
decorator, sep, source = source.partition('\n')
decorators += 1
source_file = getattr(source_module, '__file__', '<unknown file>')
module_ast = compile(source, source_file, "exec", PyCF_ONLY_AST, True)
lineoffset = func.__code__.co_firstlineno + decorators - 1
increment_lineno(module_ast, lineoffset)
assert len(module_ast.body) == 1
func_def = module_ast.body[0]
_fix_ast(func_def)
assert isinstance(func_def, FunctionDef)
# remove docstrings (really any unassigned strings)
for node in func_def.body:
if isinstance(node, Expr) and isinstance(node.value, Str):
func_def.body.remove(node)
return func_def
def get_source(self, func, name, types):
c99 = cCompiler(handler=handler.oclsHandler())
types = c99.handler.make_types(types)
decompiled = decompile_func(func)
c99.functions[name] = dict(func=func,
prefix='',
ast=decompiled,
types=types,
code=None)
return c99.convert(headers=False)
def get_source(self, func, name, types):
c99 = cCompiler(handler=handler.oclsHandler())
types = c99.handler.make_types(types)
decompiled = decompile_func(func)
c99.functions[name] = dict(func=func,
prefix='',
ast=decompiled,
types=types,
code=None)
return c99.convert(headers=False)
def eval_where(closure):
"""
Given a closure, parse and evaluate it.
Return a WHERE expression.
See ParseFilter.__doc__ for more information and examples.
"""
parser = ParseFilter()
parser.env = make_env(closure)
ast_top = decompile_func(closure)
result = parser.visit(ast_top)
return result
def _get_ast(func, flags=0):
if int(os.environ.get('NUMBA_FORCE_META_AST', 0)):
func_def = decompile_func(func)
assert isinstance(func_def, ast.FunctionDef)
return func_def
try:
source = inspect.getsource(func)
source_module = inspect.getmodule(func)
except IOError:
return decompile_func(func)
else:
# Split off decorators
# TODO: This is not quite correct, we can have comments or strings
# starting at column 0 and an indented function !
source = textwrap.dedent(source)
decorators = 0
while not source.lstrip().startswith('def'): # decorator can have multiple lines
assert source
decorator, sep, source = source.partition('\n')
decorators += 1
if (hasattr(source_module, "print_function") and
hasattr(source_module.print_function, "compiler_flag")):
flags |= source_module.print_function.compiler_flag
source_file = getattr(source_module, '__file__', '<unknown file>')
module_ast = compile(source, source_file, "exec",
ast.PyCF_ONLY_AST | flags, True)
# fix line numbering
lineoffset = func.__code__.co_firstlineno + decorators
ast.increment_lineno(module_ast, lineoffset)
assert len(module_ast.body) == 1
func_def = module_ast.body[0]
_fix_ast(func_def)
assert isinstance(func_def, ast.FunctionDef)
return func_def
def wrap(func):
nonlocal name,types,_prefix
prefix=_prefix
if name is None:
name = func.__name__
types = self.handler.make_types(types)
if signature:
classToStr={int:"int",float:"float",str:"char*",_empty:""}
params=OrderedDict([(k,(lambda x:classToStr[x] if isclass(x) else x)(v.annotation)) for k,v in signature(func).parameters.items()])#copy.deepcopy presently not working on mappingproxy
params.update(types)
types=params
print(types)
else:pass
types = self.handler.make_types(types)
decompiled = decompile_func(func)
self.functions[name] = dict(func=func,
prefix=prefix,
ast=decompiled,
types=types,
code=None)
return func
def blitz(queue, func, out=None):
'''
lets get blitzed!
'''
func_ast = decompile_func(func)
func_globals = func.func_globals.copy()
if func.func_closure:
func_globals.update({
name: cell.cell_contents
for name, cell in zip(func.func_code.co_freevars,
func.func_closure)
})
blitzer = BlitzVisitor(func.func_code.co_filename, func_globals)
blitzed = ast.Expression(blitzer.visit(func_ast))
blitzed_code = compile(blitzed, func.func_code.co_filename, 'eval')
blitzed_func = eval(blitzed_code)
blitz_kernel = create_n_arg_kernel(sorted(blitzer.locls.keys()))
args = {}
for key, var in blitzer.locls.items():
if not isinstance(var, cl.DeviceMemoryView):
var = cl.from_host(queue.context, var)
args[key] = var
shape = broadcast_shapes([var.shape for var in args.values()])
print "shape", shape
for key, var in args.items():
args[key] = cl.broadcast(var, shape)
print "out, **args", out, args
blitz_kernel(queue, blitzed_func, out, **args)
def call_python_function(self, node, func, args, keywords):
func_ast = decompile_func(func)
argtypes = {}
for keyword in keywords:
argtypes[keyword.arg] = keyword.ctype
for param, arg in zip(func_ast.args.args, args):
argtypes[param.id] = arg.ctype
func_dict = self.function_calls.setdefault(func, {})
hsh = dict2hashable(argtypes)
if hsh not in func_dict:
try:
typed_ast = Typify(func.func_name, argtypes,
func.func_globals).make_cfunction(func_ast)
except CTypeError as err:
argid = err.args[0]
ids = [arg.id for arg in func_ast.args.args]
if argid in ids:
pos = ids.index(argid)
else:
pos = '?'
raise cast.CError(node, TypeError,
err.args[1] + ' at position %s' % (pos))
key = (func, hsh)
plchldr = FuncPlaceHolder(func.func_name, key, typed_ast)
typed_ast.name = plchldr
func_dict[hsh] = typed_ast
else:
typed_ast = func_dict[hsh]
plchldr = typed_ast.name
return cast.CCall(plchldr, args, keywords, typed_ast.return_type)
def call_python_function(self, node, func, args, keywords):
func_ast = decompile_func(func)
argtypes = {}
for keyword in keywords:
argtypes[keyword.arg] = keyword.ctype
for param, arg in zip(func_ast.args.args, args):
argtypes[param.id] = arg.ctype
func_dict = self.function_calls.setdefault(func, {})
hsh = dict2hashable(argtypes)
if hsh not in func_dict:
try:
typed_ast = Typify(func.func_name, argtypes, func.func_globals).make_cfunction(func_ast)
except CTypeError as err:
argid = err.args[0]
ids = [arg.id for arg in func_ast.args.args]
if argid in ids:
pos = ids.index(argid)
else:
pos = '?'
raise cast.CError(node, TypeError, err.args[1] + ' at position %s' % (pos))
key = (func, hsh)
plchldr = FuncPlaceHolder(func.func_name, key, typed_ast)
typed_ast.name = plchldr
func_dict[hsh] = typed_ast
else:
typed_ast = func_dict[hsh]
plchldr = typed_ast.name
return cast.CCall(plchldr, args, keywords, typed_ast.return_type)
def blitz(queue, func, out=None):
'''
lets get blitzed!
'''
func_ast = decompile_func(func)
func_globals = func.func_globals.copy()
if func.func_closure:
func_globals.update({name:cell.cell_contents for name, cell in zip(func.func_code.co_freevars, func.func_closure)})
blitzer = BlitzVisitor(func.func_code.co_filename, func_globals)
blitzed = ast.Expression(blitzer.visit(func_ast))
blitzed_code = compile(blitzed, func.func_code.co_filename, 'eval')
blitzed_func = eval(blitzed_code)
blitz_kernel = create_n_arg_kernel(sorted(blitzer.locls.keys()))
args = {}
for key, var in blitzer.locls.items():
if not isinstance(var, cl.DeviceMemoryView):
var = cl.from_host(queue.context, var)
args[key] = var
shape = broadcast_shapes([var.shape for var in args.values()])
print "shape", shape
for key, var in args.items():
args[key] = cl.broadcast(var, shape)
print "out, **args", out, args
blitz_kernel(queue, blitzed_func, out, **args)
def _get_ast(func):
return decompile_func(func)
# coding:utf-8 import inspect, ast # source = inspect.getsource(func) # tree = ast.parse(source) # pip install from meta.decompiler import decompile_func tree = decompile_func(lambda x: x + 1)
def decompile(fn):
from meta.decompiler import decompile_func
assert isinstance(fn, (FunctionType, LambdaType)), \
'Can only decompilefunction type'
return pformat_ast(decompile_func(fn))
def create_kernel_source(function, argtypes):
'''
Create OpenCL source code from a Python function.
:param function: A pure python function
:param argtypes: A dict of name:type for the compiler to use in optimizing the function.
Steps:
* Decompile to AST.:
Get AST from python bytecode
* Typify AST:
This transforms the AST in to a partially typed OpenCL AST.
It will recursively dive into pure Python functions and add thoes to the OpenCL module.
Function names will be replace with placeholders for the namespace conflict resolution stage.
* Replace Constants:
Replace Python constants with values. e.g. `math.e` -> `2.7182`
* Unpack memobjects:
To support multi-dimensional indexing and non contiguous memory arrays.
CLyther adds a uint8 to the function signature to store this information.
* Replace calls of types to type casts e.g. int(i) -> ((int)(i))
* Format for loops:
only `range` or a explicit Python list is currently supported as the iterator in a for loop.
* Remove arguments to functions that are constant. e.g. python functions.
* Move keyword arguments in function calls to positional arguments.
* Resolve function place-holders
* Make printf statements from print
* Replace Types:
Replace python ctype objects with OpenCL type names.
This will also define structs in the module if required.
* Generate Source
'''
func_ast = decompile_func(function)
globls = function.func_globals
mod_ast, func_ast = typify_function(function.func_name, argtypes, globls, func_ast)
mod_ast = replace_constants(mod_ast)
unpack_mem_args(mod_ast, argtypes)
# convert type calls to casts
# eg int(i) -> ((int) (i))
call_type2type_cast(mod_ast)
format_for_loops(mod_ast)
# Remove arguments to functions that are constant
# eg. functions modules. etc
remove_const_params(mod_ast)
#C/opencl do not accept keword arguments.
#This moves them to positional arguments
move_keywords_to_args(mod_ast)
#typify created function placeholders. resolve them here
resolve_functions(mod_ast)
make_printf(mod_ast)
defaults = function.func_defaults
args = [(arg.id, arg.ctype) for arg in func_ast.args.args]
#replace python type objects with strings
replace_types(mod_ast)
# mod_ast.body.insert(0, ast.Exec(cast.CStr('#pragma OPENCL EXTENSION cl_amd_printf : enable', str), None, None))
#generate source
return args, defaults, opencl_source(mod_ast), func_ast.name
def create_kernel_source(function, argtypes):
'''
Create OpenCL source code from a Python function.
:param function: A pure python function
:param argtypes: A dict of name:type for the compiler to use in optimizing the function.
Steps:
* Decompile to AST.:
Get AST from python bytecode
* Typify AST:
This transforms the AST in to a partially typed OpenCL AST.
It will recursively dive into pure Python functions and add thoes to the OpenCL module.
Function names will be replace with placeholders for the namespace conflict resolution stage.
* Replace Constants:
Replace Python constants with values. e.g. `math.e` -> `2.7182`
* Unpack memobjects:
To support multi-dimensional indexing and non contiguous memory arrays.
CLyther adds a uint8 to the function signature to store this information.
* Replace calls of types to type casts e.g. int(i) -> ((int)(i))
* Format for loops:
only `range` or a explicit Python list is currently supported as the iterator in a for loop.
* Remove arguments to functions that are constant. e.g. python functions.
* Move keyword arguments in function calls to positional arguments.
* Resolve function place-holders
* Make printf statements from print
* Replace Types:
Replace python ctype objects with OpenCL type names.
This will also define structs in the module if required.
* Generate Source
'''
func_ast = decompile_func(function)
globls = function.func_globals
mod_ast, func_ast = typify_function(function.func_name, argtypes, globls,
func_ast)
mod_ast = replace_constants(mod_ast)
unpack_mem_args(mod_ast, argtypes)
# convert type calls to casts
# eg int(i) -> ((int) (i))
call_type2type_cast(mod_ast)
format_for_loops(mod_ast)
# Remove arguments to functions that are constant
# eg. functions modules. etc
remove_const_params(mod_ast)
#C/opencl do not accept keword arguments.
#This moves them to positional arguments
move_keywords_to_args(mod_ast)
#typify created function placeholders. resolve them here
resolve_functions(mod_ast)
make_printf(mod_ast)
defaults = function.func_defaults
args = [(arg.id, arg.ctype) for arg in func_ast.args.args]
#replace python type objects with strings
replace_types(mod_ast)
# mod_ast.body.insert(0, ast.Exec(cast.CStr('#pragma OPENCL EXTENSION cl_amd_printf : enable', str), None, None))
#generate source
return args, defaults, opencl_source(mod_ast), func_ast.name
def decompile(fn):
from meta.decompiler import decompile_func
assert isinstance(fn, (FunctionType, LambdaType)), \
'Can only decompilefunction type'
return pformat_ast(decompile_func(fn))