def exec_weld(values, ty, threads, weld_conf):
file_path = '%s.weld' % ty
weld_code = None
with open(file_path, 'r') as content_file:
weld_code = content_file.read()
enc = NumpyArrayEncoder()
names = ['x', 'y']
argtypes = [enc.py_to_weld_type(x).ctype_class for x in values]
encoded = [enc.encode(x) for x in values]
Args = args_factory(zip(names, argtypes))
weld_args = Args()
for name, value in zip(names, encoded):
setattr(weld_args, name, value)
void_ptr = ctypes.cast(ctypes.byref(weld_args), ctypes.c_void_p)
arg = cweld.WeldValue(void_ptr)
# Compile the module
err = cweld.WeldError()
conf = cweld.WeldConf()
conf.set("weld.threads", str(threads))
conf.set("weld.memory.limit", "20000000000")
if weld_conf is not None:
for key, val in weld_conf.iteritems():
conf.set(key, val)
comp_start = timer()
module = cweld.WeldModule(weld_code, conf, err)
comp_time = timer() - comp_start
if err.code() != 0:
raise ValueError("Could not compile function {}: {}".format(
weld_code, err.message()))
# Run the module
dec = ScalarDecoder()
restype = WeldVec(WeldInt())
err = cweld.WeldError()
exec_start = timer()
weld_ret = module.run(conf, arg, err)
exec_time = timer() - exec_start
if err.code() != 0:
raise ValueError(("Error while running function,\n{}\n\n"
"Error message: {}").format(
weld_code, err.message()))
ptrtype = POINTER(restype.ctype_class)
data = ctypes.cast(weld_ret.data(), ptrtype)
result = None#dec.decode(data, restype)
weld_ret.free()
arg.free()
return (result, comp_time, exec_time)
def join_weld(values, adaptive, lazy, threads, weld_conf):
weld_code = None
with open('join.weld', 'r') as content_file:
weld_code = content_file.read()
enc = NumpyArrayEncoder()
names = ['Ra', 'Rz', 'Sa', 'Sb', 'Tb', 'Tc', 'Uc', 'Ud']
argtypes = [enc.py_to_weld_type(x).ctype_class for x in values]
encoded = [enc.encode(x) for x in values]
Args = args_factory(zip(names, argtypes))
weld_args = Args()
for name, value in zip(names, encoded):
setattr(weld_args, name, value)
void_ptr = ctypes.cast(ctypes.byref(weld_args), ctypes.c_void_p)
arg = cweld.WeldValue(void_ptr)
# Compile the module
err = cweld.WeldError()
conf = cweld.WeldConf()
conf.set("weld.optimization.applyAdaptiveTransforms", "true" if adaptive else "false")
conf.set("weld.adaptive.lazyCompilation", "true" if lazy else "false")
conf.set("weld.threads", str(threads))
conf.set("weld.memory.limit", "20000000000")
if weld_conf is not None:
for key, val in weld_conf:
conf.set(key, val)
comp_start = timer()
module = cweld.WeldModule(weld_code, conf, err)
comp_time = timer() - comp_start
if err.code() != 0:
raise ValueError("Could not compile function {}: {}".format(
weld_code, err.message()))
# Run the module
dec = ScalarDecoder()
restype = WeldLong()
err = cweld.WeldError()
exec_start = timer()
weld_ret = module.run(conf, arg, err)
exec_time = timer() - exec_start
if err.code() != 0:
raise ValueError(("Error while running function,\n{}\n\n"
"Error message: {}").format(
weld_code, err.message()))
ptrtype = POINTER(restype.ctype_class)
data = ctypes.cast(weld_ret.data(), ptrtype)
result = dec.decode(data, restype)
weld_ret.free()
arg.free()
return (result, comp_time, exec_time)
def _gen_weldobj(self, arr):
'''
Generating a new weldarray from a given arr for self.
@arr: weldarray or ndarray.
- weldarray: Just update the weldobject with the context from the
weldarray.
- ndarray: Add the given array to the context of the weldobject.
Sets self.name and self.weldobj.
'''
self.weldobj = WeldObject(NumpyArrayEncoder(), NumpyArrayDecoder())
if isinstance(arr, weldarray):
self.weldobj.update(arr.weldobj)
self.weldobj.weld_code = arr.weldobj.weld_code
self.name = arr.name
else:
# general case for arr being numpy scalar or ndarray
# weldobj returns the name bound to the given array. That is also
# the array that future ops will act on, so set weld_code to it.
self.name = self.weldobj.weld_code = self.weldobj.update(
arr, SUPPORTED_DTYPES[str(arr.dtype)])
from weld.weldobject import *
from weld.types import *
from weld.encoders import NumpyArrayEncoder, NumpyArrayDecoder, ScalarDecoder
_encoder = NumpyArrayEncoder()
_decoder = NumpyArrayDecoder()
class HelloWeldVector(object):
def __init__(self, vector):
self.vector = vector
self.weldobj = WeldObject(_encoder, _decoder)
name = self.weldobj.update(vector, WeldVec(WeldInt()))
self.weldobj.weld_code = name
self.cached = None
def add(self, number):
self.cached = None
template = "map({0}, |e| e + {1})"
self.weldobj.weld_code = template.format(self.weldobj.weld_code,
str(number))
return self
def sum(self):
self.cached = None
template = "result(for({0}, merger[i64,+], |b,i,e| merge(b, i64(e))))"
prev_code = self.weldobj.weld_code
self.weldobj.weld_code = template.format(self.weldobj.weld_code)
self.weldobj.decoder = ScalarDecoder()
result = self.weldobj.evaluate(WeldLong(), verbose=False)
self.weldobj.decoder = _decoder
def benchmark(data, type, threads, weld_conf):
adaptive = type == 'Adaptive'
lazy = False
code_path = None
if (type == 'Filter->Map1'):
code_path = 'filter_then_map.weld'
elif (type == 'Map->Filter'):
code_path = 'map_then_filter.weld'
else:
code_path = 'fused.weld'
weld_code = None
with open(code_path, 'r') as content_file:
weld_code = content_file.read()
enc = NumpyArrayEncoder()
names = [c for c in sorted(data)]
argtypes = [enc.py_to_weld_type(data[c]).ctype_class for c in names]
encoded = [enc.encode(data[c]) for c in names]
Args = args_factory(zip(names, argtypes))
weld_args = Args()
for name, value in zip(names, encoded):
setattr(weld_args, name, value)
void_ptr = ctypes.cast(ctypes.byref(weld_args), ctypes.c_void_p)
arg = cweld.WeldValue(void_ptr)
# Compile the module
err = cweld.WeldError()
conf = cweld.WeldConf()
conf.set("weld.optimization.applyAdaptiveTransforms",
"true" if adaptive else "false")
conf.set("weld.adaptive.lazyCompilation", "true" if lazy else "false")
conf.set("weld.threads", str(threads))
conf.set("weld.memory.limit", "30000000000")
if type == 'Map->Filter' or type == 'Filter->Map1' or type == 'Filter->Map2':
conf.set(
'weld.optimization.passes',
'unroll-static-loop,infer-size,adapt-reorder-filter-projection,adapt-bloomfilter,adapt-predicate,short-circuit-booleans,predicate,vectorize,fix-iterate'
)
if weld_conf is not None:
for key, val in weld_conf.iteritems():
conf.set(key, val)
comp_start = timer()
module = cweld.WeldModule(weld_code, conf, err)
comp_time = timer() - comp_start
if err.code() != 0:
raise ValueError("Could not compile function {}: {}".format(
weld_code, err.message()))
# Run the module
dec = ScalarDecoder()
restype = WeldLong()
err = cweld.WeldError()
exec_start = timer()
weld_ret = module.run(conf, arg, err)
exec_time = timer() - exec_start
if err.code() != 0:
raise ValueError(("Error while running function,\n{}\n\n"
"Error message: {}").format(weld_code,
err.message()))
ptrtype = POINTER(restype.ctype_class)
data = ctypes.cast(weld_ret.data(), ptrtype)
result = dec.decode(data, restype)
weld_ret.free()
arg.free()
return (result, comp_time, exec_time)
def __init__(self, value, weldty):
self.value = value
self.weldty = weldty
self.weldobj = WeldObject(NumpyArrayEncoder(), NumpyArrayDecoder())
name = self.weldobj.update(value, WeldInt)