def merge_kernels(config):
"""
Creates a shared library named 'bh_libsij.so' containing
all the functions defined in the 'kernel_path'.
A 'bh_libsij.idx' is also produced containing all the bohrium
functions in the '.so' file.
"""
times = [('start', time.time())]
krn_path = config.get('cpu', 'kernel_path')
obj_path = config.get('cpu', 'object_path')
idx_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.idx")
lib_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.so")
if not os.path.exists(krn_path):
return (None, "kernel_path(%s) does not exist." % krn_path)
if not os.path.exists(obj_path):
return (None, "obj_path(%s) does not exist." % obj_path)
cmd = [c for c in config.get('cpu',
'compiler_cmd').replace('"','').split(' ')
if c] + [lib_path]
symbols = [] # Find the source-files
sources = []
files = []
for fn in glob.glob("%s%sKRN_*.c" % (krn_path, os.sep)):
m = re.match('.*KRN_(\d+)_([a-zA-Z0-9]{6}).c', fn)
if m:
symbol, instance = m.groups()
if symbol not in symbols: # Ignore duplicates
sources.append(open(fn, 'r').read())
symbols.append(symbol)
files.append(fn)
source = "\n".join(sources) # Compile them
times.append(('merged', time.time()))
p = subprocess.Popen(
cmd,
stdin = subprocess.PIPE,
stdout = subprocess.PIPE
)
out, err = p.communicate(input=source)
times.append(('compiled', time.time()))
with open(idx_path, 'w+') as fd: # Create the index-file
symbols.sort()
fd.write("\n".join(symbols))
times.append(('done', time.time()))
bhutils.print_timings(times)
return (out, err)
def merge_kernels(config):
"""
Creates a shared library named 'bh_libsij.so' containing
all the functions defined in the 'kernel_path'.
A 'bh_libsij.idx' is also produced containing all the bohrium
functions in the '.so' file.
"""
times = [('start', time.time())]
krn_path = config.get('cpu', 'kernel_path')
obj_path = config.get('cpu', 'object_path')
idx_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.idx")
lib_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.so")
if not os.path.exists(krn_path):
return (None, "kernel_path(%s) does not exist." % krn_path)
if not os.path.exists(obj_path):
return (None, "obj_path(%s) does not exist." % obj_path)
cmd = [
c
for c in config.get('cpu', 'compiler_cmd').replace('"', '').split(' ')
if c
] + [lib_path]
symbols = [] # Find the source-files
sources = []
files = []
for fn in glob.glob("%s%sKRN_*.c" % (krn_path, os.sep)):
m = re.match('.*KRN_(\d+)_([a-zA-Z0-9]{6}).c', fn)
if m:
symbol, instance = m.groups()
if symbol not in symbols: # Ignore duplicates
sources.append(open(fn, 'r').read())
symbols.append(symbol)
files.append(fn)
source = "\n".join(sources) # Compile them
times.append(('merged', time.time()))
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
out, err = p.communicate(input=source)
times.append(('compiled', time.time()))
with open(idx_path, 'w+') as fd: # Create the index-file
symbols.sort()
fd.write("\n".join(symbols))
times.append(('done', time.time()))
bhutils.print_timings(times)
return (out, err)
def genesis(bytecodes, types):
times=[('start', time.time())] # Why this? Well because it is always fun to know
# how long it took to do everything in the
# world of bytecode through the ether of Python/NumPy
# 1) Grab Bohrium/NumPy
(np, flush) = bhutils.import_bohrium()
def copy_operands(source, destination):
destination[:] = source
destination[:] = 1
destination[:] = True
def wrap_real(source, destination):
destination[:] = np.real(source)
def wrap_imag(source, destination):
destination[:] = np.imag(source)
times.append(('import', time.time()))
dimensions = [1,2,3,4]
# Filter out the unknown type
types = [t for t in types if t['enum'] not in suppress_types]
# Filter out system opcodes
bytecodes = [bytecode for bytecode in bytecodes if not bytecode['system_opcode']]
# Get a map from enum to numpy type
typemap = dict([(t['enum'], eval("np.{0}".format(t['numpy']))) for t in types])
type_sh = dict([(t['enum'], t['shorthand']) for t in types])
#
# Setup operands of every type, layout and dimensions 1-4
#
operands = {} # Create operands
for t in types:
tn = t['enum']
if tn not in operands:
operands[tn] = {}
for ndim in dimensions: # Of different dimenions
if ndim not in operands[tn]:
operands[tn][ndim] = {}
for op in [0,1,2]: # Create three
operands[tn][ndim][op] = { # Of different layout
'C': np.ones([3]*ndim, dtype = typemap[tn] ),
'S': np.ones(pow(3,ndim)*2, dtype = typemap[tn])[::2].reshape([3]*ndim),
'K': typemap[tn](3)
}
times.append(('setup', time.time()))
earth = [] # Flatten bytecode
for bytecode in (bytecode for bytecode in bytecodes):
opcode = bytecode['opcode']
if "BH_RANDOM" == opcode: # Hardcoded specialcase for BH_RANDOM
for layout in bytecode["layout"]:
earth.append([opcode, ["BH_FLOAT32"], layout])
earth.append([opcode, ["BH_FLOAT64"], layout])
else:
for typesig in bytecode['types']:
for layout in bytecode['layout']:
earth.append([opcode, typesig, layout])
#
# Persist the flattened bytecode
#
with tempfile.NamedTemporaryFile(delete=False) as fd:
for opcode, typesig, layout in earth:
bytecode_str = "%s_%s_%s\n" % (
opcode, ''.join([type_sh[t] for t in typesig]), ''.join(layout)
)
fd.write(bytecode_str)
print "When done", len(earth), "kernels should be ready in kernel-path."
print "See the list of function-names in the file [%s]" % fd.name
times.append(('flatten', time.time()))
#
# Execute it
#
for opcode, typesig, layout in earth:
func = eval(numpy_map[opcode]) # Grab the NumPy functions
# Ignore functions with signatures containing ignored types
broken = len([t for t in typesig if t in ignore_types])>0
if broken:
continue
for ndim in [1,2,3,4]: # Setup operands
if "BH_RANGE" in opcode: # Specialcases
op_setup = [
1, 10, 1,
typemap[typesig[0]],
True
]
elif "BH_RANDOM" in opcode:
op_setup = [
[3]*ndim,
typemap[typesig[0]],
True
]
elif "_REDUCE" in opcode:
ndim = 2 if ndim == 1 else ndim
op_setup = [
operands[typesig[1]][ndim][1][layout[1]],
0,
typemap[typesig[0]],
operands[typesig[0]][ndim-1][0][layout[0]]
]
elif "_ACCUMULATE" in opcode:
op_setup = [
operands[typesig[1]][ndim][1][layout[1]],
0,
typemap[typesig[0]],
operands[typesig[0]][ndim][1][layout[1]]
]
else:
if len(typesig) == 3:
op_setup = [
operands[typesig[1]][ndim][1][layout[1]],
operands[typesig[2]][ndim][2][layout[2]],
operands[typesig[0]][ndim][0][layout[0]]
]
elif len(typesig) == 2:
op_setup = [
operands[typesig[1]][ndim][1][layout[1]],
operands[typesig[0]][ndim][0][layout[0]]
]
elif len(typesig) == 1:
op_setup = [
operands[typesig[0]][ndim][0][layout[0]]
]
else:
print "WTF!"
try:
flush()
func(*op_setup)
flush()
except Exception as e:
print "Error when executing: %s {%s}_%s, err[%s]." % (
opcode, ','.join(typesig), ''.join(layout), e
)
times.append(('execute', time.time()))
bhutils.print_timings(times)
print "Run 'bohrium --merge_kernels' to create a stand-alone library."
return (None, None)
def genesis(bytecodes, types):
times = [('start', time.time())
] # Why this? Well because it is always fun to know
# how long it took to do everything in the
# world of bytecode through the ether of Python/NumPy
# 1) Grab Bohrium/NumPy
(np, flush) = bhutils.import_bohrium()
def copy_operands(source, destination):
destination[:] = source
destination[:] = 1
destination[:] = True
def wrap_real(source, destination):
destination[:] = np.real(source)
def wrap_imag(source, destination):
destination[:] = np.imag(source)
times.append(('import', time.time()))
dimensions = [1, 2, 3, 4]
# Filter out the unknown type
types = [t for t in types if t['enum'] not in suppress_types]
# Filter out system opcodes
bytecodes = [
bytecode for bytecode in bytecodes if not bytecode['system_opcode']
]
# Get a map from enum to numpy type
typemap = dict([(t['enum'], eval("np.{0}".format(t['numpy'])))
for t in types])
type_sh = dict([(t['enum'], t['shorthand']) for t in types])
#
# Setup operands of every type, layout and dimensions 1-4
#
operands = {} # Create operands
for t in types:
tn = t['enum']
if tn not in operands:
operands[tn] = {}
for ndim in dimensions: # Of different dimenions
if ndim not in operands[tn]:
operands[tn][ndim] = {}
for op in [0, 1, 2]: # Create three
operands[tn][ndim][op] = { # Of different layout
'C':
np.ones([3] * ndim, dtype=typemap[tn]),
'S':
np.ones(pow(3, ndim) * 2,
dtype=typemap[tn])[::2].reshape([3] * ndim),
'K':
typemap[tn](3)
}
times.append(('setup', time.time()))
earth = [] # Flatten bytecode
for bytecode in (bytecode for bytecode in bytecodes):
opcode = bytecode['opcode']
if "BH_RANDOM" == opcode: # Hardcoded specialcase for BH_RANDOM
for layout in bytecode["layout"]:
earth.append([opcode, ["BH_FLOAT32"], layout])
earth.append([opcode, ["BH_FLOAT64"], layout])
else:
for typesig in bytecode['types']:
for layout in bytecode['layout']:
earth.append([opcode, typesig, layout])
#
# Persist the flattened bytecode
#
with tempfile.NamedTemporaryFile(delete=False) as fd:
for opcode, typesig, layout in earth:
bytecode_str = "%s_%s_%s\n" % (opcode, ''.join(
[type_sh[t] for t in typesig]), ''.join(layout))
fd.write(bytecode_str)
print "When done", len(
earth), "kernels should be ready in kernel-path."
print "See the list of function-names in the file [%s]" % fd.name
times.append(('flatten', time.time()))
#
# Execute it
#
for opcode, typesig, layout in earth:
func = eval(numpy_map[opcode]) # Grab the NumPy functions
# Ignore functions with signatures containing ignored types
broken = len([t for t in typesig if t in ignore_types]) > 0
if broken:
continue
for ndim in [1, 2, 3, 4]: # Setup operands
if "BH_RANGE" in opcode: # Specialcases
op_setup = [1, 10, 1, typemap[typesig[0]], True]
elif "BH_RANDOM" in opcode:
op_setup = [[3] * ndim, typemap[typesig[0]], True]
elif "_REDUCE" in opcode:
ndim = 2 if ndim == 1 else ndim
op_setup = [
operands[typesig[1]][ndim][1][layout[1]], 0,
typemap[typesig[0]],
operands[typesig[0]][ndim - 1][0][layout[0]]
]
elif "_ACCUMULATE" in opcode:
op_setup = [
operands[typesig[1]][ndim][1][layout[1]], 0,
typemap[typesig[0]],
operands[typesig[0]][ndim][1][layout[1]]
]
else:
if len(typesig) == 3:
op_setup = [
operands[typesig[1]][ndim][1][layout[1]],
operands[typesig[2]][ndim][2][layout[2]],
operands[typesig[0]][ndim][0][layout[0]]
]
elif len(typesig) == 2:
op_setup = [
operands[typesig[1]][ndim][1][layout[1]],
operands[typesig[0]][ndim][0][layout[0]]
]
elif len(typesig) == 1:
op_setup = [operands[typesig[0]][ndim][0][layout[0]]]
else:
print "WTF!"
try:
flush()
func(*op_setup)
flush()
except Exception as e:
print "Error when executing: %s {%s}_%s, err[%s]." % (
opcode, ','.join(typesig), ''.join(layout), e)
times.append(('execute', time.time()))
bhutils.print_timings(times)
print "Run 'bohrium --merge_kernels' to create a stand-alone library."
return (None, None)
