def merge_kernels(config): """ Creates a shared library named 'bh_libsij.so' containing all the functions defined in the 'kernel_path'. A 'bh_libsij.idx' is also produced containing all the bohrium functions in the '.so' file. """ times = [('start', time.time())] krn_path = config.get('cpu', 'kernel_path') obj_path = config.get('cpu', 'object_path') idx_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.idx") lib_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.so") if not os.path.exists(krn_path): return (None, "kernel_path(%s) does not exist." % krn_path) if not os.path.exists(obj_path): return (None, "obj_path(%s) does not exist." % obj_path) cmd = [c for c in config.get('cpu', 'compiler_cmd').replace('"','').split(' ') if c] + [lib_path] symbols = [] # Find the source-files sources = [] files = [] for fn in glob.glob("%s%sKRN_*.c" % (krn_path, os.sep)): m = re.match('.*KRN_(\d+)_([a-zA-Z0-9]{6}).c', fn) if m: symbol, instance = m.groups() if symbol not in symbols: # Ignore duplicates sources.append(open(fn, 'r').read()) symbols.append(symbol) files.append(fn) source = "\n".join(sources) # Compile them times.append(('merged', time.time())) p = subprocess.Popen( cmd, stdin = subprocess.PIPE, stdout = subprocess.PIPE ) out, err = p.communicate(input=source) times.append(('compiled', time.time())) with open(idx_path, 'w+') as fd: # Create the index-file symbols.sort() fd.write("\n".join(symbols)) times.append(('done', time.time())) bhutils.print_timings(times) return (out, err)
def merge_kernels(config): """ Creates a shared library named 'bh_libsij.so' containing all the functions defined in the 'kernel_path'. A 'bh_libsij.idx' is also produced containing all the bohrium functions in the '.so' file. """ times = [('start', time.time())] krn_path = config.get('cpu', 'kernel_path') obj_path = config.get('cpu', 'object_path') idx_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.idx") lib_path = "%s%s%s" % (obj_path, os.sep, "LIB_libsij_aaaaaa.so") if not os.path.exists(krn_path): return (None, "kernel_path(%s) does not exist." % krn_path) if not os.path.exists(obj_path): return (None, "obj_path(%s) does not exist." % obj_path) cmd = [ c for c in config.get('cpu', 'compiler_cmd').replace('"', '').split(' ') if c ] + [lib_path] symbols = [] # Find the source-files sources = [] files = [] for fn in glob.glob("%s%sKRN_*.c" % (krn_path, os.sep)): m = re.match('.*KRN_(\d+)_([a-zA-Z0-9]{6}).c', fn) if m: symbol, instance = m.groups() if symbol not in symbols: # Ignore duplicates sources.append(open(fn, 'r').read()) symbols.append(symbol) files.append(fn) source = "\n".join(sources) # Compile them times.append(('merged', time.time())) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE) out, err = p.communicate(input=source) times.append(('compiled', time.time())) with open(idx_path, 'w+') as fd: # Create the index-file symbols.sort() fd.write("\n".join(symbols)) times.append(('done', time.time())) bhutils.print_timings(times) return (out, err)
def genesis(bytecodes, types): times=[('start', time.time())] # Why this? Well because it is always fun to know # how long it took to do everything in the # world of bytecode through the ether of Python/NumPy # 1) Grab Bohrium/NumPy (np, flush) = bhutils.import_bohrium() def copy_operands(source, destination): destination[:] = source destination[:] = 1 destination[:] = True def wrap_real(source, destination): destination[:] = np.real(source) def wrap_imag(source, destination): destination[:] = np.imag(source) times.append(('import', time.time())) dimensions = [1,2,3,4] # Filter out the unknown type types = [t for t in types if t['enum'] not in suppress_types] # Filter out system opcodes bytecodes = [bytecode for bytecode in bytecodes if not bytecode['system_opcode']] # Get a map from enum to numpy type typemap = dict([(t['enum'], eval("np.{0}".format(t['numpy']))) for t in types]) type_sh = dict([(t['enum'], t['shorthand']) for t in types]) # # Setup operands of every type, layout and dimensions 1-4 # operands = {} # Create operands for t in types: tn = t['enum'] if tn not in operands: operands[tn] = {} for ndim in dimensions: # Of different dimenions if ndim not in operands[tn]: operands[tn][ndim] = {} for op in [0,1,2]: # Create three operands[tn][ndim][op] = { # Of different layout 'C': np.ones([3]*ndim, dtype = typemap[tn] ), 'S': np.ones(pow(3,ndim)*2, dtype = typemap[tn])[::2].reshape([3]*ndim), 'K': typemap[tn](3) } times.append(('setup', time.time())) earth = [] # Flatten bytecode for bytecode in (bytecode for bytecode in bytecodes): opcode = bytecode['opcode'] if "BH_RANDOM" == opcode: # Hardcoded specialcase for BH_RANDOM for layout in bytecode["layout"]: earth.append([opcode, ["BH_FLOAT32"], layout]) earth.append([opcode, ["BH_FLOAT64"], layout]) else: for typesig in bytecode['types']: for layout in bytecode['layout']: earth.append([opcode, typesig, layout]) # # Persist the flattened bytecode # with tempfile.NamedTemporaryFile(delete=False) as fd: for opcode, typesig, layout in earth: bytecode_str = "%s_%s_%s\n" % ( opcode, ''.join([type_sh[t] for t in typesig]), ''.join(layout) ) fd.write(bytecode_str) print "When done", len(earth), "kernels should be ready in kernel-path." print "See the list of function-names in the file [%s]" % fd.name times.append(('flatten', time.time())) # # Execute it # for opcode, typesig, layout in earth: func = eval(numpy_map[opcode]) # Grab the NumPy functions # Ignore functions with signatures containing ignored types broken = len([t for t in typesig if t in ignore_types])>0 if broken: continue for ndim in [1,2,3,4]: # Setup operands if "BH_RANGE" in opcode: # Specialcases op_setup = [ 1, 10, 1, typemap[typesig[0]], True ] elif "BH_RANDOM" in opcode: op_setup = [ [3]*ndim, typemap[typesig[0]], True ] elif "_REDUCE" in opcode: ndim = 2 if ndim == 1 else ndim op_setup = [ operands[typesig[1]][ndim][1][layout[1]], 0, typemap[typesig[0]], operands[typesig[0]][ndim-1][0][layout[0]] ] elif "_ACCUMULATE" in opcode: op_setup = [ operands[typesig[1]][ndim][1][layout[1]], 0, typemap[typesig[0]], operands[typesig[0]][ndim][1][layout[1]] ] else: if len(typesig) == 3: op_setup = [ operands[typesig[1]][ndim][1][layout[1]], operands[typesig[2]][ndim][2][layout[2]], operands[typesig[0]][ndim][0][layout[0]] ] elif len(typesig) == 2: op_setup = [ operands[typesig[1]][ndim][1][layout[1]], operands[typesig[0]][ndim][0][layout[0]] ] elif len(typesig) == 1: op_setup = [ operands[typesig[0]][ndim][0][layout[0]] ] else: print "WTF!" try: flush() func(*op_setup) flush() except Exception as e: print "Error when executing: %s {%s}_%s, err[%s]." % ( opcode, ','.join(typesig), ''.join(layout), e ) times.append(('execute', time.time())) bhutils.print_timings(times) print "Run 'bohrium --merge_kernels' to create a stand-alone library." return (None, None)
def genesis(bytecodes, types): times = [('start', time.time()) ] # Why this? Well because it is always fun to know # how long it took to do everything in the # world of bytecode through the ether of Python/NumPy # 1) Grab Bohrium/NumPy (np, flush) = bhutils.import_bohrium() def copy_operands(source, destination): destination[:] = source destination[:] = 1 destination[:] = True def wrap_real(source, destination): destination[:] = np.real(source) def wrap_imag(source, destination): destination[:] = np.imag(source) times.append(('import', time.time())) dimensions = [1, 2, 3, 4] # Filter out the unknown type types = [t for t in types if t['enum'] not in suppress_types] # Filter out system opcodes bytecodes = [ bytecode for bytecode in bytecodes if not bytecode['system_opcode'] ] # Get a map from enum to numpy type typemap = dict([(t['enum'], eval("np.{0}".format(t['numpy']))) for t in types]) type_sh = dict([(t['enum'], t['shorthand']) for t in types]) # # Setup operands of every type, layout and dimensions 1-4 # operands = {} # Create operands for t in types: tn = t['enum'] if tn not in operands: operands[tn] = {} for ndim in dimensions: # Of different dimenions if ndim not in operands[tn]: operands[tn][ndim] = {} for op in [0, 1, 2]: # Create three operands[tn][ndim][op] = { # Of different layout 'C': np.ones([3] * ndim, dtype=typemap[tn]), 'S': np.ones(pow(3, ndim) * 2, dtype=typemap[tn])[::2].reshape([3] * ndim), 'K': typemap[tn](3) } times.append(('setup', time.time())) earth = [] # Flatten bytecode for bytecode in (bytecode for bytecode in bytecodes): opcode = bytecode['opcode'] if "BH_RANDOM" == opcode: # Hardcoded specialcase for BH_RANDOM for layout in bytecode["layout"]: earth.append([opcode, ["BH_FLOAT32"], layout]) earth.append([opcode, ["BH_FLOAT64"], layout]) else: for typesig in bytecode['types']: for layout in bytecode['layout']: earth.append([opcode, typesig, layout]) # # Persist the flattened bytecode # with tempfile.NamedTemporaryFile(delete=False) as fd: for opcode, typesig, layout in earth: bytecode_str = "%s_%s_%s\n" % (opcode, ''.join( [type_sh[t] for t in typesig]), ''.join(layout)) fd.write(bytecode_str) print "When done", len( earth), "kernels should be ready in kernel-path." print "See the list of function-names in the file [%s]" % fd.name times.append(('flatten', time.time())) # # Execute it # for opcode, typesig, layout in earth: func = eval(numpy_map[opcode]) # Grab the NumPy functions # Ignore functions with signatures containing ignored types broken = len([t for t in typesig if t in ignore_types]) > 0 if broken: continue for ndim in [1, 2, 3, 4]: # Setup operands if "BH_RANGE" in opcode: # Specialcases op_setup = [1, 10, 1, typemap[typesig[0]], True] elif "BH_RANDOM" in opcode: op_setup = [[3] * ndim, typemap[typesig[0]], True] elif "_REDUCE" in opcode: ndim = 2 if ndim == 1 else ndim op_setup = [ operands[typesig[1]][ndim][1][layout[1]], 0, typemap[typesig[0]], operands[typesig[0]][ndim - 1][0][layout[0]] ] elif "_ACCUMULATE" in opcode: op_setup = [ operands[typesig[1]][ndim][1][layout[1]], 0, typemap[typesig[0]], operands[typesig[0]][ndim][1][layout[1]] ] else: if len(typesig) == 3: op_setup = [ operands[typesig[1]][ndim][1][layout[1]], operands[typesig[2]][ndim][2][layout[2]], operands[typesig[0]][ndim][0][layout[0]] ] elif len(typesig) == 2: op_setup = [ operands[typesig[1]][ndim][1][layout[1]], operands[typesig[0]][ndim][0][layout[0]] ] elif len(typesig) == 1: op_setup = [operands[typesig[0]][ndim][0][layout[0]]] else: print "WTF!" try: flush() func(*op_setup) flush() except Exception as e: print "Error when executing: %s {%s}_%s, err[%s]." % ( opcode, ','.join(typesig), ''.join(layout), e) times.append(('execute', time.time())) bhutils.print_timings(times) print "Run 'bohrium --merge_kernels' to create a stand-alone library." return (None, None)