def get_return_package(function_name, argument_package_list, work_range, output_halo): data_package = Data_package() def get_unique_id(): global unique_id unique_id += 1 return unique_id data_package.unique_id = get_unique_id() data_package.data_dtype = numpy.ndarray data_package.data_halo = output_halo def get_return_dtype(function_name, argument_package_list): from Vivaldi_translator_layer import get_return_dtype function_code = function_code_dict[function_name] return_dtype = get_return_dtype(function_name, argument_package_list, function_code) if return_dtype.endswith('_volume'): print "Vivaldi_warning" print "---------------------------------" print "Check your function" print "you are trying to return a volume" print "return_dtype: ", return_dtype print "---------------------------------" return return_dtype return_dtype = get_return_dtype(function_name, argument_package_list) data_package.set_data_contents_dtype(return_dtype) data_package.set_full_data_range(work_range) data_package.set_data_range(work_range) data_package.halo = output_halo data_package.split = output_split data_package.shared = True return data_package
def load_data_2d(file_full_name, dtype=None, out_of_core=False): dtype = python_dtype_to_Vivaldi_dtype(dtype) file_name, extension = split_file_name_and_extension(file_full_name) el = extension.lower() if out_of_core: if extension == 'dat': file_info = matrixio.read_dat(filename, out_of_core=True) file_name = file_info['FileName'] assert (file_name != None) if 'Path' in file_info: # user defined path path = file_info['Path'] else: # default is find same folder idx = file_full_name.rfind('/') path = file_full_name[:idx + 1] shape = file_info['Resolution'].split() shape[0], shape[1] = int(shape[1]), int(shape[0]) contents_dtype = file_info['Format'].lower() chan = 1 temp = Data_package() temp.out_of_core = True temp.data_dtype = numpy.ndarray temp.file_name = path + file_name temp.file_dtype = python_dtype_to_Vivaldi_dtype(contents_dtype) temp.buffer_dtype = numpy.ndarray if dtype != None: dtype = dtype else: dtype = contents_dtype if chan != 1: dtype += str(chan) temp.set_buffer_contents_dtype(dtype) data_range = shape_to_range(shape) temp.set_data_range(data_range) temp.set_buffer_range(data_range) temp.set_full_buffer_range(data_range) return temp else: st = time.time() data = matrixio.read_2d_data(file_full_name) dtype = Vivaldi_dtype_to_python_dtype(dtype) if dtype != None: data = data.astype(dtype) else: dtype = data.dtype print 'load_data_2d_rgb:', file_full_name, 'dtype:', data.dtype, 'shape:', data.shape, 'loading time:', 1000 * ( time.time() - st), 'ms' return data return None
def load_data_2d(file_full_name, dtype=None, out_of_core=False): dtype = python_dtype_to_Vivaldi_dtype(dtype) file_name, extension = split_file_name_and_extension(file_full_name) el = extension.lower() if out_of_core: if extension == 'dat': file_info = matrixio.read_dat(file_full_name, out_of_core=True) file_name = file_info['FileName'] assert(file_name != None) if 'Path' in file_info: # user defined path path = file_info['Path'] else: # default is find same folder idx = file_full_name.rfind('/') path = file_full_name[:idx+1] shape = file_info['Resolution'].split() shape[0],shape[1] = int(shape[1]),int(shape[0]) contents_dtype = file_info['Format'].lower() chan = 1 temp = Data_package() temp.out_of_core = True temp.data_dtype = numpy.ndarray temp.file_name = path + file_name temp.file_dtype = python_dtype_to_Vivaldi_dtype(contents_dtype) temp.buffer_dtype = numpy.ndarray if dtype != None: dtype = dtype else: dtype = contents_dtype if chan != 1: dtype += str(chan) temp.set_buffer_contents_dtype(dtype) data_range = shape_to_range(shape) temp.set_data_range(data_range) temp.set_buffer_range(data_range) temp.set_full_buffer_range(data_range) return temp else: st = time.time() data = matrixio.read_2d_data(file_full_name) dtype = Vivaldi_dtype_to_python_dtype(dtype) if dtype != None: data = data.astype(dtype) else: dtype = data.dtype #print 'load_data_2d_rgb:', file_full_name, 'dtype:', data.dtype, 'shape:', data.shape,'loading time:',1000*(time.time()-st),'ms' return data return None
def get_return_package(function_name, argument_package_list, work_range, output_halo, merge_func=''): data_package = Data_package() def get_unique_id(): global unique_id unique_id += 1 return unique_id data_package.unique_id = get_unique_id() data_package.data_dtype = numpy.ndarray data_package.data_halo = output_halo # Find return type of worker function def get_return_dtype(function_name, argument_package_list): from Vivaldi_translator_layer import get_return_dtype function_code = function_code_dict[function_name] return_dtype = get_return_dtype(function_name, argument_package_list, function_code) for elem in argument_package_list: if isinstance(elem, Data_package): if elem.data_source in ["hdfs", "local"] and merge_func == '': return elem.data_contents_dtype if return_dtype.endswith('_volume'): print "Vivaldi_warning" print "---------------------------------" print "Check your function" print "you are trying to return a volume" print "return_dtype: ", return_dtype print "---------------------------------" return return_dtype def get_return_source(argument_package_list): return_source = None for elem in argument_package_list: if elem.data_source in ["hdfs", "local"] and merge_func== '': return_source = "local" return return_source return_dtype = get_return_dtype(function_name, argument_package_list) data_package.set_data_contents_dtype(return_dtype) data_package.set_full_data_range(work_range) data_package.set_data_range(work_range) data_package.halo = output_halo data_package.split = output_split data_package.shared = True # FREYJA STREAMING data_package.set_data_source(get_return_source(argument_package_list)) return data_package
def manage_as_data_package(data_name): # bring data from global variable dictionary data = globals()[data_name] # make data package for manage temp = Data_package() if type(data) == numpy.ndarray: # case of data is numpy.ndarray dtype = data.dtype shape = list(data.shape) n = len(shape) last = shape[n - 1] # for make Vivaldi data type name type_name, shape = make_type_name(dtype, shape, last) temp.data_dtype = numpy.ndarray temp.set_data_contents_dtype(type_name) data_range = shape_to_range(shape) temp.set_data_range(data_range) temp.set_full_data_range(data_range) temp.data = copy.copy(data) elif isinstance(data, Data_package): temp = data.copy() temp.data_name = data_name global unique_id u = unique_id unique_id += 1 temp.unique_id = u data_list[data_name] = u data_package_list[data_name] = temp
def manage_as_data_package(data_name): # bring data from global variable dictionary data = globals()[data_name] # make data package for manage temp = Data_package() if type(data) == numpy.ndarray: # case of data is numpy.ndarray dtype = data.dtype shape = list(data.shape) n = len(shape) last = shape[n-1] # for make Vivaldi data type name type_name,shape = make_type_name(dtype, shape, last) temp.data_dtype = numpy.ndarray temp.set_data_contents_dtype(type_name) data_range = shape_to_range(shape) temp.set_data_range(data_range) temp.set_full_data_range(data_range) temp.data = copy.copy(data) elif isinstance(data, Data_package): temp = data.copy() temp.data_name = data_name global unique_id u = unique_id unique_id += 1 temp.unique_id = u data_list[data_name] = u data_package_list[data_name] = temp
def load_data_3d(file_full_name, dtype=None, out_of_core=False): dtype = python_dtype_to_Vivaldi_dtype(dtype) file_name, extension = split_file_name_and_extension(file_full_name) el = extension.lower() if out_of_core: if el == 'dat': # read dat file # dat file have FileName and # It may have AbsolutePath or RelativePath file_info = matrixio.read_dat(file_full_name, out_of_core=True) file_name = file_info['FileName'] assert (file_name != None) if 'Path' in file_info: # user defined path path = file_info['Path'] else: # default is find same folder idx = file_full_name.rfind('/') path = file_full_name[:idx + 1] shape = file_info['Resolution'].split() shape[0], shape[1], shape[2] = int(shape[2]), int(shape[1]), int( shape[0]) contents_dtype = file_info['Format'].lower() chan = file_info['Channel'] if 'Channel' in file_info else 1 temp = Data_package() temp.out_of_core = True temp.data_dtype = numpy.ndarray temp.file_name = path + file_name temp.file_dtype = python_dtype_to_Vivaldi_dtype(contents_dtype) temp.buffer_dtype = numpy.ndarray if dtype != None: dtype = dtype else: dtype = contents_dtype if chan != 1: dtype += str(chan) temp.set_data_contents_dtype(dtype) data_range = shape_to_range(shape) temp.set_data_range(data_range) temp.set_buffer_range(data_range) temp.set_full_data_range(data_range) return temp else: print "NOT PROVIDE DATA TYPE" assert (False) else: st = time.time() data = matrixio.read_3d_data(file_full_name) if dtype != None: dtype = python_dtype_to_Vivaldi_dtype(dtype) dtype = Vivaldi_dtype_to_python_dtype(dtype) data = data.astype(dtype) if data == None: print "======================================" print "VIVALDI ERROR, data load is failed" print "Are you sure, exension is correct?" print "Extension: %s" % (extension) print "======================================" assert (False) print 'load_data_3d:', file_full_name, 'dtype:', data.dtype, 'shape:', data.shape, 'loading time:', 1000 * ( time.time() - st), 'ms' return data return None
def run_function(return_name=None, func_name='', execid=[], work_range=None, args=[], dtype_dict={}, output_halo=0, halo_dict={}, split_dict={}, merge_func='', merge_order=''): def input_argument_check(): if type(func_name) != str: print "Function_name error" print "Func_name: ", func_name if type(dtype_dict) != dict: print "Dtype_dict error" print "Dtype_dict: ", dtype_dict assert (False) if type(split_dict) != dict: print "Split_dict error" print "Split_dict: ", split_dict assert (False) if type(halo_dict) != dict: print "Halo_dict error" print "Halo_dict: ", halo_dict assert (False) if type(merge_func) != str: print "Merge function_name error" print "Merge_function name: ", merge_func # compatibility ############################################################ function_name = func_name args_list = args modifier_dict = {} # input argument error check input_argument_check() # initialization ############################################################## global mmtx, inv_mmtx global unique_id function_package = Function_package() fp = function_package if Vivaldi_viewer.v != None: mmtx = Vivaldi_viewer.mmtx inv_mmtx = Vivaldi_viewer.inv_mmtx Debug = False # arguments ################################################################################## new_args = [] for data_name in args_list: if data_name in data_package_list: # data_name, is managed as data package in the main manager dp = data_package_list[data_name] dtype = str(dp.data_contents_dtype) dtype = dtype.replace('_volume', '') function_name += dtype # we should give access to data to reader, before running function # check it is already available from reader or not data = globals()[data_name] u = dp.unique_id flag = False if dp.out_of_core and u not in retain_list: flag = True # out of core and Not informed to memory manager if not dp.out_of_core and u not in retain_list: flag = True # in core and Didn't informed ( function output already informed) if flag: manage_as_data_package(data_name) dp = data_package_list[data_name] u = dp.unique_id if u not in retain_list: retain_list[u] = [] retain_list[u].append(dp.copy()) reader_give_access_to_data(data_name) dp = dp.copy() dp.data = None dp.devptr = None elif data_name in globals(): # There are two kinds of data here # 1. volume # 2. values data = globals()[data_name] if type(data) == numpy.ndarray: # this is volume # now the data is also managed as data package manage_as_data_package(data_name) # now we have data_package correspond to the data dp = data_package_list[data_name] u = dp.unique_id if u not in retain_list: retain_list[u] = [] retain_list[u].append(dp.copy()) # Vivaldi reader have access to this data reader_give_access_to_data(data_name) # than make a new function name using the existing function name, the data_name and data dtype dtype = str(dp.data_contents_dtype) dtype = dtype.replace('_volume', '') function_name += str(dp.data_contents_dtype) dp = dp.copy() dp.data = None dp.devptr = None else: # this is constant dp = Data_package() dtype = type(data_name) dp.data_name = data_name dp.unique_id = -1 dp.data_dtype = dtype dp.data_contents_dtype = dtype dp.data_contents_memory_dtype = dtype dp.data = data else: # data_name not in the globals list # it is usually AXIS or constant like x,y not previously defined if isinstance(data_name, Data_package): dp = data_name else: data = None dp = Data_package() dtype = type(data_name) dp.data_name = data_name dp.unique_id = -1 dp.data_dtype = dtype dp.data_contents_dtype = dtype dp.data_contents_memory_dtype = dtype dp.data = data_name new_args.append(dp) args_list = new_args # get Vivaldi functions ###################################################################################### global parsed_Vivaldi_functions func_args = args_list return_dtype = parsed_Vivaldi_functions.get_return_dtype(function_name) fp.set_function_name(function_name) fp.output.unique_id = unique_id fp.mmtx = mmtx fp.inv_mmtx = inv_mmtx fp.output.data_dtype = numpy.ndarray fp.output.data_name = return_name if return_dtype == '': print "=======================================================" print "VIVALDI ERROR, can not find return dtype" print "function_name:", function_name print "return name:", return_name print "return dtype:", return_dtype print "=======================================================" assert (False) fp.output.set_data_contents_dtype(return_dtype) v = Vivaldi_viewer.v trans_on = Vivaldi_viewer.trans_on transN = Vivaldi_viewer.transN if trans_on == True: if v.getIsTFupdated() == 1: fp.trans_tex = v.getTFF() fp.update_tf = 1 fp.update_tf2 = 0 v.TFF.widget.updated = 0 elif v.getIsTFupdated2() == 1: fp.trans_tex = v.getTFF2() fp.update_tf = 0 fp.update_tf2 = 1 v.TFF2.widget.updated = 0 fp.TF_bandwidth = v.getTFBW() fp.CRG = v.window.CRG output_halo = 0 if type(work_range) == dict: if 'work_range' in work_range: work_range = work_range['work_range'] if return_name != None: # merge_func ############################################################### func_args = ['front', 'back'] func_dtypes = {} for elem in func_args: func_dtypes[elem] = return_dtype new_name = make_func_name_with_dtypes(merge_func, func_args, func_dtypes) merge_func = new_name # execid ################################################################################### if isinstance(execid, Data_package): execid = execid.data if type(execid) != list: execid = [execid] execid_list = execid fp.execid_list = execid # work range ################################################################################## if type(work_range) == dict and work_range == {}: for data_name in args_list: if isinstance(data_name, Data_package): if dp.unique_id == -1: continue data_name = dp.data_name dp = data_package_list[data_name] work_range = dp.full_data_range break work_range = to_range(work_range) if return_name == '': return_name = None work_range = {'work_range': work_range} args = [ return_name, function_name, execid, work_range, args_list, dtype_dict, output_halo, halo_dict, split_dict, merge_func, merge_order ] return None, run_function, args #return None # local functions ############################################################################ def make_tasks2(arg_packages, i): global unique_id if i == len(args_list): # common variables global unique_id fp.function_args = arg_packages modifier = modifier_dict['output'] if decom == 'in_and_out_split1': num = modifier['num'] work_range = modifier['range_list'][num - 1] fp.work_range = work_range split = modifier['split'] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] full_data_range = modifier['data_range'] fp.output.data_halo = data_halo split_position = make_split_position(split, num) fp.output.split_position = str(split_position) data_range = apply_halo(work_range, data_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) modifier['num'] += 1 elif decom == 'in_and_out_split2': num = modifier['num'] work_range = modifier['range_list'][num - 1] fp.work_range = work_range split = modifier['split'] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] full_data_range = modifier['data_range'] fp.output.data_halo = data_halo split_position = make_split_position(split, num) fp.output.split_position = str(split_position) data_range = apply_halo(work_range, data_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) modifier['num'] += 1 elif decom == 'in': fp.output.unique_id = unique_id output_range = apply_halo(output_range_list[0], output_halo) fp.output.set_data_range(output_range) fp.output.split_shape = str(SPLIT_BASE) fp.output.split_position = str(SPLIT_BASE) fp.work_range = output_range # buffer modifier = modifier_dict['output'] buffer_halo = modifier['buffer_halo'] fp.output.set_buffer_range(buffer_halo) elif decom == 'out': num = modifier['num'] work_range = modifier['range_list'][num - 1] fp.work_range = work_range split = modifier['split'] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] full_data_range = modifier['data_range'] fp.output.data_halo = data_halo split_position = make_split_position(split, num) fp.output.split_position = str(split_position) data_range = apply_halo(work_range, data_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) modifier['num'] += 1 u = fp.output.unique_id unique_id += 1 mem_retain(fp.output) if u not in retain_list: retain_list[u] = [] retain_list[u].append(fp.output.copy()) register_function(execid, fp) return dp = args_list[i] data_name = dp.data_name dp.memory_type = 'memory' # normal variables if dp.unique_id != -1: # setting about full data dp.split_shape = str(SPLIT_BASE) buf = dp.data # replace copy of original dp = dp.copy() if decom == 'in_and_out_split1': """ input output decomposition """ global in_and_out_n u = dp.unique_id data_name = dp.data_name modifier = modifier_dict[ data_name] if data_name in modifier_dict else {} # split shape split_shape = modifier['split'] dp.split_shape = str(split_shape) range_list = data_range_list_dict[data_name] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] cnt = modifier['cnt'] split_position = make_split_position(split_shape, in_and_out_n) dp.split_position = str(split_position) # set data_range data_range = apply_halo(range_list[in_and_out_n - 1], data_halo, dp.full_data_range) dp.set_data_range(data_range) dp.data_halo = data_halo # set buffer halo buffer_range = apply_halo(range_list[in_and_out_n - 1], buffer_halo) dp.set_buffer_range(buffer_range) dp.buffer_halo = buffer_halo if Debug: print "In and out DP", dp make_tasks2(arg_packages + [dp], i + 1) elif decom == 'in_and_out_split2': u = dp.unique_id data_name = dp.data_name modifier = modifier_dict[ data_name] if data_name in modifier_dict else {} # split shape split_shape = modifier['split'] dp.split_shape = str(split_shape) # make splited data and go to next argument data_name = dp.data_name data_halo = modifier['data_halo'] data_range_list = data_range_list_dict[data_name] buffer_halo = modifier['buffer_halo'] n = 1 dp.data_halo = data_halo for data_range in data_range_list: data_range = apply_halo(data_range, data_halo) dp.data_dtype = numpy.ndarray dp.set_data_range(data_range) dp.data_halo = data_halo memory_shape = dp.data_memory_shape shape = dp.data_shape bytes = dp.data_bytes # make depth depth = make_depth(data_range, mmtx) dp.depth = depth fp.output.depth = depth mem_depth(u, str(SPLIT_BASE), str(SPLIT_BASE), depth) split_position = make_split_position(split_shape, n) n += 1 dp.split_position = str(split_position) mem_depth(data_list[data_name], str(split_shape), dp.split_position, depth) dp.set_buffer_range(buffer_halo) make_tasks2(arg_packages + [dp], i + 1) elif decom == 'in': """ input decomposition data range is same """ u = dp.unique_id data_name = dp.data_name modifier = modifier_dict[ data_name] if data_name in modifier_dict else {} # split shape split_shape = modifier['split'] dp.split_shape = str(split_shape) # make splited data and go to next argument data_name = dp.data_name data_halo = modifier['data_halo'] data_range_list = data_range_list_dict[data_name] buffer_halo = modifier['buffer_halo'] n = 1 dp.data_halo = data_halo for data_range in data_range_list: data_range = apply_halo(data_range, data_halo) dp.data_dtype = numpy.ndarray dp.set_data_range(data_range) dp.data_halo = data_halo memory_shape = dp.data_memory_shape shape = dp.data_shape bytes = dp.data_bytes # make depth depth = make_depth(data_range, mmtx) dp.depth = depth fp.output.depth = depth mem_depth(u, str(SPLIT_BASE), str(SPLIT_BASE), depth) split_position = make_split_position(split_shape, n) n += 1 dp.split_position = str(split_position) mem_depth(data_list[data_name], str(split_shape), dp.split_position, depth) dp.set_buffer_range(buffer_halo) if Debug: print "DP", dp make_tasks2(arg_packages + [dp], i + 1) elif decom == 'out': u = dp.unique_id # basic package setting dp.split_shape = str(SPLIT_BASE) dp.split_position = str(SPLIT_BASE) data_name = dp.data_name modifier = modifier_dict[ data_name] if data_name in modifier_dict else {} range_list = data_range_list_dict[data_name] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] # data_range data_range = apply_halo(range_list[0], data_halo) dp.set_data_range(data_range) dp.data_halo = data_halo dp.set_full_data_range(data_range) # buffer range buffer_range = apply_halo(range_list[0], buffer_halo) dp.set_buffer_range(buffer_range) dp.buffer_halo = buffer_halo make_tasks2(arg_packages + [dp], i + 1) else: make_tasks2(arg_packages + [dp], i + 1) def check_in_and_out(modifier_dict, input_cnt, output_cnt): flag = False in_and_out_split = True # in_and_out spilt version1 test # all split shape is identical for data_name in modifier_dict: modifier = modifier_dict[data_name] if flag: if split == modifier['split']: pass else: in_and_out_split = False break else: # first one skip split = modifier['split'] flag = True if in_and_out_split: return 'in_and_out_split1' # in_and_out split version2 test # same number of input and output count if output_cnt == input_cnt: return 'in_and_out_split2' return False ############################################################################ # make argument name list args_name_list = [] for elem in args_list: if isinstance(elem, Data_package): args_name_list.append(elem.data_name) if return_name == None: return_name = 'output' # set output information modifier_dict['output'] = {} output_split = split_dict[ return_name] if return_name in split_dict else SPLIT_BASE output_data_range = work_range output_data_halo = 0 buffer_halo = 10 output_dtype = return_dtype output_range = to_range(output_data_range) output_split = to_split(output_split) output_range_list = make_range_list(output_range, output_split) cnt = shape_to_count(output_split) modifier_dict['output']['split'] = output_split modifier_dict['output']['data_range'] = output_range modifier_dict['output']['data_halo'] = output_halo modifier_dict['output']['cnt'] = cnt modifier_dict['output']['buffer_halo'] = buffer_halo modifier_dict['output']['num'] = 1 modifier_dict['output']['range_list'] = output_range_list output_cnt = cnt # temp data package temp = Data_package() temp.data_name = return_name temp.unique_id = unique_id temp.data_dtype = numpy.ndarray temp.data_halo = output_halo temp.set_data_contents_dtype(return_dtype) # modifier information about input input_cnt = 1 data_range_list_dict = {} # make modifiers_list for each argument for args in args_list: name = args.data_name if args.unique_id != -1: modifier = {} modifier['data_range'] = args.data_range modifier['dtype'] = args.data_contents_dtype data_range = args.data_range data_halo = args.data_halo data_range = apply_halo(data_range, -data_halo) split = split_dict[name] if name in split_dict else SPLIT_BASE for axis in AXIS: if axis not in split: split[axis] = 1 data_range_list = make_range_list(data_range, split) data_range_list_dict[name] = data_range_list cnt = shape_to_count(split) modifier_dict[name] = {} modifier_dict[name]['split'] = split modifier_dict[name]['data_range'] = data_range modifier_dict[name][ 'data_halo'] = halo_dict[name] if name in halo_dict else 0 modifier_dict[name]['buffer_halo'] = buffer_halo modifier_dict[name]['cnt'] = cnt input_cnt *= cnt in_and_out_split = check_in_and_out(modifier_dict, input_cnt, output_cnt) if in_and_out_split == 'in_and_out_split1': decom = 'in_and_out_split1' # this is special case called in&out split fp.output.split_shape = str(output_split) global in_and_out_n in_and_out_n = 1 for work_range in output_range_list: make_tasks2([], 0) in_and_out_n += 1 modifier = modifier_dict['output'] data_halo = modifier['data_halo'] full_data_range = apply_halo(output_range, data_halo) temp.set_data_range(str(full_data_range)) temp.set_full_data_range(str(full_data_range)) temp.set_buffer_range(str(full_data_range)) temp.data_halo = data_halo unique_id += 1 # print "TEMP", temp return temp elif in_and_out_split == 'in_and_out_split2': decom = 'in_and_out_split2' fp.output.split_shape = str(output_split) make_tasks2([], 0) modifier = modifier_dict['output'] data_halo = modifier['data_halo'] full_data_range = apply_halo(output_range, data_halo) temp.set_data_range(str(full_data_range)) temp.set_full_data_range(str(full_data_range)) temp.set_buffer_range(str(full_data_range)) temp.data_halo = data_halo unique_id += 1 return temp elif input_cnt > 1: """ input decomposition """ decom = 'in' count = input_cnt # set function package output full_data_range = apply_halo(output_range, output_halo) fp.output.set_data_range(dict(full_data_range)) fp.output.set_full_data_range(dict(full_data_range)) fp.output.data_halo = output_halo # set output package temp.set_data_range(str(full_data_range)) temp.set_full_data_range(str(full_data_range)) temp.set_buffer_range(str(full_data_range)) # register intermediate merge function u = unique_id inter = range(unique_id + 1, unique_id + count - 1) for inter_id in inter: temp.unique_id = inter_id mem_retain(temp) unique_id += count - 1 temp.unique_id = u # make input functions make_tasks2([], 0) out_range = range(unique_id, unique_id + count) # intermediate merge functions dimension = len(output_range) scheduler_request_merge(temp, out_range, merge_func, merge_order, dimension) #mem_inform(temp) return temp elif output_cnt > 1: """ output decomposition """ decom = 'out' fp.output.split_shape = str(output_split) fp.output.data_halo = output_halo full_data_range = apply_halo(output_range, output_halo) n = 1 for work_range in output_range_list: split_position = make_split_position(output_split, n) n += 1 fp.output.split_position = str(split_position) data_range = apply_halo(work_range, output_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) make_tasks2([], 0) temp.set_data_range(str(full_data_range)) temp.set_full_data_range(str(full_data_range)) temp.set_buffer_range(str(full_data_range)) unique_id += 1 return temp else: # split input and output both, but not in&out split print "===============================" print "VIVALDI ERROR" print "tried to split input and output together but number of input split and output split is different" print "input_cnt: ", input_cnt print "output_cnt: ", output_cnt print "===============================" assert (False) assert (False)
def run_function(return_name=None, func_name='', execid=[], work_range=None, args=[], dtype_dict={}, output_halo=0, halo_dict={}, split_dict={}, merge_func='', merge_order=''): def input_argument_check(): if type(func_name) != str: print "Function_name error" print "Func_name: ", func_name if type(dtype_dict) != dict: print "Dtype_dict error" print "Dtype_dict: ", dtype_dict assert(False) if type(split_dict) != dict: print "Split_dict error" print "Split_dict: ", split_dict assert(False) if type(halo_dict) != dict: print "Halo_dict error" print "Halo_dict: ", halo_dict assert(False) if type(merge_func) != str: print "Merge function_name error" print "Merge_function name: ", merge_func # compatibility ############################################################ function_name = func_name args_list = args modifier_dict={} # input argument error check input_argument_check() # initialization ############################################################## global mmtx, inv_mmtx global unique_id function_package = Function_package() fp = function_package if Vivaldi_viewer.v != None: mmtx = Vivaldi_viewer.mmtx inv_mmtx = Vivaldi_viewer.inv_mmtx Debug = False # arguments ################################################################################## new_args = [] for data_name in args_list: if data_name in data_package_list: # data_name, is managed as data package in the main manager dp = data_package_list[data_name] dtype = str(dp.data_contents_dtype) dtype = dtype.replace('_volume','') function_name += dtype # we should give access to data to reader, before running function # check it is already available from reader or not data = globals()[data_name] u = dp.unique_id flag = False if dp.out_of_core and u not in retain_list: flag = True # out of core and Not informed to memory manager if not dp.out_of_core and u not in retain_list: flag = True # in core and Didn't informed ( function output already informed) if flag: manage_as_data_package(data_name) dp = data_package_list[data_name] u = dp.unique_id if u not in retain_list: retain_list[u] = [] retain_list[u].append(dp.copy()) reader_give_access_to_data(data_name) dp = dp.copy() dp.data = None dp.devptr = None elif data_name in globals(): # There are two kinds of data here # 1. volume # 2. values data = globals()[data_name] if type(data) == numpy.ndarray: # this is volume # now the data is also managed as data package manage_as_data_package(data_name) # now we have data_package correspond to the data dp = data_package_list[data_name] u = dp.unique_id if u not in retain_list: retain_list[u] = [] retain_list[u].append(dp.copy()) # Vivaldi reader have access to this data reader_give_access_to_data(data_name) # than make a new function name using the existing function name, the data_name and data dtype dtype = str(dp.data_contents_dtype) dtype = dtype.replace('_volume','') function_name += str(dp.data_contents_dtype) dp = dp.copy() dp.data = None dp.devptr = None else: # this is constant dp = Data_package() dtype = type(data_name) dp.data_name = data_name dp.unique_id = -1 dp.data_dtype = dtype dp.data_contents_dtype = dtype dp.data_contents_memory_dtype = dtype dp.data = data else: # data_name not in the globals list # it is usually AXIS or constant like x,y not previously defined if isinstance(data_name, Data_package): dp = data_name else: data = None dp = Data_package() dtype = type(data_name) dp.data_name = data_name dp.unique_id = -1 dp.data_dtype = dtype dp.data_contents_dtype = dtype dp.data_contents_memory_dtype = dtype dp.data = data_name new_args.append(dp) args_list = new_args # get Vivaldi functions ###################################################################################### global parsed_Vivaldi_functions func_args = args_list return_dtype = parsed_Vivaldi_functions.get_return_dtype(function_name) fp.set_function_name(function_name) fp.output.unique_id = unique_id fp.mmtx = mmtx fp.inv_mmtx = inv_mmtx fp.output.data_dtype = numpy.ndarray fp.output.data_name = return_name if return_dtype == '': print "=======================================================" print "VIVALDI ERROR, can not find return dtype" print "function_name:", function_name print "return name:", return_name print "return dtype:", return_dtype print "=======================================================" assert(False) fp.output.set_data_contents_dtype(return_dtype) v = Vivaldi_viewer.v trans_on = Vivaldi_viewer.trans_on transN = Vivaldi_viewer.transN if trans_on == True: if v.getIsTFupdated() == 1: fp.trans_tex = v.getTFF() fp.update_tf = 1 fp.update_tf2 = 0 v.TFF.widget.updated = 0 elif v.getIsTFupdated2() == 1: fp.trans_tex = v.getTFF2() fp.update_tf = 0 fp.update_tf2 = 1 v.TFF2.widget.updated = 0 fp.TF_bandwidth = v.getTFBW() fp.CRG = v.window.CRG output_halo = 0 if type(work_range) == dict: if 'work_range' in work_range: work_range = work_range['work_range'] if return_name != None: # merge_func ############################################################### func_args = ['front','back'] func_dtypes = {} for elem in func_args: func_dtypes[ elem ] = return_dtype new_name = make_func_name_with_dtypes(merge_func, func_args, func_dtypes) merge_func = new_name # execid ################################################################################### if isinstance(execid, Data_package): execid = execid.data if type(execid) != list: execid = [execid] execid_list = execid fp.execid_list = execid # work range ################################################################################## if type(work_range) == dict and work_range == {}: for data_name in args_list: if isinstance(data_name, Data_package): if dp.unique_id == -1:continue data_name = dp.data_name dp = data_package_list[data_name] work_range = dp.full_data_range break work_range = to_range(work_range) if return_name == '': return_name = None work_range = {'work_range':work_range} args = [return_name, function_name, execid, work_range, args_list, dtype_dict, output_halo, halo_dict, split_dict, merge_func, merge_order] return None, run_function, args #return None # local functions ############################################################################ def make_tasks2(arg_packages, i): global unique_id if i == len(args_list): # common variables global unique_id fp.function_args = arg_packages modifier = modifier_dict['output'] if decom == 'in_and_out_split1': num = modifier['num'] work_range = modifier['range_list'][num-1] fp.work_range = work_range split = modifier['split'] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] full_data_range = modifier['data_range'] fp.output.data_halo = data_halo split_position = make_split_position(split, num) fp.output.split_position = str(split_position) data_range = apply_halo(work_range, data_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) modifier['num'] += 1 elif decom == 'in_and_out_split2': num = modifier['num'] work_range = modifier['range_list'][num-1] fp.work_range = work_range split = modifier['split'] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] full_data_range = modifier['data_range'] fp.output.data_halo = data_halo split_position = make_split_position(split, num) fp.output.split_position = str(split_position) data_range = apply_halo(work_range, data_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) modifier['num'] += 1 elif decom == 'in': fp.output.unique_id = unique_id output_range = apply_halo(output_range_list[0], output_halo) fp.output.set_data_range(output_range) fp.output.split_shape = str(SPLIT_BASE) fp.output.split_position = str(SPLIT_BASE) fp.work_range = output_range # buffer modifier = modifier_dict['output'] buffer_halo = modifier['buffer_halo'] fp.output.set_buffer_range(buffer_halo) elif decom == 'out': num = modifier['num'] work_range = modifier['range_list'][num-1] fp.work_range = work_range split = modifier['split'] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] full_data_range = modifier['data_range'] fp.output.data_halo = data_halo split_position = make_split_position(split, num) fp.output.split_position = str(split_position) data_range = apply_halo(work_range, data_halo) fp.output.set_data_range(str(data_range)) fp.output.set_full_data_range(str(full_data_range)) fp.output.set_buffer_range(buffer_halo) modifier['num'] += 1 u = fp.output.unique_id unique_id += 1 mem_retain(fp.output) if u not in retain_list: retain_list[u] = [] retain_list[u].append(fp.output.copy()) register_function(execid, fp) return dp = args_list[i] data_name = dp.data_name dp.memory_type = 'memory' # normal variables if dp.unique_id != -1: # setting about full data dp.split_shape = str(SPLIT_BASE) buf = dp.data # replace copy of original dp = dp.copy() if decom == 'in_and_out_split1': """ input output decomposition """ global in_and_out_n u = dp.unique_id data_name = dp.data_name modifier = modifier_dict[data_name] if data_name in modifier_dict else {} # split shape split_shape = modifier['split'] dp.split_shape = str(split_shape) range_list = data_range_list_dict[data_name] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] cnt = modifier['cnt'] split_position = make_split_position(split_shape, in_and_out_n) dp.split_position = str(split_position) # set data_range data_range = apply_halo(range_list[in_and_out_n-1], data_halo, dp.full_data_range) dp.set_data_range(data_range) dp.data_halo = data_halo # set buffer halo buffer_range = apply_halo(range_list[in_and_out_n-1], buffer_halo) dp.set_buffer_range(buffer_range) dp.buffer_halo = buffer_halo if Debug: print "In and out DP", dp make_tasks2(arg_packages + [dp], i + 1) elif decom == 'in_and_out_split2': u = dp.unique_id data_name = dp.data_name modifier = modifier_dict[data_name] if data_name in modifier_dict else {} # split shape split_shape = modifier['split'] dp.split_shape = str(split_shape) # make splited data and go to next argument data_name = dp.data_name data_halo = modifier['data_halo'] data_range_list = data_range_list_dict[data_name] buffer_halo = modifier['buffer_halo'] n = 1 dp.data_halo = data_halo for data_range in data_range_list: data_range = apply_halo(data_range, data_halo) dp.data_dtype = numpy.ndarray dp.set_data_range(data_range) dp.data_halo = data_halo memory_shape = dp.data_memory_shape shape = dp.data_shape bytes = dp.data_bytes # make depth depth = make_depth(data_range, mmtx) dp.depth = depth fp.output.depth = depth mem_depth(u, str(SPLIT_BASE), str(SPLIT_BASE), depth) split_position = make_split_position(split_shape, n) n += 1 dp.split_position = str(split_position) mem_depth(data_list[data_name], str(split_shape), dp.split_position, depth) dp.set_buffer_range(buffer_halo) make_tasks2(arg_packages + [dp], i + 1) elif decom == 'in': """ input decomposition data range is same """ u = dp.unique_id data_name = dp.data_name modifier = modifier_dict[data_name] if data_name in modifier_dict else {} # split shape split_shape = modifier['split'] dp.split_shape = str(split_shape) # make splited data and go to next argument data_name = dp.data_name data_halo = modifier['data_halo'] data_range_list = data_range_list_dict[data_name] buffer_halo = modifier['buffer_halo'] n = 1 dp.data_halo = data_halo for data_range in data_range_list: data_range = apply_halo(data_range, data_halo) dp.data_dtype = numpy.ndarray dp.set_data_range(data_range) dp.data_halo = data_halo memory_shape = dp.data_memory_shape shape = dp.data_shape bytes = dp.data_bytes # make depth depth = make_depth(data_range, mmtx) dp.depth = depth fp.output.depth = depth mem_depth(u, str(SPLIT_BASE), str(SPLIT_BASE), depth) split_position = make_split_position(split_shape, n) n += 1 dp.split_position = str(split_position) mem_depth(data_list[data_name], str(split_shape), dp.split_position, depth) dp.set_buffer_range(buffer_halo) if Debug: print "DP", dp make_tasks2(arg_packages + [dp], i + 1) elif decom == 'out': u = dp.unique_id # basic package setting dp.split_shape = str(SPLIT_BASE) dp.split_position = str(SPLIT_BASE) data_name = dp.data_name modifier = modifier_dict[data_name] if data_name in modifier_dict else {} range_list = data_range_list_dict[data_name] data_halo = modifier['data_halo'] buffer_halo = modifier['buffer_halo'] # data_range data_range = apply_halo(range_list[0], data_halo) dp.set_data_range(data_range) dp.data_halo = data_halo dp.set_full_data_range(data_range) # buffer range buffer_range = apply_halo(range_list[0], buffer_halo) dp.set_buffer_range(buffer_range) dp.buffer_halo = buffer_halo make_tasks2(arg_packages + [dp], i + 1) else: make_tasks2(arg_packages + [dp], i + 1) def check_in_and_out(modifier_dict, input_cnt, output_cnt): flag = False in_and_out_split = True # in_and_out spilt version1 test # all split shape is identical for data_name in modifier_dict: modifier = modifier_dict[data_name] if flag: if split == modifier['split']: pass else: in_and_out_split = False break else: # first one skip split = modifier['split'] flag = True if in_and_out_split: return 'in_and_out_split1' # in_and_out split version2 test # same number of input and output count if output_cnt == input_cnt: return 'in_and_out_split2' return False ############################################################################ # make argument name list args_name_list = [] for elem in args_list: if isinstance(elem, Data_package): args_name_list.append(elem.data_name) if return_name == None: return_name = 'output' # set output information modifier_dict['output'] = {} output_split = split_dict[return_name] if return_name in split_dict else SPLIT_BASE output_data_range = work_range output_data_halo = 0 buffer_halo = 10 output_dtype = return_dtype output_range = to_range(output_data_range) output_split = to_split(output_split) output_range_list = make_range_list(output_range, output_split) cnt = shape_to_count(output_split) modifier_dict['output']['split'] = output_split modifier_dict['output']['data_range'] = output_range modifier_dict['output']['data_halo'] = output_halo modifier_dict['output']['cnt'] = cnt modifier_dict['output']['buffer_halo'] = buffer_halo modifier_dict['output']['num'] = 1 modifier_dict['output']['range_list'] = output_range_list output_cnt = cnt # temp data package temp = Data_package() temp.data_name = return_name temp.unique_id = unique_id temp.data_dtype = numpy.ndarray temp.data_halo = output_halo temp.set_data_contents_dtype(return_dtype) # modifier information about input input_cnt = 1 data_range_list_dict = {} # make modifiers_list for each argument for args in args_list: name = args.data_name if args.unique_id != -1: modifier = {} modifier['data_range'] = args.data_range modifier['dtype'] = args.data_contents_dtype data_range = args.data_range data_halo = args.data_halo data_range = apply_halo(data_range, -data_halo) split = split_dict[name] if name in split_dict else SPLIT_BASE for axis in AXIS: if axis not in split: split[axis] = 1 data_range_list = make_range_list(data_range, split) data_range_list_dict[name] = data_range_list cnt = shape_to_count(split) modifier_dict[name] = {} modifier_dict[name]['split'] = split modifier_dict[name]['data_range'] = data_range modifier_dict[name]['data_halo'] = halo_dict[name] if name in halo_dict else 0 modifier_dict[name]['buffer_halo'] = buffer_halo modifier_dict[name]['cnt'] = cnt input_cnt *= cnt in_and_out_split = check_in_and_out(modifier_dict, input_cnt, output_cnt) if in_and_out_split == 'in_and_out_split1': decom = 'in_and_out_split1' # this is special case called in&out split fp.output.split_shape = str(output_split) global in_and_out_n in_and_out_n = 1 for work_range in output_range_list: make_tasks2([], 0) in_and_out_n += 1 modifier = modifier_dict['output'] data_halo = modifier['data_halo'] full_data_range = apply_halo(output_range, data_halo) temp.set_data_range(str(full_data_range)) temp.set_full_data_range( str(full_data_range)) temp.set_buffer_range(str(full_data_range)) temp.data_halo = data_halo unique_id += 1 # print "TEMP", temp return temp elif in_and_out_split == 'in_and_out_split2': decom = 'in_and_out_split2' fp.output.split_shape = str(output_split) make_tasks2([], 0) modifier = modifier_dict['output'] data_halo = modifier['data_halo'] full_data_range = apply_halo(output_range, data_halo) temp.set_data_range(str(full_data_range)) temp.set_full_data_range( str(full_data_range)) temp.set_buffer_range(str(full_data_range)) temp.data_halo = data_halo unique_id += 1 return temp elif input_cnt > 1: """ input decomposition """ decom = 'in' count = input_cnt # set function package output full_data_range = apply_halo(output_range, output_halo) fp.output.set_data_range( dict(full_data_range)) fp.output.set_full_data_range( dict(full_data_range)) fp.output.data_halo = output_halo # set output package temp.set_data_range( str(full_data_range)) temp.set_full_data_range( str(full_data_range)) temp.set_buffer_range(str(full_data_range)) # register intermediate merge function u = unique_id inter = range(unique_id+1, unique_id+count-1) for inter_id in inter: temp.unique_id = inter_id mem_retain(temp) unique_id += count-1 temp.unique_id = u # make input functions make_tasks2([], 0) out_range = range(unique_id, unique_id+count) # intermediate merge functions dimension = len(output_range) scheduler_request_merge(temp, out_range, merge_func, merge_order, dimension) #mem_inform(temp) return temp elif output_cnt > 1: """ output decomposition """ decom = 'out' fp.output.split_shape = str(output_split) fp.output.data_halo = output_halo full_data_range = apply_halo(output_range, output_halo) n = 1 for work_range in output_range_list: split_position = make_split_position(output_split, n) n += 1 fp.output.split_position = str(split_position) data_range = apply_halo( work_range, output_halo) fp.output.set_data_range( str(data_range)) fp.output.set_full_data_range( str(full_data_range)) fp.output.set_buffer_range(buffer_halo) make_tasks2([], 0) temp.set_data_range(str(full_data_range)) temp.set_full_data_range( str(full_data_range)) temp.set_buffer_range(str(full_data_range)) unique_id += 1 return temp else: # split input and output both, but not in&out split print "===============================" print "VIVALDI ERROR" print "tried to split input and output together but number of input split and output split is different" print "input_cnt: ", input_cnt print "output_cnt: ", output_cnt print "===============================" assert(False) assert(False)
def load_data_3d(file_full_name, dtype=None, out_of_core=False): dtype = python_dtype_to_Vivaldi_dtype(dtype) file_name, extension = split_file_name_and_extension(file_full_name) el = extension.lower() if out_of_core: if el == 'dat': # read dat file # dat file have FileName and # It may have AbsolutePath or RelativePath file_info = matrixio.read_dat(file_full_name, out_of_core=True) file_name = file_info['FileName'] assert(file_name != None) if 'Path' in file_info: # user defined path path = file_info['Path'] else: # default is find same folder idx = file_full_name.rfind('/') path = file_full_name[:idx+1] shape = file_info['Resolution'].split() shape[0],shape[1],shape[2] = int(shape[2]),int(shape[1]),int(shape[0]) contents_dtype = file_info['Format'].lower() chan = file_info['Channel'] if 'Channel' in file_info else 1 temp = Data_package() temp.out_of_core = True temp.data_dtype = numpy.ndarray temp.file_name = path + file_name temp.file_dtype = python_dtype_to_Vivaldi_dtype(contents_dtype) temp.buffer_dtype = numpy.ndarray if dtype != None: dtype = dtype else: dtype = contents_dtype if chan != 1: dtype += str(chan) temp.set_data_contents_dtype(dtype) data_range = shape_to_range(shape) temp.set_data_range(data_range) temp.set_buffer_range(data_range) temp.set_full_data_range(data_range) return temp else: print "NOT PROVIDE DATA TYPE" assert(False) else: st = time.time() data = matrixio.read_3d_data(file_full_name) if dtype != None: dtype = python_dtype_to_Vivaldi_dtype(dtype) dtype = Vivaldi_dtype_to_python_dtype(dtype) data = data.astype(dtype) if data is None: print "======================================" print "VIVALDI ERROR, data load is failed" print "Are you sure, exension is correct?" print "Extension: %s"%(extension) print "======================================" assert(False) #print 'load_data_3d:', file_full_name, 'dtype:', data.dtype, 'shape:', data.shape,'loading time:',1000*(time.time()-st),'ms' return data return None