def get_argument_package_list(args, arg_names, split_dict, halo_dict): i = 0 argument_package_list = [] for data_name in arg_names: arg = args[i] if data_name in AXIS: argument_package = Data_package(data_name) argument_package.shared = False argument_package_list.append(argument_package) else: argument_package = None # get modifier split and halo split = split_dict[data_name] if data_name in split_dict else {} halo = halo_dict[data_name] if data_name in halo_dict else 0 # apply to data_package if isinstance(arg, Data_package): argument_package = arg argument_package.split = split argument_package.halo = halo if argument_package.unique_id == None: def get_unique_id(arg): aid = id(arg) if aid in data_package_list: return data_package_list[aid].get_unique_id() else: global unique_id unique_id += 1 return unique_id argument_package.unique_id = get_unique_id(arg) argument_package.shared = False else: argument_package = Data_package(arg,split=split,halo=halo) def get_unique_id(arg): if type(arg) != numpy.ndarray: return -1 aid = id(arg) if aid in data_package_list: return data_package_list[aid].get_unique_id() else: global unique_id unique_id += 1 return unique_id argument_package.unique_id = get_unique_id(arg) def add_to_data_package_list(data_package, data): if type(data) == numpy.ndarray: key = id(data) data_package_list[key] = data_package add_to_data_package_list(argument_package, arg) argument_package.shared = False argument_package_list.append(argument_package) i += 1 return argument_package_list
def get_return_package(function_name, argument_package_list, work_range, output_halo): data_package = Data_package() def get_unique_id(): global unique_id unique_id += 1 return unique_id data_package.unique_id = get_unique_id() data_package.data_dtype = numpy.ndarray data_package.data_halo = output_halo def get_return_dtype(function_name, argument_package_list): from Vivaldi_translator_layer import get_return_dtype function_code = function_code_dict[function_name] return_dtype = get_return_dtype(function_name, argument_package_list, function_code) if return_dtype.endswith('_volume'): print "Vivaldi_warning" print "---------------------------------" print "Check your function" print "you are trying to return a volume" print "return_dtype: ", return_dtype print "---------------------------------" return return_dtype return_dtype = get_return_dtype(function_name, argument_package_list) data_package.set_data_contents_dtype(return_dtype) data_package.set_full_data_range(work_range) data_package.set_data_range(work_range) data_package.halo = output_halo data_package.split = output_split data_package.shared = True return data_package
def get_return_package(function_name, argument_package_list, work_range, output_halo, merge_func=''): data_package = Data_package() def get_unique_id(): global unique_id unique_id += 1 return unique_id data_package.unique_id = get_unique_id() data_package.data_dtype = numpy.ndarray data_package.data_halo = output_halo # Find return type of worker function def get_return_dtype(function_name, argument_package_list): from Vivaldi_translator_layer import get_return_dtype function_code = function_code_dict[function_name] return_dtype = get_return_dtype(function_name, argument_package_list, function_code) for elem in argument_package_list: if isinstance(elem, Data_package): if elem.data_source in ["hdfs", "local"] and merge_func == '': return elem.data_contents_dtype if return_dtype.endswith('_volume'): print "Vivaldi_warning" print "---------------------------------" print "Check your function" print "you are trying to return a volume" print "return_dtype: ", return_dtype print "---------------------------------" return return_dtype def get_return_source(argument_package_list): return_source = None for elem in argument_package_list: if elem.data_source in ["hdfs", "local"] and merge_func== '': return_source = "local" return return_source return_dtype = get_return_dtype(function_name, argument_package_list) data_package.set_data_contents_dtype(return_dtype) data_package.set_full_data_range(work_range) data_package.set_data_range(work_range) data_package.halo = output_halo data_package.split = output_split data_package.shared = True # FREYJA STREAMING data_package.set_data_source(get_return_source(argument_package_list)) return data_package
def get_argument_package_list(args, arg_names, split_dict, halo_dict): i = 0 argument_package_list = [] for data_name in arg_names: arg = args[i] if data_name in AXIS: argument_package = Data_package(data_name) argument_package.shared = False argument_package_list.append(argument_package) else: argument_package = None # get modifier split and halo split = split_dict[data_name] if data_name in split_dict else {} #FREYJA STREAMING if isinstance(arg, Data_package): if split_dict != {}: pass elif isinstance(work_range, Data_package): if arg.stream and (arg.data_shape == work_range.data_shape): split = {'z':arg.stream_count} split_dict[data_name] = split split_dict[return_name] = split elif isinstance(work_range, dict): if arg.stream: split = {'z':arg.stream_count} split_dict[data_name] = split halo = halo_dict[data_name] if data_name in halo_dict else 0 # apply to data_package if isinstance(arg, Data_package): argument_package = arg argument_package.split = split argument_package.halo = halo if argument_package.unique_id == None: def get_unique_id(arg): aid = id(arg) if aid in data_package_list: return data_package_list[aid].get_unique_id() else: global unique_id unique_id += 1 return unique_id argument_package.unique_id = get_unique_id(arg) argument_package.shared = False else: argument_package = Data_package(arg,split=split,halo=halo) def get_unique_id(arg): if type(arg) != numpy.ndarray: return -1 aid = id(arg) if aid in data_package_list: return data_package_list[aid].get_unique_id() else: global unique_id unique_id += 1 return unique_id argument_package.unique_id = get_unique_id(arg) def add_to_data_package_list(data_package, data): if type(data) == numpy.ndarray: key = id(data) data_package_list[key] = data_package add_to_data_package_list(argument_package, arg) argument_package.shared = False argument_package_list.append(argument_package) i += 1 return argument_package_list