def get_argument_package_list(args, arg_names, split_dict, halo_dict):
i = 0
argument_package_list = []
for data_name in arg_names:
arg = args[i]
if data_name in AXIS:
argument_package = Data_package(data_name)
argument_package.shared = False
argument_package_list.append(argument_package)
else:
argument_package = None
# get modifier split and halo
split = split_dict[data_name] if data_name in split_dict else {}
halo = halo_dict[data_name] if data_name in halo_dict else 0
# apply to data_package
if isinstance(arg, Data_package):
argument_package = arg
argument_package.split = split
argument_package.halo = halo
if argument_package.unique_id == None:
def get_unique_id(arg):
aid = id(arg)
if aid in data_package_list:
return data_package_list[aid].get_unique_id()
else:
global unique_id
unique_id += 1
return unique_id
argument_package.unique_id = get_unique_id(arg)
argument_package.shared = False
else:
argument_package = Data_package(arg,split=split,halo=halo)
def get_unique_id(arg):
if type(arg) != numpy.ndarray:
return -1
aid = id(arg)
if aid in data_package_list:
return data_package_list[aid].get_unique_id()
else:
global unique_id
unique_id += 1
return unique_id
argument_package.unique_id = get_unique_id(arg)
def add_to_data_package_list(data_package, data):
if type(data) == numpy.ndarray:
key = id(data)
data_package_list[key] = data_package
add_to_data_package_list(argument_package, arg)
argument_package.shared = False
argument_package_list.append(argument_package)
i += 1
return argument_package_list
def get_argument_package_list(args, arg_names, split_dict, halo_dict):
i = 0
argument_package_list = []
for data_name in arg_names:
arg = args[i]
if data_name in AXIS:
argument_package = Data_package(data_name)
argument_package.shared = False
argument_package_list.append(argument_package)
else:
argument_package = None
# get modifier split and halo
split = split_dict[data_name] if data_name in split_dict else {}
halo = halo_dict[data_name] if data_name in halo_dict else 0
# apply to data_package
if isinstance(arg, Data_package):
argument_package = arg
argument_package.split = split
argument_package.halo = halo
if argument_package.unique_id == None:
def get_unique_id(arg):
aid = id(arg)
if aid in data_package_list:
return data_package_list[aid].get_unique_id()
else:
global unique_id
unique_id += 1
return unique_id
argument_package.unique_id = get_unique_id(arg)
argument_package.shared = False
else:
argument_package = Data_package(arg,split=split,halo=halo)
def get_unique_id(arg):
if type(arg) != numpy.ndarray:
return -1
aid = id(arg)
if aid in data_package_list:
return data_package_list[aid].get_unique_id()
else:
global unique_id
unique_id += 1
return unique_id
argument_package.unique_id = get_unique_id(arg)
def add_to_data_package_list(data_package, data):
if type(data) == numpy.ndarray:
key = id(data)
data_package_list[key] = data_package
add_to_data_package_list(argument_package, arg)
argument_package.shared = False
argument_package_list.append(argument_package)
i += 1
return argument_package_list
def get_return_package(function_name, argument_package_list, work_range, output_halo):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
return data_package
def get_return_package(function_name, argument_package_list, work_range, output_halo):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
return data_package
def get_return_package(function_name, argument_package_list, work_range, output_halo, merge_func=''):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
# Find return type of worker function
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
for elem in argument_package_list:
if isinstance(elem, Data_package):
if elem.data_source in ["hdfs", "local"] and merge_func == '':
return elem.data_contents_dtype
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
def get_return_source(argument_package_list):
return_source = None
for elem in argument_package_list:
if elem.data_source in ["hdfs", "local"] and merge_func== '':
return_source = "local"
return return_source
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
# FREYJA STREAMING
data_package.set_data_source(get_return_source(argument_package_list))
return data_package
def get_argument_package_list(args, arg_names, split_dict, halo_dict):
i = 0
argument_package_list = []
for data_name in arg_names:
arg = args[i]
if data_name in AXIS:
argument_package = Data_package(data_name)
argument_package.shared = False
argument_package_list.append(argument_package)
else:
argument_package = None
# get modifier split and halo
split = split_dict[data_name] if data_name in split_dict else {}
#FREYJA STREAMING
if isinstance(arg, Data_package):
if split_dict != {}:
pass
elif isinstance(work_range, Data_package):
if arg.stream and (arg.data_shape == work_range.data_shape):
split = {'z':arg.stream_count}
split_dict[data_name] = split
split_dict[return_name] = split
elif isinstance(work_range, dict):
if arg.stream:
split = {'z':arg.stream_count}
split_dict[data_name] = split
halo = halo_dict[data_name] if data_name in halo_dict else 0
# apply to data_package
if isinstance(arg, Data_package):
argument_package = arg
argument_package.split = split
argument_package.halo = halo
if argument_package.unique_id == None:
def get_unique_id(arg):
aid = id(arg)
if aid in data_package_list:
return data_package_list[aid].get_unique_id()
else:
global unique_id
unique_id += 1
return unique_id
argument_package.unique_id = get_unique_id(arg)
argument_package.shared = False
else:
argument_package = Data_package(arg,split=split,halo=halo)
def get_unique_id(arg):
if type(arg) != numpy.ndarray:
return -1
aid = id(arg)
if aid in data_package_list:
return data_package_list[aid].get_unique_id()
else:
global unique_id
unique_id += 1
return unique_id
argument_package.unique_id = get_unique_id(arg)
def add_to_data_package_list(data_package, data):
if type(data) == numpy.ndarray:
key = id(data)
data_package_list[key] = data_package
add_to_data_package_list(argument_package, arg)
argument_package.shared = False
argument_package_list.append(argument_package)
i += 1
return argument_package_list
