def get_return_package(function_name, argument_package_list, work_range, output_halo):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
return data_package
def get_return_package(function_name, argument_package_list, work_range, output_halo):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
return data_package
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
def get_return_package(function_name, argument_package_list, work_range, output_halo, merge_func=''):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
# Find return type of worker function
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
for elem in argument_package_list:
if isinstance(elem, Data_package):
if elem.data_source in ["hdfs", "local"] and merge_func == '':
return elem.data_contents_dtype
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
def get_return_source(argument_package_list):
return_source = None
for elem in argument_package_list:
if elem.data_source in ["hdfs", "local"] and merge_func== '':
return_source = "local"
return return_source
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
# FREYJA STREAMING
data_package.set_data_source(get_return_source(argument_package_list))
return data_package
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
for elem in argument_package_list:
if isinstance(elem, Data_package):
if elem.data_source in ["hdfs", "local"] and merge_func == '':
return elem.data_contents_dtype
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
def parallel(function_name='', argument_package_list=[], work_range={}, execid=[], output_halo=0, output_split={}, merge_func='', merge_order=''):
# compatibility to old versions
############################################################
function_name = function_name.strip()
def to_range(input):
dtype = type(input)
if type(input) == numpy.ndarray:
input = list(input.shape)
dtype = type(input)
n = len(input)
if input[n-1] in [1,2,3]:
input.pop()
if dtype in [tuple, list]:
return shape_to_range(input)
if isinstance(input, Data_package):
dp = input
work_range = apply_halo(dp.data_range, -dp.data_halo)
return work_range
if dtype == dict:
return input
return {}
work_range = to_range(work_range)
# input argument error check
def input_argument_check():
if type(function_name) != str or function_name == '':
print "Function_name error"
print "function_name: ", function_name
if function_name not in function_code_dict:
print "======================================"
print "Vivaldi Warning"
print "the function: " + function_name + " not exist"
print "======================================"
assert(False)
if type(merge_func) != str:
print "Merge function_name error"
print "Merge_function name: ", merge_func
if type(work_range) != dict:
print "work_range error"
print "work_range: ", work_range
assert(False)
input_argument_check()
# initialization
##############################################################
global unique_id
# share argument packages
# and send data to reader
def share_argument_package_list(arugment_package_list):
def share_argument_package(argument_package):
if argument_package.get_unique_id() == '-1': # skip, small variables
pass
elif argument_package.shared == False: # not registered variables
def reader_give_access(data_package):
#scheduler_inform(data_package, 2)
u = data_package.unique_id
scheduler_retain(data_package)
out_of_core = data_package.out_of_core
if out_of_core:
scheduler_notice_data_out_of_core(data_package)
else:
send_data(2, data_package.data, data_package)
reader_give_access(argument_package)
argument_package.shared = True
for argument_package in argument_package_list:
share_argument_package(argument_package)
share_argument_package_list(argument_package_list)
# get return package
def get_return_package(function_name, argument_package_list, work_range, output_halo):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
return data_package
return_package = get_return_package(function_name, argument_package_list, work_range, output_halo)
# register return package to data_package_list
def register_return_package(key, return_package):
if key in data_package_list: # cannot happen
pass
else:
data_package_list[key] = return_package
register_return_package(id(return_package), return_package)
# register function to scheduler
def get_function_package(function_name, argument_package_list, return_package, work_range, merge_func='', merge_order=''):
fp = Function_package()
fp.set_function_name(function_name)
fp.set_function_args(argument_package_list)
from OpenGL.GL import glGetFloatv, GL_MODELVIEW_MATRIX
fp.mmtx = glGetFloatv(GL_MODELVIEW_MATRIX)
def flip_diagonal(mmtx):
new_mmtx = numpy.empty((4,4),dtype=numpy.float32)
for i in range(4):
for j in range(4):
new_mmtx[i][j] = mmtx[j][i]
return new_mmtx
fp.mmtx = flip_diagonal(fp.mmtx)
fp.work_range = work_range
fp.output = return_package
return fp
function_package = get_function_package(function_name, argument_package_list, return_package, work_range, merge_func, merge_order)
# setting viewer-src
if Vivaldi_viewer.v != None:
mmtx = Vivaldi_viewer.mmtx
inv_mmtx = Vivaldi_viewer.inv_mmtx
v = Vivaldi_viewer.v
trans_on = Vivaldi_viewer.trans_on
transN = 0
fp = function_package
if v != None:
trans_on = Vivaldi_viewer.trans_on
fp.transN = Vivaldi_viewer.transN
if v.slider != None:
fp.Sliders = v.get_sliders()
fp.Slider_opacity = v.get_slider_opacity()
#
if trans_on == True:
if v.getIsTFupdated() == 1:
fp.trans_tex = v.getTFF()
fp.update_tf = 1
fp.update_tf2 = 0
v.window.TFF.updated = 0
elif v.getIsTFupdated2() == 1:
fp.trans_tex = v.getTFF2()
fp.update_tf = 0
fp.update_tf2 = 1
v.window.TFF2.updated = 0
#
fp.TF_bandwidth = v.getTFBW()
register_function_package(function_package)
if merge_func != '':
input_package = return_package.copy()
input_package.set_data_range(input_package.full_data_range)
# function name check
if merge_func not in function_code_dict:
print "Vivaldi warning"
print "================================="
print "function: ",merge_func,"not exist"
print "================================="
assert(False)
# make function package
merge_function_package = Function_package()
# set function name
merge_function_package.set_function_name(merge_func)
# set work_range
merge_function_package.work_range = input_package.full_data_range
def get_merge_package_args(input_package, merge_func):
def get_argument_list(function_name):
function_code = function_code_dict[merge_func]
def get_args(name, code):
idx_start = code.find(name) + len(name) + 1
idx_end = code.find(')', idx_start)
args = code[idx_start:idx_end]
return args.strip()
function_args = get_args(function_name, function_code)
if function_args == '':
print "Vivaldi warning"
print "================================="
print "There are no function argument"
print "================================="
assert(False)
argument_list = []
for arg in function_args.split(','):
argument_list.append(arg.strip())
return argument_list
argument_list = get_argument_list(merge_func)
argument_package_list = []
for arg in argument_list:
argument_package = None
if arg in AXIS:
argument_package = Data_package(arg)
else:
argument_package = input_package.copy()
argument_package_list.append(argument_package)
return argument_package_list
merge_argument_package_list = get_merge_package_args(input_package, merge_func)
# set argument
merge_function_package.set_args(merge_argument_package_list)
# set return package
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(merge_func, merge_argument_package_list)
return_package.set_dtype(return_dtype)
merge_function_package.output = return_package
# split count
def get_split_count(argument_package_list):
cnt = 1
for argument_package in argument_package_list:
uid = argument_package.get_unique_id()
if uid != '-1':
split = argument_package.split
for axis in split:
cnt *= split[axis]
return cnt
n = get_split_count(argument_package_list)
# ask scheduler to merge
scheduler_merge(merge_function_package, n)
scheduler_retain(return_package)
return return_package
def parallel(function_name='', argument_package_list=[], work_range={}, execid=[], output_halo=0, output_split={}, merge_func='', merge_order=''):
# compatibility to old versions
############################################################
function_name = function_name.strip()
def to_range(input):
dtype = type(input)
if type(input) == numpy.ndarray:
input = list(input.shape)
dtype = type(input)
n = len(input)
if input[n-1] in [1,2,3]:
input.pop()
if dtype in [tuple, list]:
return shape_to_range(input)
if isinstance(input, Data_package):
dp = input
work_range = apply_halo(dp.data_range, -dp.data_halo)
return work_range
if dtype == dict:
return input
return {}
work_range = to_range(work_range)
# input argument error check
def input_argument_check():
if type(function_name) != str or function_name == '':
print "Function_name error"
print "function_name: ", function_name
if function_name not in function_code_dict:
print "======================================"
print "Vivaldi Warning"
print "the function: " + function_name + " not exist"
print "======================================"
assert(False)
if type(merge_func) != str:
print "Merge function_name error"
print "Merge_function name: ", merge_func
if type(work_range) != dict:
print "work_range error"
print "work_range: ", work_range
assert(False)
input_argument_check()
# initialization
##############################################################
global unique_id
# share argument packages
# and send data to reader
def share_argument_package_list(arugment_package_list):
def share_argument_package(argument_package):
if argument_package.get_unique_id() == '-1': # skip, small variables
pass
elif argument_package.shared == False: # not registered variables
def reader_give_access(data_package):
#scheduler_inform(data_package, 2)
u = data_package.unique_id
scheduler_retain(data_package)
out_of_core = data_package.out_of_core
if out_of_core:
scheduler_notice_data_out_of_core(data_package)
else:
send_data(2, data_package.data, data_package)
reader_give_access(argument_package)
argument_package.shared = True
for argument_package in argument_package_list:
share_argument_package(argument_package)
share_argument_package_list(argument_package_list)
# get return package
def get_return_package(function_name, argument_package_list, work_range, output_halo):
data_package = Data_package()
def get_unique_id():
global unique_id
unique_id += 1
return unique_id
data_package.unique_id = get_unique_id()
data_package.data_dtype = numpy.ndarray
data_package.data_halo = output_halo
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(function_name, argument_package_list)
data_package.set_data_contents_dtype(return_dtype)
data_package.set_full_data_range(work_range)
data_package.set_data_range(work_range)
data_package.halo = output_halo
data_package.split = output_split
data_package.shared = True
return data_package
return_package = get_return_package(function_name, argument_package_list, work_range, output_halo)
# register return package to data_package_list
def register_return_package(key, return_package):
if key in data_package_list: # cannot happen
pass
else:
data_package_list[key] = return_package
register_return_package(id(return_package), return_package)
# register function to scheduler
def get_function_package(function_name, argument_package_list, return_package, work_range, merge_func='', merge_order=''):
fp = Function_package()
fp.set_function_name(function_name)
fp.set_function_args(argument_package_list)
from OpenGL.GL import glGetFloatv, GL_MODELVIEW_MATRIX
fp.mmtx = glGetFloatv(GL_MODELVIEW_MATRIX)
def flip_diagonal(mmtx):
new_mmtx = numpy.empty((4,4),dtype=numpy.float32)
for i in range(4):
for j in range(4):
new_mmtx[i][j] = mmtx[j][i]
return new_mmtx
fp.mmtx = flip_diagonal(fp.mmtx)
fp.work_range = work_range
fp.output = return_package
return fp
function_package = get_function_package(function_name, argument_package_list, return_package, work_range, merge_func, merge_order)
# setting viewer-src
if Vivaldi_viewer.v != None:
mmtx = Vivaldi_viewer.mmtx
inv_mmtx = Vivaldi_viewer.inv_mmtx
v = Vivaldi_viewer.v
trans_on = Vivaldi_viewer.trans_on
transN = 0
fp = function_package
if v != None:
trans_on = Vivaldi_viewer.trans_on
fp.transN = Vivaldi_viewer.transN
if v.slider != None:
fp.Sliders = v.get_sliders()
fp.Slider_opacity = v.get_slider_opacity()
if trans_on == True:
if v.getIsTFupdated() == 1:
fp.trans_tex = v.getTFF()
fp.update_tf = 1
fp.update_tf2 = 0
v.window.TFF.updated = 0
elif v.getIsTFupdated2() == 1:
fp.trans_tex = v.getTFF2()
fp.update_tf = 0
fp.update_tf2 = 1
v.window.TFF2.updated = 0
fp.TF_bandwidth = v.getTFBW()
register_function_package(function_package)
if merge_func != '':
input_package = return_package.copy()
input_package.set_data_range(input_package.full_data_range)
# function name check
if merge_func not in function_code_dict:
print "Vivaldi warning"
print "================================="
print "function: ",merge_func,"not exist"
print "================================="
assert(False)
# make function package
merge_function_package = Function_package()
# set function name
merge_function_package.set_function_name(merge_func)
# set work_range
merge_function_package.work_range = input_package.full_data_range
def get_merge_package_args(input_package, merge_func):
def get_argument_list(function_name):
function_code = function_code_dict[merge_func]
def get_args(name, code):
idx_start = code.find(name) + len(name) + 1
idx_end = code.find(')', idx_start)
args = code[idx_start:idx_end]
return args.strip()
function_args = get_args(function_name, function_code)
if function_args == '':
print "Vivaldi warning"
print "================================="
print "There are no function argument"
print "================================="
assert(False)
argument_list = []
for arg in function_args.split(','):
argument_list.append(arg.strip())
return argument_list
argument_list = get_argument_list(merge_func)
argument_package_list = []
for arg in argument_list:
argument_package = None
if arg in AXIS:
argument_package = Data_package(arg)
else:
argument_package = input_package.copy()
argument_package_list.append(argument_package)
return argument_package_list
merge_argument_package_list = get_merge_package_args(input_package, merge_func)
# set argument
merge_function_package.set_args(merge_argument_package_list)
# set return package
def get_return_dtype(function_name, argument_package_list):
from Vivaldi_translator_layer import get_return_dtype
function_code = function_code_dict[function_name]
return_dtype = get_return_dtype(function_name, argument_package_list, function_code)
if return_dtype.endswith('_volume'):
print "Vivaldi_warning"
print "---------------------------------"
print "Check your function"
print "you are trying to return a volume"
print "return_dtype: ", return_dtype
print "---------------------------------"
return return_dtype
return_dtype = get_return_dtype(merge_func, merge_argument_package_list)
return_package.set_dtype(return_dtype)
merge_function_package.output = return_package
# split count
def get_split_count(argument_package_list):
cnt = 1
for argument_package in argument_package_list:
uid = argument_package.get_unique_id()
if uid != '-1':
split = argument_package.split
for axis in split:
cnt *= split[axis]
return cnt
n = get_split_count(argument_package_list)
# ask scheduler to merge
scheduler_merge(merge_function_package, n)
scheduler_retain(return_package)
return return_package
