def load_data_3d(file_full_name, dtype=None, out_of_core=False):
dtype = python_dtype_to_Vivaldi_dtype(dtype)
file_name, extension = split_file_name_and_extension(file_full_name)
el = extension.lower()
if out_of_core or el == 'stream':
if el == 'dat':
# read dat file
# dat file have FileName and
# It may have AbsolutePath or RelativePath
file_info = matrixio.read_dat(file_full_name, out_of_core=True)
file_name = file_info['FileName']
assert(file_name != None)
if 'Path' in file_info:
# user defined path
path = file_info['Path']
else:
# default is find same folder
idx = file_full_name.rfind('/')
path = file_full_name[:idx+1]
shape = file_info['Resolution'].split()
shape[0],shape[1],shape[2] = int(shape[2]),int(shape[1]),int(shape[0])
contents_dtype = file_info['Format'].lower()
chan = file_info['Channel'] if 'Channel' in file_info else 1
temp = Data_package()
temp.out_of_core = True
temp.data_dtype = numpy.ndarray
temp.file_name = path + file_name
temp.file_dtype = python_dtype_to_Vivaldi_dtype(contents_dtype)
temp.buffer_dtype = numpy.ndarray
if dtype != None: dtype = dtype
else: dtype = contents_dtype
if chan != 1: dtype += str(chan)
temp.set_data_contents_dtype(dtype)
data_range = shape_to_range(shape)
temp.data_source = "hdd"
temp.set_data_range(data_range)
temp.set_buffer_range(data_range)
temp.set_full_data_range(data_range)
return temp
# FREYJA STREAMING
elif el == 'stream':
file_info = matrixio.read_stream(file_full_name)
shape = file_info['Resolution'].split()
shape[0],shape[1],shape[2] = int(shape[2]),int(shape[1]),int(shape[0])
contents_dtype = file_info['Format'].lower()
chan = file_info['Channel'] if 'Channel' in file_info else 1
if 'Path' in file_info:
# user defined path
path = file_info['Path']
else:
# default is find same folder
idx = file_full_name.rfind('/')
path = file_full_name[:idx+1]
temp = Data_package()
temp.out_of_core = True
temp.stream = True
temp.disk_file_size = int(file_info['Filesize']) if 'Filesize' in file_info else temp.disk_file_size
temp.disk_block_size = int(file_info['Blocksize']) if 'Blocksize' in file_info else temp.disk_block_size
temp.data_dtype = numpy.ndarray
temp.file_name = path
import re
temp.stream_hdfs_file_name = file_info['Hdfs_path']
temp.stream_count = shape[0] / temp.disk_file_size
#print temp.stream_count, shape[0], temp.disk_file_size
temp.file_dtype = python_dtype_to_Vivaldi_dtype(contents_dtype)
temp.buffer_dtype = numpy.ndarray
if dtype != None: dtype = dtype
else: dtype = contents_dtype
if chan != 1: dtype += str(chan)
temp.set_data_contents_dtype(dtype)
data_range = shape_to_range(shape)
temp.data_source = "hdfs"
temp.set_data_range(data_range)
temp.set_buffer_range(data_range)
temp.set_full_data_range(data_range)
return temp
else:
print "NOT PROVIDE DATA TYPE"
assert(False)
else:
st = time.time()
data = matrixio.read_3d_data(file_full_name)
if dtype != None:
dtype = python_dtype_to_Vivaldi_dtype(dtype)
dtype = Vivaldi_dtype_to_python_dtype(dtype)
data = data.astype(dtype)
if data == None:
print "======================================"
print "VIVALDI ERROR, data load is failed"
print "Are you sure, exension is correct?"
print "Extension: %s"%(extension)
print "======================================"
assert(False)
print 'load_data_3d:', file_full_name, 'dtype:', data.dtype, 'shape:', data.shape,'loading time:',1000*(time.time()-st),'ms'
return data
return None
