def check_all_valid_str_keys(tp, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, options=options)
with h5py.File(filename) as f:
for k in key_value_names:
assert escape_path(k) not in f[name]
for k in data:
assert escape_path(k) in f[name]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def save(self):
logger.info("Saving ...")
df = pd.DataFrame(self._stats)
df.to_csv(os.path.join(self._output_dir,
'{}_measures.csv'.format(self._id)),
index=False)
sub_ids = np.array([
diffusion[0] for diffusion in self._diffusion_trials
]).flatten().astype(float)
all_trial_resp = np.array(
[diffusion[1] for diffusion in self._diffusion_trials])
all_trial_conds = np.array(
[diffusion[2] for diffusion in self._diffusion_trials])
data = {
'subjList': sub_ids,
'rt': all_trial_resp,
'subList': all_trial_conds,
}
hdf5storage.write(data,
'.',
os.path.join(
self._output_dir,
'{}_diffusion_trials'.format(self._id) + '.mat'),
matlab_compatible=True)
def check_int_key(tp, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
key = random_int()
data[key] = random_int()
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, options=options)
with h5py.File(filename, mode='r') as f:
assert_equal_nose(set(key_value_names), set(f[name].keys()))
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def test_plugin_marshaller_SubList():
mc = hdf5storage.MarshallerCollection(load_plugins=True,
lazy_loading=True)
options = hdf5storage.Options(store_python_metadata=True,
matlab_compatible=False,
marshaller_collection=mc)
ell = [1, 2, 'b1', b'3991', True, None]
data = example_hdf5storage_marshaller_plugin.SubList(ell)
f = None
name = '/a'
try:
f = tempfile.mkstemp()
os.close(f[0])
filename = f[1]
hdf5storage.write(data, path=name, filename=filename,
options=options)
out = hdf5storage.read(path=name, filename=filename,
options=options)
except:
raise
finally:
if f is not None:
os.remove(f[1])
assert_equal_nose(ell, list(out))
assert_equal_nose(type(out),
example_hdf5storage_marshaller_plugin.SubList)
def saveApprovedData(subjectPath):
approvedFile = os.path.join(subjectPath,'ea_coreg_approved.mat')
matfiledata = {}
if os.path.isfile(approvedFile):
try:
# read file and copy data except for glanat
with h5py.File(approvedFile,'r') as f:
for k in f.keys():
if k != 'glanat':
keyValue = f[k][()]
matfiledata[k] = keyValue
# now add approved glanat
matfiledata[u'glanat'] = np.array([2])
except: # use other reader for .mat file
f = io.loadmat(approvedFile)
for k in f.keys():
if k != 'glanat':
keyValue = f[k]
matfiledata[k] = keyValue
matfiledata['glanat'] = np.array([[2]],dtype='uint8')
io.savemat(approvedFile,matfiledata)
return
else:
matfiledata[u'glanat'] = np.array([2])
# save
# for some reason putting subject path into hdf5storage.write doesnt work
currentDir = os.getcwd()
os.chdir(subjectPath)
hdf5storage.write(matfiledata, '.', 'ea_coreg_approved.mat', matlab_compatible=True)
os.chdir(currentDir)
def test_multi_read():
# Makes a random dict of random paths and variables (random number
# of randomized paths with random numpy arrays as values).
data = dict()
for i in range(0, random.randint(min_dict_keys, \
max_dict_keys)):
name = random_name()
data[name] = \
random_numpy(random_numpy_shape( \
dict_value_subarray_dimensions, \
max_dict_value_subarray_axis_length), \
dtype=random.choice(dtypes))
paths = data.keys()
# Write it item by item and then read it back in one unit.
if os.path.exists(filename):
os.remove(filename)
try:
for p in paths:
hdf5storage.write(data=data[p], path=p, filename=filename)
out = hdf5storage.reads(paths=list(data.keys()),
filename=filename)
except:
raise
finally:
if os.path.exists(filename):
os.remove(filename)
# Compare data and out.
for i, p in enumerate(paths):
assert_equal(out[i], data[p])
def check_write_filters(filters):
# Read out the filter arguments.
filts = {
'compression': 'gzip',
'shuffle': True,
'fletcher32': True,
'gzip_level': 7
}
for k, v in filters.items():
filts[k] = v
# Make some random data. The dtype must be restricted so that it can
# be read back reliably.
dims = random.randint(1, 4)
dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
'complex128']))
data = random_numpy(shape=random_numpy_shape(dims, max_array_axis_length),
dtype=random.choice(dts))
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided filters and read it backt. The file needs to be deleted
# after to keep junk from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, \
store_python_metadata=False, matlab_compatible=False, \
compress=True, compress_size_threshold=0, \
compression_algorithm=filts['compression'], \
gzip_compression_level=filts['gzip_level'], \
shuffle_filter=filts['shuffle'], \
compressed_fletcher32_filter=filts['fletcher32'])
with h5py.File(filename) as f:
d = f[name]
fletcher32 = d.fletcher32
shuffle = d.shuffle
compression = d.compression
gzip_level = d.compression_opts
out = d[...]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Check the filters
assert_equal_nose(fletcher32, filts['fletcher32'])
assert_equal_nose(shuffle, filts['shuffle'])
assert_equal_nose(compression, filts['compression'])
if filts['compression'] == 'gzip':
assert_equal_nose(gzip_level, filts['gzip_level'])
# Compare
assert_equal(out, data)
def generate_histograms(self, hist_range=None):
print("Generating data for plotting examples")
testd = dataloader(dataset=self.dataset,
norm_method=self.norm_method,
val_fold=self.val_fold,
crop=self.crop,
inv_thresh=self.inv_thresh,
custom_data=None,
format='Hierarchical',
verbose=0)
testX = testd.data_test
testY = testd.label_test
testI = testd.inv_test
testP = testd.pid_test
hist_data = []
hist_label = []
hist_inv = []
hist_pid = []
for pid in range(len(testX)):
print("Creating histogram for patient ID " + str(pid) + " of " + str(len(testX)))
for core in range(len(testX[pid])):
core_loss_vec = tf.keras.losses.mean_squared_error(np.reshape(testX[pid][core], (testX[pid][core].shape[0], testX[pid][core].shape[1])),
np.reshape(self.model.predict(testX[pid][core], batch_size=None), (testX[pid][core].shape[0], testX[pid][core].shape[1])))
hist_data.append(np.histogram(core_loss_vec, bins=50, range=hist_range, density=True))
hist_label.append(testY[pid][core])
hist_inv.append(testI[pid][core])
hist_pid.append(testP[pid])
save_dict = {'data': hist_data, 'label': hist_label, 'inv': hist_inv, 'PatientId': hist_pid}
hdf5storage.write(save_dict)
def save2mat73(filename,
out_filename,
small_file=False,
dla_nhi_cut=False,
sample_file="dla_samples_a03.mat"):
'''
convert HDF5 saved by h5py to MATLAB compatible format
Parameters:
----
small_file (bool) : True if you don't want to save sample_log_likelihoods
'''
import hdf5storage
f = h5py.File(filename)
processed_file = {}
for key in f.keys():
if small_file:
if "sample_log_likelihoods" in key or "base_sample_inds" in key:
continue
processed_file[u'{}'.format(key)] = np.transpose(f[key][()])
hdf5storage.write(processed_file,
'.',
out_filename,
matlab_compatible=True)
def check_all_valid_str_keys(tp, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
options=options)
with h5py.File(filename) as f:
for k in key_value_names:
assert escape_path(k) not in f[name]
for k in data:
assert escape_path(k) in f[name]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def test_plugin_marshaller_SubList():
mc = hdf5storage.MarshallerCollection(load_plugins=True,
lazy_loading=True)
options = hdf5storage.Options(store_python_metadata=True,
matlab_compatible=False,
marshaller_collection=mc)
ell = [1, 2, 'b1', b'3991', True, None]
data = example_hdf5storage_marshaller_plugin.SubList(ell)
f = None
name = '/a'
try:
f = tempfile.mkstemp()
os.close(f[0])
filename = f[1]
hdf5storage.write(data, path=name, filename=filename,
options=options)
out = hdf5storage.read(path=name, filename=filename,
options=options)
except:
raise
finally:
if f is not None:
os.remove(f[1])
assert_equal_nose(ell, list(out))
assert_equal_nose(type(out),
example_hdf5storage_marshaller_plugin.SubList)
def tang_save_features(data, labels, groundtruthfilename='100p'):
"""temp kludge
"""
[height, width, nbands] = data.shape
x = tf.placeholder(tf.float32, shape=(19,19,nbands+18))
feat = tang_net(x)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
padded_data = np.pad(data, ((9,9),(9,9),(9,9)), 'reflect')
all_pixels = np.array(list(itertools.product(range(width),range(height))))
labelled_pixels = all_pixels[:10]
print('requesting %d MB memory' % (labelled_pixels.shape[0] * 271*nbands * 4 / 1000000.0))
labelled_pix_feat = np.zeros((labelled_pixels.shape[0], 271*nbands), dtype=np.float32)
for pixel_i, pixel in enumerate(tqdm(labelled_pixels)):
# this iterates through columns first
[pixel_x, pixel_y] = pixel
subimg = padded_data[pixel_y:(pixel_y+19), pixel_x:(pixel_x+19), :]
feed_dict = {x: subimg}
labelled_pix_feat[pixel_i,:] = sess.run(feat, feed_dict)
pdb.set_trace()
flat_labels = labels.transpose().reshape(height*width)
trainY = flat_labels[flat_labels!=0]
print('starting training')
start = time.time()
clf = SVC(kernel='linear')
clf.fit(labelled_pix_feat, trainY)
end = time.time()
print(end - start)
# now start predicting the full image, 1 column at a time
col_feat = np.zeros((height, 271*nbands), dtype=np.float32)
pred_image = np.zeros((height,width), dtype=int)
test_flags = '-q'
for pixel_x in tqdm(range(width)):
# get feat
for pixel_y in range(height):
subimg = padded_data[pixel_y:(pixel_y+19), pixel_x:(pixel_x+19), :]
feed_dict = {x: subimg}
col_feat[pixel_y,:] = sess.run(feat, feed_dict)
# get pred for feat
# dontcare = [0] * height
p_label = clf.predict(col_feat);
pred_image[:,pixel_x] = np.array(p_label).astype(int)
imgmatfiledata = {}
imgmatfiledata[u'imgHat'] = pred_image
imgmatfiledata[u'groundtruthfilename'] = groundtruthfilename
hdf5storage.write(imgmatfiledata, filename=groundtruthfilename+'_100p_tang_fullimg.mat', matlab_compatible=True)
print('done making img, run hundredpercent_img_figures.m')
def save2mat73(filename,
out_filename,
small_file=False,
dla_nhi_cut=False,
sample_file="dla_samples_a03.mat"):
'''
convert HDF5 saved by h5py to MATLAB compatible format
Parameters:
----
small_file (bool) : True if you don't want to save sample_log_likelihoods
'''
import hdf5storage
f = h5py.File(filename)
processed_file = {}
# small file checking conditions, discard vars if they are in this list
conds = ["sample_log_", "base_sample_inds", "sample_kim_log_likelihoods"]
check_conds = lambda key: any([cond in key for cond in conds])
for key in f.keys():
if small_file:
if check_conds(key):
continue
processed_file[u'{}'.format(key)] = np.transpose(f[key][()])
hdf5storage.write(processed_file,
'.',
out_filename,
matlab_compatible=True)
def mock_svhn_dataset(tmpdir_factory, mock_image_stream):
root = tmpdir_factory.mktemp("datasets")
svhn_root = root.mkdir("svhn")
file = BytesIO(mock_image_stream)
# ascii image names
first = np.array([[49], [46], [112], [110], [103]], dtype=np.int16) # 1.png
second = np.array([[50], [46], [112], [110], [103]], dtype=np.int16) # 2.png
third = np.array([[51], [46], [112], [110], [103]], dtype=np.int16) # 3.png
# labels: label is also ascii
label = {
"height": [35, 35, 35, 35],
"label": [1, 1, 3, 7],
"left": [116, 128, 137, 151],
"top": [27, 29, 29, 26],
"width": [15, 10, 17, 17],
}
matcontent = {"digitStruct": {"name": [first, second, third], "bbox": [label, label, label]}}
# Mock train data
train_root = svhn_root.mkdir("train")
hdf5storage.write(matcontent, filename=train_root.join("digitStruct.mat"))
for i in range(3):
fn = train_root.join(f"{i+1}.png")
with open(fn, "wb") as f:
f.write(file.getbuffer())
# Packing data into an archive to simulate the real data set and bypass archive extraction
archive_path = root.join("svhn_train.tar")
shutil.make_archive(root.join("svhn_train"), "tar", str(svhn_root))
return str(archive_path)
def test_multi_read():
# Makes a random dict of random paths and variables (random number
# of randomized paths with random numpy arrays as values).
data = dict()
for i in range(0, random.randint(min_dict_keys, \
max_dict_keys)):
name = random_name()
data[name] = \
random_numpy(random_numpy_shape( \
dict_value_subarray_dimensions, \
max_dict_value_subarray_axis_length), \
dtype=random.choice(dtypes))
paths = data.keys()
# Write it item by item and then read it back in one unit.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
for p in paths:
hdf5storage.write(data=data[p], path=p, filename=filename)
out = hdf5storage.reads(paths=list(data.keys()), filename=filename)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Compare data and out.
for i, p in enumerate(paths):
assert_equal(out[i], data[p])
def check_write_filters(filters):
# Read out the filter arguments.
filts = {'compression': 'gzip',
'shuffle': True,
'fletcher32': True,
'gzip_level': 7}
for k, v in filters.items():
filts[k] = v
# Make some random data. The dtype must be restricted so that it can
# be read back reliably.
dims = random.randint(1, 4)
dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
'complex128']))
data = random_numpy(shape=random_numpy_shape(dims,
max_array_axis_length),
dtype=random.choice(dts))
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided filters and read it backt. The file needs to be deleted
# after to keep junk from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, \
store_python_metadata=False, matlab_compatible=False, \
compress=True, compress_size_threshold=0, \
compression_algorithm=filts['compression'], \
gzip_compression_level=filts['gzip_level'], \
shuffle_filter=filts['shuffle'], \
compressed_fletcher32_filter=filts['fletcher32'])
with h5py.File(filename) as f:
d = f[name]
fletcher32 = d.fletcher32
shuffle = d.shuffle
compression = d.compression
gzip_level = d.compression_opts
out = d[...]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
# Check the filters
assert_equal_nose(fletcher32, filts['fletcher32'])
assert_equal_nose(shuffle, filts['shuffle'])
assert_equal_nose(compression, filts['compression'])
if filts['compression'] == 'gzip':
assert_equal_nose(gzip_level, filts['gzip_level'])
# Compare
assert_equal(out, data)
def FISR_for_video_Warp_Img(args, flow_file_name):
""" (when preparing 'FISR_for_video') Example to make 'E:/FISR_Github/FISR_test_folder/scene1/scene1_ss1_fr5_warp.mat' from 'E:/FISR_Github/FISR_test_folder/scene1/scene1_test_ss1_fr5.flo' '"""
""" Please check '# check' marks in this code to fit your usage. """
num_fr = args.frame_num # check, in our test set (=5)
# Read data from mat file
data_list = glob.glob(os.path.join(args.frame_folder_path,
'*.png')) # read YUV format images.
ss = 1
h = args.FISR_input_size[0] # check, assumption: 2K input, then, 1080
w = args.FISR_input_size[1] # check, assumption: 2K input, then, 1920
pred = np.zeros((num_fr - 1, 2, h, w, 3), dtype=np.float32)
flow_path = flow_file_name
flow = read_flo_file_5dim(flow_path)
for fr in range(num_fr - 1):
rgb_1 = Image.open(data_list[ss * fr])
rgb_1 = np.array(rgb_1, dtype=np.float32)
rgb_1 = YUV2RGB(
rgb_1
) # since PWC-Net works on RGB images, we have to convert YUV img into RGB img.
rgb_2 = Image.open(data_list[ss * (fr + 1)])
rgb_2 = np.array(rgb_2, dtype=np.float32)
rgb_2 = YUV2RGB(
rgb_2
) # since PWC-Net works on RGB images, we have to convert YUV img into RGB img.
### 1 -> 2 ###
flow_sample = flow[fr, 0, :, :, :]
warped_img_1 = warp_flow(rgb_2, flow_sample * 0.5)
pred[fr, 0, :, :, :] = RGB2YUV(warped_img_1)
### 2 -> 1 ###
flow_sample = flow[fr, 1, :, :, :]
warped_img_2 = warp_flow(rgb_1, flow_sample * 0.5)
pred[fr, 1, :, :, :] = RGB2YUV(warped_img_2)
print("Processing for warping imgs [%5d/%5d]" % (fr + 1, num_fr))
pred_warp = {}
pred_warp[u'pred'] = pred
warp_file_name = args.frame_folder_path + '/' + args.frame_folder_path.split(
'/')[-1] + '_ss{}_fr{}_warp.mat'.format(ss, num_fr)
hdf5storage.write(pred_warp, '.', warp_file_name, matlab_compatible=True)
print('[*] Warp file saved!')
'''
plt.figure(2, figsize=(5*4, 5))
plt.subplot(1, 4, 1)
plt.imshow(np.uint8(rgb_1))
plt.subplot(1, 4, 2)
plt.imshow(np.uint8(warped_img_1))
plt.subplot(1, 4, 3)
plt.imshow(np.uint8(rgb_2))
plt.subplot(1, 4, 4)
plt.imshow(np.uint8(warped_img_2))
plt.show()
'''
return warp_file_name
def check_write_filters(filters):
# Read out the filter arguments.
filts = {"compression": "gzip", "shuffle": True, "fletcher32": True, "gzip_level": 7}
for k, v in filters.items():
filts[k] = v
# Make some random data. The dtype must be restricted so that it can
# be read back reliably.
dims = random.randint(1, 4)
dts = tuple(set(dtypes) - set(["U", "S", "bool", "complex64", "complex128"]))
data = random_numpy(shape=random_numpy_shape(dims, max_array_axis_length), dtype=random.choice(dts))
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided filters and read it backt. The file needs to be deleted
# before and after to keep junk from building up.
if os.path.exists(filename):
os.remove(filename)
try:
hdf5storage.write(
data,
path=name,
filename=filename,
store_python_metadata=False,
matlab_compatible=False,
compress=True,
compress_size_threshold=0,
compression_algorithm=filts["compression"],
gzip_compression_level=filts["gzip_level"],
shuffle_filter=filts["shuffle"],
compressed_fletcher32_filter=filts["fletcher32"],
)
with h5py.File(filename) as f:
d = f[name]
fletcher32 = d.fletcher32
shuffle = d.shuffle
compression = d.compression
gzip_level = d.compression_opts
out = d[...]
except:
raise
finally:
if os.path.exists(filename):
os.remove(filename)
# Check the filters
assert fletcher32 == filts["fletcher32"]
assert shuffle == filts["shuffle"]
assert compression == filts["compression"]
if filts["compression"] == "gzip":
assert gzip_level == filts["gzip_level"]
# Compare
assert_equal(out, data)
def saveh5(self, file_name, path='/', truncate_existing=False,
matlab_compatible=False, **kwargs):
"""Save to an HDF5 file
:param file_name: output file name
:param path: group path to store fields to
"""
hdf5storage.write(self, path, file_name, truncate_existing=truncate_existing,
marshaller_collection=self.__mc(),
matlab_compatible=matlab_compatible)
def fit(self, X, y):
currdir = os.path.dirname(__file__)
basedir = os.path.abspath(os.path.join(currdir, os.pardir))
m_path = os.path.join(basedir, 'external', 'octave')
os.chdir(m_path)
self.__tmpdir = tempfile.mkdtemp()
y = numpy.array(y)
Xc = X[y == 0]
Xs = X[y == 1]
if len(Xc) > len(Xs):
Xs = Xs[:len(Xc)]
if len(Xs) > len(Xc):
Xc = Xc[:len(Xs)]
pcover = self.__tmpdir + "/F_train_cover.mat"
#savemat(pcover, mdict={'F': numpy.array(Xc)}, oned_as='column')
hdf5storage.write({u'F': numpy.array(Xc)},
'.',
pcover,
matlab_compatible=True)
pstego = self.__tmpdir + "/F_train_stego.mat"
#savemat(pstego, mdict={'F': numpy.array(Xs)}, oned_as='column')
hdf5storage.write({u'F': numpy.array(Xs)},
'.',
pstego,
matlab_compatible=True)
pclf = self.__tmpdir + "/clf.mat"
del Xc
del Xs
del X
m_code = ""
m_code += "cd " + self.__tmpdir + ";"
m_code += "addpath('" + m_path + "');"
m_code += "warning('off');"
m_code += "ensemble_fit('" + pcover + "', '" + pstego + "', '" + pclf + "');"
m_code += "exit"
p = subprocess.Popen(M_BIN + " \"" + m_code + "\"",
stdout=subprocess.PIPE,
shell=True)
# output, err = p.communicate()
status = p.wait()
self.__mat_clf = loadmat(pclf)
shutil.rmtree(self.__tmpdir)
def train(epochs):
nm = []
ep = []
start_time = datetime.datetime.now()
for epoch in range(epochs):
print("-----\nEPOCH:", epoch)
# train
for bi, (target, input_image) in enumerate(load_image_train(path)):
elapsed_time = datetime.datetime.now() - start_time
gen_loss, disc_loss = train_step(input_image, target)
print("B/E:", bi, '/', epoch, ", Generator loss:",
gen_loss.numpy(), ", Discriminator loss:", disc_loss.numpy(),
', time:', elapsed_time)
# generated and see the progress
for bii, (tar, inp) in enumerate(load_image_test(path)):
if bii == 100:
generated_image(generator, inp, tar, t=epoch + 1)
# save checkpoint
# if (epoch + 1) % 2 == 0:
ep.append(epoch + 1)
#generator.save_weights(os.path.join(BASE_PATH, "weights/generator_"+str(epoch)+".h5"))
#discriminator.save_weights(os.path.join(BASE_PATH, "weights/discriminator_"+str(epoch)+".h5"))
realim, inpuim = load_image_test_y(path)
prediction = generator(inpuim)
nm.append(fuzz.nmse(np.squeeze(realim), np.squeeze(prediction)))
if epoch == epochs - 1:
nmse_epoch = TemporaryFile()
np.save(nmse_epoch, nm)
# Save the predicted Channel
matfiledata = {} # make a dictionary to store the MAT data in
matfiledata[
u'predict_Gan_0_dB_Indoor2p4_64ant_32users_8pilot'] = np.array(
prediction
) # *** u prefix for variable name = unicode format, no issues thru Python 3.5; advise keeping u prefix indicator format based on feedback despite docs ***
hdf5storage.write(matfiledata,
'.',
'Results\Eest_cGAN_' + str(epoch + 1) +
'_0db_Indoor2p4_64ant_32users_8pilot.mat',
matlab_compatible=True)
plt.figure()
plt.plot(ep, nm, '^-r')
plt.xlabel('Epoch')
plt.ylabel('NMSE')
plt.show()
return nm, ep
def predict(args):
bs = args.batch_size
network = network_dict[args.network]
n_classes = nclass_dict[args.dataset]
trainimgname, trainlabelname = dset_filenames_dict[args.dataset]
trainimgfield, trainlabelfield = dset_fieldnames_dict[args.dataset]
st_net_spec = sts_dict[args.st_type]
data, labels = load_data(trainimgname, trainimgfield, trainlabelname, trainlabelfield, dataset_path=args.data_root)
if args.fst_preprocessing:
data = load_or_preprocess_data(data, args.preprocessed_data_path, args.model_root, st_net_spec=st_net_spec, st_patch_size=args.st_patch_size)
elif args.npca_components is not None:
data = pca_embedding(data, n_components=args.npca_components)
if args.attribute_profile:
data = build_profile(data)
height, width, bands = dset_dims[trainimgname]
# if there are multiple saved *pb files get the newest
best_models_dir = os.path.join(args.model_root, PB_EXPORT_DIR)
subdirs = [x for x in os.listdir(best_models_dir) if os.path.isdir(os.path.join(best_models_dir, x)) and 'temp' not in str(x)]
latest = sorted(subdirs)[-1]
full_model_dir = os.path.join(best_models_dir, latest)
with tf.Session() as sess:
tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], full_model_dir)
predictor = tf.contrib.predictor.from_saved_model(full_model_dir)
y_predicted = []
nbatches = (height * width // bs) + 1
# TODO do this for fst preprocessed data
s = args.network_spatial_size - 1
for bi, batchX in enumerate(tqdm(cube_iter(data, bs, addl_padding=(s,s,0)), desc='Predicting', total=nbatches)):
y_predicted += list(predictor({"subimages": batchX })['output'])
pred_image = np.array(y_predicted).reshape((width, height)).T
imgmatfiledata = {}
imgmatfiledata[u'imgHat'] = pred_image
groundtruthfilename = os.path.splitext(trainlabelname)[0]
imgmatfiledata[u'groundtruthfilename'] = '%s_%s.mat' % (groundtruthfilename, args.network)
hdf5storage.write(imgmatfiledata,
filename=os.path.join(args.model_root, imgmatfiledata[u'groundtruthfilename']),
matlab_compatible=True)
print('Saved %s' % os.path.join(args.model_root, imgmatfiledata[u'groundtruthfilename']))
npz_path = os.path.join(args.model_root, '%s_%s.npz' % (groundtruthfilename, args.network))
np.savez(npz_path, pred_image=pred_image)
print('Saved %s' % npz_path)
def write_dict_to_mat(mat_file_path,
dict_to_write,
version='7.3'): # field must be a dict
assert HAVE_HDF5STORAGE, "To use the MATSortingExtractor write_dict_to_mat function install hdf5storage: " \
"\n\n pip install hdf5storage\n\n"
if version == '7.3':
hdf5storage.write(dict_to_write,
'/',
mat_file_path,
matlab_compatible=True,
options='w')
elif version < '7.3' and version > '4':
savemat(mat_file_path, dict_to_write)
def WriteMatlab(data_np, VarName, FileName):
"""
Function:
WriteMatlab
Parameters:
data_np: input the path need to clean
VarName:
FileName:
Returns:
PIL format image
"""
matcontent = {}
matcontent[VarName] = data_np
hdf5storage.write(matcontent, filename=FileName, matlab_compatible=True)
def write_readback(self, data, name, options, read_options=None):
# Randomly convert the name to other path types.
path_choices = (str, bytes, pathlib.PurePath, pathlib.PurePosixPath,
pathlib.PureWindowsPath, pathlib.Path)
name_type_w = random.choice(path_choices)
name_type_r = random.choice(path_choices)
# Name to write with.
if name_type_w == bytes:
name_w = name.encode('utf-8')
elif name_type_w in (pathlib.PurePath, pathlib.PurePosixPath,
pathlib.PosixPath):
name_w = name_type_w(name)
elif name_type_w != str:
name_w = name_type_w(name[posixpath.isabs(name):])
else:
name_w = name
# Name to read with.
if name_type_r == bytes:
name_r = name.encode('utf-8')
elif name_type_r in (pathlib.PurePath, pathlib.PurePosixPath,
pathlib.PosixPath):
name_r = name_type_r(name)
elif name_type_r != str:
name_r = name_type_r(name[posixpath.isabs(name):])
else:
name_r = name
# Write the data to the proper file with the given name, read it
# back, and return the result. The file needs to be deleted
# after to keep junk from building up. Different options can be
# used for reading the data back.
f = None
try:
f = tempfile.mkstemp()
os.close(f[0])
filename = f[1]
hdf5storage.write(data,
path=name_w,
filename=filename,
options=options)
out = hdf5storage.read(path=name_r,
filename=filename,
options=read_options)
except:
raise
finally:
if f is not None:
os.remove(f[1])
return out
def quads_to_traintestfile(quads, selection, gt, name):
train = np.zeros((height, width))
for i, q in enumerate(quads):
if selection[i] == 1:
train[q.y:q.y_end, q.x:q.x_end] = 1
train_mask = (gt != 0) * train
train_mask = train_mask.transpose().reshape(height * width)
test_mask = (gt != 0) * (1 - train)
test_mask = test_mask.transpose().reshape(height * width)
mat_outdata = {}
mat_outdata[u'test_mask'] = test_mask
mat_outdata[u'train_mask'] = train_mask
hdf5storage.write(mat_outdata,
filename=os.path.join(DATASET_PATH, name),
matlab_compatible=True)
def write_readback(self, data, name, options):
# Write the data to the proper file with the given name, read it
# back, and return the result. The file needs to be deleted
# before and after to keep junk from building up.
if os.path.exists(self.filename):
os.remove(self.filename)
try:
hdf5storage.write(data, path=name, filename=self.filename,
options=options)
out = hdf5storage.read(path=name, filename=self.filename,
options=options)
except:
raise
finally:
if os.path.exists(self.filename):
os.remove(self.filename)
return out
def predict_proba(self, X):
currdir = os.path.dirname(__file__)
basedir = os.path.abspath(os.path.join(currdir, os.pardir))
m_path = os.path.join(basedir, 'external', 'octave')
os.chdir(m_path)
self.__tmpdir = tempfile.mkdtemp()
prob = []
path = self.__tmpdir + "/F_test.mat"
#savemat(path, mdict={'F': numpy.array(X)}, oned_as='column')
hdf5storage.write({u'F': numpy.array(X)},
'.',
path,
matlab_compatible=True)
pclf = self.__tmpdir + "/clf.mat"
savemat(pclf, self.__mat_clf)
pvotes = self.__tmpdir + "/votes.txt"
m_code = ""
m_code += "cd " + self.__tmpdir + ";"
m_code += "addpath('" + m_path + "');"
m_code += "warning('off');"
m_code += "ensemble_predict('" + pclf + "', '" + path + "', '" + pvotes + "');"
m_code += "exit"
p = subprocess.Popen(M_BIN + " \"" + m_code + "\"",
stdout=subprocess.PIPE,
shell=True)
#output, err = p.communicate()
status = p.wait()
with open(pvotes, 'r') as f:
lines = f.readlines()
f.close()
shutil.rmtree(self.__tmpdir)
for l in lines:
votes = (1 + float(l) / 500) / 2
prob.append([1 - votes, votes])
return prob
def write_array(self):
# Check if there is anything to save
if len(self.__dataset['config']) == 0:
self.__dataset['config'] = [['config'], 'missing']
config = {}
else:
config = self.__dataset['config']
if len(self.__dataset['device']) == 0:
self.__dataset['device'] = [['device'], 'missing']
device = {}
else:
device = self.__dataset['device']
#copy buffer
data_to_write = self.__dataset
#Reset dataset
self.set_data_structure(config, device)
file_name = self.__file_out
if self.files_written:
if '.mat' in self.__file_out[len(self.__file_out) -
4:len(self.__file_out)]:
file_name = self.__file_out[0:len(self.__file_out) -
4] + '_' + str(self.files_written +
1)
else:
file_name = self.__file_out + '_' + str(self.files_written + 1)
if not file_name.endswith('.mat'):
file_name = file_name + '.mat'
data_to_write['IXDATA'] = self.convert_list_to_numpy_list(
data_to_write['IXDATA'])
data_to_write['IXDATA'][u'markers'] = self.__markers.markers()
if 'elements' in data_to_write:
data_to_write['elements'] = self.convert_list_to_numpy_list(
data_to_write['elements'])
h5.write(data_to_write,
path='/',
filename=file_name,
truncate_existing=True,
store_python_metadata=True,
matlab_compatible=True)
self.files_written += 1
def test_O_field_compound():
name = '/a'
data = np.empty(shape=(1, ), dtype=[('O', 'int8'), ('a', 'uint16')])
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
matlab_compatible=False,
structured_numpy_ndarray_as_struct=False)
with h5py.File(filename) as f:
assert isinstance(f[name], h5py.Dataset)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def test_O_field_compound():
name = '/a'
data = np.empty(shape=(1, ), dtype=[('O', 'int8'), ('a', 'uint16')])
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
matlab_compatible=False,
structured_numpy_ndarray_as_struct=False)
with h5py.File(filename, mode='r') as f:
assert isinstance(f[name], h5py.Dataset)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def test_conv_utf16():
name = '/a'
data = np.unicode_('abcdefghijklmnopqrstuvwxyz')
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
matlab_compatible=False,
store_python_metadata=False,
convert_numpy_str_to_utf16=True)
with h5py.File(filename) as f:
assert f[name].dtype.type == np.uint16
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def saveApprovedDataLegacy(normalizationMethodFile):
try:
import h5py
except:
slicer.util.pip_install('h5py')
import h5py
try:
import hdf5storage
except:
slicer.util.pip_install('hdf5storage')
import hdf5storage
matfiledata = {}
if os.path.isfile(normalizationMethodFile):
try:
# read file and copy data except for glanat
with h5py.File(normalizationMethodFile,'r') as f:
for k in f.keys():
if k != 'glanat':
keyValue = f[k][()]
matfiledata[k] = keyValue
# now add approved glanat
matfiledata[u'glanat'] = np.array([2])
except: # use other reader for .mat file
f = io.loadmat(normalizationMethodFile)
for k in f.keys():
if k != 'glanat':
keyValue = f[k]
matfiledata[k] = keyValue
matfiledata['glanat'] = np.array([[2]],dtype='uint8')
io.savemat(normalizationMethodFile,matfiledata)
return
else:
matfiledata[u'glanat'] = np.array([2])
# save
# for some reason putting subject path into hdf5storage.write doesnt work
currentDir = os.getcwd()
os.chdir(os.path.dirname(normalizationMethodFile))
hdf5storage.write(matfiledata, '.', 'ea_coreg_approved.mat', matlab_compatible=True)
os.chdir(currentDir)
def test_conv_utf16():
name = '/a'
data = np.unicode_('abcdefghijklmnopqrstuvwxyz')
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
matlab_compatible=False,
store_python_metadata=False,
convert_numpy_str_to_utf16=True)
with h5py.File(filename) as f:
assert_equal_nose(f[name].dtype.type, np.uint16)
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def check_string_type_non_str_key(tp, other_tp, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
keys = list(data.keys())
key_gen = random_str_some_unicode(max_dict_key_length)
if other_tp == 'numpy.bytes_':
key = np.bytes_(key_gen.encode('UTF-8'))
elif other_tp == 'numpy.unicode_':
key = np.unicode_(key_gen)
elif other_tp == 'bytes':
key = key_gen.encode('UTF-8')
data[key] = random_int()
keys.append(key_gen)
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
options=options)
with h5py.File(filename) as f:
assert_equal_nose(set(keys), set(f[name].keys()))
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def test(filedir, filenames, flat_labels=None):
C = np.zeros((nlabels, nlabels))
chunk_size = height * width
print('testing now')
mat_outdata = {}
mat_outdata[u'metrics'] = {}
with open(outfilename + '.csv', 'a') as fd:
fd.write('id,rle_mask\n')
for file_i, image_name in enumerate(tqdm(filenames)):
filename = os.path.join(filedir, 'images', image_name)
im = Image.open(filename).convert('L')
npim = np.array(im, dtype=np.float32) / 255.0
image_padded = np.pad(npim, ((ap[0] / 2, ap[0] / 2),
(ap[1] / 2, ap[1] / 2)),
'reflect')
subimg = np.expand_dims(image_padded, 0)
subimg = np.expand_dims(subimg, -1)
feed_dict = {x: subimg}
net_out = sess.run(feat, feed_dict).reshape(
(height * width, feat_size))
p_label, p_acc, p_val = svm_predict(
np.zeros(height * width).tolist(), net_out.tolist(), m,
'-q')
if flat_labels is not None:
C += confusion_matrix(
flat_labels[(file_i * chunk_size):((file_i + 1) *
chunk_size)],
p_label,
labels=[0, 1])
mat_outdata[u'metrics'][u'CM'] = C
hdf5storage.write(mat_outdata,
filename=outfilename + '.mat',
matlab_compatible=True)
else:
pred = np.array(p_label).reshape(
(height, width)).transpose().reshape(chunk_size)
fileid, file_extension = os.path.splitext(image_name)
fd.write('%s,%s\n' % (fileid, myrlestring(pred)))
def check_string_type_non_str_key(tp, other_tp, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
keys = list(data.keys())
key_gen = random_str_some_unicode(max_dict_key_length)
if other_tp == 'numpy.bytes_':
key = np.bytes_(key_gen.encode('UTF-8'))
elif other_tp == 'numpy.unicode_':
key = np.unicode_(key_gen)
elif other_tp == 'bytes':
key = key_gen.encode('UTF-8')
data[key] = random_int()
keys.append(key_gen)
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename, options=options)
with h5py.File(filename) as f:
assert_equal_nose(set(keys), set(f[name].keys()))
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def write_readback(self, data, name, options):
# Write the data to the proper file with the given name, read it
# back, and return the result. The file needs to be deleted
# before and after to keep junk from building up.
if os.path.exists(self.filename):
os.remove(self.filename)
try:
hdf5storage.write(data,
path=name,
filename=self.filename,
options=options)
out = hdf5storage.read(path=name,
filename=self.filename,
options=options)
except:
raise
finally:
if os.path.exists(self.filename):
os.remove(self.filename)
return out
def save_masks_matlab_style(data_path, name_prefix, train_mask, val_mask):
"""
"""
matfiledata = {}
# matlab style is to collapse a matrix with columns first
matfiledata[u'train_mask'] = train_mask.T.flatten()
matfiledata[u'test_mask'] = val_mask.T.flatten()
id = str(
hash(
tuple(
np.concatenate(
[matfiledata[u'train_mask'], matfiledata[u'test_mask']]))))
outfilename = '%s_%s.mat' % (name_prefix, id[-6:])
outfile = os.path.join(data_path, outfilename)
hdf5storage.write(matfiledata, filename=outfile, matlab_compatible=True)
print('Saved %s' % outfile)
def write_readback(self, data, name, options, read_options=None):
# Write the data to the proper file with the given name, read it
# back, and return the result. The file needs to be deleted
# after to keep junk from building up. Different options can be
# used for reading the data back.
f = None
try:
f = tempfile.mkstemp()
os.close(f[0])
filename = f[1]
hdf5storage.write(data, path=name, filename=filename,
options=options)
out = hdf5storage.read(path=name, filename=filename,
options=read_options)
except:
raise
finally:
if f is not None:
os.remove(f[1])
return out
def check_str_key_previously_invalid_char(tp, ch, option_keywords):
options = hdf5storage.Options(**option_keywords)
key_value_names = (options.dict_like_keys_name,
options.dict_like_values_name)
data = random_dict(tp)
for k in key_value_names:
if k in data:
del data[k]
# Add a random invalid str key using the provided character
key = key_value_names[0]
while key in key_value_names:
key = ch.join([random_str_ascii(max_dict_key_length)
for i in range(2)])
data[key] = random_int()
# Make a random name.
name = random_name()
# Write the data to the proper file with the given name with the
# provided options. The file needs to be deleted after to keep junk
# from building up.
fld = None
try:
fld = tempfile.mkstemp()
os.close(fld[0])
filename = fld[1]
hdf5storage.write(data, path=name, filename=filename,
options=options)
with h5py.File(filename) as f:
for k in key_value_names:
assert escape_path(k) not in f[name]
for k in data:
assert escape_path(k) in f[name]
except:
raise
finally:
if fld is not None:
os.remove(fld[1])
def write_array(self):
# Check if there is anything to save
if len(self.__dataset['config']) == 0:
self.__dataset['config'] = [['config'], 'missing']
if len(self.__dataset['device']) == 0:
self.__dataset['device'] = [['device'], 'missing']
file_name = self.__file_out
if self.files_written:
if '.mat' in self.__file_out[len(self.__file_out)-4:len(self.__file_out)]:
file_name = self.__file_out[0:len(self.__file_out)-4] + '_' + str(self.files_written+1)
else:
file_name = self.__file_out + '_' + str(self.files_written+1)
self.__dataset['IXDATA'] = self.convert_list_to_numpy_list(self.__dataset['IXDATA'])
self.__dataset['IXDATA'][u'markers'] = self.__markers.markers()
if 'elements' in self.__dataset:
self.__dataset['elements'] = self.convert_list_to_numpy_list(self.__dataset['elements'])
h5.write(self.__dataset, path='/', filename=file_name, truncate_existing=True, store_python_metadata=True, matlab_compatible=True)
self.files_written += 1
def check_uncompressed_write_filters(method,
uncompressed_fletcher32_filter,
filters):
# Read out the filter arguments.
filts = {'compression': 'gzip',
'shuffle': True,
'fletcher32': True,
'gzip_level': 7}
for k, v in filters.items():
filts[k] = v
# Make some random data. The dtype must be restricted so that it can
# be read back reliably.
dims = random.randint(1, 4)
dts = tuple(set(dtypes) - set(['U', 'S', 'bool', 'complex64', \
'complex128']))
data = random_numpy(shape=random_numpy_shape(dims,
max_array_axis_length),
dtype=random.choice(dts))
# Make a random name.
name = random_name()
# Make the options to disable compression by the method specified,
# which is either that it is outright disabled or that the data is
# smaller than the compression threshold.
if method == 'compression_disabled':
opts = {'compress': False, 'compress_size_threshold': 0}
else:
opts = {'compress': True,
'compress_size_threshold': data.nbytes + 1}
# Write the data to the proper file with the given name with the
# provided filters and read it backt. The file needs to be deleted
# before and after to keep junk from building up.
if os.path.exists(filename):
os.remove(filename)
try:
hdf5storage.write(data, path=name, filename=filename, \
store_python_metadata=False, matlab_compatible=False, \
compression_algorithm=filts['compression'], \
gzip_compression_level=filts['gzip_level'], \
shuffle_filter=filts['shuffle'], \
compressed_fletcher32_filter=filts['fletcher32'], \
uncompressed_fletcher32_filter= \
uncompressed_fletcher32_filter, \
**opts)
with h5py.File(filename) as f:
d = f[name]
fletcher32 = d.fletcher32
shuffle = d.shuffle
compression = d.compression
gzip_level = d.compression_opts
out = d[...]
except:
raise
finally:
if os.path.exists(filename):
os.remove(filename)
# Check the filters
assert compression == None
assert shuffle == False
assert fletcher32 == uncompressed_fletcher32_filter
# Compare
assert_equal(out, data)
def __init__(self, filename,
Version='2.0',
data_type='double',
data_storage_type='single',
compression='gzip',
compression_opts=9,
shuffle=True,
fletcher32=True,
chunks=(1024, None),
Info=dict(),
**keywords):
# Set this first before anything else so that nothing goes wrong
# on deletion.
self._file = None
# Check that the Version is valid, and get the Version string we
# will be using.
if not isinstance(Version, str):
raise ValueError('Version must be a bytes.')
new_version = _get_supported_version(Version).encode()
if new_version is None:
raise ValueError('Unsupported Version.')
# First, if any additional keyword arguments were given, they
# need to be stuffed into Info.
for k, v in keywords.items():
Info[k] = v
# Validate inputs.
# All the simple arguments must be the right type, and that the
# right things are there.
if type(Version) != str \
or type(data_type) != str \
or type(data_storage_type) != str:
raise ValueError('At least one input arguments is not of '
+ 'right type.')
# Various parameters in info that need to be checked. In each
# tuple, the first element is the name, the second is a buple of
# types it must be one of, and the third is a default value to
# give if present (if no default value is present, the parameter
# is required to be given). Also, all string types need to be
# converted to numpy bytes_.
if sys.hexversion >= 0x03000000:
string_types = (str, bytes, np.bytes_, np.str_)
else:
string_types = (unicode, str, np.bytes_, np.unicode_)
params = [ \
('VendorDriverDescription', string_types, b''), \
('DeviceName', string_types, b''), \
('ID', string_types, b''), \
('TriggerType', string_types, b''), \
('SampleFrequency', (np.float64,)), \
('InputType', string_types, b''), \
('NumberChannels', (np.int64,)), \
('NumberSamplesBinned', (np.int64,), 1), \
('Bits', (np.int64,), np.int64(-1)), \
('ChannelMappings', (np.ndarray, type(None)), None), \
('ChannelNames', (np.ndarray, type(None)), None), \
('ChannelInputRanges', (np.ndarray, type(None)), None), \
('Offsets', (np.ndarray, type(None)), None), \
('Scalings', (np.ndarray, type(None)), None), \
('Units', (np.ndarray, type(None)), None)]
for param in params:
if param[0] in Info:
if not isinstance(Info[param[0]], param[1]):
raise ValueError("Info['" + param[0] + "'] is "
+ 'not the right type.')
elif len(param) > 2:
Info[param[0]] = param[2]
else:
raise ValueError("Info is missing field '"
+ param[0] + "'.")
if param[1] == string_types:
Info[param[0]] = _convert_to_numpy_bytes(Info[param[0]])
# Check that we have a positive number of channels.
if Info['NumberChannels'] < 1:
raise ValueError('There must be at least one channel.')
# If the channel mappings aren't given, make it the default
# (incrementing integers from 0). If it is given, check it.
if Info['ChannelMappings'] is None:
Info['ChannelMappings'] = np.int64( \
np.r_[0:Info['NumberChannels']])
elif type(Info['ChannelMappings']) != np.ndarray \
or Info['ChannelMappings'].dtype.name != 'int64' \
or Info['ChannelMappings'].shape \
!= (Info['NumberChannels'], ):
raise ValueError('ChannelMappings isn''t a numpy.int64 '
+ 'row array with an element for each '
+ 'channel.')
# If the channel names aren't given, make it the default (all
# b''). If it is given, check it.
if Info['ChannelNames'] is None:
Info['ChannelNames'] = \
np.zeros(shape=(Info['NumberChannels'], ), \
dtype='bytes')
elif type(Info['ChannelNames']) != np.ndarray \
or not Info['ChannelNames'].dtype.name.startswith( \
'bytes') \
or Info['ChannelNames'].shape \
!= (Info['NumberChannels'], ):
raise ValueError('ChannelNames isn''t a numpy.bytes_ '
+ 'row array with an element for each '
+ 'channel.')
# If the channel input ranges aren't given, make it the default
# (array from zeros). If it is given, check it.
if Info['ChannelInputRanges'] is None:
Info['ChannelInputRanges'] = np.zeros(\
shape=(Info['NumberChannels'], 2), dtype='float64')
elif type(Info['ChannelInputRanges']) != np.ndarray \
or Info['ChannelInputRanges'].dtype.name != 'float64' \
or Info['ChannelInputRanges'].shape \
!= (Info['NumberChannels'], 2):
raise ValueError('ChannelInputRanges isn''t a numpy '
+ 'float64 array with 2 columns and a ' \
+ 'row for each channel.')
# If the Offsets aren't given, make it the default (row of
# zeros). If it is given, check it.
if Info['Offsets'] is None:
Info['Offsets'] = np.zeros( \
shape=(Info['NumberChannels'],), dtype='float64')
elif type(Info['Offsets']) != np.ndarray \
or Info['Offsets'].dtype.name != 'float64' \
or Info['Offsets'].shape \
!= (Info['NumberChannels'], ):
raise ValueError('Offsets isn''t a numpy.float64 '
+ 'row array with an element for each '
+ 'channel.')
# If the Scalings aren't given, make it the default (row of
# ones). If it is given, check it.
if Info['Scalings'] is None:
Info['Scalings'] = np.ones(shape=(Info['NumberChannels'],),
dtype='float64')
elif type(Info['Scalings']) != np.ndarray \
or Info['Scalings'].dtype.name != 'float64' \
or Info['Scalings'].shape != (Info['NumberChannels'], ):
raise ValueError('Scalings isn''t a numpy.float64 '
+ 'row array with an element for each '
+ 'channel.')
# If the Units aren't given, make it the default (all b''). If
# it is given, check it.
if Info['Units'] is None:
Info['Units'] = np.zeros(shape=(Info['NumberChannels'], ),
dtype='bytes')
elif type(Info['Units']) != np.ndarray \
or not Info['Units'].dtype.name.startswith('bytes') \
or Info['Units'].shape != (Info['NumberChannels'], ):
raise ValueError('Units isn''t a numpy.bytes_ '
+ 'row array with an element for each '
+ 'channel.')
# NumberSamplesBinned must be a positive integer.
if Info['NumberSamplesBinned'] < 1:
raise ValueError('NumberSamplesBinned must be a positive '
+ 'numpy.int64.')
# data_type and data_storage_types must be in the lookup.
if data_type not in _data_types:
raise ValueError('data_type must be one of ('
+ ', '.join(list(_data_types.keys()))
+ ').')
# data_type and data_storage_types must be in the lookup.
if data_storage_type not in _data_types:
raise ValueError('data_storage_type must be one of ('
+ ', '.join(list(_data_types.keys()))
+ ').')
# Validate chunks to make sure it is None, True, a tuple of two
# ints, or a tuple of an int and None. All integers must be
# positive.
if chunks is None:
chunks = (1024, int(Info['NumberChannels']))
elif chunks is True:
pass
elif not isinstance(chunks, tuple) or len(chunks) != 2:
raise ValueError('chunks must be None, True, or a tuple '
+ 'an integer as the first element and '
+ 'either an integer or None in the '
+ 'second.')
elif not isinstance(chunks[0], int) or chunks[0] < 1:
raise ValueError('chunks must be None, True, or a tuple '
+ 'an integer as the first element and '
+ 'either an integer or None in the '
+ 'second.')
elif chunks[1] is None:
chunks = (chunks[0], int(Info['NumberChannels']))
elif not isinstance(chunks[1], int) or chunks[1] < 1:
raise ValueError('chunks must be None, True, or a tuple '
+ 'an integer as the first element and '
+ 'either an integer or None in the '
+ 'second.')
# All inputs are validated.
# Pack all of the information together, including putting in
# placeholders for the start time and the number of samples
# taken. The file type and software information is also put
# in.
software = __name__ + ' ' + __version__ + ' on ' \
+ platform.python_implementation() + ' ' \
+ platform.python_version()
self._file_data = { \
'Type': np.bytes_('Acquisition HDF5'), \
'Version': np.bytes_(new_version), \
'Software': np.bytes_(software), \
'Info': Info, \
'Data': { \
'Type': _convert_to_numpy_bytes(data_type), \
'StorageType': _convert_to_numpy_bytes(data_storage_type)}}
self._file_data['Info']['StartTime'] = np.zeros(shape=(6,),
dtype='float64')
self._file_data['Info']['NumberSamples'] = np.int64(0)
# Write it all to file, truncating it if it exists. Python
# information should not be stored, and matlab compatibility
# should not be done. While the former would make it easier to
# read the strings back in the same format
hdf5storage.write(self._file_data, path='/', filename=filename,
truncate_existing=True,
store_python_metadata=False,
matlab_compatible=False)
# Create a growable empty DataSet for the data with all the
# storage options set, and then keep the file handle around for
# later. If an exception occurs, the file needs to be closed if
# it was opened and the exception re-raised so that the caller
# knows about it. Nothing other than that needs to be done.
try:
self._file = h5py.File(filename)
self._file['/Data'].create_dataset('Data', \
shape=(0, Info['NumberChannels']), \
dtype=_data_types[data_storage_type], \
maxshape=(None, Info['NumberChannels']), \
compression=compression, \
compression_opts=compression_opts, \
shuffle=shuffle, \
fletcher32=fletcher32, \
chunks=chunks)
self._file.flush()
except:
if self._file is not None:
self._file.close()
raise