def test_store_and_load_custom_array():
a = np.array([[1, 2, 3, 4], [6, 7]])
h5w.save(fn, {'a': a}, overwrite_dataset=True)
# loading the whole data
res = h5w.load(fn)
for i in range(len(a)):
assert_array_equal(a[i], res['a'][i])
# loading path directly
res = h5w.load(fn, path='a/')
for i in range(len(a)):
assert_array_equal(a[i], res[i])
def test_store_and_load_custom_array():
a = np.array([[1, 2, 3, 4], [6, 7]])
h5w.save(fn, {'a': a}, overwrite_dataset=True)
# loading the whole data
res = h5w.load(fn)
for i in range(len(a)):
assert_array_equal(a[i], res['a'][i])
# loading path directly
res = h5w.load(fn, path='a/')
for i in range(len(a)):
assert_array_equal(a[i], res[i])
def test_handle_nonexisting_path():
res = {}
stest = 'this is a test'
h5w.save(fn, res, write_mode='w')
try:
res = h5w.load(fn, path='test/')
raise Exception() # should not get until here
except KeyError:
res['test'] = stest
h5w.save(fn, res)
res.clear()
res = h5w.load(fn, path='test/')
assert (res == stest)
def test_handle_nonexisting_path():
res = {}
stest = 'this is a test'
h5w.save(fn, res, write_mode='w')
try:
res = h5w.load(fn, path='test/')
raise Exception() # should not get until here
except KeyError:
res['test'] = stest
h5w.save(fn, res)
res.clear()
res = h5w.load(fn, path='test/')
assert(res == stest)
def test_store_and_load_with_compression():
data = {
'a': 1,
'test1': {
'b': 2
},
'test2': {
'test3': {
'c': np.array([1, 2, 3])
}
}
}
h5w.save(fn, data, write_mode='w', compression='gzip')
h5w.load(fn)
def test_load_lazy_simple():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn, lazy=True)
for key, obj in res.items():
assert(obj is None)
def test_conversion_script():
try:
import h5py_wrapper_001.wrapper as h5w_001
except ImportError:
h5w_lib.get_previous_version('0.0.1')
import h5py_wrapper_001.wrapper as h5w_001
res = {key: value for key, value in zip(simpledata_str, simpledata_val)}
res.update(
{key: value
for key, value in zip(arraydata_str, arraydata_val)})
h5w_001.add_to_h5(fn, res)
h5w_001.add_to_h5(fn2, res)
with open('conversion_list.txt', 'w') as f:
f.write(fn)
f.write('\n')
f.write(fn2)
# Specify list on command line
os.system('./convert_h5file {} {} --release=0.0.1'.format(fn, fn2))
# Read list of files from file and pipe into conversion script
os.system('cat conversion_list.txt | ./convert_h5file --release=0.0.1')
os.remove('conversion_list.txt')
# Find files based on pattern using `find` and pipe into conversion script
os.system('find -name "data*.h5" | ./convert_h5file --release=0.0.1')
res2 = h5w.load(fn)
for key, value in res.items():
if isinstance(res2[key], np.ndarray):
assert_array_equal(res2[key], value)
else:
assert (res2[key] == value)
if isinstance(res2[key], str):
assert (isinstance(res2[key], type(str(value))))
else:
assert (isinstance(res2[key], type(value)))
def test_read_empty_array_via_path():
res = {'a': np.array([[], []])}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn, path='a')
assert_array_equal(res, [[], []])
assert (np.shape(res) == (2, 0))
def test_store_and_load_quantities_array():
data = {'times': np.array([1, 2, 3]) * pq.ms, 'positions':
np.array([1, 2, 3]) * pq.cm}
h5w.save(fn, data, overwrite_dataset=True)
# loading the whole data
res = h5w.load(fn)
assert(res['times'].dimensionality == data['times'].dimensionality)
def test_write_empty_array():
res = {'a': [], 'b': np.array([])}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert_array_equal(res['a'], [])
assert_array_equal(res['b'], [])
def test_write_empty_array():
res = {'a': [], 'b': np.array([])}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert_array_equal(res['a'], [])
assert_array_equal(res['b'], [])
def test_store_and_load_simpledata():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(simpledata_str, simpledata_val):
assert (res[key] == val)
def test_old_store_and_load_simpledata():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(simpledata_str, simpledata_val):
assert(res[key] == val)
def test_save_something(self, network, tmpdir, std_results):
temp = tmpdir.mkdir('temp')
with temp.as_cwd():
network.save(file='test.h5')
loaded = h5.load('test.h5')
firing_rates = loaded['results']['firing_rates']['val']
assert_array_equal(firing_rates, network.temp_results)
def test_read_empty_array_via_path():
res = {'a': np.array([[], []])}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn, path='a')
assert_array_equal(res, [[], []])
assert(np.shape(res) == (2, 0))
def test_load_lazy_simple():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn, lazy=True)
for key, obj in res.items():
assert (obj is None)
def test_overwrite_dataset():
res = {'a': 5}
h5w.save(fn, res, write_mode='w')
res.clear()
res = {'a': 6}
with pytest.raises(KeyError):
h5w.save(fn, res, write_mode='a', overwrite_dataset=False)
res.clear()
res = h5w.load(fn)
assert (res['a'] == 5) # dataset should still contain old value
res.clear()
res = {'a': 6}
h5w.save(fn, res, write_mode='a', overwrite_dataset=True)
res.clear()
res = h5w.load(fn)
assert (res['a'] == 6) # dataset should contain new value
def test_overwrite_dataset():
res = {'a': 5}
h5w.save(fn, res, write_mode='w')
res.clear()
res = {'a': 6}
with pytest.raises(KeyError):
h5w.save(fn, res, write_mode='a', overwrite_dataset=False)
res.clear()
res = h5w.load(fn)
assert(res['a'] == 5) # dataset should still contain old value
res.clear()
res = {'a': 6}
h5w.save(
fn, res, write_mode='a', overwrite_dataset=True)
res.clear()
res = h5w.load(fn)
assert(res['a'] == 6) # dataset should contain new value
def test_save_something(self, network, tmpdir):
temp = tmpdir.mkdir('temp')
with temp.as_cwd():
network.save(file='test.h5')
loaded = h5.load('test.h5')
firing_rates = loaded['results']['lif.exp.firing_rates']['val']
with pytest.raises(AssertionError):
assert_array_equal(firing_rates, network.temp_results)
def test_write_nested_empty_array():
res = {'a': [[], []], 'b': np.array([[], []])}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert_array_equal(res['a'], [[], []])
assert (np.shape(res['a']) == (2, 0))
assert_array_equal(res['b'], [[], []])
assert (np.shape(res['b']) == (2, 0))
def test_load_lazy_nested():
res = {'a': 1, 'test1': {'b': 2}, 'test2': {
'test3': {'c': np.array([1, 2, 3])}}}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn, lazy=True)
assert(res['a'] is None)
assert(res['test1']['b'] is None)
assert(res['test2']['test3']['c'] is None)
def test_store_and_test_key_types():
data = {'a': 1, (1, 2): {4: 2.}, 4.: 3.}
h5w.save(fn, data, write_mode='w')
res = h5w.load(fn)
keys = ['a', (1, 2), 4.]
for k in keys:
assert(k in res)
assert(4 in res[(1, 2)])
def test_store_and_load_numpy_datatypes():
res = {}
res['float64'] = np.float64(f0)
res['int64'] = np.int64(i0)
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert (isinstance(res['float64'], np.float64))
assert (isinstance(res['int64'], np.int64))
def test_store_and_test_key_types():
data = {'a': 1, (1, 2): {4: 2.}, 4.: 3.}
h5w.save(fn, data, write_mode='w')
res = h5w.load(fn)
keys = ['a', (1, 2), 4.]
for k in keys:
assert (k in res)
assert (4 in res[(1, 2)])
def test_store_and_load_numpy_datatypes():
res = {}
res['float64'] = np.float64(f0)
res['int64'] = np.int64(i0)
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert(isinstance(res['float64'], np.float64))
assert(isinstance(res['int64'], np.int64))
def test_store_and_load_arraydata():
res = {}
for key, val in zip(arraydata_str, arraydata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(arraydata_str, arraydata_val):
assert_array_equal(res[key], val)
def test_write_nested_empty_array():
res = {'a': [[], []], 'b': np.array([[], []])}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert_array_equal(res['a'], [[], []])
assert(np.shape(res['a']) == (2, 0))
assert_array_equal(res['b'], [[], []])
assert(np.shape(res['b']) == (2, 0))
def test_store_and_load_arraydata():
res = {}
for key, val in zip(arraydata_str, arraydata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(arraydata_str, arraydata_val):
assert_array_equal(res[key], val)
def test_store_and_load_quantities_array():
data = {
'times': np.array([1, 2, 3]) * pq.ms,
'positions': np.array([1, 2, 3]) * pq.cm
}
h5w.save(fn, data, overwrite_dataset=True)
# loading the whole data
res = h5w.load(fn)
assert (res['times'].dimensionality == data['times'].dimensionality)
def load_from_h5(network_params={}, param_keys=[], input_name=''):
"""
Load existing results and analysis_params for given parameters from h5 file.
Loads results from h5 files named with the standard format
<label>_<hash>.h5, if this file already exists. Or uses given list of
parameters to create hash to find file. Or reads from file specified in
input_name.
Parameters:
-----------
network_params : dict
Dictionary containing network parameters as quantities.
param_keys: list
List of parameters used in file hash.
input_name: str
optional string specifying input file name (default: <label>_<hash>.h5).
Returns:
--------
analysis_params: dict
Dictionary containing all found analysis_params.
results: dict
Dictionary containing all found results.
"""
# if no input file name is specified
if not input_name:
# create hash from given parameters
# collect all parameters reflected by hash in one dictionary
hash_params = {}
hash_params.update(network_params)
# if user did not specify which parameters to use for hash
if not param_keys:
# take all parameters sorted alphabetically
param_keys = sorted(list(hash_params.keys()))
# crate hash from param_keys
hash = create_hash(hash_params, param_keys)
# default input name
input_name = '{}_{}.h5'.format(network_params['label'], str(hash))
# try to load file with standard name
try:
input_file = h5.load(input_name)
# if not existing OSError is raised by h5py_wrapper, then return empty dict
except OSError:
return {}, {}
# read in whats already stored
analysis_params = input_file['analysis_params']
results = input_file['results']
# convert results to quantitites
analysis_params = val_unit_to_quantities(analysis_params)
results = val_unit_to_quantities(results)
return analysis_params, results
def test_store_and_load_tupledata():
res = {}
for key, val in zip(tupledata_str, tupledata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(tupledata_str, tupledata_val):
assert (isinstance(res[key], tuple))
assert_array_equal(res[key], np.array(val))
def test_store_and_load_dictdata():
res = {}
for key, val in zip(dictdata_str, dictdata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for dkey, dval in zip(dictdata_str, dictdata_val):
for key, val in dval.items():
assert(res[dkey][key] == val)
def test_store_and_load_tupledata():
res = {}
for key, val in zip(tupledata_str, tupledata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(tupledata_str, tupledata_val):
assert(isinstance(res[key], tuple))
assert_array_equal(res[key], np.array(val))
def test_store_and_load_dictdata():
res = {}
for key, val in zip(dictdata_str, dictdata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for dkey, dval in zip(dictdata_str, dictdata_val):
for key, val in dval.items():
assert (res[dkey][key] == val)
def test_store_and_rehash_h5py():
d0 = {
'a': 'asd',
'b': 0.12,
'c': [3, 4, 5],
'd': np.array([[3, 4, 5], [3, 4, 5]]),
'e': True
}
hash0 = dicthash.generate_hash_from_dict(d0)
h5w.save('store_and_rehash_h5py.h5', {'d0': d0}, 'w')
d1 = h5w.load('store_and_rehash_h5py.h5', 'd0')
hash1 = dicthash.generate_hash_from_dict(d1)
assert(hash0 == hash1)
def test_store_and_rehash_h5py():
d0 = {
'a': 'asd',
'b': 0.12,
'c': [3, 4, 5],
'd': np.array([[3, 4, 5], [3, 4, 5]]),
'e': True
}
hash0 = dicthash.generate_hash_from_dict(d0)
h5w.save('store_and_rehash_h5py.h5', {'d0': d0}, 'w')
d1 = h5w.load('store_and_rehash_h5py.h5', 'd0')
hash1 = dicthash.generate_hash_from_dict(d1)
assert (hash0 == hash1)
def load_h5(filename):
"""
filename: str
default filename format is ''<label>_<hash>.h5'
"""
try:
raw_data = h5.load(filename)
except OSError:
raw_data = {}
data = {}
for key in sorted(raw_data.keys()):
data[key] = val_unit_to_quantities(raw_data[key])
return data
def test_store_and_load_listdata():
res = {}
for key, val in zip(listdata_str, listdata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(listdata_str, listdata_val):
if isinstance(val[0], list):
for ii in range(len(val)):
assert(isinstance(res[key][ii], list))
assert_array_equal(res[key][ii], val[ii])
else:
assert(isinstance(res[key], type(val)))
assert_array_equal(res[key], val)
def test_store_and_load_listdata():
res = {}
for key, val in zip(listdata_str, listdata_val):
res[key] = val
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
for key, val in zip(listdata_str, listdata_val):
if isinstance(val[0], list):
for ii in range(len(val)):
assert (isinstance(res[key][ii], list))
assert_array_equal(res[key][ii], val[ii])
else:
assert (isinstance(res[key], type(val)))
assert_array_equal(res[key], val)
def test_load_lazy_nested():
res = {
'a': 1,
'test1': {
'b': 2
},
'test2': {
'test3': {
'c': np.array([1, 2, 3])
}
}
}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn, lazy=True)
assert (res['a'] is None)
assert (res['test1']['b'] is None)
assert (res['test2']['test3']['c'] is None)
def load_val_unit_dict_from_h5(file):
"""
Load and convert val unit dict from h5 file to dict of quantities.
The val unit dictionary is loaded from the h5 file and then converted to
a dictionary containing quantities.
Parameters:
-----------
file: str
String specifying input file name.
"""
try:
loaded = h5.load(file)
except OSError:
raise IOError(f'{file} does not exist!')
converted = val_unit_to_quantities(loaded)
return converted
def test_store_and_load_dataset_directly():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w')
for key, val in zip(simpledata_str, simpledata_val):
assert(h5w.load(fn, '/' + key) == val)
def test_conversion_script(tmpdir):
# convert LocalPath object to str to ensure that path can be
# handled by os.path.join()
path = str(tmpdir)
previous_versions = ['0.0.1', '1.0.1']
for version in previous_versions:
print(version)
version_stripped = version.replace('.', '')
release_base_name = '_'.join(('h5py_wrapper', version_stripped))
module = '.'.join((release_base_name, 'wrapper'))
try:
h5w_prev = importlib.import_module(module)
except ImportError:
h5w_lib.get_previous_version(version, tmpdir)
sys.path.append(path)
print(sys.path[-1])
print(module)
h5w_prev = importlib.import_module(module)
tmp_fn = os.path.join(path, fn)
tmp_fn2 = os.path.join(path, fn2)
res = {key: value for key, value in zip(simpledata_str, simpledata_val)}
res.update({key: value for key, value in zip(arraydata_str, arraydata_val)})
h5w_prev.add_to_h5(tmp_fn, res)
h5w_prev.add_to_h5(tmp_fn2, res)
with open('conversion_list.txt', 'w') as f:
f.write(tmp_fn)
f.write('\n')
f.write(tmp_fn2)
# Specify list on command line
conversion_file_cmd = 'convert_h5file'
cmd = 'python {}/../{} {} {} --release={}'.format(os.path.dirname(__file__),
conversion_file_cmd,
tmp_fn, tmp_fn2,
version)
os.system(cmd)
# Read list of files from file and pipe into conversion script
cmd = 'cat conversion_list.txt | python {}/../{} {}={}'.format(os.path.dirname(__file__),
conversion_file_cmd,
'--release',
version)
os.system(cmd)
os.remove('conversion_list.txt')
# Find files based on pattern using `find` and pipe into conversion script
cmd = 'find -name "data*.h5" | python {}/../{} {}={}'.format(os.path.dirname(__file__),
conversion_file_cmd,
'--release',
version)
os.system(cmd)
res2 = h5w.load(tmp_fn)
for key, value in res.items():
if isinstance(res2[key], np.ndarray):
assert_array_equal(res2[key], value)
else:
assert(res2[key] == value)
if isinstance(res2[key], str):
assert(isinstance(res2[key], type(str(value))))
else:
assert(isinstance(res2[key], type(value)))
os.remove(tmp_fn)
os.remove(tmp_fn2)
def test_write_and_load_with_label():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w', path='test_label')
for key, val in zip(simpledata_str, simpledata_val):
assert(h5w.load(fn, 'test_label/' + key) == val)
def test_handle_nonexisting_file():
try:
h5w.load('asdasd.h5')
raise Exception() # should not get until here
except IOError:
pass
def test_store_none():
res = {'a1': None}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert (res['a1'] is None)
def test_store_and_load_with_compression():
data = {'a': 1, 'test1': {'b': 2}, 'test2': {
'test3': {'c': np.array([1, 2, 3])}}}
h5w.save(fn, data, write_mode='w', compression='gzip')
h5w.load(fn)
def ground_truth_result():
results = h5.load(fix_path + 'Schuecker2015_data.h5')
return results
def test_write_and_load_with_label():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w', path='test_label')
for key, val in zip(simpledata_str, simpledata_val):
assert (h5w.load(fn, 'test_label/' + key) == val)
def test_store_and_load_dataset_directly():
res = _construct_simpledata()
h5w.save(fn, res, write_mode='w')
for key, val in zip(simpledata_str, simpledata_val):
assert (h5w.load(fn, '/' + key) == val)
def test_handle_nonexisting_file():
try:
h5w.load('asdasd.h5')
raise Exception() # should not get until here
except IOError:
pass
def test_store_none():
res = {'a1': None}
h5w.save(fn, res, write_mode='w')
res.clear()
res = h5w.load(fn)
assert(res['a1'] is None)
def test_conversion_script(tmpdir):
# convert LocalPath object to str to ensure that path can be
# handled by os.path.join()
path = str(tmpdir)
previous_versions = ['0.0.1', '1.0.1']
for version in previous_versions:
print(version)
version_stripped = version.replace('.', '')
release_base_name = '_'.join(('h5py_wrapper', version_stripped))
module = '.'.join((release_base_name, 'wrapper'))
try:
h5w_prev = importlib.import_module(module)
except ImportError:
h5w_lib.get_previous_version(version, tmpdir)
sys.path.append(path)
print(sys.path[-1])
print(module)
h5w_prev = importlib.import_module(module)
tmp_fn = os.path.join(path, fn)
tmp_fn2 = os.path.join(path, fn2)
res = {
key: value
for key, value in zip(simpledata_str, simpledata_val)
}
res.update(
{key: value
for key, value in zip(arraydata_str, arraydata_val)})
h5w_prev.add_to_h5(tmp_fn, res)
h5w_prev.add_to_h5(tmp_fn2, res)
with open('conversion_list.txt', 'w') as f:
f.write(tmp_fn)
f.write('\n')
f.write(tmp_fn2)
# Specify list on command line
conversion_file_cmd = 'convert_h5file'
cmd = 'python {}/../{} {} {} --release={}'.format(
os.path.dirname(__file__), conversion_file_cmd, tmp_fn, tmp_fn2,
version)
os.system(cmd)
# Read list of files from file and pipe into conversion script
cmd = 'cat conversion_list.txt | python {}/../{} {}={}'.format(
os.path.dirname(__file__), conversion_file_cmd, '--release',
version)
os.system(cmd)
os.remove('conversion_list.txt')
# Find files based on pattern using `find` and pipe into conversion script
cmd = 'find -name "data*.h5" | python {}/../{} {}={}'.format(
os.path.dirname(__file__), conversion_file_cmd, '--release',
version)
os.system(cmd)
res2 = h5w.load(tmp_fn)
for key, value in res.items():
if isinstance(res2[key], np.ndarray):
assert_array_equal(res2[key], value)
else:
assert (res2[key] == value)
if isinstance(res2[key], str):
assert (isinstance(res2[key], type(str(value))))
else:
assert (isinstance(res2[key], type(value)))
os.remove(tmp_fn)
os.remove(tmp_fn2)
