def test_store_and_load_custom_array(): a = np.array([[1, 2, 3, 4], [6, 7]]) h5w.save(fn, {'a': a}, overwrite_dataset=True) # loading the whole data res = h5w.load(fn) for i in range(len(a)): assert_array_equal(a[i], res['a'][i]) # loading path directly res = h5w.load(fn, path='a/') for i in range(len(a)): assert_array_equal(a[i], res[i])
def test_handle_nonexisting_path(): res = {} stest = 'this is a test' h5w.save(fn, res, write_mode='w') try: res = h5w.load(fn, path='test/') raise Exception() # should not get until here except KeyError: res['test'] = stest h5w.save(fn, res) res.clear() res = h5w.load(fn, path='test/') assert (res == stest)
def test_handle_nonexisting_path(): res = {} stest = 'this is a test' h5w.save(fn, res, write_mode='w') try: res = h5w.load(fn, path='test/') raise Exception() # should not get until here except KeyError: res['test'] = stest h5w.save(fn, res) res.clear() res = h5w.load(fn, path='test/') assert(res == stest)
def test_store_and_load_with_compression(): data = { 'a': 1, 'test1': { 'b': 2 }, 'test2': { 'test3': { 'c': np.array([1, 2, 3]) } } } h5w.save(fn, data, write_mode='w', compression='gzip') h5w.load(fn)
def test_load_lazy_simple(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn, lazy=True) for key, obj in res.items(): assert(obj is None)
def test_conversion_script(): try: import h5py_wrapper_001.wrapper as h5w_001 except ImportError: h5w_lib.get_previous_version('0.0.1') import h5py_wrapper_001.wrapper as h5w_001 res = {key: value for key, value in zip(simpledata_str, simpledata_val)} res.update( {key: value for key, value in zip(arraydata_str, arraydata_val)}) h5w_001.add_to_h5(fn, res) h5w_001.add_to_h5(fn2, res) with open('conversion_list.txt', 'w') as f: f.write(fn) f.write('\n') f.write(fn2) # Specify list on command line os.system('./convert_h5file {} {} --release=0.0.1'.format(fn, fn2)) # Read list of files from file and pipe into conversion script os.system('cat conversion_list.txt | ./convert_h5file --release=0.0.1') os.remove('conversion_list.txt') # Find files based on pattern using `find` and pipe into conversion script os.system('find -name "data*.h5" | ./convert_h5file --release=0.0.1') res2 = h5w.load(fn) for key, value in res.items(): if isinstance(res2[key], np.ndarray): assert_array_equal(res2[key], value) else: assert (res2[key] == value) if isinstance(res2[key], str): assert (isinstance(res2[key], type(str(value)))) else: assert (isinstance(res2[key], type(value)))
def test_read_empty_array_via_path(): res = {'a': np.array([[], []])} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn, path='a') assert_array_equal(res, [[], []]) assert (np.shape(res) == (2, 0))
def test_store_and_load_quantities_array(): data = {'times': np.array([1, 2, 3]) * pq.ms, 'positions': np.array([1, 2, 3]) * pq.cm} h5w.save(fn, data, overwrite_dataset=True) # loading the whole data res = h5w.load(fn) assert(res['times'].dimensionality == data['times'].dimensionality)
def test_write_empty_array(): res = {'a': [], 'b': np.array([])} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert_array_equal(res['a'], []) assert_array_equal(res['b'], [])
def test_store_and_load_simpledata(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(simpledata_str, simpledata_val): assert (res[key] == val)
def test_old_store_and_load_simpledata(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(simpledata_str, simpledata_val): assert(res[key] == val)
def test_save_something(self, network, tmpdir, std_results): temp = tmpdir.mkdir('temp') with temp.as_cwd(): network.save(file='test.h5') loaded = h5.load('test.h5') firing_rates = loaded['results']['firing_rates']['val'] assert_array_equal(firing_rates, network.temp_results)
def test_read_empty_array_via_path(): res = {'a': np.array([[], []])} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn, path='a') assert_array_equal(res, [[], []]) assert(np.shape(res) == (2, 0))
def test_load_lazy_simple(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn, lazy=True) for key, obj in res.items(): assert (obj is None)
def test_overwrite_dataset(): res = {'a': 5} h5w.save(fn, res, write_mode='w') res.clear() res = {'a': 6} with pytest.raises(KeyError): h5w.save(fn, res, write_mode='a', overwrite_dataset=False) res.clear() res = h5w.load(fn) assert (res['a'] == 5) # dataset should still contain old value res.clear() res = {'a': 6} h5w.save(fn, res, write_mode='a', overwrite_dataset=True) res.clear() res = h5w.load(fn) assert (res['a'] == 6) # dataset should contain new value
def test_overwrite_dataset(): res = {'a': 5} h5w.save(fn, res, write_mode='w') res.clear() res = {'a': 6} with pytest.raises(KeyError): h5w.save(fn, res, write_mode='a', overwrite_dataset=False) res.clear() res = h5w.load(fn) assert(res['a'] == 5) # dataset should still contain old value res.clear() res = {'a': 6} h5w.save( fn, res, write_mode='a', overwrite_dataset=True) res.clear() res = h5w.load(fn) assert(res['a'] == 6) # dataset should contain new value
def test_save_something(self, network, tmpdir): temp = tmpdir.mkdir('temp') with temp.as_cwd(): network.save(file='test.h5') loaded = h5.load('test.h5') firing_rates = loaded['results']['lif.exp.firing_rates']['val'] with pytest.raises(AssertionError): assert_array_equal(firing_rates, network.temp_results)
def test_write_nested_empty_array(): res = {'a': [[], []], 'b': np.array([[], []])} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert_array_equal(res['a'], [[], []]) assert (np.shape(res['a']) == (2, 0)) assert_array_equal(res['b'], [[], []]) assert (np.shape(res['b']) == (2, 0))
def test_load_lazy_nested(): res = {'a': 1, 'test1': {'b': 2}, 'test2': { 'test3': {'c': np.array([1, 2, 3])}}} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn, lazy=True) assert(res['a'] is None) assert(res['test1']['b'] is None) assert(res['test2']['test3']['c'] is None)
def test_store_and_test_key_types(): data = {'a': 1, (1, 2): {4: 2.}, 4.: 3.} h5w.save(fn, data, write_mode='w') res = h5w.load(fn) keys = ['a', (1, 2), 4.] for k in keys: assert(k in res) assert(4 in res[(1, 2)])
def test_store_and_load_numpy_datatypes(): res = {} res['float64'] = np.float64(f0) res['int64'] = np.int64(i0) h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert (isinstance(res['float64'], np.float64)) assert (isinstance(res['int64'], np.int64))
def test_store_and_test_key_types(): data = {'a': 1, (1, 2): {4: 2.}, 4.: 3.} h5w.save(fn, data, write_mode='w') res = h5w.load(fn) keys = ['a', (1, 2), 4.] for k in keys: assert (k in res) assert (4 in res[(1, 2)])
def test_store_and_load_numpy_datatypes(): res = {} res['float64'] = np.float64(f0) res['int64'] = np.int64(i0) h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert(isinstance(res['float64'], np.float64)) assert(isinstance(res['int64'], np.int64))
def test_store_and_load_arraydata(): res = {} for key, val in zip(arraydata_str, arraydata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(arraydata_str, arraydata_val): assert_array_equal(res[key], val)
def test_write_nested_empty_array(): res = {'a': [[], []], 'b': np.array([[], []])} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert_array_equal(res['a'], [[], []]) assert(np.shape(res['a']) == (2, 0)) assert_array_equal(res['b'], [[], []]) assert(np.shape(res['b']) == (2, 0))
def test_store_and_load_quantities_array(): data = { 'times': np.array([1, 2, 3]) * pq.ms, 'positions': np.array([1, 2, 3]) * pq.cm } h5w.save(fn, data, overwrite_dataset=True) # loading the whole data res = h5w.load(fn) assert (res['times'].dimensionality == data['times'].dimensionality)
def load_from_h5(network_params={}, param_keys=[], input_name=''): """ Load existing results and analysis_params for given parameters from h5 file. Loads results from h5 files named with the standard format <label>_<hash>.h5, if this file already exists. Or uses given list of parameters to create hash to find file. Or reads from file specified in input_name. Parameters: ----------- network_params : dict Dictionary containing network parameters as quantities. param_keys: list List of parameters used in file hash. input_name: str optional string specifying input file name (default: <label>_<hash>.h5). Returns: -------- analysis_params: dict Dictionary containing all found analysis_params. results: dict Dictionary containing all found results. """ # if no input file name is specified if not input_name: # create hash from given parameters # collect all parameters reflected by hash in one dictionary hash_params = {} hash_params.update(network_params) # if user did not specify which parameters to use for hash if not param_keys: # take all parameters sorted alphabetically param_keys = sorted(list(hash_params.keys())) # crate hash from param_keys hash = create_hash(hash_params, param_keys) # default input name input_name = '{}_{}.h5'.format(network_params['label'], str(hash)) # try to load file with standard name try: input_file = h5.load(input_name) # if not existing OSError is raised by h5py_wrapper, then return empty dict except OSError: return {}, {} # read in whats already stored analysis_params = input_file['analysis_params'] results = input_file['results'] # convert results to quantitites analysis_params = val_unit_to_quantities(analysis_params) results = val_unit_to_quantities(results) return analysis_params, results
def test_store_and_load_tupledata(): res = {} for key, val in zip(tupledata_str, tupledata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(tupledata_str, tupledata_val): assert (isinstance(res[key], tuple)) assert_array_equal(res[key], np.array(val))
def test_store_and_load_dictdata(): res = {} for key, val in zip(dictdata_str, dictdata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for dkey, dval in zip(dictdata_str, dictdata_val): for key, val in dval.items(): assert(res[dkey][key] == val)
def test_store_and_load_tupledata(): res = {} for key, val in zip(tupledata_str, tupledata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(tupledata_str, tupledata_val): assert(isinstance(res[key], tuple)) assert_array_equal(res[key], np.array(val))
def test_store_and_load_dictdata(): res = {} for key, val in zip(dictdata_str, dictdata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for dkey, dval in zip(dictdata_str, dictdata_val): for key, val in dval.items(): assert (res[dkey][key] == val)
def test_store_and_rehash_h5py(): d0 = { 'a': 'asd', 'b': 0.12, 'c': [3, 4, 5], 'd': np.array([[3, 4, 5], [3, 4, 5]]), 'e': True } hash0 = dicthash.generate_hash_from_dict(d0) h5w.save('store_and_rehash_h5py.h5', {'d0': d0}, 'w') d1 = h5w.load('store_and_rehash_h5py.h5', 'd0') hash1 = dicthash.generate_hash_from_dict(d1) assert(hash0 == hash1)
def test_store_and_rehash_h5py(): d0 = { 'a': 'asd', 'b': 0.12, 'c': [3, 4, 5], 'd': np.array([[3, 4, 5], [3, 4, 5]]), 'e': True } hash0 = dicthash.generate_hash_from_dict(d0) h5w.save('store_and_rehash_h5py.h5', {'d0': d0}, 'w') d1 = h5w.load('store_and_rehash_h5py.h5', 'd0') hash1 = dicthash.generate_hash_from_dict(d1) assert (hash0 == hash1)
def load_h5(filename): """ filename: str default filename format is ''<label>_<hash>.h5' """ try: raw_data = h5.load(filename) except OSError: raw_data = {} data = {} for key in sorted(raw_data.keys()): data[key] = val_unit_to_quantities(raw_data[key]) return data
def test_store_and_load_listdata(): res = {} for key, val in zip(listdata_str, listdata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(listdata_str, listdata_val): if isinstance(val[0], list): for ii in range(len(val)): assert(isinstance(res[key][ii], list)) assert_array_equal(res[key][ii], val[ii]) else: assert(isinstance(res[key], type(val))) assert_array_equal(res[key], val)
def test_store_and_load_listdata(): res = {} for key, val in zip(listdata_str, listdata_val): res[key] = val h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) for key, val in zip(listdata_str, listdata_val): if isinstance(val[0], list): for ii in range(len(val)): assert (isinstance(res[key][ii], list)) assert_array_equal(res[key][ii], val[ii]) else: assert (isinstance(res[key], type(val))) assert_array_equal(res[key], val)
def test_load_lazy_nested(): res = { 'a': 1, 'test1': { 'b': 2 }, 'test2': { 'test3': { 'c': np.array([1, 2, 3]) } } } h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn, lazy=True) assert (res['a'] is None) assert (res['test1']['b'] is None) assert (res['test2']['test3']['c'] is None)
def load_val_unit_dict_from_h5(file): """ Load and convert val unit dict from h5 file to dict of quantities. The val unit dictionary is loaded from the h5 file and then converted to a dictionary containing quantities. Parameters: ----------- file: str String specifying input file name. """ try: loaded = h5.load(file) except OSError: raise IOError(f'{file} does not exist!') converted = val_unit_to_quantities(loaded) return converted
def test_store_and_load_dataset_directly(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w') for key, val in zip(simpledata_str, simpledata_val): assert(h5w.load(fn, '/' + key) == val)
def test_conversion_script(tmpdir): # convert LocalPath object to str to ensure that path can be # handled by os.path.join() path = str(tmpdir) previous_versions = ['0.0.1', '1.0.1'] for version in previous_versions: print(version) version_stripped = version.replace('.', '') release_base_name = '_'.join(('h5py_wrapper', version_stripped)) module = '.'.join((release_base_name, 'wrapper')) try: h5w_prev = importlib.import_module(module) except ImportError: h5w_lib.get_previous_version(version, tmpdir) sys.path.append(path) print(sys.path[-1]) print(module) h5w_prev = importlib.import_module(module) tmp_fn = os.path.join(path, fn) tmp_fn2 = os.path.join(path, fn2) res = {key: value for key, value in zip(simpledata_str, simpledata_val)} res.update({key: value for key, value in zip(arraydata_str, arraydata_val)}) h5w_prev.add_to_h5(tmp_fn, res) h5w_prev.add_to_h5(tmp_fn2, res) with open('conversion_list.txt', 'w') as f: f.write(tmp_fn) f.write('\n') f.write(tmp_fn2) # Specify list on command line conversion_file_cmd = 'convert_h5file' cmd = 'python {}/../{} {} {} --release={}'.format(os.path.dirname(__file__), conversion_file_cmd, tmp_fn, tmp_fn2, version) os.system(cmd) # Read list of files from file and pipe into conversion script cmd = 'cat conversion_list.txt | python {}/../{} {}={}'.format(os.path.dirname(__file__), conversion_file_cmd, '--release', version) os.system(cmd) os.remove('conversion_list.txt') # Find files based on pattern using `find` and pipe into conversion script cmd = 'find -name "data*.h5" | python {}/../{} {}={}'.format(os.path.dirname(__file__), conversion_file_cmd, '--release', version) os.system(cmd) res2 = h5w.load(tmp_fn) for key, value in res.items(): if isinstance(res2[key], np.ndarray): assert_array_equal(res2[key], value) else: assert(res2[key] == value) if isinstance(res2[key], str): assert(isinstance(res2[key], type(str(value)))) else: assert(isinstance(res2[key], type(value))) os.remove(tmp_fn) os.remove(tmp_fn2)
def test_write_and_load_with_label(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w', path='test_label') for key, val in zip(simpledata_str, simpledata_val): assert(h5w.load(fn, 'test_label/' + key) == val)
def test_handle_nonexisting_file(): try: h5w.load('asdasd.h5') raise Exception() # should not get until here except IOError: pass
def test_store_none(): res = {'a1': None} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert (res['a1'] is None)
def test_store_and_load_with_compression(): data = {'a': 1, 'test1': {'b': 2}, 'test2': { 'test3': {'c': np.array([1, 2, 3])}}} h5w.save(fn, data, write_mode='w', compression='gzip') h5w.load(fn)
def ground_truth_result(): results = h5.load(fix_path + 'Schuecker2015_data.h5') return results
def test_write_and_load_with_label(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w', path='test_label') for key, val in zip(simpledata_str, simpledata_val): assert (h5w.load(fn, 'test_label/' + key) == val)
def test_store_and_load_dataset_directly(): res = _construct_simpledata() h5w.save(fn, res, write_mode='w') for key, val in zip(simpledata_str, simpledata_val): assert (h5w.load(fn, '/' + key) == val)
def test_store_none(): res = {'a1': None} h5w.save(fn, res, write_mode='w') res.clear() res = h5w.load(fn) assert(res['a1'] is None)
def test_conversion_script(tmpdir): # convert LocalPath object to str to ensure that path can be # handled by os.path.join() path = str(tmpdir) previous_versions = ['0.0.1', '1.0.1'] for version in previous_versions: print(version) version_stripped = version.replace('.', '') release_base_name = '_'.join(('h5py_wrapper', version_stripped)) module = '.'.join((release_base_name, 'wrapper')) try: h5w_prev = importlib.import_module(module) except ImportError: h5w_lib.get_previous_version(version, tmpdir) sys.path.append(path) print(sys.path[-1]) print(module) h5w_prev = importlib.import_module(module) tmp_fn = os.path.join(path, fn) tmp_fn2 = os.path.join(path, fn2) res = { key: value for key, value in zip(simpledata_str, simpledata_val) } res.update( {key: value for key, value in zip(arraydata_str, arraydata_val)}) h5w_prev.add_to_h5(tmp_fn, res) h5w_prev.add_to_h5(tmp_fn2, res) with open('conversion_list.txt', 'w') as f: f.write(tmp_fn) f.write('\n') f.write(tmp_fn2) # Specify list on command line conversion_file_cmd = 'convert_h5file' cmd = 'python {}/../{} {} {} --release={}'.format( os.path.dirname(__file__), conversion_file_cmd, tmp_fn, tmp_fn2, version) os.system(cmd) # Read list of files from file and pipe into conversion script cmd = 'cat conversion_list.txt | python {}/../{} {}={}'.format( os.path.dirname(__file__), conversion_file_cmd, '--release', version) os.system(cmd) os.remove('conversion_list.txt') # Find files based on pattern using `find` and pipe into conversion script cmd = 'find -name "data*.h5" | python {}/../{} {}={}'.format( os.path.dirname(__file__), conversion_file_cmd, '--release', version) os.system(cmd) res2 = h5w.load(tmp_fn) for key, value in res.items(): if isinstance(res2[key], np.ndarray): assert_array_equal(res2[key], value) else: assert (res2[key] == value) if isinstance(res2[key], str): assert (isinstance(res2[key], type(str(value)))) else: assert (isinstance(res2[key], type(value))) os.remove(tmp_fn) os.remove(tmp_fn2)