def test_append(self): fn = tmp_path + 'append.hdf5' d = np.arange(48, dtype=np.float32).reshape(24, 2) d[2, 0] = np.nan gtsdf.save(fn, d) _, data, _ = gtsdf.load(fn) np.testing.assert_array_almost_equal(data, d, 3) gtsdf.append_block(fn, d) _, data, _ = gtsdf.load(fn) self.assertEqual(data.shape, (48, 2)) np.testing.assert_array_almost_equal(data, np.append(d, d, 0), 3) f = h5py.File(fn) self.assertIn('gains', f['block0001']) f.close()
def test_outlier(self): fn = tmp_path + 'outlier.hdf5' d = np.arange(12, dtype=np.float32).reshape(6, 2) d[2, 0] = 10**4 d[3, 1] = 10**4 self.assertRaises(Warning, gtsdf.save, fn, d) _, data, _ = gtsdf.load(fn)
def test_float_time(self): fn = tmp_path + 'time.hdf5' gtsdf.save(fn, np.arange(12).reshape(6, 2), time=np.arange(4, 5.5, .25)) time, _, _ = gtsdf.load(fn) np.testing.assert_array_equal(time, np.arange(4, 5.5, .25))
def test_gtsdf_compress2stat(self): time, data, info = gtsdf.load(tfp+'test.hdf5') fn = tmp_path + "test_compress2stat.hdf5" gtsdf.save(fn, data, time=time, **info) gtsdf.save(tmp_path + "test_compress2stat2.hdf5", data, time=time, dtype=np.float, **info) gtsdf.compress2statistics(fn) self.assertLess(os.path.getsize(fn)*50, os.path.getsize(tfp+'test.hdf5'))
def test_loadhdf5File(self): f = h5py.File(tfp + 'test.hdf5') time, data, info = gtsdf.load(f) self.assertEqual(time[1], 0.05) self.assertEqual(data[1, 1], 11.986652374267578) self.assertEqual(info['attribute_names'][1], "WSP gl. coo.,Vy")
def test_nan_float(self): fn = tmp_path + 'nan.hdf5' d = np.arange(12, dtype=np.float32).reshape(6, 2) d[2, 0] = np.nan gtsdf.save(fn, d) _, data, _ = gtsdf.load(fn) np.testing.assert_array_almost_equal(data, d, 4)
def test_gtsdf_stat(self): time, data, info = gtsdf.load(tfp+'test.hdf5') fn = tmp_path + "test_stat.hdf5" gtsdf.save(fn, data, time=time, **info) gtsdf.add_statistic(fn) stat_data,info = gtsdf.load_statistic(fn) self.assertEqual(data[:,0].min(), stat_data.values[0,0]) self.assertEqual(stat_data.shape, (49,10))
def test_time_offset(self): fn = tmp_path + 'time.hdf5' gtsdf.save(fn, np.arange(12).reshape(6, 2), time=range(6), time_start=4) time, _, _ = gtsdf.load(fn) np.testing.assert_array_equal(time, range(4, 10))
def test_data_float(self): fn = tmp_path + 'time.hdf5' d = np.arange(12).reshape(6, 2) gtsdf.save(fn, d, dtype=np.float32) f = h5py.File(fn) self.assertEqual(f['block0000']['data'].dtype, np.float32) f.close() _, data, _ = gtsdf.load(fn) np.testing.assert_array_equal(data, np.arange(12).reshape(6, 2))
def test_data(self): fn = tmp_path + 'data.hdf5' d = np.arange(12).reshape(6, 2) gtsdf.save(fn, d) f = h5py.File(fn) self.assertEqual(f['block0000']['data'].dtype, np.uint16) f.close() _, data, _ = gtsdf.load(fn) np.testing.assert_array_almost_equal(data, np.arange(12).reshape(6, 2), 4)
def test_minimum_requirements(self): fn = tmp_path + "minimum.hdf5" f = h5py.File(fn, "w") #no type self.assertRaises(ValueError, gtsdf.load, fn) f.attrs["type"] = "General time series data format" #no no_blocks self.assertRaises(ValueError, gtsdf.load, fn) f.attrs["no_blocks"] = 0 #no block0000 self.assertRaises(ValueError, gtsdf.load, fn) b = f.create_group("block0000") #no data self.assertRaises(ValueError, gtsdf.load, fn) b.create_dataset("data", data=np.empty((0, 0))) f.close() gtsdf.load(fn)
def test_subset_mean(self): time, data, info = gtsdf.load(tfp + "subset_mean_test.hdf5") triggers = time_trigger(time, 100) t, p = subset_mean([time, data[:, 0]], triggers).T self.assertEqual(t[1], time[2500:5000].mean()) self.assertEqual(p[1], data[2500:5000, 0].mean()) triggers[1] = 2501 t, p = subset_mean([time, data[:, 0]], triggers).T self.assertEqual(t[1], time[2501:5000].mean()) self.assertEqual(p[1], data[2501:5000, 0].mean())
def test_time_trigger(self): time = np.arange(0, 99.5, .5) np.testing.assert_array_equal(time[time_trigger(time, 20)], [0, 20, 40, 60, 80]) np.testing.assert_array_equal(time[time_trigger(time + .5, 20)], [0, 20, 40, 60, 80]) np.testing.assert_array_equal( time[time_trigger(time + 100000000.5, 20)], [0, 20, 40, 60, 80]) np.testing.assert_array_equal(time[time_trigger(time, 20, 20, 60)], [20, 40, 60]) np.testing.assert_array_equal(time_trigger(np.arange(101), 20), [0, 20, 40, 60, 80, 100]) time, data, info = gtsdf.load(tfp + "subset_mean_test.hdf5") np.testing.assert_array_equal(time_trigger(time, 200), [0, 5000, 10000, 15000])
def ReadGtsdf(self): fn = self.FileName if fn[-5:].lower() != '.hdf5': fn += '.hdf5' self.t, data, info = gtsdf.load(fn) self.Time = self.t self.ChInfo = [['Time'] + info['attribute_names'], ['s'] + info['attribute_units'], ['Time'] + info['attribute_descriptions']] self.NrCh = data.shape[1] + 1 self.NrSc = data.shape[0] self.Freq = self.NrSc / self.Time self.FileFormat = 'GTSDF' self.gtsdf_description = info['description'] data = np.hstack([self.Time[:, np.newaxis], data]) return data
def test_info(self): fn = tmp_path + "info.hdf5" gtsdf.save(fn, np.arange(12).reshape(6, 2), name='datasetname', description='datasetdescription', attribute_names=['att1', 'att2'], attribute_units=['s', 'm/s'], attribute_descriptions=['d1', 'd2']) _, _, info = gtsdf.load(fn) self.assertEqual(info['name'], "datasetname") self.assertEqual(info['type'], "General time series data format") self.assertEqual(info['description'], "datasetdescription") self.assertEqual(list(info['attribute_names']), ['att1', 'att2']) self.assertEqual(list(info['attribute_units']), ['s', 'm/s']) self.assertEqual(list(info['attribute_descriptions']), ['d1', 'd2'])
def test_all(self): fn = tmp_path + "all.hdf5" gtsdf.save(fn, np.arange(12).reshape(6, 2), name='datasetname', time=range(6), time_step=1 / 4, time_start=4, description='datasetdescription', attribute_names=['att1', 'att2'], attribute_units=['s', 'm/s'], attribute_descriptions=['d1', 'd2']) time, data, info = gtsdf.load(fn) self.assertEqual(info['name'], "datasetname") self.assertEqual(info['type'], "General time series data format") self.assertEqual(info['description'], "datasetdescription") self.assertEqual(list(info['attribute_names']), ['att1', 'att2']) self.assertEqual(list(info['attribute_units']), ['s', 'm/s']) self.assertEqual(list(info['attribute_descriptions']), ['d1', 'd2']) np.testing.assert_array_equal(time, np.arange(4, 5.5, .25)) np.testing.assert_array_almost_equal(data, np.arange(12).reshape(6, 2), 4)
Despiked data """ if plt: plt.plot(data, label='Input') data = np.array(data).copy() from wetb.signal.filters import frq_filters lp_data = frq_filters.low_pass(data, spike_length, 1) hp_data = data - lp_data hp_data = frq_filters.high_pass(data, spike_length * 4, 1, order=2) #spike_length, sample_frq/2, 1, order=1) spike_mask = spike_finder(hp_data, plt=plt) despike_data = spike_replacer(data, spike_mask) if plt: plt.plot(despike_data, 'k.', label='Output') plt.legend() return despike_data if __name__ == '__main__': from wetb import gtsdf time, data, info = gtsdf.load( r'C:\data\SWT36Hovsore\metdata/const_10hz.hdf5') wsp = data[:, 45] import matplotlib.pyplot as plt despike(wsp, .1, lambda data, plt: univeral_thresshold_finder(data, 'std', plt), plt=plt) plt.show()
def test_load_fileobject(self): fn = tmp_path + "fileobject.hdf5" gtsdf.save(fn, np.arange(12).reshape(4, 3)) _, _, info = gtsdf.load(fn) self.assertEqual(info['name'], 'fileobject')
def test_save_no_hdf5_ext(self): fn = tmp_path + "no_hdf5_ext" gtsdf.save(fn, np.arange(12).reshape(4, 3)) _, _, info = gtsdf.load(fn + ".hdf5") self.assertEqual(info['name'], 'no_hdf5_ext')