def test_append(self):
fn = tmp_path + 'append.hdf5'
d = np.arange(48, dtype=np.float32).reshape(24, 2)
d[2, 0] = np.nan
gtsdf.save(fn, d)
_, data, _ = gtsdf.load(fn)
np.testing.assert_array_almost_equal(data, d, 3)
gtsdf.append_block(fn, d)
_, data, _ = gtsdf.load(fn)
self.assertEqual(data.shape, (48, 2))
np.testing.assert_array_almost_equal(data, np.append(d, d, 0), 3)
f = h5py.File(fn)
self.assertIn('gains', f['block0001'])
f.close()
def test_outlier(self):
fn = tmp_path + 'outlier.hdf5'
d = np.arange(12, dtype=np.float32).reshape(6, 2)
d[2, 0] = 10**4
d[3, 1] = 10**4
self.assertRaises(Warning, gtsdf.save, fn, d)
_, data, _ = gtsdf.load(fn)
def test_float_time(self):
fn = tmp_path + 'time.hdf5'
gtsdf.save(fn,
np.arange(12).reshape(6, 2),
time=np.arange(4, 5.5, .25))
time, _, _ = gtsdf.load(fn)
np.testing.assert_array_equal(time, np.arange(4, 5.5, .25))
def test_gtsdf_compress2stat(self):
time, data, info = gtsdf.load(tfp+'test.hdf5')
fn = tmp_path + "test_compress2stat.hdf5"
gtsdf.save(fn, data, time=time, **info)
gtsdf.save(tmp_path + "test_compress2stat2.hdf5", data, time=time, dtype=np.float, **info)
gtsdf.compress2statistics(fn)
self.assertLess(os.path.getsize(fn)*50, os.path.getsize(tfp+'test.hdf5'))
def test_loadhdf5File(self):
f = h5py.File(tfp + 'test.hdf5')
time, data, info = gtsdf.load(f)
self.assertEqual(time[1], 0.05)
self.assertEqual(data[1, 1], 11.986652374267578)
self.assertEqual(info['attribute_names'][1], "WSP gl. coo.,Vy")
def test_nan_float(self):
fn = tmp_path + 'nan.hdf5'
d = np.arange(12, dtype=np.float32).reshape(6, 2)
d[2, 0] = np.nan
gtsdf.save(fn, d)
_, data, _ = gtsdf.load(fn)
np.testing.assert_array_almost_equal(data, d, 4)
def test_gtsdf_stat(self):
time, data, info = gtsdf.load(tfp+'test.hdf5')
fn = tmp_path + "test_stat.hdf5"
gtsdf.save(fn, data, time=time, **info)
gtsdf.add_statistic(fn)
stat_data,info = gtsdf.load_statistic(fn)
self.assertEqual(data[:,0].min(), stat_data.values[0,0])
self.assertEqual(stat_data.shape, (49,10))
def test_time_offset(self):
fn = tmp_path + 'time.hdf5'
gtsdf.save(fn,
np.arange(12).reshape(6, 2),
time=range(6),
time_start=4)
time, _, _ = gtsdf.load(fn)
np.testing.assert_array_equal(time, range(4, 10))
def test_data_float(self):
fn = tmp_path + 'time.hdf5'
d = np.arange(12).reshape(6, 2)
gtsdf.save(fn, d, dtype=np.float32)
f = h5py.File(fn)
self.assertEqual(f['block0000']['data'].dtype, np.float32)
f.close()
_, data, _ = gtsdf.load(fn)
np.testing.assert_array_equal(data, np.arange(12).reshape(6, 2))
def test_data(self):
fn = tmp_path + 'data.hdf5'
d = np.arange(12).reshape(6, 2)
gtsdf.save(fn, d)
f = h5py.File(fn)
self.assertEqual(f['block0000']['data'].dtype, np.uint16)
f.close()
_, data, _ = gtsdf.load(fn)
np.testing.assert_array_almost_equal(data,
np.arange(12).reshape(6, 2), 4)
def test_minimum_requirements(self):
fn = tmp_path + "minimum.hdf5"
f = h5py.File(fn, "w")
#no type
self.assertRaises(ValueError, gtsdf.load, fn)
f.attrs["type"] = "General time series data format"
#no no_blocks
self.assertRaises(ValueError, gtsdf.load, fn)
f.attrs["no_blocks"] = 0
#no block0000
self.assertRaises(ValueError, gtsdf.load, fn)
b = f.create_group("block0000")
#no data
self.assertRaises(ValueError, gtsdf.load, fn)
b.create_dataset("data", data=np.empty((0, 0)))
f.close()
gtsdf.load(fn)
def test_subset_mean(self):
time, data, info = gtsdf.load(tfp + "subset_mean_test.hdf5")
triggers = time_trigger(time, 100)
t, p = subset_mean([time, data[:, 0]], triggers).T
self.assertEqual(t[1], time[2500:5000].mean())
self.assertEqual(p[1], data[2500:5000, 0].mean())
triggers[1] = 2501
t, p = subset_mean([time, data[:, 0]], triggers).T
self.assertEqual(t[1], time[2501:5000].mean())
self.assertEqual(p[1], data[2501:5000, 0].mean())
def test_time_trigger(self):
time = np.arange(0, 99.5, .5)
np.testing.assert_array_equal(time[time_trigger(time, 20)],
[0, 20, 40, 60, 80])
np.testing.assert_array_equal(time[time_trigger(time + .5, 20)],
[0, 20, 40, 60, 80])
np.testing.assert_array_equal(
time[time_trigger(time + 100000000.5, 20)], [0, 20, 40, 60, 80])
np.testing.assert_array_equal(time[time_trigger(time, 20, 20, 60)],
[20, 40, 60])
np.testing.assert_array_equal(time_trigger(np.arange(101), 20),
[0, 20, 40, 60, 80, 100])
time, data, info = gtsdf.load(tfp + "subset_mean_test.hdf5")
np.testing.assert_array_equal(time_trigger(time, 200),
[0, 5000, 10000, 15000])
def ReadGtsdf(self):
fn = self.FileName
if fn[-5:].lower() != '.hdf5':
fn += '.hdf5'
self.t, data, info = gtsdf.load(fn)
self.Time = self.t
self.ChInfo = [['Time'] + info['attribute_names'],
['s'] + info['attribute_units'],
['Time'] + info['attribute_descriptions']]
self.NrCh = data.shape[1] + 1
self.NrSc = data.shape[0]
self.Freq = self.NrSc / self.Time
self.FileFormat = 'GTSDF'
self.gtsdf_description = info['description']
data = np.hstack([self.Time[:, np.newaxis], data])
return data
def test_info(self):
fn = tmp_path + "info.hdf5"
gtsdf.save(fn,
np.arange(12).reshape(6, 2),
name='datasetname',
description='datasetdescription',
attribute_names=['att1', 'att2'],
attribute_units=['s', 'm/s'],
attribute_descriptions=['d1', 'd2'])
_, _, info = gtsdf.load(fn)
self.assertEqual(info['name'], "datasetname")
self.assertEqual(info['type'], "General time series data format")
self.assertEqual(info['description'], "datasetdescription")
self.assertEqual(list(info['attribute_names']), ['att1', 'att2'])
self.assertEqual(list(info['attribute_units']), ['s', 'm/s'])
self.assertEqual(list(info['attribute_descriptions']), ['d1', 'd2'])
def test_all(self):
fn = tmp_path + "all.hdf5"
gtsdf.save(fn,
np.arange(12).reshape(6, 2),
name='datasetname',
time=range(6),
time_step=1 / 4,
time_start=4,
description='datasetdescription',
attribute_names=['att1', 'att2'],
attribute_units=['s', 'm/s'],
attribute_descriptions=['d1', 'd2'])
time, data, info = gtsdf.load(fn)
self.assertEqual(info['name'], "datasetname")
self.assertEqual(info['type'], "General time series data format")
self.assertEqual(info['description'], "datasetdescription")
self.assertEqual(list(info['attribute_names']), ['att1', 'att2'])
self.assertEqual(list(info['attribute_units']), ['s', 'm/s'])
self.assertEqual(list(info['attribute_descriptions']), ['d1', 'd2'])
np.testing.assert_array_equal(time, np.arange(4, 5.5, .25))
np.testing.assert_array_almost_equal(data,
np.arange(12).reshape(6, 2), 4)
Despiked data
"""
if plt:
plt.plot(data, label='Input')
data = np.array(data).copy()
from wetb.signal.filters import frq_filters
lp_data = frq_filters.low_pass(data, spike_length, 1)
hp_data = data - lp_data
hp_data = frq_filters.high_pass(data, spike_length * 4, 1, order=2)
#spike_length, sample_frq/2, 1, order=1)
spike_mask = spike_finder(hp_data, plt=plt)
despike_data = spike_replacer(data, spike_mask)
if plt:
plt.plot(despike_data, 'k.', label='Output')
plt.legend()
return despike_data
if __name__ == '__main__':
from wetb import gtsdf
time, data, info = gtsdf.load(
r'C:\data\SWT36Hovsore\metdata/const_10hz.hdf5')
wsp = data[:, 45]
import matplotlib.pyplot as plt
despike(wsp,
.1,
lambda data, plt: univeral_thresshold_finder(data, 'std', plt),
plt=plt)
plt.show()
def test_load_fileobject(self):
fn = tmp_path + "fileobject.hdf5"
gtsdf.save(fn, np.arange(12).reshape(4, 3))
_, _, info = gtsdf.load(fn)
self.assertEqual(info['name'], 'fileobject')
def test_save_no_hdf5_ext(self):
fn = tmp_path + "no_hdf5_ext"
gtsdf.save(fn, np.arange(12).reshape(4, 3))
_, _, info = gtsdf.load(fn + ".hdf5")
self.assertEqual(info['name'], 'no_hdf5_ext')
