def test_read_silixa_files_single_loadinmemory():
filepath = data_dir_single_ended
# False
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=False)
for k in ['ST', 'AST']:
assert isinstance(ds[k].data, da.Array)
# auto -> True
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory='auto')
for k in ['ST', 'AST']:
assert isinstance(ds[k].data, np.ndarray)
# True
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=True)
for k in ['ST', 'AST']:
assert isinstance(ds[k].data, np.ndarray)
pass
def test_to_mf_netcdf_open_mf_datastore():
filepath = data_dir_single_ended
ds = read_silixa_files(directory=filepath, file_ext='*.xml')
with tempfile.TemporaryDirectory() as tmpdirname:
print('created temporary directory', tmpdirname)
# work around the effects of deafault encoding.
path = os.path.join(tmpdirname, 'ds_merged.nc')
with read_silixa_files(directory=filepath, file_ext='*.xml') as ds:
ds.to_netcdf(path)
time.sleep(5) # to ensure all is written on Windows and file released
with open_datastore(path, load_in_memory=True) as ds1:
# Test saving
ds1 = ds1.chunk({'time': 1})
ds1.to_mf_netcdf(folder_path=tmpdirname,
filename_preamble='file_',
filename_extension='.nc')
correct_val = float(ds1.st.sum())
time.sleep(2) # to ensure all is written on Windows and file released
# Test loading
path = os.path.join(tmpdirname, 'file_*.nc')
with open_mf_datastore(path=path, load_in_memory=True) as ds2:
test_val = float(ds2.st.sum())
np.testing.assert_equal(correct_val, test_val)
pass
def test_read_silixa_files_double_loadinmemory():
filepath = data_dir_double_ended
# False
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=False)
for k in ['st', 'ast', 'rst', 'rast']:
assert isinstance(ds[k].data, da.Array)
# auto -> True Because small amount of data
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory='auto')
for k in ['st', 'ast', 'rst', 'rast']:
assert isinstance(ds[k].data, np.ndarray)
# True
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=True)
for k in ['st', 'ast', 'rst', 'rast']:
assert isinstance(ds[k].data, np.ndarray)
pass
def test_read_long_silixa_files():
filepath = data_dir_silixa_long
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
np.testing.assert_almost_equal(ds.st.sum(), 133223729.17096, decimal=0)
pass
def test_read_silixa_files_double_ended():
filepath = data_dir_double_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
np.testing.assert_almost_equal(ds.st.sum(), 19613502.2617, decimal=3)
pass
def test_read_silixa_files_single_ended():
filepath = data_dir_single_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
np.testing.assert_almost_equal(ds.ST.sum(), 11387947.857, decimal=3)
pass
def test_read_long_silixa_files():
filepath = data_dir_silixa_long
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
assert ds._initialized
pass
def test_merge_double_ended():
# Checking if alignment keeps working as designed and if the expected
# result changed
filepath_fw = data_dir_double_single_ch1
filepath_bw = data_dir_double_single_ch2
ds_fw = read_silixa_files(directory=filepath_fw)
ds_bw = read_silixa_files(directory=filepath_bw)
cable_length = 2017.7
ds = merge_double_ended(ds_fw,
ds_bw,
cable_length=cable_length,
plot_result=True)
result = (ds.isel(time=0).st - ds.isel(time=0).rst).sum().values
np.testing.assert_approx_equal(result, -3712866.0382, significant=10)
def test_read_single_silixa_v7():
filepath = data_dir_single_silixa_v7
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=False)
for k in ['st', 'ast']:
assert isinstance(ds[k].data, da.Array)
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=True)
for k in ['st', 'ast']:
assert isinstance(ds[k].data, np.ndarray)
pass
def test_suggest_cable_shift_double_ended():
# need more measurements for proper testing. Therefore only checking if
# no errors occur
filepath = data_dir_double_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
irange = np.arange(-4, 4)
suggest_cable_shift_double_ended(ds, irange, plot_result=True)
pass
def test_resample_datastore():
filepath = data_dir_single_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
assert ds.time.size == 3
ds_resampled = ds.resample_datastore(how='mean', time="47S")
assert ds_resampled._initialized
assert ds_resampled.time.size == 2
pass
def test_read_single_silixa_v45():
filepath = data_dir_single_silixa_v45
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=False)
assert ds._initialized
for k in ['ST', 'AST']:
assert isinstance(ds[k].data, da.Array)
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=True)
assert ds._initialized
for k in ['ST', 'AST']:
assert isinstance(ds[k].data, np.ndarray)
pass
def test_read_silixa_zipped():
files = [(data_dir_zipped_single_ended, 11387947.857184),
(data_dir_zipped_double_ended, 19613502.26171),
(data_dir_zipped_double_ended2, 28092965.5188),
(data_dir_zipped_silixa_long, 2.88763942e+08)]
for file, stsum in files:
with zipf(file) as fh:
ds = read_silixa_files(zip_handle=fh,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=True)
np.testing.assert_almost_equal(ds.st.sum(), stsum, decimal=0)
ds.close()
pass
def test_resample_datastore():
filepath = data_dir_single_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
assert ds.time.size == 3
ds_resampled = ds.resample_datastore(how='mean', time="47S")
assert ds_resampled.time.size == 2
assert ds_resampled.st.dims == ('x', 'time'), 'The dimension have to ' \
'be manually transposed ' \
'after resampling. To ' \
'guarantee the order'
pass
def test_timeseries_keys():
filepath = data_dir_single_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
k = ds.timeseries_keys
# no false positive
for ki in k:
assert ds[ki].dims == ('time', )
# no false negatives
k_not = [ki for ki in ds.data_vars if ki not in k]
for kni in k_not:
assert ds[kni].dims != ('time', )
pass
def test_read_silixa_zipped():
files = [
data_dir_zipped_single_ended,
data_dir_zipped_double_ended,
data_dir_zipped_double_ended2,
data_dir_zipped_silixa_long,
# data_dir_zipped_sensornet_single_ended
]
for file in files:
with zipf(file) as fh:
ds = read_silixa_files(zip_handle=fh,
timezone_netcdf='UTC',
file_ext='*.xml',
load_in_memory=True)
assert ds._initialized
pass
def test_variance_of_stokes():
correct_var = 9.045
filepath = data_dir_double_ended2
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
sections = {
'probe1Temperature': [slice(7.5, 17.),
slice(70., 80.)], # cold bath
'probe2Temperature': [slice(24., 34.),
slice(85., 95.)], # warm bath
}
I_var, _ = ds.variance_stokes(st_label='ST', sections=sections)
np.testing.assert_almost_equal(I_var, correct_var, decimal=1)
ds_dask = ds.chunk(chunks={})
I_var, _ = ds_dask.variance_stokes(st_label='ST', sections=sections)
np.testing.assert_almost_equal(I_var, correct_var, decimal=1)
pass
def test_shift_double_ended_shift_backforward():
# shifting it back and forward, should result in the same
filepath = data_dir_double_ended
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
dsmin1 = shift_double_ended(ds, -1)
ds2 = shift_double_ended(dsmin1, 1)
np.testing.assert_allclose(ds.x[1:-1], ds2.x)
for k in ds2:
if 'x' not in ds2[k].dims:
continue
old = ds[k].isel(x=slice(1, -1))
new = ds2[k]
np.testing.assert_allclose(old, new)
pass
def test_calibration_ols():
"""Testing ordinary least squares procedure. And compare with device calibrated temperature.
The measurements were calibrated by the device using only section 8--17.m. Those temperatures
are compared up to 2 decimals. Silixa only uses a single calibration constant (I think they
fix gamma), or a different formulation, see Shell primer.
"""
filepath = data_dir_double_ended2
ds = read_silixa_files(directory=filepath,
timezone_netcdf='UTC',
file_ext='*.xml')
ds100 = ds.sel(x=slice(0, 100))
sections_ultima = {
'probe1Temperature': [slice(8., 17.)], # cold bath
}
st_label = 'ST'
ast_label = 'AST'
rst_label = 'REV-ST'
rast_label = 'REV-AST'
ds100.calibration_double_ended(sections=sections_ultima,
st_label=st_label,
ast_label=ast_label,
rst_label=rst_label,
rast_label=rast_label,
store_tmpw='TMPW',
method='ols')
np.testing.assert_array_almost_equal(ds100['TMPW'].data,
ds100.TMP.data,
decimal=1)
ds009 = ds100.sel(x=sections_ultima['probe1Temperature'][0])
np.testing.assert_array_almost_equal(ds009['TMPW'].data,
ds009.TMP.data,
decimal=2)
pass
