def test_sections_property():
ds = DataStore({
'st': (['x', 'time'], np.ones((5, 5))),
'ast': (['x', 'time'], np.ones((5, 5))),
'probe1Temperature': (['time'], range(5)),
'probe2Temperature': (['time'], range(5))
},
coords={
'x': range(5),
'time': range(5)})
sections1 = {
'probe1Temperature': [slice(7.5, 17.), slice(70., 80.)], # cold bath
'probe2Temperature': [slice(24., 34.), slice(85., 95.)], # warm bath
}
sections2 = {
'probe1Temperature': [slice(0., 17.), slice(70., 80.)], # cold bath
'probe2Temperature': [slice(24., 34.), slice(85., 95.)], # warm bath
}
ds.sections = sections1
assert isinstance(ds._sections, str)
assert ds.sections == sections1
assert ds.sections != sections2
# delete property
del ds.sections
assert ds.sections is None
pass
def test_io_sections_property():
ds = DataStore({
'st': (['x', 'time'], np.ones((5, 5))),
'ast': (['x', 'time'], np.ones((5, 5))),
'probe1Temperature': (['time'], range(5)),
'probe2Temperature': (['time'], range(5))
},
coords={
'x': range(5),
'time': range(5)})
sections = {
'probe1Temperature': [slice(7.5, 17.), slice(70., 80.)], # cold bath
'probe2Temperature': [slice(24., 34.), slice(85., 95.)], # warm bath
}
ds.sections = sections
with tempfile.NamedTemporaryFile() as tmp:
ds.to_netcdf(path=tmp.name)
ds2 = open_datastore(tmp.name)
assert ds.sections == ds2.sections
pass
def test_io_sections_property():
ds = DataStore(
{
'st': (['x', 'time'], np.ones((100, 5))),
'ast': (['x', 'time'], np.ones((100, 5))),
'probe1Temperature': (['time'], range(5)),
'probe2Temperature': (['time'], range(5))
},
coords={
'x': ('x', range(100), {
'units': 'm'
}),
'time': range(5)
})
sections = {
'probe1Temperature': [slice(7.5, 17.),
slice(70., 80.)], # cold bath
'probe2Temperature': [slice(24., 34.),
slice(85., 95.)], # warm bath
}
ds['x'].attrs['units'] = 'm'
ds.sections = sections
# Create a temporary file to write data to.
# 'with' method is used so the file is closed by tempfile
# and free to be overwritten.
with tempfile.NamedTemporaryFile('w') as tmp:
temppath = tmp.name
# Write the datastore to the temp file
ds.to_netcdf(path=temppath)
try:
ds2 = open_datastore(temppath)
except ValueError as e:
if str(e) != 'cannot guess the engine, try passing one explicitly':
raise
else:
warnings.warn('Could not guess engine, defaulted to netcdf4')
ds2 = open_datastore(temppath, engine='netcdf4')
assert ds.sections == ds2.sections
# Close the datastore so the temp file can be removed
ds2.close()
ds2 = None
# Remove the temp file once the test is done
if os.path.exists(temppath):
os.remove(temppath)
pass
def test_io_sections_property():
ds = DataStore(
{
'st': (['x', 'time'], np.ones((100, 5))),
'ast': (['x', 'time'], np.ones((100, 5))),
'probe1Temperature': (['time'], range(5)),
'probe2Temperature': (['time'], range(5))
},
coords={
'x': ('x', range(100), {
'units': 'm'
}),
'time': range(5)
})
sections = {
'probe1Temperature': [slice(7.5, 17.),
slice(70., 80.)], # cold bath
'probe2Temperature': [slice(24., 34.),
slice(85., 95.)], # warm bath
}
ds['x'].attrs['units'] = 'm'
ds.sections = sections
# Create a temporary file to write data to.
# 'with' method is used so the file is closed by tempfile
# and free to be overwritten.
with tempfile.NamedTemporaryFile('w') as tmp:
temppath = tmp.name
# Write the datastore to the temp file
ds.to_netcdf(path=temppath)
ds2 = open_datastore(temppath)
assert ds.sections == ds2.sections
# Close the datastore so the temp file can be removed
ds2.close()
ds2 = None
# Remove the temp file once the test is done
if os.path.exists(temppath):
os.remove(temppath)
pass
def test_variance_of_stokes_synthetic():
"""
Produces a synthetic Stokes measurement with a known noise distribution. Check if same
variance is obtained.
Returns
-------
"""
yvar = 5.
nx = 50
x = np.linspace(0., 20., nx)
nt = 1000
beta = np.linspace(3000, 4000, nt)[None]
y = beta * np.exp(-0.001 * x[:, None])
y += stats.norm.rvs(size=y.size, scale=yvar**0.5).reshape(y.shape)
ds = DataStore(
{
'test_ST': (['x', 'time'], y),
'probe1Temperature': (['time'], range(nt)),
'userAcquisitionTimeFW': (['time'], np.ones(nt)),
},
coords={
'x': x,
'time': range(nt)
},
attrs={'customData:isDoubleEnded': '0'})
sections = {
'probe1Temperature': [
slice(0., 20.),
]
}
test_ST_var, _ = ds.variance_stokes(st_label='test_ST',
sections=sections,
suppress_info=True)
np.testing.assert_almost_equal(test_ST_var, yvar, decimal=1)
def shift_double_ended(ds, i_shift):
"""
The cable length was initially configured during the DTS measurement. For double ended
measurements it is important to enter the correct length so that the forward channel and the
backward channel are aligned.
This function can be used to shift the backward channel so that the forward channel and the
backward channel are aligned. The backward channel is shifted per index
along the x dimension.
The DataStore object that is returned only has values for the backscatter that are measured
for both the forward channel and the backward channel in the shifted object
There is no interpolation, as this would alter the accuracy.
Parameters
----------
ds : DataSore object
DataStore object that needs to be shifted
i_shift : int
if i_shift < 0, the cable was configured to be too long and there is too much data
recorded. If i_shift > 0, the cable was configured to be too short and part of the cable is
not measured.
Returns
-------
ds2 : DataStore oobject
With a shifted x-axis
"""
from dtscalibration import DataStore
assert isinstance(i_shift, (int, np.integer))
nx = ds.x.size
nx2 = nx - i_shift
if i_shift < 0:
# The cable was configured to be too long.
# There is too much data recorded.
st = ds.ST.data[:i_shift]
ast = ds.AST.data[:i_shift]
rst = ds['REV-ST'].data[-i_shift:]
rast = ds['REV-AST'].data[-i_shift:]
x2 = ds.x.data[:i_shift]
# TMP2 = ds.TMP.data[:i_shift]
else:
# The cable was configured to be too short.
# Part of the cable is not measured.
st = ds.ST.data[i_shift:]
ast = ds.AST.data[i_shift:]
rst = ds['REV-ST'].data[:nx2]
rast = ds['REV-AST'].data[:nx2]
x2 = ds.x.data[i_shift:]
# TMP2 = ds.TMP.data[i_shift:]
d2_coords = dict(ds.coords)
d2_coords['x'] = (('x', ), x2, ds.x.attrs)
d2_data = dict(ds.data_vars)
for k in ds.data_vars:
if 'x' in ds[k].dims and k in d2_data:
del d2_data[k]
new_data = (('ST', st), ('AST', ast), ('REV-ST', rst), ('REV-AST', rast))
for k, v in new_data:
d2_data[k] = (ds[k].dims, v, ds[k].attrs)
not_included = [k for k in ds.data_vars if k not in d2_data]
if not_included:
print('I dont know what to do with the following data', not_included)
return DataStore(data_vars=d2_data, coords=d2_coords, attrs=ds.attrs)
def test_has_sectionattr_upon_creation():
ds = DataStore()
assert hasattr(ds, '_sections')
assert isinstance(ds._sections, str)
pass
def test_repr():
ds = DataStore()
assert str(ds).find('dtscalibration') != -1
assert str(ds).find('Sections') != -1
pass
def test_empty_construction():
ds = DataStore() # noqa: F841
pass
def test_repr():
ds = DataStore()
assert ds.__repr__().find('dtscalibration')
assert ds.__repr__().find('Sections')
pass
def test_empty_construction():
ds = DataStore()
assert ds._initialized, 'Empty obj in not initialized'
pass
def test_double_ended_variance_estimate_synthetic():
import dask.array as da
from dtscalibration import DataStore
import numpy as np
from scipy import stats
np.random.seed(0)
state = da.random.RandomState(0)
# from dtscalibration.calibrate_utils import
stokes_m_var = 40.
cable_len = 100.
nt = 500
time = np.arange(nt)
x = np.linspace(0., cable_len, 100)
ts_cold = np.ones(nt) * 4.
ts_warm = np.ones(nt) * 20.
C_p = 15246
C_m = 2400.
dalpha_r = 0.0005284
dalpha_m = 0.0004961
dalpha_p = 0.0005607
gamma = 482.6
cold_mask = x < 0.5 * cable_len
warm_mask = np.invert(cold_mask) # == False
temp_real = np.ones((len(x), nt))
temp_real[cold_mask] *= ts_cold + 273.15
temp_real[warm_mask] *= ts_warm + 273.15
st = C_p * np.exp(-dalpha_r * x[:, None]) * np.exp(
-dalpha_p * x[:, None]) * np.exp(
-gamma / temp_real) / (1 - np.exp(-gamma / temp_real))
ast = C_m * np.exp(-dalpha_r * x[:, None]) * np.exp(
-dalpha_m * x[:, None]) / (1 - np.exp(-gamma / temp_real))
rst = C_p * np.exp(-dalpha_r * (-x[:, None] + 100)) * np.exp(
-dalpha_p * (-x[:, None] + 100)) * np.exp(
-gamma / temp_real) / (1 - np.exp(-gamma / temp_real))
rast = C_m * np.exp(-dalpha_r * (-x[:, None] + 100)) * np.exp(
-dalpha_m * (-x[:, None] + 100)) / (1 - np.exp(-gamma / temp_real))
st_m = st + stats.norm.rvs(size=st.shape, scale=stokes_m_var**0.5)
ast_m = ast + stats.norm.rvs(size=ast.shape, scale=1.1 * stokes_m_var**0.5)
rst_m = rst + stats.norm.rvs(size=rst.shape, scale=0.9 * stokes_m_var**0.5)
rast_m = rast + stats.norm.rvs(size=rast.shape,
scale=0.8 * stokes_m_var**0.5)
print('alphaint', cable_len * (dalpha_p - dalpha_m))
print('alpha', dalpha_p - dalpha_m)
print('C', np.log(C_p / C_m))
print('x0', x.max())
ds = DataStore(
{
'st': (['x', 'time'], st),
'ast': (['x', 'time'], ast),
'rst': (['x', 'time'], rst),
'rast': (['x', 'time'], rast),
'mst': (['x', 'time'], st_m),
'mast': (['x', 'time'], ast_m),
'mrst': (['x', 'time'], rst_m),
'mrast': (['x', 'time'], rast_m),
'userAcquisitionTimeFW': (['time'], np.ones(nt)),
'userAcquisitionTimeBW': (['time'], np.ones(nt)),
'cold': (['time'], ts_cold),
'warm': (['time'], ts_warm)
},
coords={
'x': x,
'time': time
},
attrs={'customData:isDoubleEnded': '1'})
sections = {
'cold': [slice(0., 0.5 * cable_len)],
'warm': [slice(0.5 * cable_len, cable_len)]
}
mst_var, _ = ds.variance_stokes(st_label='mst',
sections=sections,
suppress_info=True)
mast_var, _ = ds.variance_stokes(st_label='mast',
sections=sections,
suppress_info=True)
mrst_var, _ = ds.variance_stokes(st_label='mrst',
sections=sections,
suppress_info=True)
mrast_var, _ = ds.variance_stokes(st_label='mrast',
sections=sections,
suppress_info=True)
st_label = 'mst'
ast_label = 'mast'
rst_label = 'mrst'
rast_label = 'mrast'
# MC variqnce
ds.calibration_double_ended(
sections=sections,
st_label=st_label,
ast_label=ast_label,
rst_label=rst_label,
rast_label=rast_label,
st_var=mst_var,
ast_var=mast_var,
rst_var=mrst_var,
rast_var=mrast_var,
method='wls',
# conf_ints=[0.00135, 0.025, 0.15865, 0.5, 0.84135, 0.975, 0.99865],
conf_ints=[0.025, 0.5, 0.975],
ci_avg_time_flag=0,
store_tempvar='_var',
conf_ints_size=500,
solver='sparse',
da_random_state=state)
# Calibrated variance
stdsf1 = ds.ufunc_per_section(label='TMPF',
func=np.std,
temp_err=True,
calc_per='stretch')
stdsb1 = ds.ufunc_per_section(label='TMPB',
func=np.std,
temp_err=True,
calc_per='stretch')
# Use a single timestep to better check if the parameter uncertainties propagate
ds1 = ds.isel(time=1)
# Estimated VAR
stdsf2 = ds1.ufunc_per_section(label='TMPF_MC_var',
func=np.mean,
temp_err=False,
calc_per='stretch')
stdsb2 = ds1.ufunc_per_section(label='TMPB_MC_var',
func=np.mean,
temp_err=False,
calc_per='stretch')
for (_, v1), (_, v2) in zip(stdsf1.items(), stdsf2.items()):
for v1i, v2i in zip(v1, v2):
print('Real VAR: ', v1i**2, 'Estimated VAR: ', v2i)
np.testing.assert_almost_equal(v1i**2, v2i, decimal=2)
for (_, v1), (_, v2) in zip(stdsb1.items(), stdsb2.items()):
for v1i, v2i in zip(v1, v2):
print('Real VAR: ', v1i**2, 'Estimated VAR: ', v2i)
np.testing.assert_almost_equal(v1i**2, v2i, decimal=2)
pass