def test_read_silixa_files_single_loadinmemory(): filepath = data_dir_single_ended # False ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=False) for k in ['ST', 'AST']: assert isinstance(ds[k].data, da.Array) # auto -> True ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory='auto') for k in ['ST', 'AST']: assert isinstance(ds[k].data, np.ndarray) # True ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=True) for k in ['ST', 'AST']: assert isinstance(ds[k].data, np.ndarray) pass
def test_to_mf_netcdf_open_mf_datastore(): filepath = data_dir_single_ended ds = read_silixa_files(directory=filepath, file_ext='*.xml') with tempfile.TemporaryDirectory() as tmpdirname: print('created temporary directory', tmpdirname) # work around the effects of deafault encoding. path = os.path.join(tmpdirname, 'ds_merged.nc') with read_silixa_files(directory=filepath, file_ext='*.xml') as ds: ds.to_netcdf(path) time.sleep(5) # to ensure all is written on Windows and file released with open_datastore(path, load_in_memory=True) as ds1: # Test saving ds1 = ds1.chunk({'time': 1}) ds1.to_mf_netcdf(folder_path=tmpdirname, filename_preamble='file_', filename_extension='.nc') correct_val = float(ds1.st.sum()) time.sleep(2) # to ensure all is written on Windows and file released # Test loading path = os.path.join(tmpdirname, 'file_*.nc') with open_mf_datastore(path=path, load_in_memory=True) as ds2: test_val = float(ds2.st.sum()) np.testing.assert_equal(correct_val, test_val) pass
def test_read_silixa_files_double_loadinmemory(): filepath = data_dir_double_ended # False ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=False) for k in ['st', 'ast', 'rst', 'rast']: assert isinstance(ds[k].data, da.Array) # auto -> True Because small amount of data ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory='auto') for k in ['st', 'ast', 'rst', 'rast']: assert isinstance(ds[k].data, np.ndarray) # True ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=True) for k in ['st', 'ast', 'rst', 'rast']: assert isinstance(ds[k].data, np.ndarray) pass
def test_read_long_silixa_files(): filepath = data_dir_silixa_long ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') np.testing.assert_almost_equal(ds.st.sum(), 133223729.17096, decimal=0) pass
def test_read_silixa_files_double_ended(): filepath = data_dir_double_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') np.testing.assert_almost_equal(ds.st.sum(), 19613502.2617, decimal=3) pass
def test_read_silixa_files_single_ended(): filepath = data_dir_single_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') np.testing.assert_almost_equal(ds.ST.sum(), 11387947.857, decimal=3) pass
def test_read_long_silixa_files(): filepath = data_dir_silixa_long ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') assert ds._initialized pass
def test_merge_double_ended(): # Checking if alignment keeps working as designed and if the expected # result changed filepath_fw = data_dir_double_single_ch1 filepath_bw = data_dir_double_single_ch2 ds_fw = read_silixa_files(directory=filepath_fw) ds_bw = read_silixa_files(directory=filepath_bw) cable_length = 2017.7 ds = merge_double_ended(ds_fw, ds_bw, cable_length=cable_length, plot_result=True) result = (ds.isel(time=0).st - ds.isel(time=0).rst).sum().values np.testing.assert_approx_equal(result, -3712866.0382, significant=10)
def test_read_single_silixa_v7(): filepath = data_dir_single_silixa_v7 ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=False) for k in ['st', 'ast']: assert isinstance(ds[k].data, da.Array) ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=True) for k in ['st', 'ast']: assert isinstance(ds[k].data, np.ndarray) pass
def test_suggest_cable_shift_double_ended(): # need more measurements for proper testing. Therefore only checking if # no errors occur filepath = data_dir_double_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') irange = np.arange(-4, 4) suggest_cable_shift_double_ended(ds, irange, plot_result=True) pass
def test_resample_datastore(): filepath = data_dir_single_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') assert ds.time.size == 3 ds_resampled = ds.resample_datastore(how='mean', time="47S") assert ds_resampled._initialized assert ds_resampled.time.size == 2 pass
def test_read_single_silixa_v45(): filepath = data_dir_single_silixa_v45 ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=False) assert ds._initialized for k in ['ST', 'AST']: assert isinstance(ds[k].data, da.Array) ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=True) assert ds._initialized for k in ['ST', 'AST']: assert isinstance(ds[k].data, np.ndarray) pass
def test_read_silixa_zipped(): files = [(data_dir_zipped_single_ended, 11387947.857184), (data_dir_zipped_double_ended, 19613502.26171), (data_dir_zipped_double_ended2, 28092965.5188), (data_dir_zipped_silixa_long, 2.88763942e+08)] for file, stsum in files: with zipf(file) as fh: ds = read_silixa_files(zip_handle=fh, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=True) np.testing.assert_almost_equal(ds.st.sum(), stsum, decimal=0) ds.close() pass
def test_resample_datastore(): filepath = data_dir_single_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') assert ds.time.size == 3 ds_resampled = ds.resample_datastore(how='mean', time="47S") assert ds_resampled.time.size == 2 assert ds_resampled.st.dims == ('x', 'time'), 'The dimension have to ' \ 'be manually transposed ' \ 'after resampling. To ' \ 'guarantee the order' pass
def test_timeseries_keys(): filepath = data_dir_single_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') k = ds.timeseries_keys # no false positive for ki in k: assert ds[ki].dims == ('time', ) # no false negatives k_not = [ki for ki in ds.data_vars if ki not in k] for kni in k_not: assert ds[kni].dims != ('time', ) pass
def test_read_silixa_zipped(): files = [ data_dir_zipped_single_ended, data_dir_zipped_double_ended, data_dir_zipped_double_ended2, data_dir_zipped_silixa_long, # data_dir_zipped_sensornet_single_ended ] for file in files: with zipf(file) as fh: ds = read_silixa_files(zip_handle=fh, timezone_netcdf='UTC', file_ext='*.xml', load_in_memory=True) assert ds._initialized pass
def test_variance_of_stokes(): correct_var = 9.045 filepath = data_dir_double_ended2 ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') sections = { 'probe1Temperature': [slice(7.5, 17.), slice(70., 80.)], # cold bath 'probe2Temperature': [slice(24., 34.), slice(85., 95.)], # warm bath } I_var, _ = ds.variance_stokes(st_label='ST', sections=sections) np.testing.assert_almost_equal(I_var, correct_var, decimal=1) ds_dask = ds.chunk(chunks={}) I_var, _ = ds_dask.variance_stokes(st_label='ST', sections=sections) np.testing.assert_almost_equal(I_var, correct_var, decimal=1) pass
def test_shift_double_ended_shift_backforward(): # shifting it back and forward, should result in the same filepath = data_dir_double_ended ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') dsmin1 = shift_double_ended(ds, -1) ds2 = shift_double_ended(dsmin1, 1) np.testing.assert_allclose(ds.x[1:-1], ds2.x) for k in ds2: if 'x' not in ds2[k].dims: continue old = ds[k].isel(x=slice(1, -1)) new = ds2[k] np.testing.assert_allclose(old, new) pass
def test_calibration_ols(): """Testing ordinary least squares procedure. And compare with device calibrated temperature. The measurements were calibrated by the device using only section 8--17.m. Those temperatures are compared up to 2 decimals. Silixa only uses a single calibration constant (I think they fix gamma), or a different formulation, see Shell primer. """ filepath = data_dir_double_ended2 ds = read_silixa_files(directory=filepath, timezone_netcdf='UTC', file_ext='*.xml') ds100 = ds.sel(x=slice(0, 100)) sections_ultima = { 'probe1Temperature': [slice(8., 17.)], # cold bath } st_label = 'ST' ast_label = 'AST' rst_label = 'REV-ST' rast_label = 'REV-AST' ds100.calibration_double_ended(sections=sections_ultima, st_label=st_label, ast_label=ast_label, rst_label=rst_label, rast_label=rast_label, store_tmpw='TMPW', method='ols') np.testing.assert_array_almost_equal(ds100['TMPW'].data, ds100.TMP.data, decimal=1) ds009 = ds100.sel(x=sections_ultima['probe1Temperature'][0]) np.testing.assert_array_almost_equal(ds009['TMPW'].data, ds009.TMP.data, decimal=2) pass