def test_open_converted(ek60_converted_zarr, minio_bucket): # noqa
def _check_path(zarr_path):
storage_options = {}
if zarr_path.startswith("s3://"):
storage_options = dict(
client_kwargs=dict(endpoint_url="http://localhost:9000/"),
key="minioadmin",
secret="minioadmin",
)
return storage_options
storage_options = {}
if not isinstance(ek60_converted_zarr, fsspec.FSMap):
storage_options = _check_path(str(ek60_converted_zarr))
try:
ed = open_converted(
ek60_converted_zarr, storage_options=storage_options
)
assert isinstance(ed, EchoData) is True
except Exception as e:
if (
isinstance(ek60_converted_zarr, str)
and ek60_converted_zarr.startswith("s3://")
and ek60_converted_zarr.endswith(".nc")
):
assert isinstance(e, ValueError) is True
def test_compute_Sv_ek60_echoview():
ek60_raw_path = str(ek60_path.joinpath(
'DY1801_EK60-D20180211-T164025.raw')) # constant range_bin
ek60_echoview_path = ek60_path.joinpath('from_echoview')
# Convert file
c = ep.open_raw(ek60_raw_path, sonar_model='EK60')
c.to_netcdf(overwrite=True)
# Calibrate to get Sv
echodata = ep.open_converted(converted_raw_path=c.converted_raw_path)
ds_Sv = ep.calibrate.compute_Sv(echodata)
# Compare with EchoView outputs
channels = []
for freq in [18, 38, 70, 120, 200]:
fname = str(
ek60_echoview_path.joinpath(
'DY1801_EK60-D20180211-T164025-Sv%d.csv' % freq))
channels.append(
pd.read_csv(fname, header=None, skiprows=[0]).iloc[:, 13:])
test_Sv = np.stack(channels)
# Echoview data is shifted by 1 sample along range (missing the first sample)
assert np.allclose(test_Sv[:, :, 7:],
ds_Sv.Sv.isel(ping_time=slice(None, 10),
range_bin=slice(8, None)),
atol=1e-8)
Path(c.converted_raw_path).unlink()
def __init__(self,
file_path="",
salinity=27.9,
pressure=59,
temperature=None):
warnings.warn(
"Initializing a Process object is deprecated "
"and it will be removed in the next release. "
"More information on how to use the new processing "
"functions can be found in the echopype documentation.",
DeprecationWarning, 3)
self.echodata = echopype.open_converted(file_path)
self._file_format = None
self.file_path = file_path
# Initialize AZFP environment parameters. Will error if None
if self.echodata.sonar_model == 'AZFP':
if temperature is None:
temperature = self.echodata.environment['temperature']
self._env_params = {
'salinity': salinity,
'temperature': temperature,
'pressure': pressure
}
else:
self._env_params = None
if 'backscatter_i' in self.echodata.beam:
self.waveform_mode = 'BB'
self.encode_mode = 'complex'
else:
self.waveform_mode = 'CW'
self.encode_mode = 'power'
self.calibrator = CALIBRATOR[self.echodata.sonar_model](
self.echodata,
env_params=self._env_params,
cal_params=None,
waveform_mode=self.waveform_mode,
encode_mode=self.encode_mode)
# Deprecated data attributes
self.Sv = None
self.Sv_path = None
self.Sv_clean = None
self.TS = None
self.TS_path = None
# Deprecated proc attributes
self.noise_est_range_bin_size = 5 # meters per tile for noise estimation
self.noise_est_ping_size = 30 # number of pings per tile for noise estimation
self.MVBS_range_bin_size = 5 # meters per tile for MVBS
self.MVBS_ping_size = 30 # number of pings per tile for MVBS
if self.file_path.upper().endswith('.NC'):
self._file_format = 'netcdf4'
if self.file_path.upper().endswith('.ZARR'):
self._file_format = 'zarr'
def test_compute_Sv_azfp():
azfp_01a_path = str(azfp_path.joinpath('17082117.01A'))
azfp_xml_path = str(azfp_path.joinpath('17041823.XML'))
azfp_matlab_Sv_path = str(
azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat'))
azfp_matlab_Sp_path = str(
azfp_path.joinpath('from_matlab/17082117_matlab_Output_TS.mat'))
# Convert to .nc file
c = ep.open_raw(raw_file=azfp_01a_path,
sonar_model='AZFP',
xml_path=azfp_xml_path)
c.to_netcdf(overwrite=True)
# Calibrate using identical env params as in Matlab ParametersAZFP.m
with xr.open_dataset(c.converted_raw_path, group='Environment') as ds_env:
avg_temperature = ds_env['temperature'].mean(
'ping_time').values # AZFP Matlab code uses average temperature
env_params = {
'temperature': avg_temperature,
'salinity': 27.9,
'pressure': 59
}
echodata = ep.open_converted(converted_raw_path=c.converted_raw_path)
ds_Sv = ep.calibrate.compute_Sv(echodata=echodata, env_params=env_params)
ds_Sp = ep.calibrate.compute_Sp(echodata=echodata, env_params=env_params)
# Load matlab outputs and test
# matlab outputs were saved using
# save('from_matlab/17082117_matlab_Output.mat', 'Output') # data variables
# save('from_matlab/17082117_matlab_Par.mat', 'Par') # parameters
def check_output(base_path, ds_cmp, cal_type):
ds_base = loadmat(base_path)
cal_type_in_ds_cmp = {
'Sv': 'Sv',
'TS': 'Sp', # Sp here is TS in matlab outputs
}
for fidx in range(4): # loop through all freq
assert np.alltrue(
ds_cmp.range.isel(frequency=fidx).values == ds_base['Output']
[0]['Range'][fidx])
assert np.allclose(ds_cmp[cal_type_in_ds_cmp[cal_type]].isel(
frequency=fidx).values,
ds_base['Output'][0][cal_type][fidx],
atol=1e-13,
rtol=0)
# Check Sv
check_output(base_path=azfp_matlab_Sv_path, ds_cmp=ds_Sv, cal_type='Sv')
# Check Sp
check_output(base_path=azfp_matlab_Sp_path, ds_cmp=ds_Sp, cal_type='TS')
Path(c.converted_raw_path).unlink()
def test_compute_Sv_ek80_BB_complex():
"""Test calibrate BB mode data encoded as complex sam[les.
"""
ek80_raw_path = str(
ek80_path.joinpath('ar2.0-D20201209-T235955.raw')) # CW complex
c = ep.open_raw(ek80_raw_path, sonar_model='EK80')
c.to_netcdf()
echodata = ep.open_converted(converted_raw_path=c.converted_raw_path)
assert ep.calibrate.compute_Sv(echodata,
waveform_mode='BB',
encode_mode='complex')
Path(c.converted_raw_path).unlink()
def test_compute_Sv_ek80_pc_echoview():
"""Compare pulse compressed outputs from echopype and csv exported from EchoView.
Unresolved: the difference is large and it is not clear why.
"""
ek80_raw_path = str(ek80_path.joinpath('D20170912-T234910.raw'))
ek80_bb_pc_test_path = str(
ek80_path.joinpath(
'from_echoview/70 kHz pulse-compressed power.complex.csv'))
c = ep.open_raw(ek80_raw_path, sonar_model='EK80')
c.to_netcdf(overwrite=True)
# Create a CalibrateEK80 object to perform pulse compression
echodata = ep.open_converted(converted_raw_path=c.converted_raw_path)
waveform_mode = 'BB'
cal_obj = CalibrateEK80(echodata,
env_params=None,
cal_params=None,
waveform_mode=waveform_mode)
cal_obj.compute_range_meter(waveform_mode=waveform_mode,
tvg_correction_factor=0) # compute range [m]
chirp, _, tau_effective = cal_obj.get_transmit_chirp(
waveform_mode=waveform_mode)
pc = cal_obj.compress_pulse(chirp)
pc_mean = pc.pulse_compressed_output.isel(frequency=0).mean(
dim='quadrant').dropna('range_bin')
# Read EchoView pc raw power output
df = pd.read_csv(ek80_bb_pc_test_path, header=None, skiprows=[0])
df_header = pd.read_csv(ek80_bb_pc_test_path,
header=0,
usecols=range(14),
nrows=0)
df = df.rename(columns={
cc: vv
for cc, vv in zip(df.columns, df_header.columns.values)
})
df.columns = df.columns.str.strip()
df_real = df.loc[df['Component'] == ' Real', :].iloc[:, 14:]
# Compare only values for range > 0: difference is surprisingly large
range_meter = cal_obj.range_meter.isel(frequency=0, ping_time=0).values
first_nonzero_range = np.argwhere(range_meter == 0).squeeze().max()
assert np.allclose(
df_real.values[:, first_nonzero_range:pc_mean.values.shape[1]],
pc_mean.values.real[:, first_nonzero_range:],
rtol=0,
atol=1.03e-3)
Path(c.converted_raw_path).unlink()
def test_compute_Sv_ek80_matlab():
"""Compare pulse compressed outputs from echopype and Matlab outputs.
Unresolved: there is a discrepancy between the range vector due to minRange=0.02 m set in Matlab.
"""
ek80_raw_path = str(ek80_path.joinpath('D20170912-T234910.raw'))
ek80_matlab_path = str(
ek80_path.joinpath('from_matlab/D20170912-T234910_data.mat'))
c = ep.open_raw(ek80_raw_path, sonar_model='EK80')
c.to_netcdf()
echodata = ep.open_converted(converted_raw_path=c.converted_raw_path)
ds_Sv = ep.calibrate.compute_Sv(echodata,
waveform_mode='BB',
encode_mode='complex')
# TODO: resolve discrepancy in range between echopype and Matlab code
ds_matlab = loadmat(ek80_matlab_path)
Sv_70k = ds_Sv.Sv.isel(frequency=0, ping_time=0).dropna('range_bin').values
Path(c.converted_raw_path).unlink()
def main(rdir, model):
save_dir = os.path.join(os.path.dirname(rdir), 'processed')
os.makedirs(save_dir, exist_ok=True)
rawfiles = sorted(glob.glob(os.path.join(rdir, '*.raw')))
# convert from raw file to netCDF
for filename in rawfiles:
data_tmp = ep.open_raw(filename, sonar_model=model)
data_tmp.to_netcdf(save_path=save_dir)
# process files, calculate Sv and MVBS
ncfiles = sorted(glob.glob(os.path.join(save_dir, '*.nc')))
for ncpath in ncfiles:
ed = ep.open_converted(ncpath) # create an EchoData object
# get a dataset containing Sv, range and the calibration and environmental parameters
ds_Sv = ep.calibrate.compute_Sv(ed)
# bin the data
# ds_Sv = calibrated Sv dataset
# range_meter_bin = bin size to average along range in meters
# ping_time_bin = bin size to average along ping_time in seconds
ds_MVBS = ep.preprocess.compute_MVBS(ds_Sv,
range_meter_bin=5,
ping_time_bin='20S')
# remove noise
# ds_Sv = calibrated Sv dataset
# range_bin_num = number of samples along the range_bin dimension for estimating noise
# ping_num = number of pings for estimating noise
ds_Sv_clean = ep.preprocess.remove_noise(ds_Sv,
range_bin_num=30,
ping_num=5)
# save files to disk
ed_filename = ncpath.split('/')[-1]
print('\nWriting {} Sv and MVBS datasets to file'.format(ed_filename))
ds_Sv.to_netcdf(os.path.join(save_dir, f'ds_Sv_{ed_filename}'))
#ds_Sv_clean.to_netcdf(os.path.join(save_dir, f'ds_Sv_clean_{ed_filename}'))
ds_MVBS.to_netcdf(os.path.join(save_dir, f'ds_MVBS_{ed_filename}'))
def test_compute_Sv_ek60_matlab():
ek60_raw_path = str(
ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw'))
ek60_matlab_path = str(
ek60_path.joinpath('from_matlab/DY1801_EK60-D20180211-T164025.mat'))
# Convert file
c = ep.open_raw(ek60_raw_path, sonar_model='EK60')
c.to_netcdf()
# Calibrate to get Sv
echodata = ep.open_converted(converted_raw_path=c.converted_raw_path)
ds_Sv = ep.calibrate.compute_Sv(echodata)
ds_Sp = ep.calibrate.compute_Sp(echodata)
# Load matlab outputs and test
# matlab outputs were saved using
# save('from_matlab/DY1801_EK60-D20180211-T164025.mat', 'data')
ds_base = loadmat(ek60_matlab_path)
def check_output(ds_cmp, cal_type):
for fidx in range(5): # loop through all freq
assert np.allclose(
ds_cmp[cal_type].isel(frequency=0).T.values,
ds_base['data']['pings'][0][0][cal_type][0, 0],
atol=4e-5,
rtol=0) # difference due to use of Single in matlab code
# Check Sv
check_output(ds_Sv, 'Sv')
# Check Sp
check_output(ds_Sp, 'Sp')
Path(c.converted_raw_path).unlink()
def _check_raw_output(
filepath_raw,
output_dir,
ocean_contour_export_090_dir,
ocean_contour_export_076_dir,
absolute_tolerance,
):
print("checking raw", filepath_raw)
echodata = open_converted(converted_raw_path=output_dir.joinpath(
filepath_raw.with_suffix(".nc").name))
if "090" in filepath_raw.parts:
ocean_contour_converted_config_path = (
ocean_contour_export_090_dir.joinpath(
filepath_raw.with_suffix(filepath_raw.suffix +
".00000.nc").name))
ocean_contour_converted_transmit_data_path = (
ocean_contour_converted_config_path)
ocean_contour_converted_data_path = ocean_contour_converted_config_path
else:
ocean_contour_converted_config_path = (
ocean_contour_export_076_dir / filepath_raw.with_suffix("").name /
"Raw Echo 1_1000 kHz_001.nc")
ocean_contour_converted_transmit_data_path = (
ocean_contour_export_076_dir / filepath_raw.with_suffix("").name /
"Raw Echo 1_1000 kHz Tx_001.nc")
ocean_contour_converted_data_path = (
ocean_contour_export_076_dir / filepath_raw.with_suffix("").name /
"Raw Echo 1_1000 kHz_001.nc")
if not all((
ocean_contour_converted_config_path.exists(),
ocean_contour_converted_transmit_data_path.exists(),
ocean_contour_converted_data_path.exists(),
)):
pass
else:
# check pulse compression
base = xr.open_dataset(str(ocean_contour_converted_config_path),
group="Config")
pulse_compressed = 0
for i in range(1, 4):
if "090" in filepath_raw.parts:
if base.attrs[f"echo_pulseComp{i}"]:
pulse_compressed = i
break
else:
if base.attrs[f"Instrument_echo_pulseComp{i}"]:
pulse_compressed = i
break
assert echodata.vendor.attrs["pulse_compressed"] == pulse_compressed
base.close()
# check raw data transmit samples
try:
netCDF4.Dataset(str(ocean_contour_converted_transmit_data_path)
)["Data/RawEcho1_1000kHzTx"]
except IndexError:
# no transmit data in this dataset
pass
else:
base = xr.open_dataset(
str(ocean_contour_converted_transmit_data_path),
group="Data/RawEcho1_1000kHzTx",
)
if "090" in filepath_raw.parts:
assert np.allclose(
echodata.vendor["echosounder_raw_transmit_samples_i"].data.
flatten(),
base["DataI"].data.flatten(),
atol=absolute_tolerance,
)
assert np.allclose(
echodata.vendor["echosounder_raw_transmit_samples_q"].data.
flatten(),
base["DataQ"].data.flatten(),
atol=absolute_tolerance,
)
else:
assert np.allclose(
echodata.vendor["echosounder_raw_transmit_samples_i"].data.
flatten(),
base["Data_I"].data.flatten(),
atol=absolute_tolerance,
)
assert np.allclose(
echodata.vendor["echosounder_raw_transmit_samples_q"].data.
flatten(),
base["Data_Q"].data.flatten(),
atol=absolute_tolerance,
)
base.close()
# check raw data samples
base = xr.open_dataset(
str(ocean_contour_converted_data_path),
group="Data/RawEcho1_1000kHz",
)
if "090" in filepath_raw.parts:
assert np.allclose(
echodata.vendor["echosounder_raw_samples_i"].data.flatten(),
base["DataI"].data.flatten(),
atol=absolute_tolerance,
)
assert np.allclose(
echodata.vendor["echosounder_raw_samples_q"].data.flatten(),
base["DataQ"].data.flatten(),
atol=absolute_tolerance,
)
else:
# note the transpose
assert np.allclose(
echodata.vendor["echosounder_raw_samples_i"].data.flatten(),
base["Data_I"].data.T.flatten(),
atol=absolute_tolerance,
)
assert np.allclose(
echodata.vendor["echosounder_raw_samples_q"].data.flatten(),
base["Data_Q"].data.T.flatten(),
atol=absolute_tolerance,
)
base.close()
