def test_ping_time_reversal(ek60_reversed_ping_time_test_data):
eds = [
echopype.open_raw(file, "EK60")
for file in ek60_reversed_ping_time_test_data
]
combined = echopype.combine_echodata(eds,
"overwrite_conflicts") # type: ignore
for group_name in combined.group_map:
combined_group: xr.Dataset = getattr(combined, group_name)
if combined_group is not None:
if "ping_time" in combined_group and group_name != "provenance":
assert not exist_reversed_time(combined_group, "ping_time")
if "old_ping_time" in combined_group:
assert exist_reversed_time(combined_group, "old_ping_time")
if "location_time" in combined_group and group_name not in (
"provenance",
"nmea",
):
assert not exist_reversed_time(combined_group, "location_time")
if "old_location_time" in combined_group:
assert exist_reversed_time(combined_group, "old_location_time")
if "mru_time" in combined_group and group_name != "provenance":
assert not exist_reversed_time(combined_group, "mru_time")
if "old_mru_time" in combined_group:
assert exist_reversed_time(combined_group, "old_mru_time")
def test_convert_time_encodings(sonar_model, raw_file, xml_path):
ed = open_raw(sonar_model=sonar_model,
raw_file=raw_file,
xml_path=xml_path)
ed.to_netcdf()
for group, details in ed._EchoData__group_map.items():
if hasattr(ed, group):
group_ds = getattr(ed, group)
if isinstance(group_ds, xr.Dataset):
for var, encoding in DEFAULT_ENCODINGS.items():
if var in group_ds:
da = group_ds[var]
assert da.encoding == encoding
# Combine encoding and attributes since this
# is what is shown when using decode_cf=False
# without dtype attribute
total_attrs = dict(**da.attrs, **da.encoding)
total_attrs.pop('dtype')
# Read converted file back in
file_da = xr.open_dataset(ed.converted_raw_path,
group=details['ep_group'],
decode_cf=False)[var]
assert file_da.attrs == total_attrs
assert file_da.dtype == encoding['dtype']
# Read converted file back in
decoded_da = xr.open_dataset(ed.converted_raw_path,
group=details['ep_group'],
decode_cf=True)[var]
assert da.equals(decoded_da) is True
os.unlink(ed.converted_raw_path)
def test_convert_ek60_echoview_raw():
"""Compare parsed power data (count) with csv exported by EchoView.
"""
ek60_raw_path = str(
ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw'))
ek60_csv_path = [
ek60_path.joinpath(
'from_echoview/DY1801_EK60-D20180211-T164025-Power%d.csv' % freq)
for freq in [18, 38, 70, 120, 200]
]
# Read csv files exported by EchoView
channels = []
for file in ek60_csv_path:
channels.append(
pd.read_csv(file, header=None, skiprows=[0]).iloc[:, 13:])
test_power = np.stack(channels)
# Convert to netCDF and check
echodata = open_raw(raw_file=ek60_raw_path, sonar_model='EK60')
echodata.to_netcdf(save_path=output_dir)
with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
for fidx, atol in zip(range(5), [1e-5, 1.1e-5, 1.1e-5, 1e-5, 1e-5]):
assert np.allclose(test_power[fidx, :, :],
ds_beam.backscatter_r.isel(
frequency=fidx,
ping_time=slice(None, 10),
range_bin=slice(1, None)),
atol=9e-6,
rtol=atol)
Path(echodata.converted_raw_path).unlink()
output_dir.rmdir()
def test_compute_Sv_ek60_echoview():
ek60_raw_path = str(ek60_path.joinpath(
'DY1801_EK60-D20180211-T164025.raw')) # constant range_bin
ek60_echoview_path = ek60_path.joinpath('from_echoview')
# Convert file
echodata = ep.open_raw(ek60_raw_path, sonar_model='EK60')
# Calibrate to get Sv
ds_Sv = ep.calibrate.compute_Sv(echodata)
# Compare with EchoView outputs
channels = []
for freq in [18, 38, 70, 120, 200]:
fname = str(
ek60_echoview_path.joinpath(
'DY1801_EK60-D20180211-T164025-Sv%d.csv' % freq))
channels.append(
pd.read_csv(fname, header=None, skiprows=[0]).iloc[:, 13:])
test_Sv = np.stack(channels)
# Echoview data is shifted by 1 sample along range (missing the first sample)
assert np.allclose(test_Sv[:, :, 7:],
ds_Sv.Sv.isel(ping_time=slice(None, 10),
range_bin=slice(8, None)),
atol=1e-8)
def test_convert_azfp_01a_raw_echoview(azfp_path):
"""Compare parsed power data (count) with csv exported by EchoView."""
azfp_01a_path = str(azfp_path.joinpath('17082117.01A'))
azfp_xml_path = str(azfp_path.joinpath('17041823.XML'))
# Read csv files exported by EchoView
azfp_csv_path = [
azfp_path.joinpath('from_echoview/17082117-raw%d.csv' % freq)
for freq in [38, 125, 200, 455]
]
channels = []
for file in azfp_csv_path:
channels.append(
pd.read_csv(file, header=None, skiprows=[0]).iloc[:, 6:])
test_power = np.stack(channels)
# Convert to netCDF and check
echodata = open_raw(raw_file=azfp_01a_path,
sonar_model='AZFP',
xml_path=azfp_xml_path)
assert np.array_equal(
test_power,
echodata.beam.backscatter_r.isel(beam=0).drop('beam'))
# check convention-required variables in the Platform group
check_platform_required_vars(echodata)
def test_plot_multi_get_range(
filepath,
sonar_model,
azfp_xml_path,
range_kwargs,
):
# TODO: Need to figure out how to compare the actual rendered plots
ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path)
if ed.sonar_model.lower() == 'azfp':
avg_temperature = (
ed.environment['temperature'].mean('ping_time').values)
env_params = {
'temperature': avg_temperature,
'salinity': 27.9,
'pressure': 59,
}
range_kwargs['env_params'] = env_params
plots = echopype.visualize.create_echogram(ed,
get_range=True,
range_kwargs=range_kwargs)
assert isinstance(plots, list) is True
assert all(isinstance(plot, FacetGrid) for plot in plots) is True
# Quadrant shape check
if (sonar_model.lower() == 'ek80'
and range_kwargs['encode_mode'] == 'complex'):
assert plots[0].axes.shape[-1] > 1
else:
assert plots[0].axes.shape[-1] == 1
# Frequency shape check
assert ed.beam.frequency.shape[0] == len(plots)
def test_update_platform(update_platform_samples):
raw_file, extra_platform_data_file = update_platform_samples
extra_platform_data_file_name = extra_platform_data_file.name
ed = echopype.open_raw(raw_file, "EK80")
updated = ["pitch", "roll", "latitude", "longitude", "water_level"]
for variable in updated:
assert np.isnan(ed.platform[variable].values).all()
extra_platform_data = xr.open_dataset(extra_platform_data_file)
ed.update_platform(
extra_platform_data,
extra_platform_data_file_name=extra_platform_data_file_name,
)
for variable in updated:
assert not np.isnan(ed.platform[variable].values).all()
# times have max interval of 2s
# check times are > min(ed.beam["ping_time"]) - 2s
assert (ed.platform["time1"] >
ed.beam["ping_time"].min() - np.timedelta64(2, "s")).all()
# check there is only 1 time < min(ed.beam["ping_time"])
assert (np.count_nonzero(
ed.platform["time1"] < ed.beam["ping_time"].min()) == 1)
# check times are < max(ed.beam["ping_time"]) + 2s
assert (ed.platform["time1"] <
ed.beam["ping_time"].max() + np.timedelta64(2, "s")).all()
# check there is only 1 time > max(ed.beam["ping_time"])
assert (np.count_nonzero(
ed.platform["time1"] > ed.beam["ping_time"].max()) == 1)
def test_plot_mvbs(
filepath,
sonar_model,
azfp_xml_path,
range_kwargs,
):
# TODO: Need to figure out how to compare the actual rendered plots
ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path)
if ed.sonar_model.lower() == 'azfp':
avg_temperature = (
ed.environment['temperature'].mean('ping_time').values)
env_params = {
'temperature': avg_temperature,
'salinity': 27.9,
'pressure': 59,
}
range_kwargs['env_params'] = env_params
if 'azfp_cal_type' in range_kwargs:
range_kwargs.pop('azfp_cal_type')
Sv = echopype.calibrate.compute_Sv(ed, **range_kwargs)
mvbs = echopype.preprocess.compute_MVBS(Sv, ping_time_bin='10S')
plots = []
try:
plots = echopype.visualize.create_echogram(mvbs)
except Exception as e:
assert isinstance(e, ValueError)
assert str(
e
) == "Ping time must be greater or equal to 2 data points." # noqa
if len(plots) > 0:
assert all(isinstance(plot, FacetGrid) for plot in plots) is True
def test_compute_Sv_ek60_matlab():
ek60_raw_path = str(
ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw'))
ek60_matlab_path = str(
ek60_path.joinpath('from_matlab/DY1801_EK60-D20180211-T164025.mat'))
# Convert file
echodata = ep.open_raw(ek60_raw_path, sonar_model='EK60')
# Calibrate to get Sv
ds_Sv = ep.calibrate.compute_Sv(echodata)
ds_Sp = ep.calibrate.compute_Sp(echodata)
# Load matlab outputs and test
# matlab outputs were saved using
# save('from_matlab/DY1801_EK60-D20180211-T164025.mat', 'data')
ds_base = loadmat(ek60_matlab_path)
def check_output(ds_cmp, cal_type):
for fidx in range(5): # loop through all freq
assert np.allclose(
ds_cmp[cal_type].isel(frequency=0).T.values,
ds_base['data']['pings'][0][0][cal_type][0, 0],
atol=4e-5,
rtol=0) # difference due to use of Single in matlab code
# Check Sv
check_output(ds_Sv, 'Sv')
# Check Sp
check_output(ds_Sp, 'Sp')
def test_preprocess_mvbs(test_data_samples):
"""
Test running through from open_raw to compute_MVBS.
"""
(
filepath,
sonar_model,
azfp_xml_path,
range_kwargs,
) = test_data_samples
ed = ep.open_raw(filepath, sonar_model, azfp_xml_path)
if ed.sonar_model.lower() == 'azfp':
avg_temperature = (
ed.environment['temperature'].mean('time1').values
)
env_params = {
'temperature': avg_temperature,
'salinity': 27.9,
'pressure': 59,
}
range_kwargs['env_params'] = env_params
if 'azfp_cal_type' in range_kwargs:
range_kwargs.pop('azfp_cal_type')
Sv = ep.calibrate.compute_Sv(ed, **range_kwargs)
assert ep.preprocess.compute_MVBS(Sv) is not None
def test_convert_ek80_complex_echoview():
"""Compare parsed EK80 BB data with csv exported by EchoView.
"""
ek80_raw_path_bb = ek80_path.joinpath('D20170912-T234910.raw')
ek80_echoview_bb_power_csv = ek80_path.joinpath(
'from_echoview/D20170912-T234910/70 kHz raw power.complex.csv')
# Convert file
echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80')
echodata.to_netcdf(save_path=output_dir)
# Test complex parsed data
df_bb = pd.read_csv(ek80_echoview_bb_power_csv, header=None,
skiprows=[0]) # averaged across quadrants
with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
assert np.allclose(
ds_beam.backscatter_r.sel(frequency=70e3).dropna('range_bin').mean(
dim='quadrant'),
df_bb.iloc[::2, 14:], # real rows
rtol=0,
atol=8e-6)
assert np.allclose(
ds_beam.backscatter_i.sel(frequency=70e3).dropna('range_bin').mean(
dim='quadrant'),
df_bb.iloc[1::2, 14:], # imag rows
rtol=0,
atol=4e-6)
Path(echodata.converted_raw_path).unlink()
output_dir.rmdir()
def test_convert_ek60_duplicate_frequencies(ek60_path):
"""Convert a file with duplicate frequencies"""
raw_path = (ek60_path / "DY1002_EK60-D20100318-T023008_rep_freq.raw")
ed = open_raw(raw_path, "EK60")
truth_chan_vals = np.array([
'GPT 18 kHz 009072034d45 1-1 ES18-11',
'GPT 38 kHz 009072033fa2 2-1 ES38B',
'GPT 70 kHz 009072058c6c 3-1 ES70-7C',
'GPT 70 kHz 009072058c6c 3-2 ES70-7C',
'GPT 120 kHz 00907205794e 4-1 ES120-7C',
'GPT 200 kHz 0090720346a8 5-1 ES200-7C'
],
dtype='<U37')
truth_freq_nom_vals = np.array(
[18000., 38000., 70000., 70000., 120000., 200000.], dtype=np.float64)
assert np.allclose(ed['Sonar/Beam_group1'].frequency_nominal,
truth_freq_nom_vals,
rtol=1e-05,
atol=1e-08)
assert np.all(ed['Sonar/Beam_group1'].channel.values == truth_chan_vals)
def test_convert_ek80_complex_matlab():
"""Compare parsed EK80 CW power/angle data with Matlab parsed data.
"""
ek80_raw_path_bb = str(ek80_path.joinpath('D20170912-T234910.raw'))
ek80_matlab_path_bb = str(
ek80_path.joinpath('from_matlab/D20170912-T234910_data.mat'))
# Convert file
echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80')
echodata.to_netcdf(save_path=output_dir)
# Test complex parsed data
ds_matlab = loadmat(ek80_matlab_path_bb)
with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
assert np.array_equal(
ds_beam.backscatter_r.isel(
frequency=0, ping_time=0).dropna('range_bin').values[1:, :],
np.real(ds_matlab['data']['echodata'][0][0][0, 0]
['complexsamples']) # real part
)
assert np.array_equal(
ds_beam.backscatter_i.isel(
frequency=0, ping_time=0).dropna('range_bin').values[1:, :],
np.imag(ds_matlab['data']['echodata'][0][0][0, 0]
['complexsamples']) # imag part
)
Path(echodata.converted_raw_path).unlink()
output_dir.rmdir()
def test_convert_azfp_01a_raw_echoview():
"""Compare parsed power data (count) with csv exported by EchoView.
"""
azfp_01a_path = str(azfp_path.joinpath('17082117.01A'))
azfp_xml_path = str(azfp_path.joinpath('17041823.XML'))
# Read csv files exported by EchoView
azfp_csv_path = [
azfp_path.joinpath('from_echoview/17082117-raw%d.csv' % freq)
for freq in [38, 125, 200, 455]
]
channels = []
for file in azfp_csv_path:
channels.append(
pd.read_csv(file, header=None, skiprows=[0]).iloc[:, 6:])
test_power = np.stack(channels)
# Convert to netCDF and check
echodata = open_raw(raw_file=azfp_01a_path,
sonar_model='AZFP',
xml_path=azfp_xml_path)
echodata.to_netcdf(save_path=output_dir)
with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
assert np.array_equal(test_power, ds_beam.backscatter_r)
Path(echodata.converted_raw_path).unlink()
# Convert to zarr and check
echodata.to_zarr(save_path=output_dir)
with xr.open_zarr(echodata.converted_raw_path, group='Beam') as ds_beam:
assert np.array_equal(test_power, ds_beam.backscatter_r)
shutil.rmtree(echodata.converted_raw_path)
output_dir.rmdir()
def test_convert_ek60_matlab_raw():
"""Compare parsed Beam group data with Matlab outputs.
"""
ek60_raw_path = str(
ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw'))
ek60_matlab_path = str(
ek60_path.joinpath(
'from_matlab/DY1801_EK60-D20180211-T164025_rawData.mat'))
# Convert file
echodata = open_raw(raw_file=ek60_raw_path, sonar_model='EK60')
# Compare with matlab outputs
ds_matlab = loadmat(ek60_matlab_path)
# power
assert np.allclose([
ds_matlab['rawData'][0]['pings'][0]['power'][0][fidx]
for fidx in range(5)
],
echodata.beam.backscatter_r.transpose(
'frequency', 'range_bin', 'ping_time'),
rtol=0,
atol=1.6e-5)
# angle: alongship and athwartship
for angle in ['alongship', 'athwartship']:
assert np.array_equal([
ds_matlab['rawData'][0]['pings'][0][angle][0][fidx]
for fidx in range(5)
], echodata.beam['angle_' + angle].transpose('frequency', 'range_bin',
'ping_time'))
def test_attr_storage(ek60_test_data):
# check storage of attributes before combination in provenance group
eds = [echopype.open_raw(file, "EK60") for file in ek60_test_data]
combined = echopype.combine_echodata(eds,
"overwrite_conflicts") # type: ignore
for group in combined.group_map:
if f"{group}_attrs" in combined.provenance:
group_attrs = combined.provenance[f"{group}_attrs"]
for i, ed in enumerate(eds):
for attr, value in getattr(ed, group).attrs.items():
assert str(
group_attrs.isel(echodata_filename=i).sel({
f"{group}_attr_key":
attr
}).data[()]) == str(value)
# check selection by echodata_filename
for file in ek60_test_data:
assert Path(file).name in combined.provenance["echodata_filename"]
for group in combined.group_map:
if f"{group}_attrs" in combined.provenance:
group_attrs = combined.provenance[f"{group}_attrs"]
assert np.array_equal(
group_attrs.sel(
echodata_filename=Path(ek60_test_data[0]).name),
group_attrs.isel(echodata_filename=0),
)
def test_convert_azfp_01a_matlab_raw():
"""Compare parsed raw data with Matlab outputs.
"""
azfp_01a_path = str(azfp_path.joinpath('17082117.01A'))
azfp_xml_path = str(azfp_path.joinpath('17041823.XML'))
azfp_matlab_data_path = str(
azfp_path.joinpath('from_matlab/17082117_matlab_Data.mat'))
azfp_matlab_output_path = str(
azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat'))
# Convert file
echodata = open_raw(raw_file=azfp_01a_path,
sonar_model='AZFP',
xml_path=azfp_xml_path)
echodata.to_netcdf(save_path=output_dir)
# Read in the dataset that will be used to confirm working conversions. (Generated by Matlab)
ds_matlab = loadmat(azfp_matlab_data_path)
ds_matlab_output = loadmat(azfp_matlab_output_path)
# Test beam group
with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
# frequency
assert np.array_equal(ds_matlab['Data']['Freq'][0][0].squeeze(),
ds_beam.frequency / 1000) # matlab file in kHz
# backscatter count
assert np.array_equal(
np.array([
ds_matlab_output['Output'][0]['N'][fidx] for fidx in range(4)
]), ds_beam.backscatter_r.values)
# tilt x-y
assert np.array_equal(
np.array([d[0]
for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(),
ds_beam.tilt_x_count)
assert np.array_equal(
np.array([d[1]
for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(),
ds_beam.tilt_y_count)
# Test vendor group
with xr.open_dataset(echodata.converted_raw_path,
group='Vendor') as ds_vend:
# Test temperature
assert np.array_equal(
np.array([d[4]
for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(),
ds_vend.ancillary.isel(ancillary_len=4).values)
assert np.array_equal(
np.array([d[0]
for d in ds_matlab['Data']['BatteryTx'][0]]).squeeze(),
ds_vend.battery_tx)
assert np.array_equal(
np.array([d[0]
for d in ds_matlab['Data']['BatteryMain'][0]]).squeeze(),
ds_vend.battery_main)
Path(echodata.converted_raw_path).unlink()
output_dir.rmdir()
def test_convert_ek80_freq_subset(ek80_path):
"""Make sure can convert EK80 file with multiple frequency channels off."""
ek80_raw_path_freq_subset = str(
ek80_path.joinpath('2019118 group2survey-D20191214-T081342.raw'))
echodata = open_raw(raw_file=ek80_raw_path_freq_subset, sonar_model='EK80')
# Check if converted output has only 2 frequency channels
assert echodata.beam.frequency.size == 2
def test_convert_ek80_cw_power_angle_echoview(ek80_path):
"""Compare parsed EK80 CW power/angle data with csv exported by EchoView."""
ek80_raw_path_cw = str(
ek80_path.joinpath('D20190822-T161221.raw')) # Small file (CW)
freq_list = [18, 38, 70, 120, 200]
ek80_echoview_power_csv = [
ek80_path.joinpath('from_echoview/D20190822-T161221/%dkHz.power.csv' %
freq) for freq in freq_list
]
ek80_echoview_angle_csv = [
ek80_path.joinpath(
'from_echoview/D20190822-T161221/%dkHz.angles.points.csv' % freq)
for freq in freq_list
]
# Convert file
echodata = open_raw(ek80_raw_path_cw, sonar_model='EK80')
# Test power
# single point error in original raw data. Read as -2000 by echopype and -999 by EchoView
echodata.beam.backscatter_r[3, 4, 13174] = -999
for file, freq in zip(ek80_echoview_power_csv, freq_list):
test_power = pd.read_csv(file, delimiter=';').iloc[:, 13:].values
assert np.allclose(
test_power,
echodata.beam.backscatter_r.sel(frequency=freq *
1e3).dropna('range_bin'),
rtol=0,
atol=1.1e-5,
)
# Convert from electrical angles to physical angle [deg]
major = (echodata.beam['angle_athwartship'] * 1.40625 /
echodata.beam['angle_sensitivity_athwartship'] -
echodata.beam['angle_offset_athwartship'])
minor = (echodata.beam['angle_alongship'] * 1.40625 /
echodata.beam['angle_sensitivity_alongship'] -
echodata.beam['angle_offset_alongship'])
for freq, file in zip(freq_list, ek80_echoview_angle_csv):
df_angle = pd.read_csv(file)
# NB: EchoView exported data only has 6 pings, but raw data actually has 7 pings.
# The first raw ping (ping 0) was removed in EchoView for some reason.
# Therefore the comparison will use ping 1-6.
for ping_idx in df_angle['Ping_index'].value_counts().index:
assert np.allclose(
df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Major'],
major.sel(frequency=freq *
1e3).isel(ping_time=ping_idx).dropna('range_bin'),
rtol=0,
atol=5e-5,
)
assert np.allclose(
df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Minor'],
minor.sel(frequency=freq *
1e3).isel(ping_time=ping_idx).dropna('range_bin'),
rtol=0,
atol=5e-5,
)
def test_convert_ek80_complex_matlab(ek80_path):
"""Compare parsed EK80 CW power/angle data with Matlab parsed data."""
ek80_raw_path_bb = str(ek80_path.joinpath('D20170912-T234910.raw'))
ek80_matlab_path_bb = str(
ek80_path.joinpath('from_matlab/D20170912-T234910_data.mat')
)
# Convert file
echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80')
# check water_level
assert (echodata["Platform"]["water_level"] == 0).all()
# Test complex parsed data
ds_matlab = loadmat(ek80_matlab_path_bb)
assert np.array_equal(
echodata.beam.backscatter_r.sel(channel='WBT 549762-15 ES70-7C',
ping_time='2017-09-12T23:49:10.722999808')
.dropna('range_sample')
.values[1:, :],
np.real(
ds_matlab['data']['echodata'][0][0][0, 0]['complexsamples']
), # real part
)
assert np.array_equal(
echodata.beam.backscatter_i.sel(channel='WBT 549762-15 ES70-7C',
ping_time='2017-09-12T23:49:10.722999808')
.dropna('range_sample')
.values[1:, :],
np.imag(
ds_matlab['data']['echodata'][0][0][0, 0]['complexsamples']
), # imag part
)
check_env_xml(echodata)
# check platform
nan_plat_vars = [
"MRU_offset_x",
"MRU_offset_y",
"MRU_offset_z",
"MRU_rotation_x",
"MRU_rotation_y",
"MRU_rotation_z",
"position_offset_x",
"position_offset_y",
"position_offset_z"
]
for plat_var in nan_plat_vars:
assert plat_var in echodata["Platform"]
assert np.isnan(echodata["Platform"][plat_var]).all()
zero_plat_vars = [
"transducer_offset_x",
"transducer_offset_y",
"transducer_offset_z",
]
for plat_var in zero_plat_vars:
assert plat_var in echodata["Platform"]
assert (echodata["Platform"][plat_var] == 0).all()
def test_convert_ek80_complex_echoview(ek80_path):
"""Compare parsed EK80 BB data with csv exported by EchoView."""
ek80_raw_path_bb = ek80_path.joinpath('D20170912-T234910.raw')
ek80_echoview_bb_power_csv = ek80_path.joinpath(
'from_echoview/D20170912-T234910/70 kHz raw power.complex.csv'
)
# Convert file
echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80')
# check water_level
assert (echodata["Platform"]["water_level"] == 0).all()
# Test complex parsed data
df_bb = pd.read_csv(
ek80_echoview_bb_power_csv, header=None, skiprows=[0]
) # averaged across beams
assert np.allclose(
echodata.beam.backscatter_r.sel(channel='WBT 549762-15 ES70-7C')
.dropna('range_sample')
.mean(dim='beam'),
df_bb.iloc[::2, 14:], # real rows
rtol=0,
atol=8e-6,
)
assert np.allclose(
echodata.beam.backscatter_i.sel(channel='WBT 549762-15 ES70-7C')
.dropna('range_sample')
.mean(dim='beam'),
df_bb.iloc[1::2, 14:], # imag rows
rtol=0,
atol=4e-6,
)
check_env_xml(echodata)
# check platform
nan_plat_vars = [
"MRU_offset_x",
"MRU_offset_y",
"MRU_offset_z",
"MRU_rotation_x",
"MRU_rotation_y",
"MRU_rotation_z",
"position_offset_x",
"position_offset_y",
"position_offset_z"
]
for plat_var in nan_plat_vars:
assert plat_var in echodata["Platform"]
assert np.isnan(echodata["Platform"][plat_var]).all()
zero_plat_vars = [
"transducer_offset_x",
"transducer_offset_y",
"transducer_offset_z",
]
for plat_var in zero_plat_vars:
assert plat_var in echodata["Platform"]
assert (echodata["Platform"][plat_var] == 0).all()
def test_convert_azfp_01a_matlab_raw(azfp_path):
"""Compare parsed raw data with Matlab outputs."""
azfp_01a_path = str(azfp_path.joinpath('17082117.01A'))
azfp_xml_path = str(azfp_path.joinpath('17041823.XML'))
azfp_matlab_data_path = str(
azfp_path.joinpath('from_matlab/17082117_matlab_Data.mat'))
azfp_matlab_output_path = str(
azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat'))
# Convert file
echodata = open_raw(raw_file=azfp_01a_path,
sonar_model='AZFP',
xml_path=azfp_xml_path)
# Read in the dataset that will be used to confirm working conversions. (Generated by Matlab)
ds_matlab = loadmat(azfp_matlab_data_path)
ds_matlab_output = loadmat(azfp_matlab_output_path)
# Test beam group
# frequency
assert np.array_equal(
ds_matlab['Data']['Freq'][0][0].squeeze(),
echodata.beam.frequency_nominal / 1000,
) # matlab file in kHz
# backscatter count
assert np.array_equal(
np.array(
[ds_matlab_output['Output'][0]['N'][fidx] for fidx in range(4)]),
echodata.beam.backscatter_r.isel(beam=0).drop('beam').values,
)
# Test vendor group
# Test temperature
assert np.array_equal(
np.array([d[4] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(),
echodata.vendor.ancillary.isel(ancillary_len=4).values,
)
assert np.array_equal(
np.array([d[0] for d in ds_matlab['Data']['BatteryTx'][0]]).squeeze(),
echodata.vendor.battery_tx,
)
assert np.array_equal(
np.array([d[0]
for d in ds_matlab['Data']['BatteryMain'][0]]).squeeze(),
echodata.vendor.battery_main,
)
# tilt x-y
assert np.array_equal(
np.array([d[0] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(),
echodata.vendor.tilt_x_count,
)
assert np.array_equal(
np.array([d[1] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(),
echodata.vendor.tilt_y_count,
)
# check convention-required variables in the Platform group
check_platform_required_vars(echodata)
def test_convert_ek60_duplicate_ping_times(ek60_path):
"""Convert a file with duplicate ping times"""
raw_path = (ek60_path / "ooi" /
"CE02SHBP-MJ01C-07-ZPLSCB101_OOI-D20191201-T000000.raw")
ed = open_raw(raw_path, "EK60")
assert "duplicate_ping_times" in ed.provenance.attrs
assert "old_ping_time" in ed.provenance
def test_convert_ek80_cw_bb_in_single_file(ek80_path):
"""Make sure can convert a single EK80 file containing both CW and BB mode data."""
ek80_raw_path_bb_cw = str(
ek80_path.joinpath('Summer2018--D20180905-T033113.raw'))
echodata = open_raw(raw_file=ek80_raw_path_bb_cw, sonar_model='EK80')
# Check there are both Beam and Beam_power groups in the converted file
assert echodata.beam_power is not None
assert echodata.beam is not None
def test_convert(ek80_new_file, dump_output_dir):
print("converting", ek80_new_file)
echodata = open_raw(raw_file=str(ek80_new_file), sonar_model="EK80")
echodata.to_netcdf(save_path=dump_output_dir, overwrite=True)
nc_file = (dump_output_dir / ek80_new_file.name).with_suffix('.nc')
assert nc_file.is_file() is True
nc_file.unlink()
def test_compute_Sv_ek80_CW_complex():
"""Test calibrate CW mode data encoded as complex sam[les.
"""
ek80_raw_path = str(
ek80_path.joinpath('ar2.0-D20201210-T000409.raw')) # CW complex
echodata = ep.open_raw(ek80_raw_path, sonar_model='EK80')
assert ep.calibrate.compute_Sv(echodata,
waveform_mode='CW',
encode_mode='complex')
def test_compute_Sv_ek80_BB_complex():
"""Test calibrate BB mode data encoded as complex sam[les.
"""
ek80_raw_path = str(
ek80_path.joinpath('ar2.0-D20201209-T235955.raw')) # CW complex
echodata = ep.open_raw(ek80_raw_path, sonar_model='EK80')
assert ep.calibrate.compute_Sv(echodata,
waveform_mode='BB',
encode_mode='complex')
def test_convert_ek80_cw_power_echoview():
"""Compare parsed EK80 CW power/angle data with csv exported by EchoView.
"""
ek80_raw_path_cw = str(
ek80_path.joinpath('D20190822-T161221.raw')) # Small file (CW)
freq_list = [18, 38, 70, 120, 200]
ek80_echoview_power_csv = [
ek80_path.joinpath('from_echoview/D20190822-T161221/%dkHz.power.csv' %
freq) for freq in freq_list
]
# ek80_echoview_angle_csv = [
# ek80_path.joinpath('from_echoview/D20190822-T161221/%dkHz.angles.points.csv' % freq)
# for freq in freq_list
# ]
# Convert file
echodata = open_raw(ek80_raw_path_cw, sonar_model='EK80')
echodata.to_netcdf(save_path=output_dir)
# Test power
with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
# single point error in original raw data. Read as -2000 by echopype and -999 by EchoView
ds_beam.backscatter_r[3, 4, 13174] = -999
for file, freq in zip(ek80_echoview_power_csv, freq_list):
test_power = pd.read_csv(file, delimiter=';').iloc[:, 13:].values
assert np.allclose(test_power,
ds_beam.backscatter_r.sel(
frequency=freq * 1e3).dropna('range_bin'),
rtol=0,
atol=1.1e-5)
# # Test angle TODO: fix angle test: bug in parser
# with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam:
# # Convert from electrical angles to physical angle [deg]
# major = (ds_beam['angle_athwartship'] * 1.40625
# / ds_beam['angle_sensitivity_athwartship']
# - ds_beam['angle_offset_athwartship'])
# minor = (ds_beam['angle_alongship'] * 1.40625
# / ds_beam['angle_sensitivity_alongship']
# - ds_beam['angle_offset_alongship'])
# for file, freq in zip(ek80_echoview_angle_csv, freq_list):
# df_angle = pd.read_csv(file)
# for ping_idx in df_angle['Ping_index'].value_counts().index:
# assert np.allclose(
# df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Major'],
# major.isel(frequency=0, ping_time=0),
# rtol=0, atol=1e-5
# )
# assert np.allclose(
# df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Minor'],
# minor.isel(frequency=0, ping_time=0),
# rtol=0, atol=1e-5
# )
Path(echodata.converted_raw_path).unlink()
output_dir.rmdir()
def test_compute_Sv_ek80_pc_echoview(ek80_path):
"""Compare pulse compressed outputs from echopype and csv exported from EchoView.
Unresolved: the difference is large and it is not clear why.
"""
ek80_raw_path = str(ek80_path.joinpath('D20170912-T234910.raw'))
ek80_bb_pc_test_path = str(
ek80_path.joinpath(
'from_echoview/70 kHz pulse-compressed power.complex.csv'))
echodata = ep.open_raw(ek80_raw_path, sonar_model='EK80')
# Create a CalibrateEK80 object to perform pulse compression
cal_obj = CalibrateEK80(
echodata,
env_params=None,
cal_params=None,
waveform_mode="BB",
encode_mode="complex",
)
cal_obj.compute_range_meter(waveform_mode="BB",
encode_mode="complex") # compute range [m]
chirp, _, tau_effective = cal_obj.get_transmit_chirp(waveform_mode="BB")
freq_center = (
echodata.beam["frequency_start"] +
echodata.beam["frequency_end"]).dropna(
dim="channel"
) / 2 # drop those that contain CW samples (nan in freq start/end)
pc = cal_obj.compress_pulse(chirp, chan_BB=freq_center.channel)
pc_mean = (pc.pulse_compressed_output.isel(channel=1).mean(
dim='beam').dropna('range_sample'))
# Read EchoView pc raw power output
df = pd.read_csv(ek80_bb_pc_test_path, header=None, skiprows=[0])
df_header = pd.read_csv(ek80_bb_pc_test_path,
header=0,
usecols=range(14),
nrows=0)
df = df.rename(columns={
cc: vv
for cc, vv in zip(df.columns, df_header.columns.values)
})
df.columns = df.columns.str.strip()
df_real = df.loc[df['Component'] == ' Real', :].iloc[:, 14:]
# Compare only values for range > 0: difference is surprisingly large
range_meter = cal_obj.range_meter.sel(
channel='WBT 549762-15 ES70-7C',
ping_time='2017-09-12T23:49:10.722999808').values
first_nonzero_range = np.argwhere(range_meter == 0).squeeze().max()
assert np.allclose(
df_real.values[:, first_nonzero_range:pc_mean.values.shape[1]],
pc_mean.values.real[:, first_nonzero_range:],
rtol=0,
atol=1.03e-3,
)
def test_convert_azfp_01a_different_ranges():
"""Test converting files with different range settings across frequency.
"""
azfp_01a_path = str(azfp_path.joinpath('17031001.01A'))
azfp_xml_path = str(azfp_path.joinpath('17030815.XML'))
# Convert file
echodata = open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path)
assert echodata.beam.backscatter_r.isel(frequency=0).dropna('range_bin').shape == (360, 438)
assert echodata.beam.backscatter_r.isel(frequency=3).dropna('range_bin').shape == (360, 135)
