def test_ping_time_reversal(ek60_reversed_ping_time_test_data): eds = [ echopype.open_raw(file, "EK60") for file in ek60_reversed_ping_time_test_data ] combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore for group_name in combined.group_map: combined_group: xr.Dataset = getattr(combined, group_name) if combined_group is not None: if "ping_time" in combined_group and group_name != "provenance": assert not exist_reversed_time(combined_group, "ping_time") if "old_ping_time" in combined_group: assert exist_reversed_time(combined_group, "old_ping_time") if "location_time" in combined_group and group_name not in ( "provenance", "nmea", ): assert not exist_reversed_time(combined_group, "location_time") if "old_location_time" in combined_group: assert exist_reversed_time(combined_group, "old_location_time") if "mru_time" in combined_group and group_name != "provenance": assert not exist_reversed_time(combined_group, "mru_time") if "old_mru_time" in combined_group: assert exist_reversed_time(combined_group, "old_mru_time")
def test_convert_time_encodings(sonar_model, raw_file, xml_path): ed = open_raw(sonar_model=sonar_model, raw_file=raw_file, xml_path=xml_path) ed.to_netcdf() for group, details in ed._EchoData__group_map.items(): if hasattr(ed, group): group_ds = getattr(ed, group) if isinstance(group_ds, xr.Dataset): for var, encoding in DEFAULT_ENCODINGS.items(): if var in group_ds: da = group_ds[var] assert da.encoding == encoding # Combine encoding and attributes since this # is what is shown when using decode_cf=False # without dtype attribute total_attrs = dict(**da.attrs, **da.encoding) total_attrs.pop('dtype') # Read converted file back in file_da = xr.open_dataset(ed.converted_raw_path, group=details['ep_group'], decode_cf=False)[var] assert file_da.attrs == total_attrs assert file_da.dtype == encoding['dtype'] # Read converted file back in decoded_da = xr.open_dataset(ed.converted_raw_path, group=details['ep_group'], decode_cf=True)[var] assert da.equals(decoded_da) is True os.unlink(ed.converted_raw_path)
def test_convert_ek60_echoview_raw(): """Compare parsed power data (count) with csv exported by EchoView. """ ek60_raw_path = str( ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw')) ek60_csv_path = [ ek60_path.joinpath( 'from_echoview/DY1801_EK60-D20180211-T164025-Power%d.csv' % freq) for freq in [18, 38, 70, 120, 200] ] # Read csv files exported by EchoView channels = [] for file in ek60_csv_path: channels.append( pd.read_csv(file, header=None, skiprows=[0]).iloc[:, 13:]) test_power = np.stack(channels) # Convert to netCDF and check echodata = open_raw(raw_file=ek60_raw_path, sonar_model='EK60') echodata.to_netcdf(save_path=output_dir) with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: for fidx, atol in zip(range(5), [1e-5, 1.1e-5, 1.1e-5, 1e-5, 1e-5]): assert np.allclose(test_power[fidx, :, :], ds_beam.backscatter_r.isel( frequency=fidx, ping_time=slice(None, 10), range_bin=slice(1, None)), atol=9e-6, rtol=atol) Path(echodata.converted_raw_path).unlink() output_dir.rmdir()
def test_compute_Sv_ek60_echoview(): ek60_raw_path = str(ek60_path.joinpath( 'DY1801_EK60-D20180211-T164025.raw')) # constant range_bin ek60_echoview_path = ek60_path.joinpath('from_echoview') # Convert file echodata = ep.open_raw(ek60_raw_path, sonar_model='EK60') # Calibrate to get Sv ds_Sv = ep.calibrate.compute_Sv(echodata) # Compare with EchoView outputs channels = [] for freq in [18, 38, 70, 120, 200]: fname = str( ek60_echoview_path.joinpath( 'DY1801_EK60-D20180211-T164025-Sv%d.csv' % freq)) channels.append( pd.read_csv(fname, header=None, skiprows=[0]).iloc[:, 13:]) test_Sv = np.stack(channels) # Echoview data is shifted by 1 sample along range (missing the first sample) assert np.allclose(test_Sv[:, :, 7:], ds_Sv.Sv.isel(ping_time=slice(None, 10), range_bin=slice(8, None)), atol=1e-8)
def test_convert_azfp_01a_raw_echoview(azfp_path): """Compare parsed power data (count) with csv exported by EchoView.""" azfp_01a_path = str(azfp_path.joinpath('17082117.01A')) azfp_xml_path = str(azfp_path.joinpath('17041823.XML')) # Read csv files exported by EchoView azfp_csv_path = [ azfp_path.joinpath('from_echoview/17082117-raw%d.csv' % freq) for freq in [38, 125, 200, 455] ] channels = [] for file in azfp_csv_path: channels.append( pd.read_csv(file, header=None, skiprows=[0]).iloc[:, 6:]) test_power = np.stack(channels) # Convert to netCDF and check echodata = open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path) assert np.array_equal( test_power, echodata.beam.backscatter_r.isel(beam=0).drop('beam')) # check convention-required variables in the Platform group check_platform_required_vars(echodata)
def test_plot_multi_get_range( filepath, sonar_model, azfp_xml_path, range_kwargs, ): # TODO: Need to figure out how to compare the actual rendered plots ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( ed.environment['temperature'].mean('ping_time').values) env_params = { 'temperature': avg_temperature, 'salinity': 27.9, 'pressure': 59, } range_kwargs['env_params'] = env_params plots = echopype.visualize.create_echogram(ed, get_range=True, range_kwargs=range_kwargs) assert isinstance(plots, list) is True assert all(isinstance(plot, FacetGrid) for plot in plots) is True # Quadrant shape check if (sonar_model.lower() == 'ek80' and range_kwargs['encode_mode'] == 'complex'): assert plots[0].axes.shape[-1] > 1 else: assert plots[0].axes.shape[-1] == 1 # Frequency shape check assert ed.beam.frequency.shape[0] == len(plots)
def test_update_platform(update_platform_samples): raw_file, extra_platform_data_file = update_platform_samples extra_platform_data_file_name = extra_platform_data_file.name ed = echopype.open_raw(raw_file, "EK80") updated = ["pitch", "roll", "latitude", "longitude", "water_level"] for variable in updated: assert np.isnan(ed.platform[variable].values).all() extra_platform_data = xr.open_dataset(extra_platform_data_file) ed.update_platform( extra_platform_data, extra_platform_data_file_name=extra_platform_data_file_name, ) for variable in updated: assert not np.isnan(ed.platform[variable].values).all() # times have max interval of 2s # check times are > min(ed.beam["ping_time"]) - 2s assert (ed.platform["time1"] > ed.beam["ping_time"].min() - np.timedelta64(2, "s")).all() # check there is only 1 time < min(ed.beam["ping_time"]) assert (np.count_nonzero( ed.platform["time1"] < ed.beam["ping_time"].min()) == 1) # check times are < max(ed.beam["ping_time"]) + 2s assert (ed.platform["time1"] < ed.beam["ping_time"].max() + np.timedelta64(2, "s")).all() # check there is only 1 time > max(ed.beam["ping_time"]) assert (np.count_nonzero( ed.platform["time1"] > ed.beam["ping_time"].max()) == 1)
def test_plot_mvbs( filepath, sonar_model, azfp_xml_path, range_kwargs, ): # TODO: Need to figure out how to compare the actual rendered plots ed = echopype.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( ed.environment['temperature'].mean('ping_time').values) env_params = { 'temperature': avg_temperature, 'salinity': 27.9, 'pressure': 59, } range_kwargs['env_params'] = env_params if 'azfp_cal_type' in range_kwargs: range_kwargs.pop('azfp_cal_type') Sv = echopype.calibrate.compute_Sv(ed, **range_kwargs) mvbs = echopype.preprocess.compute_MVBS(Sv, ping_time_bin='10S') plots = [] try: plots = echopype.visualize.create_echogram(mvbs) except Exception as e: assert isinstance(e, ValueError) assert str( e ) == "Ping time must be greater or equal to 2 data points." # noqa if len(plots) > 0: assert all(isinstance(plot, FacetGrid) for plot in plots) is True
def test_compute_Sv_ek60_matlab(): ek60_raw_path = str( ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw')) ek60_matlab_path = str( ek60_path.joinpath('from_matlab/DY1801_EK60-D20180211-T164025.mat')) # Convert file echodata = ep.open_raw(ek60_raw_path, sonar_model='EK60') # Calibrate to get Sv ds_Sv = ep.calibrate.compute_Sv(echodata) ds_Sp = ep.calibrate.compute_Sp(echodata) # Load matlab outputs and test # matlab outputs were saved using # save('from_matlab/DY1801_EK60-D20180211-T164025.mat', 'data') ds_base = loadmat(ek60_matlab_path) def check_output(ds_cmp, cal_type): for fidx in range(5): # loop through all freq assert np.allclose( ds_cmp[cal_type].isel(frequency=0).T.values, ds_base['data']['pings'][0][0][cal_type][0, 0], atol=4e-5, rtol=0) # difference due to use of Single in matlab code # Check Sv check_output(ds_Sv, 'Sv') # Check Sp check_output(ds_Sp, 'Sp')
def test_preprocess_mvbs(test_data_samples): """ Test running through from open_raw to compute_MVBS. """ ( filepath, sonar_model, azfp_xml_path, range_kwargs, ) = test_data_samples ed = ep.open_raw(filepath, sonar_model, azfp_xml_path) if ed.sonar_model.lower() == 'azfp': avg_temperature = ( ed.environment['temperature'].mean('time1').values ) env_params = { 'temperature': avg_temperature, 'salinity': 27.9, 'pressure': 59, } range_kwargs['env_params'] = env_params if 'azfp_cal_type' in range_kwargs: range_kwargs.pop('azfp_cal_type') Sv = ep.calibrate.compute_Sv(ed, **range_kwargs) assert ep.preprocess.compute_MVBS(Sv) is not None
def test_convert_ek80_complex_echoview(): """Compare parsed EK80 BB data with csv exported by EchoView. """ ek80_raw_path_bb = ek80_path.joinpath('D20170912-T234910.raw') ek80_echoview_bb_power_csv = ek80_path.joinpath( 'from_echoview/D20170912-T234910/70 kHz raw power.complex.csv') # Convert file echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80') echodata.to_netcdf(save_path=output_dir) # Test complex parsed data df_bb = pd.read_csv(ek80_echoview_bb_power_csv, header=None, skiprows=[0]) # averaged across quadrants with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: assert np.allclose( ds_beam.backscatter_r.sel(frequency=70e3).dropna('range_bin').mean( dim='quadrant'), df_bb.iloc[::2, 14:], # real rows rtol=0, atol=8e-6) assert np.allclose( ds_beam.backscatter_i.sel(frequency=70e3).dropna('range_bin').mean( dim='quadrant'), df_bb.iloc[1::2, 14:], # imag rows rtol=0, atol=4e-6) Path(echodata.converted_raw_path).unlink() output_dir.rmdir()
def test_convert_ek60_duplicate_frequencies(ek60_path): """Convert a file with duplicate frequencies""" raw_path = (ek60_path / "DY1002_EK60-D20100318-T023008_rep_freq.raw") ed = open_raw(raw_path, "EK60") truth_chan_vals = np.array([ 'GPT 18 kHz 009072034d45 1-1 ES18-11', 'GPT 38 kHz 009072033fa2 2-1 ES38B', 'GPT 70 kHz 009072058c6c 3-1 ES70-7C', 'GPT 70 kHz 009072058c6c 3-2 ES70-7C', 'GPT 120 kHz 00907205794e 4-1 ES120-7C', 'GPT 200 kHz 0090720346a8 5-1 ES200-7C' ], dtype='<U37') truth_freq_nom_vals = np.array( [18000., 38000., 70000., 70000., 120000., 200000.], dtype=np.float64) assert np.allclose(ed['Sonar/Beam_group1'].frequency_nominal, truth_freq_nom_vals, rtol=1e-05, atol=1e-08) assert np.all(ed['Sonar/Beam_group1'].channel.values == truth_chan_vals)
def test_convert_ek80_complex_matlab(): """Compare parsed EK80 CW power/angle data with Matlab parsed data. """ ek80_raw_path_bb = str(ek80_path.joinpath('D20170912-T234910.raw')) ek80_matlab_path_bb = str( ek80_path.joinpath('from_matlab/D20170912-T234910_data.mat')) # Convert file echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80') echodata.to_netcdf(save_path=output_dir) # Test complex parsed data ds_matlab = loadmat(ek80_matlab_path_bb) with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: assert np.array_equal( ds_beam.backscatter_r.isel( frequency=0, ping_time=0).dropna('range_bin').values[1:, :], np.real(ds_matlab['data']['echodata'][0][0][0, 0] ['complexsamples']) # real part ) assert np.array_equal( ds_beam.backscatter_i.isel( frequency=0, ping_time=0).dropna('range_bin').values[1:, :], np.imag(ds_matlab['data']['echodata'][0][0][0, 0] ['complexsamples']) # imag part ) Path(echodata.converted_raw_path).unlink() output_dir.rmdir()
def test_convert_azfp_01a_raw_echoview(): """Compare parsed power data (count) with csv exported by EchoView. """ azfp_01a_path = str(azfp_path.joinpath('17082117.01A')) azfp_xml_path = str(azfp_path.joinpath('17041823.XML')) # Read csv files exported by EchoView azfp_csv_path = [ azfp_path.joinpath('from_echoview/17082117-raw%d.csv' % freq) for freq in [38, 125, 200, 455] ] channels = [] for file in azfp_csv_path: channels.append( pd.read_csv(file, header=None, skiprows=[0]).iloc[:, 6:]) test_power = np.stack(channels) # Convert to netCDF and check echodata = open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path) echodata.to_netcdf(save_path=output_dir) with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: assert np.array_equal(test_power, ds_beam.backscatter_r) Path(echodata.converted_raw_path).unlink() # Convert to zarr and check echodata.to_zarr(save_path=output_dir) with xr.open_zarr(echodata.converted_raw_path, group='Beam') as ds_beam: assert np.array_equal(test_power, ds_beam.backscatter_r) shutil.rmtree(echodata.converted_raw_path) output_dir.rmdir()
def test_convert_ek60_matlab_raw(): """Compare parsed Beam group data with Matlab outputs. """ ek60_raw_path = str( ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw')) ek60_matlab_path = str( ek60_path.joinpath( 'from_matlab/DY1801_EK60-D20180211-T164025_rawData.mat')) # Convert file echodata = open_raw(raw_file=ek60_raw_path, sonar_model='EK60') # Compare with matlab outputs ds_matlab = loadmat(ek60_matlab_path) # power assert np.allclose([ ds_matlab['rawData'][0]['pings'][0]['power'][0][fidx] for fidx in range(5) ], echodata.beam.backscatter_r.transpose( 'frequency', 'range_bin', 'ping_time'), rtol=0, atol=1.6e-5) # angle: alongship and athwartship for angle in ['alongship', 'athwartship']: assert np.array_equal([ ds_matlab['rawData'][0]['pings'][0][angle][0][fidx] for fidx in range(5) ], echodata.beam['angle_' + angle].transpose('frequency', 'range_bin', 'ping_time'))
def test_attr_storage(ek60_test_data): # check storage of attributes before combination in provenance group eds = [echopype.open_raw(file, "EK60") for file in ek60_test_data] combined = echopype.combine_echodata(eds, "overwrite_conflicts") # type: ignore for group in combined.group_map: if f"{group}_attrs" in combined.provenance: group_attrs = combined.provenance[f"{group}_attrs"] for i, ed in enumerate(eds): for attr, value in getattr(ed, group).attrs.items(): assert str( group_attrs.isel(echodata_filename=i).sel({ f"{group}_attr_key": attr }).data[()]) == str(value) # check selection by echodata_filename for file in ek60_test_data: assert Path(file).name in combined.provenance["echodata_filename"] for group in combined.group_map: if f"{group}_attrs" in combined.provenance: group_attrs = combined.provenance[f"{group}_attrs"] assert np.array_equal( group_attrs.sel( echodata_filename=Path(ek60_test_data[0]).name), group_attrs.isel(echodata_filename=0), )
def test_convert_azfp_01a_matlab_raw(): """Compare parsed raw data with Matlab outputs. """ azfp_01a_path = str(azfp_path.joinpath('17082117.01A')) azfp_xml_path = str(azfp_path.joinpath('17041823.XML')) azfp_matlab_data_path = str( azfp_path.joinpath('from_matlab/17082117_matlab_Data.mat')) azfp_matlab_output_path = str( azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat')) # Convert file echodata = open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path) echodata.to_netcdf(save_path=output_dir) # Read in the dataset that will be used to confirm working conversions. (Generated by Matlab) ds_matlab = loadmat(azfp_matlab_data_path) ds_matlab_output = loadmat(azfp_matlab_output_path) # Test beam group with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: # frequency assert np.array_equal(ds_matlab['Data']['Freq'][0][0].squeeze(), ds_beam.frequency / 1000) # matlab file in kHz # backscatter count assert np.array_equal( np.array([ ds_matlab_output['Output'][0]['N'][fidx] for fidx in range(4) ]), ds_beam.backscatter_r.values) # tilt x-y assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), ds_beam.tilt_x_count) assert np.array_equal( np.array([d[1] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), ds_beam.tilt_y_count) # Test vendor group with xr.open_dataset(echodata.converted_raw_path, group='Vendor') as ds_vend: # Test temperature assert np.array_equal( np.array([d[4] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), ds_vend.ancillary.isel(ancillary_len=4).values) assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['BatteryTx'][0]]).squeeze(), ds_vend.battery_tx) assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['BatteryMain'][0]]).squeeze(), ds_vend.battery_main) Path(echodata.converted_raw_path).unlink() output_dir.rmdir()
def test_convert_ek80_freq_subset(ek80_path): """Make sure can convert EK80 file with multiple frequency channels off.""" ek80_raw_path_freq_subset = str( ek80_path.joinpath('2019118 group2survey-D20191214-T081342.raw')) echodata = open_raw(raw_file=ek80_raw_path_freq_subset, sonar_model='EK80') # Check if converted output has only 2 frequency channels assert echodata.beam.frequency.size == 2
def test_convert_ek80_cw_power_angle_echoview(ek80_path): """Compare parsed EK80 CW power/angle data with csv exported by EchoView.""" ek80_raw_path_cw = str( ek80_path.joinpath('D20190822-T161221.raw')) # Small file (CW) freq_list = [18, 38, 70, 120, 200] ek80_echoview_power_csv = [ ek80_path.joinpath('from_echoview/D20190822-T161221/%dkHz.power.csv' % freq) for freq in freq_list ] ek80_echoview_angle_csv = [ ek80_path.joinpath( 'from_echoview/D20190822-T161221/%dkHz.angles.points.csv' % freq) for freq in freq_list ] # Convert file echodata = open_raw(ek80_raw_path_cw, sonar_model='EK80') # Test power # single point error in original raw data. Read as -2000 by echopype and -999 by EchoView echodata.beam.backscatter_r[3, 4, 13174] = -999 for file, freq in zip(ek80_echoview_power_csv, freq_list): test_power = pd.read_csv(file, delimiter=';').iloc[:, 13:].values assert np.allclose( test_power, echodata.beam.backscatter_r.sel(frequency=freq * 1e3).dropna('range_bin'), rtol=0, atol=1.1e-5, ) # Convert from electrical angles to physical angle [deg] major = (echodata.beam['angle_athwartship'] * 1.40625 / echodata.beam['angle_sensitivity_athwartship'] - echodata.beam['angle_offset_athwartship']) minor = (echodata.beam['angle_alongship'] * 1.40625 / echodata.beam['angle_sensitivity_alongship'] - echodata.beam['angle_offset_alongship']) for freq, file in zip(freq_list, ek80_echoview_angle_csv): df_angle = pd.read_csv(file) # NB: EchoView exported data only has 6 pings, but raw data actually has 7 pings. # The first raw ping (ping 0) was removed in EchoView for some reason. # Therefore the comparison will use ping 1-6. for ping_idx in df_angle['Ping_index'].value_counts().index: assert np.allclose( df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Major'], major.sel(frequency=freq * 1e3).isel(ping_time=ping_idx).dropna('range_bin'), rtol=0, atol=5e-5, ) assert np.allclose( df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Minor'], minor.sel(frequency=freq * 1e3).isel(ping_time=ping_idx).dropna('range_bin'), rtol=0, atol=5e-5, )
def test_convert_ek80_complex_matlab(ek80_path): """Compare parsed EK80 CW power/angle data with Matlab parsed data.""" ek80_raw_path_bb = str(ek80_path.joinpath('D20170912-T234910.raw')) ek80_matlab_path_bb = str( ek80_path.joinpath('from_matlab/D20170912-T234910_data.mat') ) # Convert file echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80') # check water_level assert (echodata["Platform"]["water_level"] == 0).all() # Test complex parsed data ds_matlab = loadmat(ek80_matlab_path_bb) assert np.array_equal( echodata.beam.backscatter_r.sel(channel='WBT 549762-15 ES70-7C', ping_time='2017-09-12T23:49:10.722999808') .dropna('range_sample') .values[1:, :], np.real( ds_matlab['data']['echodata'][0][0][0, 0]['complexsamples'] ), # real part ) assert np.array_equal( echodata.beam.backscatter_i.sel(channel='WBT 549762-15 ES70-7C', ping_time='2017-09-12T23:49:10.722999808') .dropna('range_sample') .values[1:, :], np.imag( ds_matlab['data']['echodata'][0][0][0, 0]['complexsamples'] ), # imag part ) check_env_xml(echodata) # check platform nan_plat_vars = [ "MRU_offset_x", "MRU_offset_y", "MRU_offset_z", "MRU_rotation_x", "MRU_rotation_y", "MRU_rotation_z", "position_offset_x", "position_offset_y", "position_offset_z" ] for plat_var in nan_plat_vars: assert plat_var in echodata["Platform"] assert np.isnan(echodata["Platform"][plat_var]).all() zero_plat_vars = [ "transducer_offset_x", "transducer_offset_y", "transducer_offset_z", ] for plat_var in zero_plat_vars: assert plat_var in echodata["Platform"] assert (echodata["Platform"][plat_var] == 0).all()
def test_convert_ek80_complex_echoview(ek80_path): """Compare parsed EK80 BB data with csv exported by EchoView.""" ek80_raw_path_bb = ek80_path.joinpath('D20170912-T234910.raw') ek80_echoview_bb_power_csv = ek80_path.joinpath( 'from_echoview/D20170912-T234910/70 kHz raw power.complex.csv' ) # Convert file echodata = open_raw(raw_file=ek80_raw_path_bb, sonar_model='EK80') # check water_level assert (echodata["Platform"]["water_level"] == 0).all() # Test complex parsed data df_bb = pd.read_csv( ek80_echoview_bb_power_csv, header=None, skiprows=[0] ) # averaged across beams assert np.allclose( echodata.beam.backscatter_r.sel(channel='WBT 549762-15 ES70-7C') .dropna('range_sample') .mean(dim='beam'), df_bb.iloc[::2, 14:], # real rows rtol=0, atol=8e-6, ) assert np.allclose( echodata.beam.backscatter_i.sel(channel='WBT 549762-15 ES70-7C') .dropna('range_sample') .mean(dim='beam'), df_bb.iloc[1::2, 14:], # imag rows rtol=0, atol=4e-6, ) check_env_xml(echodata) # check platform nan_plat_vars = [ "MRU_offset_x", "MRU_offset_y", "MRU_offset_z", "MRU_rotation_x", "MRU_rotation_y", "MRU_rotation_z", "position_offset_x", "position_offset_y", "position_offset_z" ] for plat_var in nan_plat_vars: assert plat_var in echodata["Platform"] assert np.isnan(echodata["Platform"][plat_var]).all() zero_plat_vars = [ "transducer_offset_x", "transducer_offset_y", "transducer_offset_z", ] for plat_var in zero_plat_vars: assert plat_var in echodata["Platform"] assert (echodata["Platform"][plat_var] == 0).all()
def test_convert_azfp_01a_matlab_raw(azfp_path): """Compare parsed raw data with Matlab outputs.""" azfp_01a_path = str(azfp_path.joinpath('17082117.01A')) azfp_xml_path = str(azfp_path.joinpath('17041823.XML')) azfp_matlab_data_path = str( azfp_path.joinpath('from_matlab/17082117_matlab_Data.mat')) azfp_matlab_output_path = str( azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat')) # Convert file echodata = open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path) # Read in the dataset that will be used to confirm working conversions. (Generated by Matlab) ds_matlab = loadmat(azfp_matlab_data_path) ds_matlab_output = loadmat(azfp_matlab_output_path) # Test beam group # frequency assert np.array_equal( ds_matlab['Data']['Freq'][0][0].squeeze(), echodata.beam.frequency_nominal / 1000, ) # matlab file in kHz # backscatter count assert np.array_equal( np.array( [ds_matlab_output['Output'][0]['N'][fidx] for fidx in range(4)]), echodata.beam.backscatter_r.isel(beam=0).drop('beam').values, ) # Test vendor group # Test temperature assert np.array_equal( np.array([d[4] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), echodata.vendor.ancillary.isel(ancillary_len=4).values, ) assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['BatteryTx'][0]]).squeeze(), echodata.vendor.battery_tx, ) assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['BatteryMain'][0]]).squeeze(), echodata.vendor.battery_main, ) # tilt x-y assert np.array_equal( np.array([d[0] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), echodata.vendor.tilt_x_count, ) assert np.array_equal( np.array([d[1] for d in ds_matlab['Data']['Ancillary'][0]]).squeeze(), echodata.vendor.tilt_y_count, ) # check convention-required variables in the Platform group check_platform_required_vars(echodata)
def test_convert_ek60_duplicate_ping_times(ek60_path): """Convert a file with duplicate ping times""" raw_path = (ek60_path / "ooi" / "CE02SHBP-MJ01C-07-ZPLSCB101_OOI-D20191201-T000000.raw") ed = open_raw(raw_path, "EK60") assert "duplicate_ping_times" in ed.provenance.attrs assert "old_ping_time" in ed.provenance
def test_convert_ek80_cw_bb_in_single_file(ek80_path): """Make sure can convert a single EK80 file containing both CW and BB mode data.""" ek80_raw_path_bb_cw = str( ek80_path.joinpath('Summer2018--D20180905-T033113.raw')) echodata = open_raw(raw_file=ek80_raw_path_bb_cw, sonar_model='EK80') # Check there are both Beam and Beam_power groups in the converted file assert echodata.beam_power is not None assert echodata.beam is not None
def test_convert(ek80_new_file, dump_output_dir): print("converting", ek80_new_file) echodata = open_raw(raw_file=str(ek80_new_file), sonar_model="EK80") echodata.to_netcdf(save_path=dump_output_dir, overwrite=True) nc_file = (dump_output_dir / ek80_new_file.name).with_suffix('.nc') assert nc_file.is_file() is True nc_file.unlink()
def test_compute_Sv_ek80_CW_complex(): """Test calibrate CW mode data encoded as complex sam[les. """ ek80_raw_path = str( ek80_path.joinpath('ar2.0-D20201210-T000409.raw')) # CW complex echodata = ep.open_raw(ek80_raw_path, sonar_model='EK80') assert ep.calibrate.compute_Sv(echodata, waveform_mode='CW', encode_mode='complex')
def test_compute_Sv_ek80_BB_complex(): """Test calibrate BB mode data encoded as complex sam[les. """ ek80_raw_path = str( ek80_path.joinpath('ar2.0-D20201209-T235955.raw')) # CW complex echodata = ep.open_raw(ek80_raw_path, sonar_model='EK80') assert ep.calibrate.compute_Sv(echodata, waveform_mode='BB', encode_mode='complex')
def test_convert_ek80_cw_power_echoview(): """Compare parsed EK80 CW power/angle data with csv exported by EchoView. """ ek80_raw_path_cw = str( ek80_path.joinpath('D20190822-T161221.raw')) # Small file (CW) freq_list = [18, 38, 70, 120, 200] ek80_echoview_power_csv = [ ek80_path.joinpath('from_echoview/D20190822-T161221/%dkHz.power.csv' % freq) for freq in freq_list ] # ek80_echoview_angle_csv = [ # ek80_path.joinpath('from_echoview/D20190822-T161221/%dkHz.angles.points.csv' % freq) # for freq in freq_list # ] # Convert file echodata = open_raw(ek80_raw_path_cw, sonar_model='EK80') echodata.to_netcdf(save_path=output_dir) # Test power with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: # single point error in original raw data. Read as -2000 by echopype and -999 by EchoView ds_beam.backscatter_r[3, 4, 13174] = -999 for file, freq in zip(ek80_echoview_power_csv, freq_list): test_power = pd.read_csv(file, delimiter=';').iloc[:, 13:].values assert np.allclose(test_power, ds_beam.backscatter_r.sel( frequency=freq * 1e3).dropna('range_bin'), rtol=0, atol=1.1e-5) # # Test angle TODO: fix angle test: bug in parser # with xr.open_dataset(echodata.converted_raw_path, group='Beam') as ds_beam: # # Convert from electrical angles to physical angle [deg] # major = (ds_beam['angle_athwartship'] * 1.40625 # / ds_beam['angle_sensitivity_athwartship'] # - ds_beam['angle_offset_athwartship']) # minor = (ds_beam['angle_alongship'] * 1.40625 # / ds_beam['angle_sensitivity_alongship'] # - ds_beam['angle_offset_alongship']) # for file, freq in zip(ek80_echoview_angle_csv, freq_list): # df_angle = pd.read_csv(file) # for ping_idx in df_angle['Ping_index'].value_counts().index: # assert np.allclose( # df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Major'], # major.isel(frequency=0, ping_time=0), # rtol=0, atol=1e-5 # ) # assert np.allclose( # df_angle.loc[df_angle['Ping_index'] == ping_idx, ' Minor'], # minor.isel(frequency=0, ping_time=0), # rtol=0, atol=1e-5 # ) Path(echodata.converted_raw_path).unlink() output_dir.rmdir()
def test_compute_Sv_ek80_pc_echoview(ek80_path): """Compare pulse compressed outputs from echopype and csv exported from EchoView. Unresolved: the difference is large and it is not clear why. """ ek80_raw_path = str(ek80_path.joinpath('D20170912-T234910.raw')) ek80_bb_pc_test_path = str( ek80_path.joinpath( 'from_echoview/70 kHz pulse-compressed power.complex.csv')) echodata = ep.open_raw(ek80_raw_path, sonar_model='EK80') # Create a CalibrateEK80 object to perform pulse compression cal_obj = CalibrateEK80( echodata, env_params=None, cal_params=None, waveform_mode="BB", encode_mode="complex", ) cal_obj.compute_range_meter(waveform_mode="BB", encode_mode="complex") # compute range [m] chirp, _, tau_effective = cal_obj.get_transmit_chirp(waveform_mode="BB") freq_center = ( echodata.beam["frequency_start"] + echodata.beam["frequency_end"]).dropna( dim="channel" ) / 2 # drop those that contain CW samples (nan in freq start/end) pc = cal_obj.compress_pulse(chirp, chan_BB=freq_center.channel) pc_mean = (pc.pulse_compressed_output.isel(channel=1).mean( dim='beam').dropna('range_sample')) # Read EchoView pc raw power output df = pd.read_csv(ek80_bb_pc_test_path, header=None, skiprows=[0]) df_header = pd.read_csv(ek80_bb_pc_test_path, header=0, usecols=range(14), nrows=0) df = df.rename(columns={ cc: vv for cc, vv in zip(df.columns, df_header.columns.values) }) df.columns = df.columns.str.strip() df_real = df.loc[df['Component'] == ' Real', :].iloc[:, 14:] # Compare only values for range > 0: difference is surprisingly large range_meter = cal_obj.range_meter.sel( channel='WBT 549762-15 ES70-7C', ping_time='2017-09-12T23:49:10.722999808').values first_nonzero_range = np.argwhere(range_meter == 0).squeeze().max() assert np.allclose( df_real.values[:, first_nonzero_range:pc_mean.values.shape[1]], pc_mean.values.real[:, first_nonzero_range:], rtol=0, atol=1.03e-3, )
def test_convert_azfp_01a_different_ranges(): """Test converting files with different range settings across frequency. """ azfp_01a_path = str(azfp_path.joinpath('17031001.01A')) azfp_xml_path = str(azfp_path.joinpath('17030815.XML')) # Convert file echodata = open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path) assert echodata.beam.backscatter_r.isel(frequency=0).dropna('range_bin').shape == (360, 438) assert echodata.beam.backscatter_r.isel(frequency=3).dropna('range_bin').shape == (360, 135)