def test_open_converted(ek60_converted_zarr, minio_bucket): # noqa def _check_path(zarr_path): storage_options = {} if zarr_path.startswith("s3://"): storage_options = dict( client_kwargs=dict(endpoint_url="http://localhost:9000/"), key="minioadmin", secret="minioadmin", ) return storage_options storage_options = {} if not isinstance(ek60_converted_zarr, fsspec.FSMap): storage_options = _check_path(str(ek60_converted_zarr)) try: ed = open_converted( ek60_converted_zarr, storage_options=storage_options ) assert isinstance(ed, EchoData) is True except Exception as e: if ( isinstance(ek60_converted_zarr, str) and ek60_converted_zarr.startswith("s3://") and ek60_converted_zarr.endswith(".nc") ): assert isinstance(e, ValueError) is True
def test_compute_Sv_ek60_echoview(): ek60_raw_path = str(ek60_path.joinpath( 'DY1801_EK60-D20180211-T164025.raw')) # constant range_bin ek60_echoview_path = ek60_path.joinpath('from_echoview') # Convert file c = ep.open_raw(ek60_raw_path, sonar_model='EK60') c.to_netcdf(overwrite=True) # Calibrate to get Sv echodata = ep.open_converted(converted_raw_path=c.converted_raw_path) ds_Sv = ep.calibrate.compute_Sv(echodata) # Compare with EchoView outputs channels = [] for freq in [18, 38, 70, 120, 200]: fname = str( ek60_echoview_path.joinpath( 'DY1801_EK60-D20180211-T164025-Sv%d.csv' % freq)) channels.append( pd.read_csv(fname, header=None, skiprows=[0]).iloc[:, 13:]) test_Sv = np.stack(channels) # Echoview data is shifted by 1 sample along range (missing the first sample) assert np.allclose(test_Sv[:, :, 7:], ds_Sv.Sv.isel(ping_time=slice(None, 10), range_bin=slice(8, None)), atol=1e-8) Path(c.converted_raw_path).unlink()
def __init__(self, file_path="", salinity=27.9, pressure=59, temperature=None): warnings.warn( "Initializing a Process object is deprecated " "and it will be removed in the next release. " "More information on how to use the new processing " "functions can be found in the echopype documentation.", DeprecationWarning, 3) self.echodata = echopype.open_converted(file_path) self._file_format = None self.file_path = file_path # Initialize AZFP environment parameters. Will error if None if self.echodata.sonar_model == 'AZFP': if temperature is None: temperature = self.echodata.environment['temperature'] self._env_params = { 'salinity': salinity, 'temperature': temperature, 'pressure': pressure } else: self._env_params = None if 'backscatter_i' in self.echodata.beam: self.waveform_mode = 'BB' self.encode_mode = 'complex' else: self.waveform_mode = 'CW' self.encode_mode = 'power' self.calibrator = CALIBRATOR[self.echodata.sonar_model]( self.echodata, env_params=self._env_params, cal_params=None, waveform_mode=self.waveform_mode, encode_mode=self.encode_mode) # Deprecated data attributes self.Sv = None self.Sv_path = None self.Sv_clean = None self.TS = None self.TS_path = None # Deprecated proc attributes self.noise_est_range_bin_size = 5 # meters per tile for noise estimation self.noise_est_ping_size = 30 # number of pings per tile for noise estimation self.MVBS_range_bin_size = 5 # meters per tile for MVBS self.MVBS_ping_size = 30 # number of pings per tile for MVBS if self.file_path.upper().endswith('.NC'): self._file_format = 'netcdf4' if self.file_path.upper().endswith('.ZARR'): self._file_format = 'zarr'
def test_compute_Sv_azfp(): azfp_01a_path = str(azfp_path.joinpath('17082117.01A')) azfp_xml_path = str(azfp_path.joinpath('17041823.XML')) azfp_matlab_Sv_path = str( azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat')) azfp_matlab_Sp_path = str( azfp_path.joinpath('from_matlab/17082117_matlab_Output_TS.mat')) # Convert to .nc file c = ep.open_raw(raw_file=azfp_01a_path, sonar_model='AZFP', xml_path=azfp_xml_path) c.to_netcdf(overwrite=True) # Calibrate using identical env params as in Matlab ParametersAZFP.m with xr.open_dataset(c.converted_raw_path, group='Environment') as ds_env: avg_temperature = ds_env['temperature'].mean( 'ping_time').values # AZFP Matlab code uses average temperature env_params = { 'temperature': avg_temperature, 'salinity': 27.9, 'pressure': 59 } echodata = ep.open_converted(converted_raw_path=c.converted_raw_path) ds_Sv = ep.calibrate.compute_Sv(echodata=echodata, env_params=env_params) ds_Sp = ep.calibrate.compute_Sp(echodata=echodata, env_params=env_params) # Load matlab outputs and test # matlab outputs were saved using # save('from_matlab/17082117_matlab_Output.mat', 'Output') # data variables # save('from_matlab/17082117_matlab_Par.mat', 'Par') # parameters def check_output(base_path, ds_cmp, cal_type): ds_base = loadmat(base_path) cal_type_in_ds_cmp = { 'Sv': 'Sv', 'TS': 'Sp', # Sp here is TS in matlab outputs } for fidx in range(4): # loop through all freq assert np.alltrue( ds_cmp.range.isel(frequency=fidx).values == ds_base['Output'] [0]['Range'][fidx]) assert np.allclose(ds_cmp[cal_type_in_ds_cmp[cal_type]].isel( frequency=fidx).values, ds_base['Output'][0][cal_type][fidx], atol=1e-13, rtol=0) # Check Sv check_output(base_path=azfp_matlab_Sv_path, ds_cmp=ds_Sv, cal_type='Sv') # Check Sp check_output(base_path=azfp_matlab_Sp_path, ds_cmp=ds_Sp, cal_type='TS') Path(c.converted_raw_path).unlink()
def test_compute_Sv_ek80_BB_complex(): """Test calibrate BB mode data encoded as complex sam[les. """ ek80_raw_path = str( ek80_path.joinpath('ar2.0-D20201209-T235955.raw')) # CW complex c = ep.open_raw(ek80_raw_path, sonar_model='EK80') c.to_netcdf() echodata = ep.open_converted(converted_raw_path=c.converted_raw_path) assert ep.calibrate.compute_Sv(echodata, waveform_mode='BB', encode_mode='complex') Path(c.converted_raw_path).unlink()
def test_compute_Sv_ek80_pc_echoview(): """Compare pulse compressed outputs from echopype and csv exported from EchoView. Unresolved: the difference is large and it is not clear why. """ ek80_raw_path = str(ek80_path.joinpath('D20170912-T234910.raw')) ek80_bb_pc_test_path = str( ek80_path.joinpath( 'from_echoview/70 kHz pulse-compressed power.complex.csv')) c = ep.open_raw(ek80_raw_path, sonar_model='EK80') c.to_netcdf(overwrite=True) # Create a CalibrateEK80 object to perform pulse compression echodata = ep.open_converted(converted_raw_path=c.converted_raw_path) waveform_mode = 'BB' cal_obj = CalibrateEK80(echodata, env_params=None, cal_params=None, waveform_mode=waveform_mode) cal_obj.compute_range_meter(waveform_mode=waveform_mode, tvg_correction_factor=0) # compute range [m] chirp, _, tau_effective = cal_obj.get_transmit_chirp( waveform_mode=waveform_mode) pc = cal_obj.compress_pulse(chirp) pc_mean = pc.pulse_compressed_output.isel(frequency=0).mean( dim='quadrant').dropna('range_bin') # Read EchoView pc raw power output df = pd.read_csv(ek80_bb_pc_test_path, header=None, skiprows=[0]) df_header = pd.read_csv(ek80_bb_pc_test_path, header=0, usecols=range(14), nrows=0) df = df.rename(columns={ cc: vv for cc, vv in zip(df.columns, df_header.columns.values) }) df.columns = df.columns.str.strip() df_real = df.loc[df['Component'] == ' Real', :].iloc[:, 14:] # Compare only values for range > 0: difference is surprisingly large range_meter = cal_obj.range_meter.isel(frequency=0, ping_time=0).values first_nonzero_range = np.argwhere(range_meter == 0).squeeze().max() assert np.allclose( df_real.values[:, first_nonzero_range:pc_mean.values.shape[1]], pc_mean.values.real[:, first_nonzero_range:], rtol=0, atol=1.03e-3) Path(c.converted_raw_path).unlink()
def test_compute_Sv_ek80_matlab(): """Compare pulse compressed outputs from echopype and Matlab outputs. Unresolved: there is a discrepancy between the range vector due to minRange=0.02 m set in Matlab. """ ek80_raw_path = str(ek80_path.joinpath('D20170912-T234910.raw')) ek80_matlab_path = str( ek80_path.joinpath('from_matlab/D20170912-T234910_data.mat')) c = ep.open_raw(ek80_raw_path, sonar_model='EK80') c.to_netcdf() echodata = ep.open_converted(converted_raw_path=c.converted_raw_path) ds_Sv = ep.calibrate.compute_Sv(echodata, waveform_mode='BB', encode_mode='complex') # TODO: resolve discrepancy in range between echopype and Matlab code ds_matlab = loadmat(ek80_matlab_path) Sv_70k = ds_Sv.Sv.isel(frequency=0, ping_time=0).dropna('range_bin').values Path(c.converted_raw_path).unlink()
def main(rdir, model): save_dir = os.path.join(os.path.dirname(rdir), 'processed') os.makedirs(save_dir, exist_ok=True) rawfiles = sorted(glob.glob(os.path.join(rdir, '*.raw'))) # convert from raw file to netCDF for filename in rawfiles: data_tmp = ep.open_raw(filename, sonar_model=model) data_tmp.to_netcdf(save_path=save_dir) # process files, calculate Sv and MVBS ncfiles = sorted(glob.glob(os.path.join(save_dir, '*.nc'))) for ncpath in ncfiles: ed = ep.open_converted(ncpath) # create an EchoData object # get a dataset containing Sv, range and the calibration and environmental parameters ds_Sv = ep.calibrate.compute_Sv(ed) # bin the data # ds_Sv = calibrated Sv dataset # range_meter_bin = bin size to average along range in meters # ping_time_bin = bin size to average along ping_time in seconds ds_MVBS = ep.preprocess.compute_MVBS(ds_Sv, range_meter_bin=5, ping_time_bin='20S') # remove noise # ds_Sv = calibrated Sv dataset # range_bin_num = number of samples along the range_bin dimension for estimating noise # ping_num = number of pings for estimating noise ds_Sv_clean = ep.preprocess.remove_noise(ds_Sv, range_bin_num=30, ping_num=5) # save files to disk ed_filename = ncpath.split('/')[-1] print('\nWriting {} Sv and MVBS datasets to file'.format(ed_filename)) ds_Sv.to_netcdf(os.path.join(save_dir, f'ds_Sv_{ed_filename}')) #ds_Sv_clean.to_netcdf(os.path.join(save_dir, f'ds_Sv_clean_{ed_filename}')) ds_MVBS.to_netcdf(os.path.join(save_dir, f'ds_MVBS_{ed_filename}'))
def test_compute_Sv_ek60_matlab(): ek60_raw_path = str( ek60_path.joinpath('DY1801_EK60-D20180211-T164025.raw')) ek60_matlab_path = str( ek60_path.joinpath('from_matlab/DY1801_EK60-D20180211-T164025.mat')) # Convert file c = ep.open_raw(ek60_raw_path, sonar_model='EK60') c.to_netcdf() # Calibrate to get Sv echodata = ep.open_converted(converted_raw_path=c.converted_raw_path) ds_Sv = ep.calibrate.compute_Sv(echodata) ds_Sp = ep.calibrate.compute_Sp(echodata) # Load matlab outputs and test # matlab outputs were saved using # save('from_matlab/DY1801_EK60-D20180211-T164025.mat', 'data') ds_base = loadmat(ek60_matlab_path) def check_output(ds_cmp, cal_type): for fidx in range(5): # loop through all freq assert np.allclose( ds_cmp[cal_type].isel(frequency=0).T.values, ds_base['data']['pings'][0][0][cal_type][0, 0], atol=4e-5, rtol=0) # difference due to use of Single in matlab code # Check Sv check_output(ds_Sv, 'Sv') # Check Sp check_output(ds_Sp, 'Sp') Path(c.converted_raw_path).unlink()
def _check_raw_output( filepath_raw, output_dir, ocean_contour_export_090_dir, ocean_contour_export_076_dir, absolute_tolerance, ): print("checking raw", filepath_raw) echodata = open_converted(converted_raw_path=output_dir.joinpath( filepath_raw.with_suffix(".nc").name)) if "090" in filepath_raw.parts: ocean_contour_converted_config_path = ( ocean_contour_export_090_dir.joinpath( filepath_raw.with_suffix(filepath_raw.suffix + ".00000.nc").name)) ocean_contour_converted_transmit_data_path = ( ocean_contour_converted_config_path) ocean_contour_converted_data_path = ocean_contour_converted_config_path else: ocean_contour_converted_config_path = ( ocean_contour_export_076_dir / filepath_raw.with_suffix("").name / "Raw Echo 1_1000 kHz_001.nc") ocean_contour_converted_transmit_data_path = ( ocean_contour_export_076_dir / filepath_raw.with_suffix("").name / "Raw Echo 1_1000 kHz Tx_001.nc") ocean_contour_converted_data_path = ( ocean_contour_export_076_dir / filepath_raw.with_suffix("").name / "Raw Echo 1_1000 kHz_001.nc") if not all(( ocean_contour_converted_config_path.exists(), ocean_contour_converted_transmit_data_path.exists(), ocean_contour_converted_data_path.exists(), )): pass else: # check pulse compression base = xr.open_dataset(str(ocean_contour_converted_config_path), group="Config") pulse_compressed = 0 for i in range(1, 4): if "090" in filepath_raw.parts: if base.attrs[f"echo_pulseComp{i}"]: pulse_compressed = i break else: if base.attrs[f"Instrument_echo_pulseComp{i}"]: pulse_compressed = i break assert echodata.vendor.attrs["pulse_compressed"] == pulse_compressed base.close() # check raw data transmit samples try: netCDF4.Dataset(str(ocean_contour_converted_transmit_data_path) )["Data/RawEcho1_1000kHzTx"] except IndexError: # no transmit data in this dataset pass else: base = xr.open_dataset( str(ocean_contour_converted_transmit_data_path), group="Data/RawEcho1_1000kHzTx", ) if "090" in filepath_raw.parts: assert np.allclose( echodata.vendor["echosounder_raw_transmit_samples_i"].data. flatten(), base["DataI"].data.flatten(), atol=absolute_tolerance, ) assert np.allclose( echodata.vendor["echosounder_raw_transmit_samples_q"].data. flatten(), base["DataQ"].data.flatten(), atol=absolute_tolerance, ) else: assert np.allclose( echodata.vendor["echosounder_raw_transmit_samples_i"].data. flatten(), base["Data_I"].data.flatten(), atol=absolute_tolerance, ) assert np.allclose( echodata.vendor["echosounder_raw_transmit_samples_q"].data. flatten(), base["Data_Q"].data.flatten(), atol=absolute_tolerance, ) base.close() # check raw data samples base = xr.open_dataset( str(ocean_contour_converted_data_path), group="Data/RawEcho1_1000kHz", ) if "090" in filepath_raw.parts: assert np.allclose( echodata.vendor["echosounder_raw_samples_i"].data.flatten(), base["DataI"].data.flatten(), atol=absolute_tolerance, ) assert np.allclose( echodata.vendor["echosounder_raw_samples_q"].data.flatten(), base["DataQ"].data.flatten(), atol=absolute_tolerance, ) else: # note the transpose assert np.allclose( echodata.vendor["echosounder_raw_samples_i"].data.flatten(), base["Data_I"].data.T.flatten(), atol=absolute_tolerance, ) assert np.allclose( echodata.vendor["echosounder_raw_samples_q"].data.flatten(), base["Data_Q"].data.T.flatten(), atol=absolute_tolerance, ) base.close()