def process_azfp(site, data_directory, xml_file, output_directory, dates,
tilt_correction):
"""
Use echopype to convert and process the ASL AZFP bio-acoustic sonar data
(in *.01A files) to generate echograms for use by the community
:param site:
:param data_directory:
:param xml_file:
:param output_directory:
:param dates:
:param tilt_correction:
:return data:
"""
# generate a list of data files given the input dates
file_list = azfp_file_list(data_directory, dates)
# reset the file_list to a single index
file_list = [file for sub in file_list for file in sub]
if not file_list:
# if there are no files to process, exit cleanly
return None
# make sure the data output directory exists
output_directory = os.path.join(output_directory,
dates[0] + '-' + dates[1])
if not os.path.isdir(output_directory):
os.mkdir(output_directory)
# convert the list of .01A files using echopype and save the output as NetCDF files
dc = Convert(file_list, xml_file)
dc.platform_name = site # OOI site name
dc.platform_type = 'Mooring' # ICES platform type
dc.platform_code_ICES = '48' # ICES code: tethered collection of instruments at a fixed location that may
# include seafloor, mid-water or surface components
dc.raw2nc(save_path=output_directory)
# process the data, calculating the volume acoustic backscatter strength and the vertical range
echo = []
nc_files = glob.glob(output_directory + '/[12]???????.nc')
for nc in nc_files:
tmp_echo = Process(nc)
tmp_echo.calibrate() # calculate Sv
data = tmp_echo.Sv # extract the Sv dataset
echo.append(data.sortby('ping_time')) # append to the echogram list
# concatenate the data into a single dataset
data = xr.concat(echo, dim='ping_time', join='outer')
data = data.sortby(['frequency', 'ping_time'])
data['frequency'] = data['frequency'].astype(np.float32)
data['range_bin'] = data['range_bin'].astype(np.int32)
data['range'] = data['range'].sel(ping_time=data.ping_time[0], drop=True)
data = data.set_coords('range')
if tilt_correction:
range_correction(
data, tilt_correction) # apply a tilt correction, if applicable
# pass the Sv data back for further processing
return data
def test_calibrate_ek80_cw():
"""Check noise estimation and noise removal using xarray and brute force using numpy.
"""
ek80_raw_path = ek80_path.joinpath('D20190822-T161221.raw')
# Unpack data and convert to .nc file
tmp = Convert(ek80_raw_path, model="EK80")
tmp.raw2nc()
# Read .nc file into an Process object and calibrate
e_data = Process(tmp.nc_path)
e_data.calibrate(save=True)
os.remove(e_data.Sv_path)
def test_process_AZFP_matlab():
# Read in the dataset that will be used to confirm working conversions. Generated from MATLAB code.
Sv_test = loadmat(
str(azfp_path.joinpath('from_matlab/17082117_matlab_Output_Sv.mat')))
TS_test = loadmat(
str(azfp_path.joinpath('from_matlab/17082117_matlab_Output_TS.mat')))
# Convert to .nc file
tmp_convert = Convert(str(azfp_path.joinpath('17082117.01A')),
str(azfp_path.joinpath('17041823.XML')))
tmp_convert.raw2nc()
tmp_echo = Process(tmp_convert.nc_path,
salinity=27.9,
pressure=59,
temperature=None)
tmp_echo.calibrate(save=True)
tmp_echo.calibrate_TS(save=True)
tmp_echo.remove_noise()
tmp_echo.get_MVBS()
# Tolerance lowered due to temperature not being averaged as is the case in the matlab code
# Test Sv data
def check_output(ds_base, ds_cmp, cal_type):
for fidx in range(4): # loop through all freq
assert np.alltrue(
ds_cmp.range.isel(frequency=fidx).values == ds_base['Output']
[0]['Range'][fidx])
assert np.allclose(ds_cmp[cal_type].isel(frequency=fidx).values,
ds_base['Output'][0][cal_type][fidx],
atol=1e-13,
rtol=0)
# Check Sv
check_output(ds_base=Sv_test, ds_cmp=tmp_echo.Sv, cal_type='Sv')
# Check Sp
check_output(ds_base=TS_test, ds_cmp=tmp_echo.TS, cal_type='TS')
os.remove(tmp_echo.Sv_path)
os.remove(tmp_echo.TS_path)
del tmp_echo
os.remove(tmp_convert.nc_path)
def test_calibration_ek60_echoview():
ek60_raw_path = str(ek60_path.joinpath(
'DY1801_EK60-D20180211-T164025.raw')) # constant range_bin
ek60_echoview_path = ek60_path.joinpath('from_echoview')
tmp = Convert(ek60_raw_path)
tmp.raw2nc(overwrite=True)
# Read .nc file into an Process object and calibrate
e_data = Process(tmp.nc_path)
e_data.calibrate(save=True)
# Compare with EchoView outputs
channels = []
for freq in [18, 38, 70, 120, 200]:
fname = str(
ek60_echoview_path.joinpath(
'DY1801_EK60-D20180211-T164025-Sv%d.csv' % freq))
channels.append(
pd.read_csv(fname, header=None, skiprows=[0]).iloc[:, 13:])
test_Sv = np.stack(channels)
# Echoview data is missing 1 range. Also the first few ranges are handled differently
assert np.allclose(test_Sv[:, :, 7:], e_data.Sv.Sv[:, :10, 8:], atol=1e-8)
def process_ek60(site, data_directory, output_directory, dates,
tilt_correction):
"""
:param site:
:param data_directory:
:param output_directory:
:param dates:
:param tilt_correction:
:return data:
"""
# generate a list of data files given the input dates
file_list = ek60_file_list(data_directory, dates)
# reset the file_list to a single index
file_list = [file for sub in file_list for file in sub]
if not file_list:
# if there are no files to process, exit cleanly
return None
# make sure the data output directory exists
output_directory = os.path.join(output_directory,
dates[0] + '-' + dates[1])
if not os.path.isdir(output_directory):
os.mkdir(output_directory)
# convert the list of .raw files using echopype and save the output as NetCDF files
dc = Convert(file_list)
dc.platform_name = site # OOI site name
if site == 'CE02SHBP':
dc.platform_type = 'Fixed Benthic Node' # ICES platform type
dc.platform_code_ICES = '11' # ICES code
else:
dc.platform_type = 'Mooring' # ICES platform type
dc.platform_code_ICES = '48' # ICES code: tethered collection of instruments at a fixed location that may
# include seafloor, mid-water or surface components
dc.raw2nc(save_path=output_directory)
# process the data, calculating the volume acoustic backscatter strength and the vertical range
echo = []
sample_thickness = []
tvg_correction_factor = []
nc_files = glob.glob(output_directory + '/*OOI-D*.nc')
for nc in nc_files:
tmp_echo = Process(nc)
tmp_echo.calibrate() # calculate Sv
data = tmp_echo.Sv # extract the Sv dataset
echo.append(data.sortby('ping_time')) # append to the echogram list
sample_thickness.append(tmp_echo.sample_thickness.values)
tvg_correction_factor.append(tmp_echo.tvg_correction_factor)
# concatenate the data into a single dataset
data = xr.concat(echo, dim='ping_time', join='outer')
data = data.sortby(['frequency', 'ping_time'])
data['range_bin'] = data['range_bin'].astype(np.int32)
data['range'] = data['range'].sel(ping_time=data.ping_time[0], drop=True)
data = data.set_coords('range')
# recalculate the range to deal with some discrepancies caused by the xarray concat
thickness = np.max(np.array(sample_thickness), 0)
correction_factor = np.max(tvg_correction_factor)
range_meter = calc_range(data, thickness, correction_factor)
data['range'] = data['range'].fillna(range_meter)
if tilt_correction:
range_correction(
data, tilt_correction) # apply a tilt correction, if applicable
# pass the Sv data back for further processing
return data
def EchoData(nc_path):
warn("Echodata has been renamed to Process and will no longer be supported in the future.",
DeprecationWarning, 2)
return Process(nc_path)