def extract_sbe43(netCDFfile):
ds = Dataset(netCDFfile, 'r')
ds.set_auto_mask(False)
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
var_pres = ds.variables["PRES"]
# the SBE43 voltage
var_v0 = ds.variables["V0"]
dep_code = ds.deployment_code
print('deployment', dep_code)
out_file = dep_code + "-SBE43.nc"
ds_out = Dataset(out_file, 'w', data_model='NETCDF4_CLASSIC')
ds_out.createDimension("TIME", len(ds.variables['TIME']))
ncVarIn = ds.variables['TIME']
ncTimesOut = ds_out.createVariable('TIME', "f8", ("TIME", ), zlib=True)
for a in ncVarIn.ncattrs():
if a != '_FillValue':
ncTimesOut.setncattr(a, ncVarIn.getncattr(a))
ncTimesOut[:] = ds.variables['TIME'][:]
# copy old variables into new file
in_vars = set([x for x in ds.variables])
z = in_vars.intersection([
'TEMP', 'PSAL', 'PRES', 'V0', 'TEMP_quality_control',
'PSAL_quality_control', 'PRES_quality_control', 'V0_quality_control',
'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH'
])
for v in z:
print('copying', v, 'dimensions', ncVarIn.dimensions)
ncVarIn = ds.variables[v]
if '_FillValue' in ncVarIn.ncattrs():
fill = ncVarIn._FillValue
else:
fill = None
ncVarOut = ds_out.createVariable(
v, ncVarIn.dtype, ncVarIn.dimensions, fill_value=fill,
zlib=True) # fill_value=nan otherwise defaults to max
for a in ncVarIn.ncattrs():
if a != '_FillValue':
if a == 'ancillary_variables':
ncVarOut.setncattr(
a, v + '_quality_control'
) # only copying main quality control, not all individual flags
else:
ncVarOut.setncattr(a, ncVarIn.getncattr(a))
ncVarOut[:] = ncVarIn[:]
if 'DOX2' in ds.variables:
ncVarIn = ds.variables['DOX2']
ncVarOut = ds_out.createVariable(
'DOX2_SBE', "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
for a in ncVarIn.ncattrs():
if a not in ['_FillValue', 'ancillary_variables']:
ncVarOut.setncattr(a, ncVarIn.getncattr(a))
ncVarOut[:] = ncVarIn[:]
T = var_temp[:]
calibration_Soc = float(var_v0.calibration_Soc)
calibration_offset = float(var_v0.calibration_offset)
calibration_A = float(var_v0.calibration_A)
calibration_B = float(var_v0.calibration_B)
calibration_C = float(var_v0.calibration_C)
calibration_E = float(var_v0.calibration_E)
slope_correction = 1.0
if 'oxygen_correction_slope' in var_v0.ncattrs():
slope_correction = float(var_v0.oxygen_correction_slope)
lat = -47
lon = 142
try:
lat = ds.variables["LATITUDE"][0]
lon = ds.variables["LONGITUDE"][0]
except:
pass
SP = var_psal[:]
p = var_pres[:]
SA = gsw.SA_from_SP(SP, p, lon, lat)
pt = gsw.pt0_from_t(SA, T, p)
CT = gsw.CT_from_t(SA, T, p)
sigma_t0 = gsw.sigma0(SA, CT)
# calc oxygen solubility
# 0.01 % difference to sea bird calculation
#oxsol = gsw.O2sol_SP_pt(SP, pt) # umol/kg returned
# calc OXSOL in ml/l as per seabird application note 64
# this method gives a 0.2 % difference to what is calculated by sea bird (and what is calculated by TEOS-10)
A0 = 2.00907
A1 = 3.22014
A2 = 4.0501
A3 = 4.94457
A4 = -0.256847
A5 = 3.88767
B0 = -0.00624523
B1 = -0.00737614
B2 = -0.010341
B3 = -0.00817083
C0 = -0.000000488682
ts = np.log((298.15 - T) / (273.15 + T))
oxsol = np.exp(A0 + A1 * ts + A2 * (ts**2) + A3 * (ts**3) + A4 * (ts**4) +
A5 * (ts**5) +
SP * [B0 + B1 * (ts) + B2 * (ts**2) + B3 *
(ts**3)] + C0 * (SP**2))
#
# # calculate oxygen from V0
dox = slope_correction * calibration_Soc * (
var_v0[:] + calibration_offset) * oxsol * (
1 + calibration_A * T + calibration_B * T**2 +
calibration_C * T**3) * np.exp(calibration_E * p / (T + 273.15))
#
# # create SBE43 oxygen ml/l
# # ncVarOut = ds_out.createVariable("DOX", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
# #
# # ncVarOut[:] = dox
# # ncVarOut.long_name = "volume_concentration_of_dissolved_molecular_oxygen_in_sea_water"
# # ncVarOut.valid_min = 0
# # ncVarOut.valid_max = 40
# # ncVarOut.units = "ml/l"
# # ncVarOut.equation_1 = "Ox(ml/l)=Soc.(V+Voffset).(1+A.T+B.T^2+V.T^3).OxSOL(T,S).exp(E.P/K) ... SeaBird (AN64-2)"
# create SBE43 oxygen in umol/kg
ncVarOut = ds_out.createVariable(
"DOX2", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = dox * 44600 / (1000 + sigma_t0)
#ncVarOut[:] = dox * 44.6
ncVarOut.standard_name = "moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut.long_name = "moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut.coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'
ncVarOut.valid_min = 0
ncVarOut.valid_max = 400
ncVarOut.units = "umol/kg"
ncVarOut.equation_1 = "Ox[ml/l]=Soc.(V+Voffset).(1+A.T+B.T^2+C.T^3).OxSOL(T,S)[ml/l].exp(E.P/K) ... SeaBird (AN64)"
ncVarOut.equation_2 = "Ox[umol/kg]=Ox[ml/l].44660/(sigma-theta(P=0,Theta,S)+1000)"
#ncVarOut.equation_1 = "Ox[umol/kg]=Soc.(V+Voffset).(1+A.T+B.T^2+C.T^3).OxSOL(T,S)[umol/kg].exp(E.P/K) ... SeaBird (AN64)"
#ncVarOut.comment = 'OxSOL in umol/kg'
ncVarOut.ancillary_variables = "DOX2_quality_control DOX2_quality_control_in"
# quality flags
ncVarOut_qc = ds_out.createVariable(
"DOX2_quality_control", "i1", ("TIME", ), fill_value=99, zlib=True
) # fill_value=99 otherwise defaults to max, imos-toolbox uses 99
ncVarOut_qc[:] = np.zeros(ncVarOut_qc.shape)
if 'V0_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['V0_quality_control'][:]],
axis=0)
ncVarOut_qc[:] = mx
if 'TEMP_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['TEMP_quality_control'][:]],
axis=0)
print('TEMP max', mx)
ncVarOut_qc[:] = mx
if 'PSAL_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['PSAL_quality_control'][:]],
axis=0)
print('PSAL max', mx)
ncVarOut_qc[:] = mx
if 'PRES_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['PRES_quality_control'][:]],
axis=0)
print('PRES max', mx)
ncVarOut_qc[:] = mx
ncVarOut_qc.standard_name = ncVarOut.standard_name + " status_flag"
ncVarOut_qc.quality_control_conventions = "IMOS standard flags"
ncVarOut_qc.flag_values = np.array([0, 1, 2, 3, 4, 6, 7, 9], dtype=np.int8)
ncVarOut_qc.flag_meanings = 'unknown good_data probably_good_data probably_bad_data bad_data not_deployed interpolated missing_value'
ncVarOut_qc.comment = 'maximum of all flags'
# create a QC flag variable for the input data
ncVarOut_qc = ds_out.createVariable(
"DOX2_quality_control_in", "i1", ("TIME", ), fill_value=99, zlib=True
) # fill_value=99 otherwise defaults to max, imos-toolbox uses 99
ncVarOut_qc[:] = mx
ncVarOut_qc.long_name = "input data flag for moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut_qc.units = "1"
ncVarOut_qc.comment = "data flagged from input variables TEMP, PSAL, PRES"
# save the OxSOL
if 'OXSOL' in ds_out.variables:
ncVarOut = ds_out.variables['OXSOL']
else:
ncVarOut = ds_out.createVariable(
"OXSOL", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
#ncVarOut[:] = oxsol * 44600 / (1000+sigma_t0)
ncVarOut[:] = oxsol
ncVarOut.units = "umol/kg"
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html function gsw.O2sol_SP_pt"
ncVarOut.long_name = "moles_of_oxygen_per_unit_mass_in_sea_water_at_saturation"
ncVarOut.coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'
for v in ds.ncattrs():
if not v.startswith('sea_bird'):
ds_out.setncattr(v, ds.getncattr(v))
ds_out.instrument = 'Sea-Bird Electronics ; SBE43'
ds_out.instrument_model = 'SBE43'
ds_out.instrument_serial_number = '43' + var_v0.calibration_SerialNumber
ncTimeFormat = "%Y-%m-%dT%H:%M:%SZ"
attrs = ds.ncattrs()
for at in attrs:
if at not in [
'title', 'instrument', 'instrument_model',
'instrument_serial_number', 'history', 'date_created', 'title'
]:
#print('copy att', at)
ds_out.setncattr(at, ds.getncattr(at))
ds_out.deployment_code = ds.deployment_code
ds_out.instrument = 'SeaBird Electronics ; SBE43'
ds_out.instrument_model = 'SBE43'
ds_out.instrument_serial_number = ds.variables[
'V0'].calibration_SerialNumber
ds_out.title = 'Oceanographic mooring data deployment of {platform_code} at latitude {geospatial_lat_max:3.1f} longitude {geospatial_lon_max:3.1f} depth {geospatial_vertical_max:3.0f} (m) instrument {instrument} serial {instrument_serial_number}'
# add creating and history entry
ds_out.setncattr("date_created", datetime.utcnow().strftime(ncTimeFormat))
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
# keep the history so we know where it came from
ds_out.setncattr(
'history', hist + datetime.utcnow().strftime("%Y-%m-%d") +
" calculate DOX2 from file " + os.path.basename(netCDFfile))
ds.close()
ds_out.close()
return out_file
def psal2asal(self,
psal_parameter='PSAL',
pres_parameter='PRES',
lon='auto',
lat='auto',
inplace=True):
"""
This function uses the gsw library.
Adds the parameter Absolute Salinity (ASAL) from Practical Salinity (PSAL).
Since PSAL is non-negative by definition, this function changes any
negative input values of SP to be zero.
Parameters
----------
psal_parameter: str
Parameter with valus of Practical Salinity
pres_parameter: str
Parameter with values of pressure, in dBar
lon: str
Parameter with the values of the longitud, in degrees
If lon is 'auto', the value of lon will be taken from the
metadata['last_longitude_observation']
lat: str
Parameter with the values of the ñatitude, in degrees
If lat is 'auto', the value of lon will be taken from the
metadata['last_latitude_observation']
Returns
-------
wf: WaterFrame
"""
df_copy = self.data.copy()
if lat == 'auto':
lat_parameter = 'LATITUDE'
df_copy['LATITUDE'] = float(self.metadata['last_latitude_observation'])
df_copy['LATITUDE_QC'] = 1
else:
lat_parameter = lat
if lon == 'auto':
lon_parameter = 'LONGITUDE'
df_copy['LONGITUDE'] = float(
self.metadata['last_longitude_observation'])
df_copy['LONGITUDE_QC'] = 1
else:
lon_parameter = 'lon'
df_copy['ASAL'] = gsw.SA_from_SP(df_copy[psal_parameter],
df_copy[pres_parameter],
df_copy[lon_parameter],
df_copy[lat_parameter])
# Add QC
df_copy['ASAL_QC'] = 1
# QC=0
df_copy.loc[df_copy[f'{psal_parameter}_QC'] == 0, 'ASAL_QC'] = 0
df_copy.loc[df_copy[f'{pres_parameter}_QC'] == 0, 'ASAL_QC'] = 0
df_copy.loc[df_copy[f'{lon_parameter}_QC'] == 0, 'ASAL_QC'] = 0
df_copy.loc[df_copy[f'{lat_parameter}_QC'] == 0, 'ASAL_QC'] = 0
# QC=4
df_copy.loc[df_copy[f'{psal_parameter}_QC'] == 4, 'ASAL_QC'] = 4
df_copy.loc[df_copy[f'{pres_parameter}_QC'] == 4, 'ASAL_QC'] = 4
df_copy.loc[df_copy[f'{lon_parameter}_QC'] == 4, 'ASAL_QC'] = 4
df_copy.loc[df_copy[f'{lat_parameter}_QC'] == 4, 'ASAL_QC'] = 4
# Delete lat and lon
if lat == 'auto':
del df_copy[f'{lat_parameter}_QC']
del df_copy[lat_parameter]
if lon == 'auto':
del df_copy[f'{lon_parameter}_QC']
del df_copy[lon_parameter]
new_wf = self.copy()
new_wf.data = df_copy.copy()
# Add vocabulary
new_wf.vocabulary['ASAL'] = {
'long_name': 'Absolute Salinity',
'units': 'g/kg'
}
if inplace:
self.data = df_copy.copy()
self.vocabulary = new_wf.vocabulary.copy()
return new_wf
def process_all_new():
# Directory and file information
expocode = settings.cruise['expocode']
sectionID = settings.cruise['sectionid']
raw_directory = settings.ctd_processing_dir['raw_data_directory']
time_directory = settings.ctd_processing_dir['time_data_directory']
converted_directory = settings.ctd_processing_dir['converted_directory']
pressure_directory = settings.ctd_processing_dir['pressure_data_directory']
oxygen_directory = settings.ctd_processing_dir['oxygen_directory']
btl_directory = settings.ctd_processing_dir['bottle_directory']
o2flask_file = settings.ctd_processing_dir['o2flask_file']
log_directory = settings.ctd_processing_dir['log_directory']
p_log_file = settings.ctd_processing_dir['pressure_log']
hex_prefix = settings.ctd_processing_dir['hex_prefix']
hex_postfix = settings.ctd_processing_dir['hex_postfix']
xml_prefix = settings.ctd_processing_dir['xml_prefix']
xml_postfix = settings.ctd_processing_dir['xml_postfix']
# CTD Data Inputs
p_col = settings.ctd_inputs['p']
t_col = settings.ctd_inputs['t']
t1_col = settings.ctd_inputs['t1']
t2_col = settings.ctd_inputs['t2']
c_col = settings.ctd_inputs['c']
c1_col = settings.ctd_inputs['c1']
c2_col = settings.ctd_inputs['c2']
sal_col = settings.ctd_inputs['salt']
dov_col = settings.ctd_inputs['dov']
lat_col = settings.ctd_inputs['lat']
lon_col = settings.ctd_inputs['lon']
time_col = settings.ctd_inputs['scan_datetime']
# Bottle Data Inputs
p_btl_col = settings.bottle_inputs['p']
t_btl_col = settings.bottle_inputs['t']
t1_btl_col = settings.bottle_inputs['t1']
t2_btl_col = settings.bottle_inputs['t2']
c_btl_col = settings.bottle_inputs['c']
c1_btl_col = settings.bottle_inputs['c1']
c2_btl_col = settings.bottle_inputs['c2']
reft_col = settings.bottle_inputs['reft']
cond_col = settings.bottle_inputs['btl_cond']
cr_avg = settings.bottle_inputs['cond_ratio']
bath_temp = settings.bottle_inputs['bath_temp']
sal_btl_col = settings.bottle_inputs['salt']
dov_btl_col = settings.bottle_inputs['dov']
lat_btl_col = settings.bottle_inputs['lat']
lon_btl_col = settings.bottle_inputs['lon']
oxy_btl_col = settings.bottle_inputs['btl_oxy']
time_btl_col = settings.bottle_inputs['scan_datetime']
# CTD Information
sample_rate = settings.ctd_processing_constants['sample_rate']
search_time = settings.ctd_processing_constants['roll_filter_time']
ctd = settings.ctd_processing_constants['ctd_serial']
p_column_names = settings.pressure_series_output['column_name']
p_column_units = settings.pressure_series_output['column_units']
btl_data_prefix = 'data/bottle/'
btl_data_postfix = '_btl_mean.pkl'
time_data_prefix = 'data/time/'
time_data_postfix = '_time.pkl'
p_log_file = 'data/logs/ondeck_pressure.csv'
# Columns from btl and ctd file to be read:
btl_cols = settings.btl_input_array
ctd_cols = settings.ctd_input_array
ssscc = settings.ssscc
# time_start = time.perf_counter()
cnv_dir_list = os.listdir(converted_directory)
time_dir_list = os.listdir(time_directory)
btl_dir_list = os.listdir(btl_directory)
for station in ssscc:
if '{}.pkl'.format(station) in cnv_dir_list:
continue
#convert hex to ctd
hex_file = hex_prefix + station + hex_postfix
xml_file = xml_prefix + station + xml_postfix
sbe_convert.convert_sbe(station, hex_file, xml_file,
converted_directory)
print('Converted_sbe SSSCC: ' + station + ' done')
# time_convert = time.perf_counter()
for station in ssscc:
if '{}_time.pkl'.format(station) in time_dir_list:
continue
sbe_convert.sbe_metadata(station)
print('sbe_metadata SSSCC: ' + station + ' done')
for station in ssscc:
if '{}_btl_mean.pkl'.format(station) in btl_dir_list:
continue
#process bottle file
sbe_convert.process_bottle(station)
print('process_bottle SSSCC: ' + station + ' done')
# time_bottle = time.perf_counter()
###########################################################################
### File I/O
# Load in all bottle, time, ref_data files into DataFrame
btl_data_all = process_ctd.load_all_ctd_files(ssscc, btl_data_prefix,
btl_data_postfix, 'bottle',
btl_cols)
time_data_all = process_ctd.load_all_ctd_files(ssscc, time_data_prefix,
time_data_postfix, 'time',
ctd_cols)
################################################################################
### Pressure Calibration
# Determine Pressure offset from logs
pressure_log = process_ctd.load_pressure_logs(p_log_file)
p_off = process_ctd.get_pressure_offset(pressure_log.ondeck_start_p,
pressure_log.ondeck_end_p)
btl_data_all[p_btl_col] = fit_ctd.apply_pressure_offset(
btl_data_all[p_btl_col], p_off)
time_data_all[p_col] = fit_ctd.apply_pressure_offset(
time_data_all[p_col], p_off)
###########################################################################
df_ques_t1 = pd.DataFrame()
df_ques_t2 = pd.DataFrame()
df_ques_c1 = pd.DataFrame()
df_ques_c2 = pd.DataFrame()
### Temperature Calibration
for x in range(2):
# Second order calibration
df_temp_good = process_ctd.prepare_fit_data(btl_data_all, reft_col)
df_ques_reft = process_ctd.quality_check(df_temp_good[t2_btl_col],
df_temp_good[t1_btl_col],
df_temp_good[p_btl_col],
df_temp_good['SSSCC'],
df_temp_good['btl_fire_num'],
'quest')
df_ques_reft['Parameter'] = 'REF_TEMP'
if settings.do_primary == 1:
coef_temp_1, df_ques_t1 = process_ctd.calibrate_param(
df_temp_good[t1_btl_col],
df_temp_good[reft_col],
df_temp_good[p_btl_col],
'TP',
2,
df_temp_good.SSSCC,
df_temp_good.btl_fire_num,
xRange='800:6000')
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col], coef_temp_1)
time_data_all[t1_col] = fit_ctd.temperature_polyfit(
time_data_all[t1_col], time_data_all[p_col], coef_temp_1)
elif settings.do_secondary == 1:
coef_temp_2, df_ques_t2 = process_ctd.calibrate_param(
df_temp_good[t2_btl_col],
df_temp_good[reft_col],
df_temp_good[p_btl_col],
'TP',
2,
df_temp_good.SSSCC,
df_temp_good.btl_fire_num,
xRange='1500:6000')
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col], coef_temp_2)
time_data_all[t2_col] = fit_ctd.temperature_polyfit(
time_data_all[t2_col], time_data_all[p_col], coef_temp_2)
# Apply fitting coef to data
# Construct Quality Flag file
qual_flag_temp = process_ctd.combine_quality_flags(
[df_ques_reft, df_ques_t1, df_ques_t2])
## First order calibtation
df_temp_good = process_ctd.prepare_fit_data(btl_data_all, reft_col)
# df_ques_reft = process_ctd.quality_check(df_temp_good[t2_btl_col], df_temp_good[t1_btl_col], df_temp_good[p_btl_col], df_temp_good['SSSCC'], df_temp_good['btl_fire_num'], 'quest')
# df_ques_reft['Parameter'] = 'REF_TEMP'
if settings.do_primary == 1:
coef_temp_prim, df_ques_t1 = process_ctd.calibrate_param(
df_temp_good[t1_btl_col], df_temp_good[reft_col],
df_temp_good[p_btl_col], 'T', 1, df_temp_good.SSSCC,
df_temp_good.btl_fire_num)
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col],
coef_temp_prim)
time_data_all[t1_col] = fit_ctd.temperature_polyfit(
time_data_all[t1_col], time_data_all[p_col], coef_temp_prim)
elif settings.do_secondary == 1:
coef_temp_sec, df_ques_t2 = process_ctd.calibrate_param(
df_temp_good[t2_btl_col], df_temp_good[reft_col],
df_temp_good[p_btl_col], 'T', 1, df_temp_good.SSSCC,
df_temp_good.btl_fire_num)
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col],
coef_temp_sec)
time_data_all[t2_col] = fit_ctd.temperature_polyfit(
time_data_all[t2_col], time_data_all[p_col], coef_temp_sec)
# Apply fitting coef to data
qual_flag_temp = process_ctd.combine_quality_flags(
[df_ques_reft, df_ques_t1, df_ques_t2])
###########################################################################
#
### Conductivity Calibration
for x in range(2):
btl_data_all[cond_col] = fit_ctd.CR_to_cond(btl_data_all[cr_avg],
btl_data_all[bath_temp],
btl_data_all[t1_btl_col],
btl_data_all[p_btl_col])
df_cond_good = process_ctd.prepare_fit_data(btl_data_all, cond_col)
df_ques_refc = process_ctd.quality_check(df_cond_good[c2_btl_col],
df_temp_good[c1_btl_col],
df_temp_good[p_btl_col],
df_temp_good['SSSCC'],
df_temp_good['btl_fire_num'],
'quest')
df_ques_refc['Parameter'] = 'REF_COND'
# Second Order Calibration
if settings.do_primary == 1:
coef_cond_1, df_ques_c1 = process_ctd.calibrate_param(
df_cond_good[c1_btl_col],
df_cond_good[cond_col],
df_cond_good[p_btl_col],
'CP',
2,
df_cond_good['SSSCC'],
df_cond_good['btl_fire_num'],
xRange='800:6000')
btl_data_all[c1_btl_col], btl_data_all[
sal_btl_col] = fit_ctd.conductivity_polyfit(
btl_data_all[c1_btl_col], btl_data_all[t1_btl_col],
btl_data_all[p_btl_col], coef_cond_1)
time_data_all[c1_col], time_data_all[
sal_col] = fit_ctd.conductivity_polyfit(
time_data_all[c1_col], time_data_all[t1_col],
time_data_all[p_col], coef_cond_1)
elif settings.do_secondary == 1:
coef_cond_2, df_ques_c2 = process_ctd.calibrate_param(
df_cond_good[c2_btl_col],
df_cond_good[cond_col],
df_cond_good[p_btl_col],
'CP',
2,
df_cond_good['SSSCC'],
df_cond_good['btl_fire_num'],
xRange='1500:6000')
btl_data_all[c2_btl_col], sal_2 = fit_ctd.conductivity_polyfit(
btl_data_all[c2_btl_col], btl_data_all[t2_btl_col],
btl_data_all[p_btl_col], coef_cond_2)
time_data_all[c2_btl_col], sal2 = fit_ctd.conductivity_polyfit(
time_data_all[c2_col], time_data_all[t2_col],
time_data_all[p_col], coef_cond_2)
qual_flag_cond = process_ctd.combine_quality_flags(
[df_ques_c1, df_ques_c2, df_ques_refc])
btl_data_all[cond_col] = fit_ctd.CR_to_cond(btl_data_all[cr_avg],
btl_data_all[bath_temp],
btl_data_all[t1_btl_col],
btl_data_all[p_btl_col])
df_cond_good = process_ctd.prepare_fit_data(btl_data_all, cond_col)
if settings.do_primary == 1:
coef_cond_prim, df_ques_c1 = process_ctd.calibrate_param(
df_cond_good[c1_btl_col], df_cond_good[cond_col],
df_cond_good[p_btl_col], 'C', 2, df_cond_good['SSSCC'],
df_cond_good['btl_fire_num'])
btl_data_all[c1_btl_col], btl_data_all[
sal_btl_col] = fit_ctd.conductivity_polyfit(
btl_data_all[c1_btl_col], btl_data_all[t1_btl_col],
btl_data_all[p_btl_col], coef_cond_prim)
time_data_all[c1_col], time_data_all[
sal_col] = fit_ctd.conductivity_polyfit(
time_data_all[c1_col], time_data_all[t1_col],
time_data_all[p_col], coef_cond_prim)
elif settings.do_secondary == 1:
coef_cond_sec, df_ques_c2 = process_ctd.calibrate_param(
df_cond_good.CTDCOND2, df_cond_good.BTLCOND,
df_cond_good.CTDPRS, 'C', 2, df_cond_good.SSSCC,
df_cond_good.btl_fire_num)
btl_data_all[c2_btl_col], sal_2 = fit_ctd.conductivity_polyfit(
btl_data_all[c2_btl_col], btl_data_all[t2_btl_col],
btl_data_all[p_btl_col], coef_cond_sec)
time_data_all[c2_col], sal2 = fit_ctd.conductivity_polyfit(
time_data_all[c2_col], time_data_all[t2_col],
time_data_all[p_col], coef_cond_sec)
qual_flag_cond = process_ctd.combine_quality_flags(
[df_ques_c1, df_ques_c2, df_ques_refc])
###########################################################################
#
# ## Oxygen Calibration
# Calculate Sigma
btl_data_all['sigma_btl'] = oxy_fitting.sigma_from_CTD(
btl_data_all[sal_btl_col], btl_data_all[t_btl_col],
btl_data_all[p_btl_col], btl_data_all[lon_btl_col],
btl_data_all[lat_btl_col])
time_data_all['sigma_ctd'] = oxy_fitting.sigma_from_CTD(
time_data_all[sal_col], time_data_all[t_col], time_data_all[p_col],
time_data_all[lon_col], time_data_all[lat_col])
btl_data_all[oxy_btl_col] = oxy_fitting.calculate_bottle_oxygen(
ssscc, btl_data_all['SSSCC'], btl_data_all['TITR_VOL'],
btl_data_all['TITR_TEMP'], btl_data_all['FLASKNO'])
btl_data_all[oxy_btl_col] = oxy_fitting.oxy_ml_to_umolkg(
btl_data_all[oxy_btl_col], btl_data_all['sigma_btl'])
# Calculate SA and PT
btl_data_all['SA'] = gsw.SA_from_SP(btl_data_all[sal_btl_col],
btl_data_all[p_btl_col],
btl_data_all[lon_btl_col],
btl_data_all[lat_btl_col])
btl_data_all['PT'] = gsw.pt0_from_t(btl_data_all['SA'],
btl_data_all[t_btl_col],
btl_data_all[p_btl_col])
time_data_all['SA'] = gsw.SA_from_SP(time_data_all[sal_col],
time_data_all[p_col],
time_data_all[lon_col],
time_data_all[lat_col])
time_data_all['PT'] = gsw.pt0_from_t(time_data_all['SA'],
time_data_all[t_col],
time_data_all[p_col])
# Calculate OS in µmol/kg
btl_data_all['OS_btl'] = oxy_fitting.os_umol_kg(btl_data_all['SA'],
btl_data_all['PT'])
time_data_all['OS_ctd'] = oxy_fitting.os_umol_kg(time_data_all['SA'],
time_data_all['PT'])
oxy_df = pd.DataFrame()
coef_dict = {}
for station in ssscc:
btl_data = btl_data_all[btl_data_all['SSSCC'] == station]
time_data = time_data_all[time_data_all['SSSCC'] == station]
hex_file = hex_prefix + station + hex_postfix
xml_file = xml_prefix + station + xml_postfix
coef0 = oxy_fitting.get_SB_coef(hex_file, xml_file)
cfw_coef, df = oxy_fitting.oxy_fit(
btl_data[p_btl_col], btl_data[oxy_btl_col], btl_data['sigma_btl'],
time_data['sigma_ctd'], time_data['OS_ctd'], time_data[p_col],
time_data[t_col], time_data[dov_col], time_data[time_col], coef0)
df['SSSCC'] = station
coef_dict[station] = cfw_coef
oxy_df = pd.concat([oxy_df, df])
print(station, ' Completed')
coef_df = oxy_fitting.create_coef_df(coef_dict)
oxy_df = oxy_fitting.flag_oxy_data(oxy_df)
# Merge oxygen fitting DF to btl_data_all
btl_data_all = oxy_fitting.merge_oxy_df(btl_data_all, oxy_df)
# Apply coef to Time Data
time_data_all = oxy_fitting.apply_oxygen_coef_ctd(time_data_all, coef_df,
ssscc)
################ Clean and export data #######################
btl_data_all = process_ctd.merge_cond_flags(btl_data_all, qual_flag_cond)
btl_data_all = process_ctd.merge_refcond_flags(btl_data_all,
qual_flag_cond)
btl_data_all = process_ctd.merged_reftemp_flags(btl_data_all,
qual_flag_temp)
### Export Quality Flags
qual_flag_temp.to_csv('data/logs/qual_flag_temp_new.csv', index=False)
qual_flag_cond.to_csv('data/logs/qual_flag_cond_new.csv', index=False)
### Clean up Bottle Data by removing rows with no ctd data
btl_data_all = btl_data_all.dropna(subset=btl_cols)
### Add DATE and TIME
btl_data_all['DATE'] = ''
btl_data_all['TIME'] = ''
for station in ssscc:
df = btl_data_all.loc[btl_data_all['SSSCC'] == station].copy()
btl_data_all.loc[btl_data_all['SSSCC'] ==
station] = process_ctd.get_btl_time(
df, 'btl_fire_num', time_col)
### Create CT Files and HY files
process_ctd.export_btl_data(btl_data_all, expocode, sectionID, expocode)
process_ctd.export_time_data(time_data_all, ssscc, int(sample_rate),
int(search_time), expocode, sectionID, ctd,
p_column_names, p_column_units)
'salinity')
salinity.lon, salinity.lat, = lon, lat
fig, ax = plt_cross_section(salinity, cmap=cm.odv, levels=0.02)
ax.set_xlabel(u'Seção de Salinidade climatológica (WOA09)'
u' no Atlântico (em longitude %3.1f\u00B0)' % longitude)
fname = 'cross_section_salinity_woa09.%s' % fmt
fig.savefig(fname, **kfig)
os.system('convert -trim %s %s' % (fname, fname))
# Profile.
fig, ax = salinity[-18.5].plot(label=u'Salinidade Prática (SP)',
linewidth=2,
figsize=(5.5, 6))
if False: # FIXME.
SA = gsw.SA_from_SP(salinity[-18.5],
temperature.index.values.astype(float), -25.5,
-18.5)
SA = gsw.SR_from_SP(salinity[-18.5])
ax.plot(SA, salinity.index, linewidth=2, label='Salnidade Absoluta (SA')
ax.grid()
ax.set_xlabel(u"[g kg$^{-1}$]")
ax.set_ylabel("Profundidade [m]")
ax.legend(numpoints=1, loc='lower right')
fname = 'profile_temperature_woa09.%s' % fmt
fig.savefig(fname, **kfig)
os.system('convert -trim %s %s' % (fname, fname))
# Atlantic cross section -- Temperature.
lon, lat, depth, temperature = get_data('woa09_temperature_at337.5.h5',
'temperature')
def IO_argo(floatdir):
from datetime import datetime, date, timedelta
programs = os.listdir(floatdir)
programs = programs[1:]
# initialize
tot_fl = 0
maxNprof = 0
ll = 0
ll1 = 0
l0 = 0
dateFlNum = []
iniDate = datetime(1950, 1, 1)
minDate = datetime(2019, 1, 1).toordinal()
# initialize big arrays
PT_dataSO = np.nan * np.ones((822 * 310, 2000), '>f4')
SA_dataSO = np.nan * np.ones((822 * 310, 2000), '>f4')
lon_dataSO = np.nan * np.ones((822 * 310), '>f4')
lat_dataSO = np.nan * np.ones((822 * 310), '>f4')
pr_dataSO = np.nan * np.ones((822 * 2000 * 310), '>f4')
yySO = np.zeros((822 * 310), 'int32')
IDSO = np.zeros((822 * 2000 * 310), 'int32')
for pp in programs:
SO_floats = os.listdir(os.path.join(floatdir, '%s' % pp))
for ff_SO in SO_floats:
print ff_SO
# @hidden_cell
SO_prof = [
f for f in os.listdir(
os.path.join(floatdir, '%s/%s' % (pp, ff_SO)))
if 'prof.nc' in f
]
tot_fl += 1
file = os.path.join(floatdir, '%s/%s/%s' % (pp, ff_SO, SO_prof[0]))
data = nc.Dataset(file)
time = data.variables['JULD'][:]
lon = data.variables['LONGITUDE'][:]
lat = data.variables['LATITUDE'][:]
pressPre = data.variables['PRES_ADJUSTED'][:].transpose()
tempPre = data.variables['TEMP_ADJUSTED'][:].transpose()
tempQC = data.variables['TEMP_ADJUSTED_QC'][:].transpose()
psaPre = data.variables['PSAL_ADJUSTED'][:].transpose()
psaQC = data.variables['PSAL_ADJUSTED_QC'][:].transpose()
# have to add the good QC: [1,2,5,8] (not in ['1','2','5','8'])
psaPre[np.where(psaQC != '1')] = np.nan
tempPre[np.where(tempQC != '1')] = np.nan
# let's get rid of some profiles that are out of the indian sector
msk1 = np.where(np.logical_and(lon >= 0, lon < 180))[0][:]
msk2 = np.where(np.logical_and(lat[msk1] > -70,
lat[msk1] < -30))[0][:]
lon = lon[msk1][msk2]
lat = lat[msk1][msk2]
time = time[msk1][msk2]
pressPre = pressPre[:, msk1[msk2]]
tempPre = tempPre[:, msk1[msk2]]
psaPre = psaPre[:, msk1[msk2]]
if time[0] + iniDate.toordinal() < minDate:
minDate = time[0]
if len(lon) > maxNprof:
maxNprof = len(lon[~np.isnan(lon)])
flN = ff_SO
# interpolate
lat = np.ma.masked_less(lat, -90)
lon = np.ma.masked_less(lon, -500)
lon[lon > 360.] = lon[lon > 360.] - 360.
Nprof = np.linspace(1, len(lat), len(lat))
# turn the variables upside down, to have from the surface to depth and not viceversa
if any(pressPre[:10, 0] > 500.):
pressPre = pressPre[::-1, :]
psaPre = psaPre[::-1, :]
tempPre = tempPre[::-1, :]
# interpolate data on vertical grid with 1db of resolution (this is fundamental to then create profile means)
fields = [psaPre, tempPre]
press = np.nan * np.ones((2000, pressPre.shape[1]))
for kk in range(press.shape[1]):
press[:, kk] = np.arange(2, 2002, 1)
psa = np.nan * np.ones((press.shape), '>f4')
temp = np.nan * np.ones((press.shape), '>f4')
for ii, ff in enumerate(fields):
for nn in range(pressPre.shape[1]):
# only use non-nan values, otherwise it doesn't interpolate well
try:
f1 = ff[:, nn][ff[:, nn].mask ==
False] #ff[:,nn][~np.isnan(ff[:,nn])]
f2 = pressPre[:, nn][ff[:, nn].mask == False]
except:
f1 = ff[:, nn]
f2 = pressPre[:, nn]
if len(f1) == 0:
f1 = ff[:, nn]
f2 = pressPre[:, nn]
try:
sp = interpolate.interp1d(f2[~np.isnan(f1)],
f1[~np.isnan(f1)],
kind='linear',
bounds_error=False,
fill_value=np.nan)
ff_int = sp(press[:, nn])
if ii == 0:
psa[:, nn] = ff_int
elif ii == 1:
temp[:, nn] = ff_int
except:
continue
#print 'At profile number %i, the float %s has only 1 record valid'
# To compute theta, I need absolute salinity [g/kg] from practical salinity (PSS-78) [unitless] and conservative temperature.
sa = np.nan * np.ones((press.shape), '>f4')
for kk in range(press.shape[1]):
sa[:, kk] = gsw.SA_from_SP(psa[:, kk], press[:, 0], lon[kk],
lat[kk])
ptemp = gsw.pt_from_CT(sa, temp)
# mask out the profiles with :
msk = np.where(lat < -1000)
lat[msk] = np.nan
lon[msk] = np.nan
sa[msk] = np.nan
sa[sa == 0.] = np.nan
ptemp[msk] = np.nan
ptemp[temp == 0.] = np.nan
# save the nprofiles
NN = np.ones((temp.shape), 'int32')
for ii in range(len(Nprof)):
NN[:, ii] = Nprof[ii]
lon_dataSO[ll1:ll1 + len(lon)] = lon
lat_dataSO[ll1:ll1 + len(lon)] = lat
PT_dataSO[ll:ll + len(sa[0, :]), :] = ptemp.T
SA_dataSO[ll:ll + len(sa[0, :]), :] = sa.T
floatID = int(ff_SO) * np.ones((sa.shape[1]), 'int32')
IDSO[ll:ll + len(sa[0, :])] = floatID
# separate seasons
dateFl = []
for dd in time:
floatDate = iniDate + timedelta(float(dd))
dateFl.append(floatDate)
dateFlNum = np.append(dateFlNum, floatDate.toordinal())
yearsSO = np.array([int(dd.year) for dd in dateFl])
yySO[ll:ll + len(sa[0, :])] = yearsSO
ll = ll + len(sa[0, :])
ll1 = ll1 + len(lon)
# chop away the part of the array with no data
PT_dataSO = PT_dataSO[:ll, :]
SA_dataSO = SA_dataSO[:ll, :]
IDSO = IDSO[:ll]
lat_dataSO = lat_dataSO[:ll]
lon_dataSO = lon_dataSO[:ll]
mmSO = mmSO[:ll]
yySO = yySO[:ll]
# remove entire columns of NaNs
idBad = []
for ii in range(PT_dataSO.shape[0]):
f0 = PT_dataSO[ii, :]
f1 = f0[~np.isnan(f0)]
if len(f1) == 0:
idBad.append(ii)
PT_dataSO = np.delete(PT_dataSO, idBad, 0)
SA_dataSO = np.delete(SA_dataSO, idBad, 0)
IDSO = np.delete(IDSO, idBad, 0)
lon_dataSO = np.delete(lon_dataSO, idBad, 0)
lat_dataSO = np.delete(lat_dataSO, idBad, 0)
mmSO = np.delete(mmSO, idBad, 0)
yySO = np.delete(yySO, idBad, 0)
idBad = []
# Interpolate again in depth.. for some reason, some profiles have still wholes in the middle:
for ii in range(PT_dataSO.shape[0]):
f0 = PT_dataSO[ii, :]
try:
sp = interpolate.interp1d(press[~np.isnan(f0), 0],
f0[~np.isnan(f0)],
kind='linear',
bounds_error=False,
fill_value=np.nan)
PT_dataSO[ii, :] = sp(press[:, 0])
f0 = SA_dataSO[ii, :]
sp = interpolate.interp1d(press[~np.isnan(f0), 0],
f0[~np.isnan(f0)],
kind='linear',
bounds_error=False,
fill_value=np.nan)
SA_dataSO[ii, :] = sp(press[:, 0])
except:
idBad.append(ii)
#print ii, ' has only 1 number'
PT_dataSO = np.delete(PT_dataSO, idBad, 0)
SA_dataSO = np.delete(SA_dataSO, idBad, 0)
IDSO = np.delete(IDSO, idBad, 0)
lon_dataSO = np.delete(lon_dataSO, idBad, 0)
lat_dataSO = np.delete(lat_dataSO, idBad, 0)
mmSO = np.delete(mmSO, idBad, 0)
yySO = np.delete(yySO, idBad, 0)
IO_argo.profSO = [PT_dataSO, SA_dataSO, lon_dataSO, lat_dataSO, IDSO]
IO_argo.temporalSO = [yySO, mmSO]
return [IO_argo.profSO, IO_argo.temporalSO]
def read_sbe_cnv(file, lat=0, lon=0):
"""
Read Seabird SBE37 .cnv file and return as xarray.Dataset.
Parameters
----------
file : str
Complete path to .cnv file
lat : float
Latitude (used for gsw calculations). Defaults to zero.
lon : float
Longitude (used for gsw calculations). Defaults to zero.
Returns
-------
mc : xarray.Dataset
Microcat data as Dataset with some metadata in the attributes.
"""
# Read cnv file using Seabird package
cnv = fCNV(file)
# parse time
mcyday = cnv["timeJV2"]
start_time_str_all = cnv.attributes["start_time"]
start_time_str = start_time_str_all.split("[")[0]
base_year = pd.to_datetime(start_time_str).year
mctime = yday1_to_datetime64(base_year, mcyday)
# let's make sure the first time stamp we generated matches the string in the cnv file
assert pd.to_datetime(np.datetime64(mctime[0],
"s")) == pd.to_datetime(start_time_str)
# data vars
dvars = {"prdM": "p", "tv290C": "t"}
mcdata = {}
for k, di in dvars.items():
if k in cnv.keys():
# print(di, ':', k)
mcdata[di] = (["time"], cnv[k])
mc = xr.Dataset(data_vars=mcdata, coords={"time": mctime})
mc.attrs["file"] = cnv.attributes["filename"]
mc.attrs["sbe_model"] = cnv.attributes["sbe_model"]
# conductivity
cvars = {"cond0mS/cm": "c", "cond0S/m": "c"}
for k, di in cvars.items():
if k in cnv.keys():
# convert from S/m to mS/cm as this is needed for gsw.SP_from_C
if k == "cond0S/m":
conductivity = cnv[k] * 10
else:
conductivity = cnv[k]
mc[di] = (["time"], conductivity)
# calculate oceanographic variables
mc["SP"] = (["time"], gsw.SP_from_C(mc.c, mc.t, mc.p))
if lat == 0 and lon == 0:
print(
"warning: absolute salinity, conservative temperature\n",
"and density calculation may be inaccurate\n",
"due to missing latitude/longitude",
)
mc["SA"] = (["time"], gsw.SA_from_SP(mc.SP, mc.p, lat=lat, lon=lon))
mc["CT"] = (["time"], gsw.CT_from_t(mc.SA, mc.t, mc.p))
mc["sg0"] = (["time"], gsw.sigma0(mc.SA, mc.CT))
# add attributes
attributes = {
"p": dict(long_name="pressure", units="dbar"),
"t": dict(long_name="in-situ temperature", units="°C"),
"CT": dict(long_name="conservative temperature", units="°C"),
"SA": dict(long_name="absolute salinity", units=r"kg/m$^3$"),
"c": dict(long_name="conductivity", units="mS/cm"),
"SP": dict(long_name="practical salinity", units=""),
"sg0": dict(long_name=r"potential density $\sigma_0$",
units=r"kg/m$^3$"),
}
for k, att in attributes.items():
if k in list(mc.variables.keys()):
mc[k].attrs = att
return mc
def getRho(self):
SA = gsw.SA_from_SP(self.sal, self.press, self.lon, self.lat)
CT = gsw.CT_from_t(SA, self.theta,
self.press) # Is this potential temperature
rho = gsw.rho(SA, CT, self.press)
return rho
def __init__(self,
time,
sal,
temp,
pres,
lon,
lat,
ballast,
pitch,
profile,
navresource,
ADCP_vel=None,
**param):
self.timestamp = time
self.time = date2float(self.timestamp)
self.pressure = pres
self.longitude = lon
self.latitude = lat
self.profile = profile
self.ADCP_vel = ADCP_vel
self.temperature = temp
self.salinity = sal
self.ballast = ballast / 1000000 # m^3
self.pitch = np.deg2rad(pitch) # rad
self.AR = 7
self.eOsborne = 0.8
self.Cd1_hull = 2.1
self.Omega = 0.75
self.param_reference = dict({
'mass': 60.772, # Vehicle mass in kg
'vol0': 59.015 /
1000, # Reference volume in m**3, with ballast at 0 (with -500 to 500 range), at surface pressure and 20 degrees C
'area_w': 0.09, # Wing surface area, m**2
'Cd_0': 0.11781, #
'Cd_1': 2.94683, #
'Cl_w': 3.82807, #
'Cl_h': 3.41939, #
'comp_p': 4.5e-06, # Pressure dependent hull compression factor
'comp_t': -6.5e-05 # Temperature dependent hull compression factor
})
self.param = self.param_reference.copy()
for k, v in param.items():
self.param[k] = v
self.param_initial = self.param
def fillGaps(x, y):
f = interp1d(x[np.isfinite(x + y)],
y[np.isfinite(x + y)],
bounds_error=False,
fill_value=np.NaN)
return (f(x))
def RM(x, N):
big = np.full([N, len(x) + N - 1], np.nan)
for n in np.arange(N):
if n == N - 1:
big[n, n:] = x
else:
big[n, n:-N + n + 1] = x
return np.nanmedian(big[:,
int(np.floor(N /
2)):-int(np.floor(N / 2))],
axis=0)
def smooth(x, N):
return np.convolve(x, np.ones(N) / N, mode='same')
self.depth = gsw.z_from_p(
self.pressure, self.latitude
) # m . Note depth (Z) is negative, so diving is negative dZdt
self.dZdt = np.gradient(self.depth, self.time) # m.s-1
self.g = gsw.grav(self.latitude, self.pressure)
self.SA = gsw.SA_from_SP(self.salinity, self.pressure, self.longitude,
self.latitude)
self.CT = gsw.CT_from_t(self.SA, self.temperature, self.pressure)
self.rho = gsw.rho(self.SA, self.CT, self.pressure)
### Basic model
# Relies on steady state assumption that buoyancy, weight, drag and lift cancel out when not accelerating.
# F_B - cos(glide_angle)*F_L - sin(glide_angle)*F_D - F_g = 0
# cos(glide_angle)*F_L + sin(glide_angle)*F_D = 0
# Begin with an initial computation of angle of attack and speed through water:
self.model_function()
### Get good datapoints to regress over
self._valid = np.full(np.shape(self.pitch), True)
self._valid[self.pressure < 5] = False
self._valid[np.abs(self.pitch) < 0.2] = False # TODO change back to 15
self._valid[np.abs(self.pitch) > 0.6] = False # TODO change back to 15
self._valid[np.abs(np.gradient(self.dZdt, self.time)) >
0.005] = False # Accelerations
self._valid[np.gradient(self.pitch, self.time) == 0] = False
self._valid = self._valid & ((navresource == 100) |
(navresource == 117))
print('Number of valid points: ' + str(np.count_nonzero(self._valid)) +
' (out of ' + str(len(self._valid)) + ')')
# Do first pass regression on vol parameters, or volume and hydro?
self.regression_parameters = ('vol0', 'Cd_0', 'Cd_1', 'comp_p',
'comp_t')
def __init__(self, time, sal, temp, pres, lon, lat, ballast, pitch,
profile, navresource, **param):
self.timestamp = time
self.time = date2float(self.timestamp)
self.pressure = pres
self.longitude = lon
self.latitude = lat
self.profile = profile
self.temperature = temp
self.salinity = sal
self.ballast = ballast / 1000000 # m^3
self.pitch = np.deg2rad(pitch) # rad
self._dives = np.full(len(pres), True)
self.param_reference = dict({
'mass': 60.772, # Vehicle mass in kg
'vol0': 60 /
1000, # Reference volume in m**3, with ballast at 0 (with -500 to 500 range), at surface pressure and 20 degrees C
'hd_a': 0.015, # 0.003, Wing surface area, m**2
'hd_b': 0.018, # 0.0118
'hd_c': 9.85e-6, #
'hd_s': 0.2, # -0.25, #
'comp_p': 4.5e-06, # Pressure dependent hull compression factor
'comp_t': -6.5e-05 # Temperature dependent hull compression factor
})
self.param = self.param_reference.copy()
for k, v in param.items():
self.param[k] = v
self.param_initial = self.param
def fillGaps(x, y):
f = interp1d(x[np.isfinite(x + y)],
y[np.isfinite(x + y)],
bounds_error=False,
fill_value=np.NaN)
return (f(x))
def RM(x, N):
big = np.full([N, len(x) + N - 1], np.nan)
for n in np.arange(N):
if n == N - 1:
big[n, n:] = x
else:
big[n, n:-N + n + 1] = x
return np.nanmedian(big[:,
int(np.floor(N /
2)):-int(np.floor(N / 2))],
axis=0)
def smooth(x, N):
return np.convolve(x, np.ones(N) / N, mode='same')
self.depth = gsw.z_from_p(
self.pressure, self.latitude
) # m . Note depth (Z) is negative, so diving is negative dZdt
self.dZdt = np.gradient(self.depth, self.time) # m.s-1
self.g = gsw.grav(self.latitude, self.pressure)
self.SA = gsw.SA_from_SP(self.salinity, self.pressure, self.longitude,
self.latitude)
self.CT = gsw.CT_from_t(self.SA, self.temperature, self.pressure)
self.rho = gsw.rho(self.SA, self.CT, self.pressure)
# Begin with an initial computation of angle of attack and speed through water:
self.model_function()
### Get good datapoints to regress over
self._valid = np.full(np.shape(self.pitch), True)
self._valid[self.pressure < 5] = False
self._valid[np.abs(self.pitch) < 0.2] = False # TODO change back to 15
self._valid[np.abs(self.pitch) > 0.6] = False # TODO change back to 15
self._valid[np.abs(np.gradient(self.dZdt, self.time)) >
0.0005] = False # Accelerations
self._valid[np.gradient(self.pitch, self.time) == 0] = False
self._valid = self._valid & ((navresource == 100) |
(navresource == 117))
# Do first pass regression on vol parameters, or volume and hydro?
self.regression_parameters = ('vol0', 'hd_a', 'hd_b', 'hd_c', 'comp_t',
'comp_p')
1) Cabbeling ****************************************************************************""" # You make two measurements of seawater with a CTD... # T (in-situ temperature, C) T1 = 0.0 T2 = 16.45 # Sp (Practical Salinity, PSU) S1 = 31.0 S2 = 32.0 p0 = 0 # dbar pressure at surface lat = 45 # N lon = -30 # E # First convert the measurments to absolute salinity Sa1 = sw.SA_from_SP(S1,p0,lon,lat) Sa2 = sw.SA_from_SP(S2,p0,lon,lat) # ...and conservative temperature. Tc1 = sw.CT_from_t(Sa1,T1,p0) Tc2 = sw.CT_from_t(Sa2,T2,p0) # Now calculate the density of each water parcel? rho1 = sw.rho(Sa1,T1,p0) rho2 = sw.rho(Sa2,T2,p0) #Which water mass is denser? print"The measurement 1 is", round(rho1-rho2,SF),"kg/m^2 denser than measurement 2." #What is their average density? print"Their average density is",round((rho1+rho2)/2,SF),"."
def find_sigma0_z(salinity, temperature, pressure, latitude, longitude,
sigmas):
"""
Find the depth of the isopycnal using T/S from a cast
Inputs:
salinity (array) : Salinity in PSU
temperature (array) : In-situ temperature (C)
latitude (array) : Latitude of the sample
longitude (array) : Longitude of the sample
sigmas (array) : Isopycnals for which to find the depth
Outputs:
sigma_z (array) : Depth of the isopycnals
Algorithm:
1. Convert in-situ salinity and temperature to Absolute Salinity and Conservative Temperature
2. Calculate sigma0
3. Check to ensure the isopycnal surface spans the density range of the water column
4. Build interpolating functions for both T and S
5. Sweep down and find the first place where the specified density exists in between adjacent points
6. Use the EOS to find the where the zero crossing is
"""
def dens_diff(pressure, SA_f, CT_f, target_sigma0):
return np.square(
gsw.sigma0(SA_f(pressure), CT_f(pressure)) - target_sigma0)
def return_early(is_scalar):
if is_scalar:
return np.nan
else:
return np.nan * np.ones(sigmas.shape)
# Promote to single element array if only one level requested
valid = (~np.isnan(salinity)) & (~np.isnan(temperature)) & (
~np.isnan(pressure))
salinity = salinity[valid]
temperature = temperature[valid]
pressure = pressure[valid]
longitude = longitude[valid]
latitude = latitude[valid]
is_scalar = type(sigmas) == type(0.)
if is_scalar:
sigmas = np.array([sigmas])
if valid.sum() < 3:
return return_early(is_scalar)
pressure = np.array(pressure)
SA = gsw.SA_from_SP(salinity, pressure, longitude, latitude)
CT = gsw.CT_from_t(salinity, temperature, pressure)
P_sort = np.unique(pressure)
if len(P_sort) < 3:
return return_early(is_scalar)
SA_sort = np.zeros(P_sort.shape)
CT_sort = np.zeros(P_sort.shape)
# Loop through all pressures that overlap, average the temperatures and salinity accordingly
for idx, P in enumerate(P_sort):
presidx = (pressure == P)
SA_sort[idx] = SA[presidx].mean()
CT_sort[idx] = CT[presidx].mean()
SA_intp = intp.interp1d(P_sort, SA_sort, kind='quadratic')
CT_intp = intp.interp1d(P_sort, CT_sort, kind='quadratic')
sigma0 = gsw.sigma0(SA_sort, CT_sort)
sigma0_max = sigma0.max()
sigma0_min = sigma0.min()
sigma0_z = np.zeros(len(sigmas))
for sigidx, siglev in enumerate(sigmas):
if (siglev < sigma0_min) or (siglev > sigma0_max):
sigma0_z[sigidx] = np.nan
else:
for botidx in range(len(sigma0) - 1):
if (siglev >= sigma0[botidx]) & (siglev <= sigma0[botidx + 1]):
start_idx = botidx
break
out = optimize.minimize_scalar(
dens_diff,
bounds=[P_sort[botidx], P_sort[botidx + 1]],
method='Bounded',
args=(SA_intp, CT_intp, siglev))
sigma0_z[sigidx] = out.x
if is_scalar:
sigma0_z = np.squeeze(sigma0_z)
return sigma0_z
def MLD(PresOld, TempOld, SalOld, Lat, Lon, InterpFlag):
# Calculate MLD of a single profile
# Extrapolates surface values only if min pres is less than 15 dbar
MinSurfP=0
MaxSurfP=10
j=0
surf_flag=0
surf_pres_i=[]
dense_offset=.03
mld_pres=np.NaN
# Interpolate pressure and other variables
sflag = 0
if InterpFlag == True:
if np.nanmin(PresOld) < 15:
Pres = np.arange(1,np.nanmax(PresOld))
sal_int = interpolate.interp1d(PresOld, SalOld, fill_value = 'extrapolate')
temp_int = interpolate.interp1d(PresOld, TempOld, fill_value = 'extrapolate')
Sal = sal_int(Pres)
Temp = temp_int(Pres)
sflag = 1
else:
Pres = PresOld
Temp = TempOld
Sal = SalOld
sflag = 1
if sflag == 1:
if np.sum(np.isnan(Pres)) != len(Pres) and np.nanmin(Pres) <= MaxSurfP:
while (j<len(Pres) and surf_flag ==0):
if (Pres[j]>= MinSurfP and Pres[j] <= MaxSurfP):
surf_pres_i=surf_pres_i+[j]
if Pres[j] > MaxSurfP:
surf_flag=1
j=j+1
if surf_pres_i != []:
if len(surf_pres_i)>1:
s_start=surf_pres_i[0]
s_end=surf_pres_i[-1]+1
P_mean=np.nanmean(Pres[s_start:s_end])
else:
P_mean=surf_pres_i[0]
s_end=surf_pres_i[-1]+1
# Calculate density
SA=gsw.SA_from_SP(Sal,Pres,Lat,Lon)
CT=gsw.CT_from_t(SA,Temp,P_mean)
density = np.zeros(len(Pres))
density[:]=np.NaN
for k in np.arange(len(density)):
density[k]=gsw.density.sigma0(SA=SA[k],CT=CT[k])
if len(surf_pres_i)>1:
surf_dense=np.nanmean(density[s_start:s_end])
else:
surf_dense=np.nanmean(density[surf_pres_i[0]])
if np.isnan(surf_dense) == False:
#print(surf_dense)
rho_mld=surf_dense+dense_offset
#print(mld_dense)
# Find MLD
mld_flag=np.NaN
inter_flag=np.NaN
mld_pres=np.NaN
P_L=np.NaN
rho_L=np.NaN
P_D=np.NaN
rho_D=np.NaN
mld_exact = 0
j = s_end -1
# Start search at base of surface layer
while(j<len(density) and np.isnan(mld_flag)==True):
if density[j] == rho_mld:
mld_pres=Pres[j]
mld_flag=1
mld_exact = 1
elif density[j]<rho_mld:
P_L=Pres[j]
rho_L=density[j]
elif density[j]>rho_mld:
P_D=Pres[j]
rho_D=density[j]
inter_flag=1
mld_flag=1
j=j+1
#if mld_exact == 1:
# mld_pres=mld_pres
if (np.isnan(mld_flag) == True):
# Ran through entire profile and did not find mld
# i.e. MLD is deeper than deepest pressure measurement
# So make md deepest pressure measurement
# OR 2000 dbar?
mld_pres=np.nanmax(Pres)
elif (np.isnan(mld_flag)== False and inter_flag == 1):
# Need to interpolate to get MLD
#mld_pres=M_D-(((M_D-mld_dense)/(M_D-L_D))*(M_P-L_P))
mld_pres = P_D-(P_D - P_L)*((rho_D-rho_mld)/(rho_D-rho_L))
elif (np.isnan(mld_flag)== False and inter_flag == 0):
mld_pres=mld_pres
else:
print('ERROR')
return mld_pres
# Determine if the roundup or rounddown date is closer
rd_dif = abs(
(prof_date -
rounddown_date).total_seconds())
ru_dif = abs(
(prof_date -
roundup_date).total_seconds())
if ru_dif <= rd_dif:
close_date = roundup_date
else:
close_date = rounddown_date
# Unit Conversion
SA = gsw.SA_from_SP(
surf_S, P, lon[j], lat[j])
# Calculate conservative temp or temp
CT = gsw.CT_from_t(SA, surf_T, P)
# Calculate density
#dense=gsw.density.rho_t_exact(SA, T, P) # kg/m^3
surf_dense = gsw.density.sigma0(SA,
CT) + 1000
# Calculate oxygen saturation at each point
O2_sol = gsw.O2sol(SA, CT, P, lon[j],
lat[j])
oxy_sat = np.round(surf_O / O2_sol * 100,
2)
oxy_dev = surf_O - O2_sol
distlon = (lon_int - iflon) * 111e3
fdist = np.sign(lat_int - iflat) * np.sqrt((
(lat_int - iflat) * 111e3 * np.cos(lat_int))**2 +
((lon_int - iflon) * 111e3)**2)
#%% HISTO IN T-S Space 3 panel
cmap = 'gnuplot'
cl = [-5, -1]
norm = np.max(fluorppb_ts)
mask = (fluorppb_ts / norm > 10**(cl[0])) & (jday_ts < 68)
#mask = jday_ts<90
xedges = np.linspace(34.5, 37, 150)
yedges = np.linspace(13, 22, 150)
X, Y = np.meshgrid(xedges, yedges)
SA = gsw.SA_from_SP(X, 0, -66, 39)
CT = gsw.CT_from_t(SA, Y, 0)
R = gsw.rho(SA, CT, 0)
R = R - 1000
SA = gsw.SA_from_SP(fS, fP, -66, 39)
CT = gsw.CT_from_t(SA, fT, fP)
Rf = gsw.rho(SA, CT, 0)
Rf = Rf - 1000
norm = np.sum(fluorppb_ts[mask])
norm = 1
H, xedges, yedges = np.histogram2d(S_ts[mask],
T_ts[mask],
weights=(fluorppb_ts[mask] / norm),
bins=(xedges, yedges),
density=False)
def teos10(self,
vlist: list = ['SA', 'CT', 'SIG0', 'N2', 'PV', 'PTEMP'],
inplace: bool = True):
""" Add TEOS10 variables to the dataset
By default, add: 'SA', 'CT', 'SIG0', 'N2', 'PV', 'PTEMP'
Rely on the gsw library.
Parameters
----------
vlist: list(str)
List with the name of variables to add.
inplace: boolean, True by default
If True, return the input :class:`xarray.Dataset` with new TEOS10 variables added as a new :class:`xarray.DataArray`
If False, return a :class:`xarray.Dataset` with new TEOS10 variables
Returns
-------
:class:`xarray.Dataset`
"""
if not with_gsw:
raise ModuleNotFoundError(
"This functionality requires the gsw library")
this = self._obj
to_profile = False
if self._type == 'profile':
to_profile = True
this = this.argo.profile2point()
# Get base variables as numpy arrays:
psal = this['PSAL'].values
temp = this['TEMP'].values
pres = this['PRES'].values
lon = this['LONGITUDE'].values
lat = this['LATITUDE'].values
f = lat
# Coriolis
f = gsw.f(lat)
# Depth:
depth = gsw.z_from_p(pres, lat)
# Absolute salinity
sa = gsw.SA_from_SP(psal, pres, lon, lat)
# Conservative temperature
ct = gsw.CT_from_t(sa, temp, depth)
# Potential Temperature
if 'PTEMP' in vlist:
pt = gsw.pt_from_CT(sa, ct)
# Potential density referenced to surface
if 'SIG0' in vlist:
sig0 = gsw.sigma0(sa, ct)
# N2
if 'N2' in vlist or 'PV' in vlist:
n2_mid, p_mid = gsw.Nsquared(sa, ct, pres, lat)
# N2 on the CT grid:
ishallow = (slice(0, -1), Ellipsis)
ideep = (slice(1, None), Ellipsis)
def mid(x):
return 0.5 * (x[ideep] + x[ishallow])
n2 = np.zeros(ct.shape) * np.nan
n2[1:-1] = mid(n2_mid)
# PV:
if 'PV' in vlist:
pv = f * n2 / gsw.grav(lat, pres)
# Back to the dataset:
that = []
if 'SA' in vlist:
SA = xr.DataArray(sa, coords=this['PSAL'].coords, name='SA')
SA.attrs['standard_name'] = 'Absolute Salinity'
SA.attrs['unit'] = 'g/kg'
that.append(SA)
if 'CT' in vlist:
CT = xr.DataArray(ct, coords=this['TEMP'].coords, name='CT')
CT.attrs['standard_name'] = 'Conservative Temperature'
CT.attrs['unit'] = 'degC'
that.append(CT)
if 'SIG0' in vlist:
SIG0 = xr.DataArray(sig0, coords=this['TEMP'].coords, name='SIG0')
SIG0.attrs[
'long_name'] = 'Potential density anomaly with reference pressure of 0 dbar'
SIG0.attrs['standard_name'] = 'Potential Density'
SIG0.attrs['unit'] = 'kg/m^3'
that.append(SIG0)
if 'N2' in vlist:
N2 = xr.DataArray(n2, coords=this['TEMP'].coords, name='N2')
N2.attrs['standard_name'] = 'Squared buoyancy frequency'
N2.attrs['unit'] = '1/s^2'
that.append(N2)
if 'PV' in vlist:
PV = xr.DataArray(pv, coords=this['TEMP'].coords, name='PV')
PV.attrs['standard_name'] = 'Planetary Potential Vorticity'
PV.attrs['unit'] = '1/m/s'
that.append(PV)
if 'PTEMP' in vlist:
PTEMP = xr.DataArray(pt, coords=this['TEMP'].coords, name='PTEMP')
PTEMP.attrs['standard_name'] = 'Potential Temperature'
PTEMP.attrs['unit'] = 'degC'
that.append(PTEMP)
# Create a dataset with all new variables:
that = xr.merge(that)
# Add to the dataset essential Argo variables (allows to keep using the argo accessor):
that = that.assign({
k: this[k]
for k in [
'TIME', ' LATITUDE', 'LONGITUDE', 'PRES', 'PRES_ADJUSTED',
'PLATFORM_NUMBER', 'CYCLE_NUMBER', 'DIRECTION'
] if k in this
})
# Manage output:
if inplace:
# Merge previous with new variables
for v in that.variables:
this[v] = that[v]
if to_profile:
this = this.argo.point2profile()
for k in this:
if k not in self._obj:
self._obj[k] = this[k]
return self._obj
else:
if to_profile:
return that.argo.point2profile()
else:
return that
def __init__(self, time, sal, temp_ext, temp_int, pres_ext, pres_int, lon,
lat, ballast, pitch, profile, navresource, tau,
speed_through_water, **param):
# Questions:
# is dzdt spiky?
# do values need to be interpolated?
# Parse input variables:
self.timestamp = time
self.time = date2float(self.timestamp)
self.external_pressure = pres_ext
self.internal_pressure = pres_int
self.external_temperature = temp_ext
self.internal_temperature = temp_int
self.longitude = lon
self.latitude = lat
self.profile = profile
self.salinity = sal
self.ballast = ballast / 1000000 # m^3
self.pitch = np.deg2rad(pitch) # rad
self.navresource = navresource
self.tau = tau
self.speed_through_water = speed_through_water
# Rerefence parameters for scaling and initial guesses:
self.param_reference = dict({
'mass': 60, # Vehicle mass in kg
'vol0':
0.06, # Reference volume in m**3, with ballast at 0 (with -500 to 500 range), at surface pressure and 20 degrees C
'area_w': 0.24, # Wing surface area, m**2
'Cd_0': 0.046, #
'Cd_1': 2.3, #
'Cl': 2.0, # Negative because wrong convention on theta
'comp_p':
4.7e-06, #1.023279317627415e-06, #Pressure dependent hull compression factor
'comp_t':
8.9e-05, #1.5665248101730484e-04, # Temperature dependent hull compression factor
'SSStau': 18.5 #characteristic response time of the glider in sec
})
self.param = self.param_reference.copy()
for k, v in param.items():
self.param[k] = v
self.param_initial = self.param
self.depth = gsw.z_from_p(
self.external_pressure, self.latitude
) # m . Note depth (Z) is negative, so diving is negative dZdt
self.dZdt = np.gradient(self.depth, self.time) # m.s-1
self.g = gsw.grav(self.latitude, self.external_pressure)
self.SA = gsw.SA_from_SP(self.salinity, self.external_pressure,
self.longitude, self.latitude)
self.CT = gsw.CT_from_t(self.SA, self.external_temperature,
self.external_pressure)
self.rho = gsw.rho(self.SA, self.CT, self.external_pressure)
### Basic model
# Relies on steady state assumption that buoyancy, weight, drag and lift cancel out when not accelerating.
# F_B - cos(glide_angle)*F_L - sin(glide_angle)*F_D - F_g = 0
# cos(glide_angle)*F_L + sin(glide_angle)*F_D = 0
# Begin with an initial computation of angle of attack and speed through water:
self.model_function()
### Get good datapoints to regress over
self._valid = np.full(np.shape(self.pitch), True)
self._valid[self.external_pressure < 5] = False
self._valid[np.abs(self.pitch) < 0.2] = False # TODO change back to 15
self._valid[np.abs(self.pitch) > 0.6] = False # TODO change back to 15
self._valid[np.abs(np.gradient(self.dZdt, self.time)) >
0.0005] = False # Accelerations
self._valid[np.gradient(self.pitch, self.time) ==
0] = False # Rotation
self._valid = self._valid & (
(navresource == 100) | (navresource == 117)
) #100=glider going down & 117=glider going up (=> not at surface or inflecting)
# Do first pass regression on vol parameters, or volume and hydro?
self.regression_parameters = ('vol0', 'Cd_0', 'Cd_1', 'Cl', 'comp_p',
'comp_t')
def find_isopycnals(p_btl_col, t_btl_col, sal_btl_col, dov_btl_col,
lat_btl_col, lon_btl_col, btl_data, p_col, t_col, sal_col,
dov_col, lat_col, lon_col, time_data):
"""find_iscopycnals
p_btl_col: Pressure column for bottle data
t_btl_col: Temperature column for bottle data
sal_btl_col: Salinity column for bottle data
dov_btl_col: Oxygen voltage column for bottle data
lat_btl_col: Latitude bottle column for bottle data
lon_btl_col: Longitude bottle column for bottle data
btl_data: Bottle data ndarray
p_col: Pressure column for bottle data
t_col: Temperature column for bottle data
sal_col: Salinity column for bottle data
dov_col: Oxygen voltage column for bottle data
lat_col: Latitude column for bottle data
lon_col: Longitude column for bottle data
time_data: Time data ndarray
"""
time_sigma = []
CT = gsw.CT_from_t(time_data[sal_col], time_data[t_col], time_data[p_col])
SA = gsw.SA_from_SP(time_data[sal_col], time_data[p_col],
time_data[lon_col], time_data[lat_col])
time_sigma = gsw.sigma0(SA, CT)
# Pressure-bounded isopycnal search.
# Based on maximum decent rate and package size.
CT = gsw.CT_from_t(btl_data[sal_btl_col], btl_data[t_btl_col],
btl_data[p_btl_col])
SA = gsw.SA_from_SP(btl_data[sal_btl_col], btl_data[p_btl_col],
btl_data[lon_btl_col], btl_data[lat_btl_col])
btl_sigma = gsw.sigma0(SA, CT)
for i in range(0, len(btl_data[p_btl_col])):
#CT = gsw.CT_from_t(btl_data[sal_btl_col][i],btl_data[t_btl_col][i],btl_data[p_btl_col][i])
#SA = gsw.SA_from_SP(btl_data[sal_btl_col][i],btl_data[p_btl_col][i],btl_data[lon_btl_col][i],btl_data[lat_btl_col][i])
#btl_sigma = gsw.sigma0(SA,CT)
p_indx = find_nearest(time_data[p_col], btl_data[p_btl_col][i])
indx = find_nearest(
time_sigma[p_indx - int(24 * 1.5):p_indx + int(24 * 1.5)],
btl_sigma[i])
#print('Bottle:')
#print('Sigma: '+str(btl_sigma))
#print('Pres: '+str(btl_data[p_btl_col][i]))
#print('Temp: '+str(btl_data[t_btl_col][i]))
#print('Salt: '+str(btl_data[sal_btl_col][i]))
#print('Pressure: '+str(p_indx)+' '+str(indx+p_indx))
#print('Sigma: '+str(time_sigma[indx+p_indx]))
#print('Pres: '+str(time_data[p_col][indx+p_indx]))
#print('Temp: '+str(time_data[t_col][indx+p_indx]))
#print('Salt: '+str(time_data[sal_col][indx+p_indx]))
if indx + p_indx > len(time_sigma):
btl_data[t_btl_col][i] = time_data[t_col][len(time_data) - 1]
btl_data[sal_btl_col][i] = time_data[sal_col][len(time_data) - 1]
btl_data[dov_btl_col][i] = time_data[dov_col][len(time_data) - 1]
else:
btl_data[t_btl_col][i] = time_data[t_col][indx + p_indx]
btl_data[sal_btl_col][i] = time_data[sal_col][indx + p_indx]
btl_data[dov_btl_col][i] = time_data[dov_col][indx + p_indx]
return btl_data
def prepare_oxy(btl_df, time_df, ssscc_list):
"""
Calculate oxygen-related variables needed for calibration:
sigma, oxygen solubility (OS), and bottle oxygen
Parameters
----------
btl_df : DataFrame
CTD data at bottle stops
time_df : DataFrame
Continuous CTD data
ssscc_list : list of str
List of stations to process
Returns
-------
"""
# Calculate SA and CT
btl_df["SA"] = gsw.SA_from_SP(
btl_df[cfg.column["sal"]],
btl_df[cfg.column["p_btl"]],
btl_df[cfg.column["lon_btl"]],
btl_df[cfg.column["lat_btl"]],
)
btl_df["CT"] = gsw.CT_from_t(
btl_df["SA"],
btl_df[
cfg.column["t1_btl"]], # oxygen sensor is on primary line (ie t1)
btl_df[cfg.column["p_btl"]],
)
time_df["SA"] = gsw.SA_from_SP(
time_df[cfg.column["sal"]],
time_df[cfg.column["p"]],
time_df[cfg.column["lon_btl"]],
time_df[cfg.column["lat_btl"]],
)
time_df["CT"] = gsw.CT_from_t(
time_df["SA"],
time_df[cfg.column["t1"]], # oxygen sensor is on primary line (ie t1)
time_df[cfg.column["p"]],
)
# calculate sigma
btl_df["sigma_btl"] = gsw.sigma0(btl_df["SA"], btl_df["CT"])
time_df["sigma_btl"] = gsw.sigma0(time_df["SA"], time_df["CT"])
# Calculate oxygen solubility in µmol/kg
btl_df["OS"] = gsw.O2sol(
btl_df["SA"],
btl_df["CT"],
btl_df[cfg.column["p_btl"]],
btl_df[cfg.column["lon_btl"]],
btl_df[cfg.column["lat_btl"]],
)
time_df["OS"] = gsw.O2sol(
time_df["SA"],
time_df["CT"],
time_df[cfg.column["p"]],
time_df[cfg.column["lon"]],
time_df[cfg.column["lat"]],
)
# Convert CTDOXY units
btl_df["CTDOXY"] = oxy_ml_to_umolkg(btl_df["CTDOXY1"], btl_df["sigma_btl"])
# Calculate bottle oxygen
btl_df[cfg.column["oxy_btl"]] = calculate_bottle_oxygen(
ssscc_list,
btl_df["SSSCC"],
btl_df["TITR_VOL"],
btl_df["TITR_TEMP"],
btl_df["FLASKNO"],
)
btl_df[cfg.column["oxy_btl"]] = oxy_ml_to_umolkg(
btl_df[cfg.column["oxy_btl"]], btl_df["sigma_btl"])
btl_df["OXYGEN_FLAG_W"] = flagging.nan_values(
btl_df[cfg.column["oxy_btl"]])
# Load manual OXYGEN flags
if Path("data/oxygen/manual_oxy_flags.csv").exists():
manual_flags = pd.read_csv("data/oxygen/manual_oxy_flags.csv",
dtype={"SSSCC": str})
for _, flags in manual_flags.iterrows():
df_row = (btl_df["SSSCC"] == flags["SSSCC"]) & (
btl_df["btl_fire_num"] == flags["Bottle"])
btl_df.loc[df_row, "OXYGEN_FLAG_W"] = flags["Flag"]
return True
def __init__(self,eyed, data,tempunit,salunit):
##id of profiles plus info
if tempunit not in ["insitu","conservative","potential"]:
raise ValueError("This temperature unit is not supported")
if salunit not in ["practical","absolute","insitu"]:
raise ValueError("This salinity unit is not supported")
if not {"sal","temp","pres","lat","lon"}.issubset(data.keys()):
raise ValueError("This does not contain the required information")
if abs(max(data["pres"])-min(data["pres"])) <50:
print(data["pres"])
raise ValueError("This does not contain enough pressure information ")
self.eyed = eyed
self.lat = np.abs(data["lat"])
self.maplat = data["lat"]
self.lon = data["lon"]
self.f = gsw.f(self.lat)
self.gamma = (9.8)/(self.f*1025.0)
if "time" in data.keys():
self.time = data["time"]
if "cruise" in data.keys():
self.cruise = data["cruise"]#+str(self.time.year)
if "station" in data.keys():
self.station = data["station"]#+str(self.time.year)
if "knownns" in data.keys():
self.knownns = data["knownns"]
else:
self.knownns = {}
if "relcoord" in data.keys():
self.relcoord = data["relcoord"]
if "knownu" in data.keys():
self.knownu = data["knownu"]
self.knownv = data["knownv"]
if "knownpsi" in data.keys():
self.knownpsi = data["knownpsi"]
if "kapredi" in data.keys():
self.kapredi = data["kapredi"]
self.kapgm = data["kapgm"]
self.diffkr = data["diffkr"]
#Temerature Salinity and Pressure
self.temps = np.asarray(data["temp"])
self.sals = np.asarray(data["sal"])
self.pres = np.abs(np.asarray(data["pres"]))
if hasattr(self.temps,"mask"):
print("mask")
flter = np.logical_or(~self.temps.mask,~self.sals.mask)
self.temps = self.temps[flter]
self.sals = self.sals[flter]
self.pres = self.pres[flter]
if np.isnan(self.temps).any() or np.isnan(self.sals).any():
print("huh")
if salunit == "practical":
self.sals = gsw.SA_from_SP(self.sals,self.pres,self.lon,self.maplat)
if tempunit == "potential":
self.temps = gsw.CT_from_pt(self.sals,self.temps)
if tempunit == "insitu":
self.temps = gsw.CT_from_t(self.sals,self.temps,np.abs(self.pres))
s = np.argsort(self.pres)
self.temps = self.temps[s]
self.sals = self.sals[s]
self.pres = self.pres[s]
##Interpolated Temperature, Salinity, and Pressure
self.itemps = []
self.isals = []
self.ipres = []
theta = np.deg2rad(self.lat)
r = (90-self.lon) *111*1000
x = (r*np.cos(theta))
y = (r*np.sin(theta))
self.x = x
self.y = y
self.idensities = []
self.neutraldepth = {}
self.interpolate()
def createFuncs(self, X, t, lonpad=1.5, latpad=1.5, tpad=1.5):
"""
Pass data fields of satgem to generate interpolation functions
Generate Interpolation functions
Paramaters
----------
X: Position Vector
t: initial time (center of the interpolation field)
"""
if tpad < 7:
tpad = 7
# Get indicies for subset of satgem/bathy data
lonind = self.gem.temp.lon.sel(lon=slice(X[0] - lonpad, X[0] + lonpad))
clonind = self.gem.V.clon.sel(clon=slice(X[0] - lonpad, X[0] + lonpad))
latind = self.gem.temp.lat.sel(lat=slice(X[1] - latpad, X[1] + latpad))
clatind = self.gem.U.clat.sel(clat=slice(X[1] - latpad, X[1] + latpad))
tind = self.gem.temp.time.sel(time=slice(t - tpad, t + tpad))
blonind = self.bathy_file.lon.sel(lon=slice(X[0] - lonpad, X[0] +
lonpad))
blatind = self.bathy_file.lat.sel(lat=slice(X[1] - latpad, X[1] +
latpad))
bsub = self.bathy_file.elevation.sel(lon=blonind, lat=blatind)
setattr(
self, 'bathy',
LinearNDInterpolatorExt((blonind, blatind),
bsub.T,
fill_value=None))
N2 = []
rho = []
for i in range(tind.shape[0]):
SA = gsw.SA_from_SP(
self.gem.sal.sel(lon=lonind, lat=latind, time=tind[i]),
self.gem.depth[:], X[0], X[1])
CT = gsw.CT_from_t(
SA, self.gem.temp.sel(lon=lonind, lat=latind, time=tind[0]),
self.gem.depth[:])
if i == 0:
# since the pmid grid will be uniform, only save once
n2i, pmid = gsw.Nsquared(SA, CT, self.gem.depth, axis=2)
N2.append(n2i)
rho.append(gsw.sigma0(
SA,
CT,
))
else:
N2.append(gsw.Nsquared(SA, CT, self.gem.depth, axis=2)[0])
rho.append(gsw.sigma0(
SA,
CT,
))
FN2 = LinearNDInterpolatorExt(
(self.gem.lon.sel(lon=lonind), self.gem.lat.sel(lat=latind),
pmid[0, 0, :], self.gem.time.sel(time=tind)), np.stack(N2,
axis=3))
rho1 = LinearNDInterpolatorExt(
(self.gem.lon.sel(lon=lonind), self.gem.lat.sel(lat=latind),
self.gem.depth, self.gem.time.sel(time=tind)),
np.stack(rho, axis=3))
setattr(self, 'N2', FN2)
setattr(self, 'rho', rho1)
N2 = np.absolute(np.stack(N2, axis=3))
N2 = np.sqrt(N2)
# Don't actually need this as a dataArray but its used as one further down and im too lazy to change it
N2 = xr.DataArray(N2,
coords=[
self.gem.lon.sel(lon=lonind),
self.gem.lat.sel(lat=latind), pmid[0, 0, :],
self.gem.time.sel(time=tind)
],
dims=['lon', 'lat', 'depth', 'time'],
name='N2')
Usub = self.gem.U.sel(lon=lonind, clat=clatind, time=tind)
Tsub = self.gem.temp.sel(lon=lonind, lat=latind, time=tind)
Vsub = self.gem.V.sel(clon=clonind, lat=latind, time=tind)
# space and time Gradients
delt = Usub.time.diff(dim='time') * 24 * 60 * \
60 # time delta in seconds
# U gradients
dxu = gsw.distance(np.meshgrid(Usub.lon, Usub.clat)[0],
np.meshgrid(Usub.lon, Usub.clat)[1],
axis=1)
dxu = np.repeat(np.repeat(dxu.T[:, :, np.newaxis],
Usub.shape[2],
axis=2)[:, :, :, np.newaxis],
Usub.shape[3],
axis=3)
dyu = gsw.distance(np.meshgrid(Usub.lon, Usub.clat)[0],
np.meshgrid(Usub.lon, Usub.clat)[1],
axis=0)
dyu = np.repeat(np.repeat(dyu.T[:, :, np.newaxis],
Usub.shape[2],
axis=2)[:, :, :, np.newaxis],
Usub.shape[3],
axis=3)
# V gradients
dxv = gsw.distance(np.meshgrid(Vsub.clon, Vsub.lat)[0],
np.meshgrid(Vsub.clon, Vsub.lat)[1],
axis=1)
dxv = np.repeat(np.repeat(dxv.T[:, :, np.newaxis],
Usub.shape[2],
axis=2)[:, :, :, np.newaxis],
Usub.shape[3],
axis=3)
dyv = gsw.distance(np.meshgrid(Vsub.clon, Vsub.lat)[0],
np.meshgrid(Vsub.clon, Vsub.lat)[1],
axis=0)
dyv = np.repeat(np.repeat(dyv.T[:, :, np.newaxis],
Usub.shape[2],
axis=2)[:, :, :, np.newaxis],
Usub.shape[3],
axis=3)
# N2 gradient
dxn = gsw.distance(np.meshgrid(N2.lon, N2.lat)[0],
np.meshgrid(N2.lon, N2.lat)[1],
axis=1)
dxn = np.repeat(np.repeat(dxn.T[:, :, np.newaxis], N2.shape[2],
axis=2)[:, :, :, np.newaxis],
N2.shape[3],
axis=3)
dyn = gsw.distance(np.meshgrid(N2.lon, N2.lat)[0],
np.meshgrid(N2.lon, N2.lat)[1],
axis=0)
dyn = np.repeat(np.repeat(dyn.T[:, :, np.newaxis], N2.shape[2],
axis=2)[:, :, :, np.newaxis],
N2.shape[3],
axis=3)
dz = np.nanmean(np.diff(Usub.depth))
# Spatial Gradient revisied grids
clat = (Usub.clat[:-1] + np.diff(Usub.clat) / 2)
clon = (Vsub.clon[:-1] + np.diff(Vsub.clon) / 2)
lat = (N2.lat[:-1] + np.diff(N2.lat) / 2)
lon = (N2.lon[:-1] + np.diff(N2.lon) / 2)
time = Usub.time[:-1] + np.diff(Usub.time) / 2
pmid = pmid[5, 5, :]
pmidn = pmid[:-1] + np.diff(pmid) / 2
setattr(
self, 'dudx',
LinearNDInterpolatorExt((lon, Usub.clat, Usub.depth, Usub.time),
Usub.diff(dim='lon').values / dxu,
fill_value=0))
setattr(
self, 'dudy',
LinearNDInterpolatorExt((Usub.lon, clat, Usub.depth, Usub.time),
Usub.diff(dim='clat').values / dyu,
fill_value=0))
setattr(
self, 'dudz',
LinearNDInterpolatorExt((Usub.lon, Usub.clat, pmid, Usub.time),
Usub.diff(dim='depth').values / dz,
fill_value=0))
setattr(
self, 'dvdx',
LinearNDInterpolatorExt((clon, Vsub.lat, Usub.depth, Usub.time),
Vsub.diff(dim='clon').values / dxv,
fill_value=1343431))
setattr(
self, 'dvdy',
LinearNDInterpolatorExt((Vsub.clon, lat, Usub.depth, Usub.time),
Vsub.diff(dim='lat').values / dyv,
fill_value=0))
setattr(
self, 'dvdz',
LinearNDInterpolatorExt((Vsub.clon, Vsub.lat, pmid, Vsub.time),
Vsub.diff(dim='depth').values / dz,
fill_value=0))
setattr(
self, 'dndx',
LinearNDInterpolatorExt((lon, N2.lat, N2.depth, N2.time),
N2.diff(dim='lon').values / dxn,
fill_value=0))
setattr(
self, 'dndy',
LinearNDInterpolatorExt((N2.lon, lat, N2.depth, N2.time),
N2.diff(dim='lat').values / dyn,
fill_value=0))
setattr(
self, 'dndz',
LinearNDInterpolatorExt((N2.lon, N2.lat, pmidn, N2.time),
N2.diff(dim='depth').values / dz,
fill_value=0))
# Time Gradients Final
delt = Usub.time.diff(dim='time') * 24 * 60 * \
60 # time delta in seconds
dudt = []
dvdt = []
dn2dt = []
for i, dt1 in enumerate(delt):
dudt.append(Usub.diff(dim='time')[:, :, :, i] / dt1)
dvdt.append(Vsub.diff(dim='time')[:, :, :, i] / dt1)
dn2dt.append(N2.diff(dim='time')[:, :, :, i] / dt1)
setattr(
self, 'dndt',
LinearNDInterpolatorExt((N2.lon, N2.lat, N2.depth, time),
np.stack(dn2dt, axis=3),
fill_value=0))
setattr(
self, 'dudt',
LinearNDInterpolatorExt((Usub.lon, Usub.clat, Usub.depth, time),
np.stack(dudt, axis=3),
fill_value=0))
setattr(
self, 'dvdt',
LinearNDInterpolatorExt((Vsub.clon, Vsub.lat, Vsub.depth, time),
np.stack(dvdt, axis=3),
fill_value=0))
setattr(
self, 'U',
LinearNDInterpolatorExt(
(Usub.lon, Usub.clat, Usub.depth, Usub.time), Usub.values))
setattr(
self, 'V',
LinearNDInterpolatorExt(
(Vsub.clon, Vsub.lat, Vsub.depth, Vsub.time), Vsub.values))
setattr(
self, 'T',
LinearNDInterpolatorExt(
(Tsub.lon, Tsub.lat, Tsub.depth, Tsub.time), Tsub.values))
lonlim = [N2.lon.min(), N2.lon.max()]
latlim = [N2.lat.min(), N2.lat.max()]
tlim = [N2.time.min(), N2.time.max()]
return lonlim, latlim, tlim
# Calculate N$^2$ from the CTD data, smooth it and then translate it to the linear equation of state.
# $$
# N^2 = -\frac{g}{\rho} \frac{d \rho}{dz}
# $$
# $$
# \rho = \alpha_T \theta
# $$
# $$
# T_{z0} = \frac{N^2}{g \alpha_T}
# $$
# %%
zshift = -50 # shift peak by how many metres
tmp = sec9.where(sec9.station == 12, drop=True)
SA = gsw.SA_from_SP(tmp.s.squeeze(), tmp.p.squeeze(), tmp.lon, tmp.lat)
CT = gsw.CT_from_t(SA, tmp.t.squeeze(), tmp.p.squeeze())
N2, pmid = gsw.Nsquared(SA, CT, tmp.p.squeeze(), tmp.lat)
zmid = utils.mid(tmp.z.data)
good = np.isfinite(N2)
N2smooth = utils.convolve_smooth(N2[good], 100)
zs = zmid[good]
imax, _ = sp.signal.find_peaks(N2smooth, height=(3e-6, 5e-6), prominence=2e-6)
hw = 220
ispeak = (zs > zs[imax - hw]) & (zs < zs[imax + hw])
N2filled = np.interp(zs, zs[~ispeak], N2smooth[~ispeak])
N2peak = (N2smooth - N2filled)
fpeak = sp.interpolate.interp1d(zs,
N2peak,
bounds_error=False,
def teos10(
self,
vlist: list = ["SA", "CT", "SIG0", "N2", "PV", "PTEMP"],
inplace: bool = True,
):
""" Add TEOS10 variables to the dataset
By default, adds: 'SA', 'CT'
Other possible variables: 'SIG0', 'N2', 'PV', 'PTEMP', 'SOUND_SPEED'
Relies on the gsw library.
If one exists, the correct CF standard name will be added to the attrs.
Parameters
----------
vlist: list(str)
List with the name of variables to add.
Must be a list containing one or more of the following string values:
* `"SA"`
Adds an absolute salinity variable
* `"CT"`
Adds a conservative temperature variable
* `"SIG0"`
Adds a potential density anomaly variable referenced to 0 dbar
* `"N2"`
Adds a buoyancy (Brunt-Vaisala) frequency squared variable.
This variable has been regridded to the original pressure levels in the Dataset using a linear interpolation.
* `"PV"`
Adds a planetary vorticity variable calculated from :math:`\\frac{f N^2}{\\text{gravity}}`.
This is not a TEOS-10 variable from the gsw toolbox, but is provided for convenience.
This variable has been regridded to the original pressure levels in the Dataset using a linear interpolation.
* `"PTEMP"`
Adds a potential temperature variable
* `"SOUND_SPEED"`
Adds a sound speed variable
inplace: boolean, True by default
If True, return the input :class:`xarray.Dataset` with new TEOS10 variables added as a new :class:`xarray.DataArray`
If False, return a :class:`xarray.Dataset` with new TEOS10 variables
Returns
-------
:class:`xarray.Dataset`
"""
if not with_gsw:
raise ModuleNotFoundError(
"This functionality requires the gsw library")
allowed = ['SA', 'CT', 'SIG0', 'N2', 'PV', 'PTEMP', 'SOUND_SPEED']
if any(var not in allowed for var in vlist):
raise ValueError(
f"vlist must be a subset of {allowed}, instead found {vlist}")
warnings.warn(
"Default variables will be reduced to 'SA' and 'CT' in 0.1.9",
category=FutureWarning)
this = self._obj
to_profile = False
if self._type == "profile":
to_profile = True
this = this.argo.profile2point()
# Get base variables as numpy arrays:
psal = this['PSAL'].values
temp = this['TEMP'].values
pres = this['PRES'].values
lon = this['LONGITUDE'].values
lat = this['LATITUDE'].values
# Coriolis
f = gsw.f(lat)
# Absolute salinity
sa = gsw.SA_from_SP(psal, pres, lon, lat)
# Conservative temperature
ct = gsw.CT_from_t(sa, temp, pres)
# Potential Temperature
if "PTEMP" in vlist:
pt = gsw.pt_from_CT(sa, ct)
# Potential density referenced to surface
if "SIG0" in vlist:
sig0 = gsw.sigma0(sa, ct)
# N2
if "N2" in vlist or "PV" in vlist:
n2_mid, p_mid = gsw.Nsquared(sa, ct, pres, lat)
# N2 on the CT grid:
ishallow = (slice(0, -1), Ellipsis)
ideep = (slice(1, None), Ellipsis)
def mid(x):
return 0.5 * (x[ideep] + x[ishallow])
n2 = np.zeros(ct.shape) * np.nan
n2[1:-1] = mid(n2_mid)
# PV:
if "PV" in vlist:
pv = f * n2 / gsw.grav(lat, pres)
# Sound Speed:
if 'SOUND_SPEED' in vlist:
cs = gsw.sound_speed(sa, ct, pres)
# Back to the dataset:
that = []
if 'SA' in vlist:
SA = xr.DataArray(sa, coords=this['PSAL'].coords, name='SA')
SA.attrs['long_name'] = 'Absolute Salinity'
SA.attrs['standard_name'] = 'sea_water_absolute_salinity'
SA.attrs['unit'] = 'g/kg'
that.append(SA)
if 'CT' in vlist:
CT = xr.DataArray(ct, coords=this['TEMP'].coords, name='CT')
CT.attrs['long_name'] = 'Conservative Temperature'
CT.attrs['standard_name'] = 'sea_water_conservative_temperature'
CT.attrs['unit'] = 'degC'
that.append(CT)
if 'SIG0' in vlist:
SIG0 = xr.DataArray(sig0, coords=this['TEMP'].coords, name='SIG0')
SIG0.attrs[
'long_name'] = 'Potential density anomaly with reference pressure of 0 dbar'
SIG0.attrs['standard_name'] = 'sea_water_sigma_theta'
SIG0.attrs['unit'] = 'kg/m^3'
that.append(SIG0)
if 'N2' in vlist:
N2 = xr.DataArray(n2, coords=this['TEMP'].coords, name='N2')
N2.attrs['long_name'] = 'Squared buoyancy frequency'
N2.attrs['unit'] = '1/s^2'
that.append(N2)
if 'PV' in vlist:
PV = xr.DataArray(pv, coords=this['TEMP'].coords, name='PV')
PV.attrs['long_name'] = 'Planetary Potential Vorticity'
PV.attrs['unit'] = '1/m/s'
that.append(PV)
if 'PTEMP' in vlist:
PTEMP = xr.DataArray(pt, coords=this['TEMP'].coords, name='PTEMP')
PTEMP.attrs['long_name'] = 'Potential Temperature'
PTEMP.attrs['standard_name'] = 'sea_water_potential_temperature'
PTEMP.attrs['unit'] = 'degC'
that.append(PTEMP)
if 'SOUND_SPEED' in vlist:
CS = xr.DataArray(cs,
coords=this['TEMP'].coords,
name='SOUND_SPEED')
CS.attrs['long_name'] = 'Speed of sound'
CS.attrs['standard_name'] = 'speed_of_sound_in_sea_water'
CS.attrs['unit'] = 'm/s'
that.append(CS)
# Create a dataset with all new variables:
that = xr.merge(that)
# Add to the dataset essential Argo variables (allows to keep using the argo accessor):
that = that.assign({
k: this[k]
for k in [
"TIME",
" LATITUDE",
"LONGITUDE",
"PRES",
"PRES_ADJUSTED",
"PLATFORM_NUMBER",
"CYCLE_NUMBER",
"DIRECTION",
] if k in this
})
# Manage output:
if inplace:
# Merge previous with new variables
for v in that.variables:
this[v] = that[v]
if to_profile:
this = this.argo.point2profile()
for k in this:
if k not in self._obj:
self._obj[k] = this[k]
return self._obj
else:
if to_profile:
return that.argo.point2profile()
else:
return that
## Interpolate location
lat_interp_time=interpolate.interp1d(date_reform_num, lat)
lat_interp=lat_interp_time(new_dates_num)
lon_interp_time=interpolate.interp1d(date_reform_num, lon)
lon_interp=lon_interp_time(new_dates_num)
for l in np.arange(lat_array_pd.shape[0]):
# rows: Pressure, columns dates
lat_array_pd[l,:]=lat_interp.T
lon_array_pd[l,:]=lon_interp.T
P_PD=np.zeros(T_PD.shape)
for l in np.arange(len(new_dates)):
P_PD[:,l]=pres_range
SA=gsw.SA_from_SP(S_PD,P_PD,lon_array_pd,lat_array_pd)
CT=gsw.CT_from_t(SA, T_PD, P_PD)
oxy_eq=gsw.O2sol(SA, CT, P_PD, lon_array_pd, lat_array_pd)
OS_PD=O_PD/oxy_eq*100
D_PD=np.zeros(T_PD.shape)
D_PD[:]=np.NaN
for r in np.arange(D_PD.shape[0]):
for c in np.arange(D_PD.shape[1]):
if np.isnan(SA[r,c]) == False and np.isnan(CT[r,c]) == False:
D_PD[r,c]=gsw.sigma0(SA[r,c], CT[r,c])
plt.figure(figsize=(figx, figy))
plt.pcolormesh(X,Y,OS_PD ,vmin=minOsat, vmax=maxOsat, cmap=cmo.haline)
plt.gca().invert_yaxis()
def oxygen(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
var_pres = ds.variables["PRES"]
t = var_temp[:]
SP = var_psal[:]
p = var_pres[:]
SA = gsw.SA_from_SP(SP, p, ds.longitude, ds.latitude)
CT = gsw.CT_from_t(SA, t, p)
pt = gsw.pt0_from_t(SA, t, p)
sigma_theta0 = gsw.sigma0(SA, CT)
oxsol = ds.variables["OXSOL"][:]
ts = np.log((298.15 - t) / (273.15 + t))
# psal correction, from Aanderra TD-218, Gordon and Garcia oxygaen solubility salinity coefficents
B0 = -6.24097e-3
C0 = -3.11680e-7
B1 = -6.93498e-3
B2 = -6.90358e-3
B3 = -4.29155e-3
psal_correction = np.exp(SP *(B0 + B1 * ts + B2 * np.power(ts, 2) + B3 * np.power(ts, 3)) + np.power(SP, 2) * C0)
# get correction slope, offset
slope = 1.0
offset = 0.0
try:
slope = ds.variables['DOX2_RAW'].calibration_slope
offset = ds.variables['DOX2_RAW'].calibration_offset
except AttributeError:
pass
# calculate disolved oxygen, umol/kg
dox2_raw = ds.variables['DOX2_RAW'] * psal_correction * slope + offset
dox2 = 1000 * dox2_raw / (sigma_theta0 + 1000)
if 'DOX2' not in ds.variables:
ncVarOut = ds.createVariable("DOX2", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOX2']
ncVarOut[:] = dox2
ncVarOut.units = "umol/kg"
ncVarOut.comment = "calculated using DOX2 = DOX2_RAW * PSAL_CORRECTION / ((sigma_theta(P=0,Theta,S) + 1000).. Sea Bird AN 64, Aanderaa TD210 Optode Manual"
ncVarOut.comment_calibration = "calibration slope " + str(slope) + " offset " + str(offset) + " umol/l"
# calculate, and write the oxygen mass/seawater mass
if 'DOX_MG' not in ds.variables:
ncVarOut = ds.createVariable("DOXY", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOXY']
ncVarOut[:] = dox2_raw / 31.24872
ncVarOut.units = "mg/l"
ncVarOut.comment = "calculated using DOXY = * DOX2_RAW * PSAL_CORRECTION / 31.24872... Aanderaa TD210 Optode Manual"
ncVarOut.comment_calibration = "calibration slope " + str(slope) + " offset " + str(offset) + " umol/l"
# calculate and write oxygen solubility, ratio of disolved oxgen / oxygen solubility
if 'DOXS' not in ds.variables:
ncVarOut = ds.createVariable("DOXS", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOXS']
ncVarOut[:] = dox2/oxsol
ncVarOut.units = "1"
ncVarOut.comment = "calculated using DOX2/OXSOL"
# finish off, and close file
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + " : added derived oxygen, DOX2, DOXS, DOX_MG")
ds.close()
def load_seals(sealdir, bathy_data):
from datetime import datetime, date, timedelta
# initialize
maxNprof = 0
ll = 0
ll1 = 0
l0 = 0
dateFlNum = []
XC = bathy_data[0]
YC = bathy_data[1]
x1 = bathy_data[3]
x2 = bathy_data[4]
y1 = bathy_data[5]
y2 = bathy_data[6]
# initialize big arrays
CT_dataSO = np.nan * np.ones((100 * 310, 1000), '>f4')
SA_dataSO = np.nan * np.ones((100 * 310, 1000), '>f4')
SP_dataSO = np.nan * np.ones((100 * 310, 1000), '>f4')
lon_dataSO = np.nan * np.ones((100 * 310), '>f4')
lat_dataSO = np.nan * np.ones((100 * 310), '>f4')
pr_dataSO = np.nan * np.ones((100 * 1000 * 310), '>f4')
yySO = np.zeros((100 * 310), 'int32')
mmSO = np.zeros((100 * 310), 'int32')
IDSO = np.zeros((100 * 310), 'int32')
data_list = [
g for g in os.listdir(seadir)
if g.endswith('nc') and not g.startswith('WOD')
]
for ff_SO in data_list:
#printff_SO
tot_fl = 0
iniDate = datetime(1950, 1, 1)
minDate = datetime(2019, 1, 1).toordinal()
#printff_SO
file = os.path.join(sealdir, '%s' % ff_SO)
# load data
data = nc.Dataset(file)
cruise = data.variables['PLATFORM_NUMBER'][:]
if "CTD2020" in ff_SO:
cruise = np.array(['202020%0.3i' % (int(f)) for f in cruise])
elif "CTD2019" in ff_SO:
cruise = np.array(['191919%0.3i' % (int(f)) for f in cruise])
else:
cruise = np.array(['%0.9i' % f for f in cruise])
time = data.variables['JULD'][:]
lon = data.variables['LONGITUDE'][:]
lat = data.variables['LATITUDE'][:]
pressPre = data.variables['PRES_ADJUSTED'][:].transpose()
tempPre = data.variables['TEMP_ADJUSTED'][:].transpose()
tempQC = data.variables['TEMP_ADJUSTED_QC'][:].transpose()
psaPre = data.variables['PSAL_ADJUSTED'][:].transpose()
psaQC = data.variables['PSAL_ADJUSTED_QC'][:].transpose()
psaPre[np.where(psaQC == '4')] = np.nan
tempPre[np.where(tempQC == '4')] = np.nan
# let's get rid of some profiles that are out of the Amudsen Sea
tot_fl += 1
msk1 = np.where(np.logical_and(lon >= XC[x1], lon < XC[x2]))[0][:]
msk2 = np.where(np.logical_and(lat[msk1] > YC[y1],
lat[msk1] < YC[y2]))[0][:]
lon = lon[msk1][msk2]
lat = lat[msk1][msk2]
time = time[msk1][msk2]
pressPre = pressPre[:, msk1[msk2]]
tempPre = tempPre[:, msk1[msk2]]
psaPre = psaPre[:, msk1[msk2]]
cruise = cruise[msk1][msk2]
if time[0] + iniDate.toordinal() < minDate:
minDate = time[0]
if len(lon) > maxNprof:
maxNprof = len(lon[~np.isnan(lon)])
flN = ff_SO
# remove data below 1000m (there are just so few seal profiles with data below 1000m anyway)
msk = np.where(pressPre > 1000)
pressPre[msk] = np.nan
tempPre[msk] = np.nan
psaPre[msk] = np.nan
# interpolate and prepare the data
lat = np.ma.masked_less(lat, -90)
lon = np.ma.masked_less(lon, -500)
lon[lon > 360.] = lon[lon > 360.] - 360.
Nprof = np.linspace(1, len(lat), len(lat))
# turn the variables upside down, to have from the surface to depth and not viceversa
if any(pressPre[:10, 0] > 500.):
pressPre = pressPre[::-1, :]
psaPre = psaPre[::-1, :]
tempPre = tempPre[::-1, :]
# interpolate data on vertical grid with 1db of resolution (this is fundamental to then create profile means)
fields = [psaPre, tempPre]
press = np.nan * np.ones((1000, pressPre.shape[1]))
for kk in range(press.shape[1]):
press[:, kk] = np.arange(2, 1002, 1)
psa = np.nan * np.ones((press.shape), '>f4')
inst_temp = np.nan * np.ones((press.shape), '>f4')
for ii, ff in enumerate(fields):
for nn in range(pressPre.shape[1]):
# only use non-nan values, otherwise it doesn't interpolate well
try:
f1 = ff[:, nn][ff[:, nn].mask ==
False] #ff[:,nn][~np.isnan(ff[:,nn])]
f2 = pressPre[:, nn][ff[:, nn].mask == False]
except:
f1 = ff[:, nn]
f2 = pressPre[:, nn]
if len(f1) == 0:
f1 = ff[:, nn]
f2 = pressPre[:, nn]
try:
sp = interpolate.interp1d(f2[~np.isnan(f1)],
f1[~np.isnan(f1)],
kind='linear',
bounds_error=False,
fill_value=np.nan)
ff_int = sp(press[:, nn])
if ii == 0:
psa[:, nn] = ff_int
elif ii == 1:
inst_temp[:, nn] = ff_int
except:
print(
'At profile number %i, the float %s has only 1 record valid: len(f2)=%i'
% (nn, ff_SO, len(f2)))
# To compute theta, I need absolute salinity [g/kg] from practical salinity (PSS-78) [unitless] and conservative temperature.
sa = np.nan * np.ones((press.shape), '>f4')
for kk in range(press.shape[1]):
sa[:, kk] = gsw.SA_from_SP(psa[:, kk], press[:, 0], lon[kk],
lat[kk])
temp = gsw.CT_from_t(sa, inst_temp, press)
ptemp = gsw.pt_from_CT(sa, temp)
# mask out the profiles with :
msk = np.where(lat < -1000)
lat[msk] = np.nan
lon[msk] = np.nan
cruise[msk] = np.nan
psa[msk] = np.nan
sa[msk] = np.nan
psa[sa == 0.] = np.nan
sa[sa == 0.] = np.nan
# use CT instead of PT
temp[msk] = np.nan
temp[temp == 0.] = np.nan
# save the nprofiles
NN = np.ones((temp.shape), 'int32')
for ii in range(len(Nprof)):
NN[:, ii] = Nprof[ii]
lon_dataSO[ll1:ll1 + len(lon)] = lon
lat_dataSO[ll1:ll1 + len(lon)] = lat
CT_dataSO[ll:ll + len(sa[0, :]), :] = temp.T
SA_dataSO[ll:ll + len(sa[0, :]), :] = sa.T
SP_dataSO[ll:ll + len(sa[0, :]), :] = psa.T
IDSO[ll1:ll1 + len(lon)] = [int(f) for f in cruise]
# separate seasons
dateFl = []
for dd in time:
floatDate = iniDate + timedelta(float(dd))
dateFl.append(floatDate)
dateFlNum = np.append(dateFlNum, floatDate.toordinal())
yearsSO = np.array([int(dd.year) for dd in dateFl])
monthsSO = np.array([int(dd.month) for dd in dateFl])
SPR = np.where(np.logical_and(monthsSO >= 9, monthsSO <= 11))
SUM = np.where(np.logical_or(monthsSO == 12, monthsSO <= 2))
AUT = np.where(np.logical_and(monthsSO >= 3, monthsSO <= 5))
WIN = np.where(np.logical_and(monthsSO >= 6, monthsSO <= 8))
mmFlSO = monthsSO.copy()
mmFlSO[SPR] = 1
mmFlSO[SUM] = 2
mmFlSO[AUT] = 3
mmFlSO[WIN] = 4
mmSO[ll:ll + len(sa[0, :])] = mmFlSO
yySO[ll:ll + len(sa[0, :])] = yearsSO
ll = ll + len(sa[0, :])
ll1 = ll1 + len(lon)
minArgoDate = timedelta(float(minDate)) + iniDate
minArgoDate.strftime('%Y/%m/%d %H:%M:%S%z')
# chop away the part of the array with no data
CT_dataSO = CT_dataSO[:ll, :]
SA_dataSO = SA_dataSO[:ll, :]
SP_dataSO = SP_dataSO[:ll, :]
IDSO = IDSO[:ll]
lat_dataSO = lat_dataSO[:ll]
lon_dataSO = lon_dataSO[:ll]
mmSO = mmSO[:ll]
yySO = yySO[:ll]
# remove entire columns of NaNs
idBad = []
for ii in range(CT_dataSO.shape[1]):
f0 = CT_dataSO[:, ii]
f1 = f0[~np.isnan(f0)]
if len(f1) == 0:
idBad.append(ii)
CT_dataSO = np.delete(CT_dataSO, idBad, 0)
SA_dataSO = np.delete(SA_dataSO, idBad, 0)
SP_dataSO = np.delete(SP_dataSO, idBad, 0)
IDSO = np.delete(IDSO, idBad, 0)
lon_dataSO = np.delete(lon_dataSO, idBad, 0)
lat_dataSO = np.delete(lat_dataSO, idBad, 0)
mmSO = np.delete(mmSO, idBad, 0)
yySO = np.delete(yySO, idBad, 0)
load_seals.profSO = [
CT_dataSO, SA_dataSO, lon_dataSO, lat_dataSO, IDSO, SP_dataSO
]
load_seals.temporalSO = [yySO, mmSO]
return [load_seals.profSO, load_seals.temporalSO]
if l_accurate and not l2d:
vz = f_out.variables['deptht'][:]
[nt, nk, nj, ni] = nmp.shape(xsal)
xdepth = nmp.zeros((nk, nj, ni))
# building 3d arrays of depth, lon and lat:
for jk in range(nk):
xdepth[jk, :, :] = vz[jk]
# pressure should be in dbar and it's the same as the depth in metre actually:
for jt in range(nt):
print ' jt =', jt
f_out.variables[cv_sal][jt, :, :, :] = gsw.SA_from_SP(
xsal[jt, :, :, :], xdepth, -140., 0.)
else:
# Fabien says it's enough:
if l2d:
f_out.variables[cv_sal][:, :, :] = xsal[:, :, :] * SSO / 35.
else:
f_out.variables[cv_sal][:, :, :, :] = xsal[:, :, :, :] * SSO / 35.
f_out.variables[
cv_sal].long_name = 'Absolute Salinity (TEOS10) build from practical salinity (*35.16504/35)'
f_out.close()
print cf_out + ' sucessfully created!'
dfm['PRES_ADJUSTED_QC'] = dfm['PRES_ADJUSTED_QC'].str.decode("utf-8")
dfm['PSAL_ADJUSTED_QC'] = dfm['PSAL_ADJUSTED_QC'].str.decode("utf-8")
# In[3]:
dfm["JULD"] = pd.to_datetime(dfm['JULD'], infer_datetime_format=True)
# In[4]:
dfm["DEPTH"] = gsw.z_from_p(dfm["PRES_ADJUSTED"], dfm["LATITUDE"])
# uncomment the cell below if you want the entire "dfm" dataframe to be written out into a csv file
# #### Using Gibbs Sea Water Equation of State (TEOS-10) functions, we calculate the density and the conserved density/temperature
SA = gsw.SA_from_SP(dfm['PSAL_ADJUSTED'], dfm['PRES_ADJUSTED'],
dfm['LONGITUDE'], dfm['LATITUDE'])
CT = gsw.CT_from_t(SA, dfm['TEMP_ADJUSTED'], dfm['PRES_ADJUSTED'])
dfm['DENSITY_INSITU'] = gsw.density.rho(SA, CT, dfm['PRES_ADJUSTED'])
dfm['POT_DENSITY'] = gsw.density.sigma0(SA, CT)
dfm['CTEMP'] = CT
dfm['SA'] = SA
mask_notbad_temp = ~(dfm['TEMP_ADJUSTED_QC'] == 4)
mask_notbad_sal = ~(dfm['PSAL_ADJUSTED_QC'] == 4)
mask_notbad_pres = ~(dfm['PRES_ADJUSTED_QC'] == 4)
dfmg = dfm[mask_notbad_pres & mask_notbad_sal
& mask_notbad_temp] # data with only good QC + null value flags
from scipy.io import netcdf
import matplotlib.pyplot as plt
from ocean_tools import TKED
import gsw
directory = '../../Data/deployment_raw/'
outdir = '../../plots/ctd/LT_D_R/'
deployment_name = 'deploy2_'
measurement_type = '1ctd_'
file_type = 'raw_'
for i in range(10):
c_file = 'C' + ("%07d" % (i, ))
c_data = pd.read_pickle(directory + deployment_name + file_type + c_file)
CT = gsw.CT_from_t(c_data['c_sal'], c_data['c_temp'], c_data['c_pres'])
SA = gsw.SA_from_SP(c_data['c_sal'], c_data['c_pres'], 174, -43)
pdens = gsw.sigma0(SA, CT)
c_data["pdens"] = pdens
[LT, Td, Nsqu, Lo, R, x_sorted,
idxs] = TKED.thorpe_scales(c_data["c_depth"].values * -1,
c_data['pdens'].values,
full_output=True)
#plt.show()
fig2, (ax2, ax3, ax4) = plt.subplots(1, 3, sharey=True)
# Temperature
ax2.plot(c_data['pdens'], c_data["c_depth"].values, c='k')
ax2.set_ylabel('Depth (m)')
#ax2.set_ylim(ax2.get_ylim()[::-1]) #this reverses the yaxis (i.e. deep at the bottom)
ax2.set_xlabel('Density (kg/m3)')
ax2.xaxis.set_label_position('top') # this moves the label to the top
date_reform_num[:] = np.NaN
for l in np.arange(len(date_reform)):
date_reform_num[l] = date_reform_sub[
l].total_seconds()
lat_array = np.zeros(pres.shape)
lat_array[:] = np.NaN
lon_array = np.zeros(pres.shape)
lon_array[:] = np.NaN
for l in np.arange(lat_array.shape[0]):
lat_array[l, :] = lat[l]
lon_array[l, :] = lon[l]
SA = gsw.SA_from_SP(sal, pres, lon_array, lat_array)
CT = gsw.CT_from_t(SA, temp, pres)
oxy_eq = gsw.O2sol(SA, CT, pres, lon_array, lat_array)
# calculate depth
geo_strf = gsw.geo_strf_dyn_height(SA=SA.T,
CT=CT.T,
p=pres.T)
geo_strf = geo_strf.T
z = gsw.z_from_p(p=pres,
lat=lat_array,
geo_strf_dyn_height=geo_strf)
z = -1 * z
oxy_sat = doxy / oxy_eq * 100
uni['N2']['vmin'] = -8
uni['N2']['vmax'] = -2
uni['N2']['cmap'] = cm.YlGn
uni['sal_1m'] = uni['sal']
uni['tmp_1m'] = uni['tmp']
uni['turner_1m'] = uni['turner']
uni['o2'] = {}
uni['o2']['vmin'] = 4
uni['o2']['vmax'] = 9
uni['o2']['cmap'] = cm.rainbow
salvec = linspace(31, 36, 103)
tmpvec = linspace(-3, 16, 103)
salmat, tmpmat = meshgrid(salvec, tmpvec)
SA_vec = gsw.SA_from_SP(salvec, zeros(len(salvec)), -44, 59.5)
SA_vec_1000 = gsw.SA_from_SP(salvec, 1e3 * ones(len(salvec)), -44, 59.5)
CT_vec = gsw.CT_from_pt(SA_vec, tmpvec)
pdenmat = zeros((shape(salmat)))
pdenmat2 = zeros((shape(salmat)))
sigma1mat = zeros((shape(salmat)))
for ii in range(len(salvec)):
for jj in range(len(tmpvec)):
pdenmat[jj, ii] = gsw.sigma0(SA_vec[ii], CT_vec[jj])
pdenmat2[jj, ii] = gsw.pot_rho_t_exact(SA_vec[ii], tmpvec[jj], 750,
0) - 1e3
sigma1mat[jj, ii] = gsw.sigma1(SA_vec[ii], CT_vec[jj])
