def add_psal(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
var_pres = ds.variables["PRES"]
t = var_temp[:]
SP = var_psal[:]
p = var_pres[:]
SA = gsw.SA_from_SP(SP, p, ds.longitude, ds.latitude)
pt = gsw.pt0_from_t(SA, t, p)
oxsol = gsw.O2sol_SP_pt(SP, pt)
ncVarOut = ds.createVariable("OXSOL", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = oxsol
ncVarOut.units = "umol/kg"
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html function gsw.O2sol_SP_pt"
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + " : added oxygen solubility")
ds.close()
def _compute_data(self,data, units, names, p_ref = 0, baltic = False, lon=0, lat=0, isen = '0'):
""" Computes convservative temperature, absolute salinity and potential density from input data, expects a recarray with the following entries data['C']: conductivity in mS/cm, data['T']: in Situ temperature in degree Celsius (ITS-90), data['p']: in situ sea pressure in dbar
Arguments:
p_ref: Reference pressure for potential density
baltic: if True use the Baltic Sea density equation instead of open ocean
lon: Longitude of ctd cast default=0
lat: Latitude of ctd cast default=0
Returns:
list [cdata,cunits,cnames] with cdata: recarray with entries 'SP', 'SA', 'pot_rho', etc., cunits: dictionary with units, cnames: dictionary with names
"""
sen = isen + isen
# Check for units and convert them if neccessary
if(units['C' + isen] == 'S/m'):
logger.info('Converting conductivity units from S/m to mS/cm')
Cfac = 10
if(('68' in units['T' + isen]) or ('68' in names['T' + isen]) ):
logger.info('Converting IPTS-68 to T90')
T = gsw.t90_from_t68(data['T' + isen])
else:
T = data['T' + isen]
SP = gsw.SP_from_C(data['C' + isen], T, data['p'])
SA = gsw.SA_from_SP(SP,data['p'],lon = lon, lat = lat)
if(baltic == True):
SA = gsw.SA_from_SP_Baltic(SA,lon = lon, lat = lat)
PT = gsw.pt0_from_t(SA, T, data['p'])
CT = gsw.CT_from_t(SA, T, data['p'])
pot_rho = gsw.pot_rho_t_exact(SA, T, data['p'], p_ref)
names = ['SP' + sen,'SA' + sen,'pot_rho' + sen,'pt0' + sen,'CT' + sen]
formats = ['float','float','float','float','float']
cdata = {}
cdata['SP' + sen] = SP
cdata['SA' + sen] = SA
cdata['pot_rho' + sen] = pot_rho
cdata['pt' + sen] = PT
cdata['CT' + sen] = CT
cnames = {'SA' + sen:'Absolute salinity','SP' + sen: 'Practical Salinity on the PSS-78 scale',
'pot_rho' + sen: 'Potential density',
'pt' + sen:'potential temperature with reference sea pressure (p_ref) = 0 dbar',
'CT' + sen:'Conservative Temperature (ITS-90)'}
cunits = {'SA' + sen:'g/kg','SP' + sen:'PSU','pot_rho' + sen:'kg/m^3' ,'CT' + sen:'deg C','pt' + sen:'deg C'}
return [cdata,cunits,cnames]
def add_SA_CT_PT(xray):
if 'PRES' in list(xray.data_vars):
PRES_out = xray['PRES']
else:
PRES_out = -gsw.p_from_z(xray['DEPTH'])
SA_out = gsw.SA_from_SP(xray['PSAL'], PRES_out, xray.LONGITUDE,
xray.LATITUDE)
if 'PTMP' in list(xray.data_vars):
PT_out = xray['PTMP']
else:
PT_out = gsw.pt0_from_t(SA_out, xray['TEMP'], PRES_out)
CT_out = gsw.CT_from_pt(SA_out, PT_out)
PD_out = gsw.sigma0(SA_out, CT_out)
xray['ASAL'] = (('TIME', 'DEPTH'), SA_out)
xray['PTMP'] = (('TIME', 'DEPTH'), PT_out)
xray['CTMP'] = (('TIME', 'DEPTH'), CT_out)
xray['PDEN'] = (('TIME', 'DEPTH'), PD_out)
def oxygen(netCDFfile):
ds = Dataset(netCDFfile, 'a')
if 'DOX2_RAW' not in ds.variables:
ds.close()
return
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
if 'PRES' in ds.variables:
var_pres = ds.variables["PRES"]
p = var_pres[:]
elif 'NOMINAL_DEPTH' in ds.variables:
var_pres = ds.variables["NOMINAL_DEPTH"]
p = var_pres[:]
else:
p = 1.0
t = var_temp[:]
SP = var_psal[:]
SA = gsw.SA_from_SP(SP, p, ds.variables["LONGITUDE"][0],
ds.variables["LATITUDE"][0])
CT = gsw.CT_from_t(SA, t, p)
pt = gsw.pt0_from_t(SA, t, p)
sigma_theta0 = gsw.sigma0(SA, CT)
ts = np.log((298.15 - t) / (273.15 + t))
# psal correction, from Aanderra TD-218, Gordon and Garcia oxygaen solubility salinity coefficents, salinity coeff, col 5 combined fit (ml/l).
B0 = -6.24097e-3
C0 = -3.11680e-7
B1 = -6.93498e-3
B2 = -6.90358e-3
B3 = -4.29155e-3
psal_correction = np.exp(
SP * (B0 + B1 * ts + B2 * np.power(ts, 2) + B3 * np.power(ts, 3)) +
np.power(SP, 2) * C0)
# get correction slope, offset
slope = 1.0
offset = 0.0
try:
slope = ds.variables['DOX2_RAW'].calibration_slope
offset = ds.variables['DOX2_RAW'].calibration_offset
except AttributeError:
pass
# calculate disolved oxygen, umol/kg
dox2_raw = ds.variables['DOX2_RAW'] * psal_correction * slope + offset
dox2 = 1000 * dox2_raw / (sigma_theta0 + 1000)
#dox2 = dox2_raw
if 'DOX2' not in ds.variables:
ncVarOut = ds.createVariable(
"DOX2", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOX2']
ncVarOut[:] = dox2
ncVarOut.standard_name = "moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut.long_name = "moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut.units = "umol/kg"
ncVarOut.comment1 = "calculated using DOX2_uM = DOX2_RAW * PSAL_CORRECTION .. Aanderaa TD210 Optode Manual (optode/SBE63 only)"
ncVarOut.comment2 = "DOX2 = DOX2_uM / ((sigma_theta(P=0,Theta,S) + 1000).. Sea Bird AN 64 (unit conversion umol/l -> umol/kg)"
ncVarOut.comment_calibration = "calibration slope " + str(
slope) + " offset " + str(offset) + " umol/l"
ncVarOut.coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'
# finish off, and close file
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr(
'history', hist + datetime.utcnow().strftime("%Y-%m-%d") +
" added derived oxygen, DOX2")
ds.close()
return netCDFfile
def process_all_new():
ssscc_file = 'data/ssscc.csv'
iniFile = 'data/ini-files/configuration.ini'
config = configparser.RawConfigParser()
config.read(iniFile)
# maybe reformat config file?
# fold this code into config module
p_col = config['analytical_inputs']['p']
p_btl_col = config['inputs']['p']
t1_col = config['analytical_inputs']['t1']
t1_btl_col = config['inputs']['t1']
t2_col = config['analytical_inputs']['t2']
t2_btl_col = config['inputs']['t2']
c1_col = config['analytical_inputs']['c1']
c1_btl_col = config['inputs']['c1']
c2_col = config['analytical_inputs']['c2']
c2_btl_col = config['inputs']['c2']
sal_col = config['analytical_inputs']['salt']
reft_col = config['inputs']['reft']
sample_rate = config['ctd_processing']['sample_rate']
search_time = config['ctd_processing']['roll_filter_time']
expocode = config['cruise']['expocode']
sectionID = config['cruise']['sectionid']
raw_directory = config['ctd_processing']['raw_data_directory']
time_directory = config['ctd_processing']['time_data_directory']
pressure_directory = config['ctd_processing']['pressure_data_directory']
oxygen_directory = config['ctd_processing']['oxygen_directory']
btl_directory = config['ctd_processing']['bottle_directory']
o2flask_file = config['ctd_processing']['o2flask_file']
log_directory = config['ctd_processing']['log_directory']
sample_rate = config['ctd_processing']['sample_rate']
search_time = config['ctd_processing']['roll_filter_time']
ctd = config['ctd_processing']['ctd_serial']
# MK: import from settings/config instead of hard code?
btl_data_prefix = 'data/bottle/'
btl_data_postfix = '_btl_mean.pkl'
time_data_prefix = 'data/time/'
time_data_postfix = '_time.pkl'
p_log_file = 'data/logs/ondeck_pressure.csv'
t1_btl_col = 'CTDTMP1'
t2_btl_col = 'CTDTMP2'
t1_col = 'CTDTMP1'
t2_col = 'CTDTMP2'
p_btl_col = 'CTDPRS'
p_col = 'CTDPRS'
c1_btl_col = 'CTDCOND1'
c2_btl_col = 'CTDCOND2'
c1_col = 'CTDCOND1'
c2_col = 'CTDCOND2'
reft_col = 'T90'
refc_col = 'BTLCOND'
sal_col = 'CTDSAL'
#time_column_data = config['time_series_output']['data_names'].split(',')
time_column_data = config['time_series_output']['data_output']
time_column_names = config['time_series_output']['column_name'].split(',')
time_column_units = config['time_series_output']['column_units'].split(',')
time_column_format = config['time_series_output']['format']
#pressure_column_data = config['time_series_output']['data_names'].split(',')
p_column_data = config['pressure_series_output']['data'].split(',')
p_column_names = config['pressure_series_output']['column_name'].split(',')
p_column_units = config['pressure_series_output']['column_units'].split(
',')
p_column_format = config['pressure_series_output']['format']
p_column_qual = config['pressure_series_output']['qual_columns'].split(',')
p_column_one = list(
config['pressure_series_output']['q1_columns'].split(','))
# Columns from btl file to be read:
cols = [
'index', 'CTDTMP1', 'CTDTMP2', 'CTDPRS', 'CTDCOND1', 'CTDCOND2',
'CTDSAL', 'CTDOXY1', 'CTDOXYVOLTS', 'CTDXMISS', 'ALT', 'REF_PAR',
'GPSLAT', 'GPSLON', 'new_fix', 'pressure_temp_int', 'pump_on',
'btl_fire', 'scan_datetime', 'btl_fire_num'
]
# Load ssscc from file
ssscc = []
with open(ssscc_file, 'r') as filename:
ssscc = [line.strip() for line in filename]
#check for already converted files to skip later
time_start = time.perf_counter()
cnv_dir_list = os.listdir('data/converted/')
time_dir_list = os.listdir('data/time/')
for x in ssscc:
if '{}.pkl'.format(x) in cnv_dir_list:
continue
#convert hex to ctd
subprocess.run([
'odf_convert_sbe.py', 'data/raw/' + x + '.hex',
'data/raw/' + x + '.XMLCON', '-o', 'data/converted'
],
stdout=subprocess.PIPE)
print('odf_convert_sbe.py SSSCC: ' + x + ' done')
time_convert = time.perf_counter()
for x in ssscc:
if '{}_time.pkl'.format(x) in time_dir_list:
continue
subprocess.run(['odf_sbe_metadata.py', 'data/converted/' + x + '.pkl'],
stdout=subprocess.PIPE)
print('odf_sbe_metadata.py SSSCC: ' + x + ' done')
btl_dir_list = os.listdir('data/bottle/')
for x in ssscc:
if '{}_btl_mean.pkl'.format(x) in btl_dir_list:
continue
#process bottle file
subprocess.run([
'odf_process_bottle.py', 'data/converted/' + x + '.pkl', '-o',
'data/bottle/'
],
stdout=subprocess.PIPE)
print('odf_process_bottle.py SSSCC: ' + x + ' done')
time_bottle = time.perf_counter()
# generate salts file here: (salt parser)
salt_dir = './data/salt/'
import scripts.odf_salt_parser as salt_parser
file_list = os.listdir(salt_dir)
files = []
for file in file_list:
if '.' not in file:
files.append(file)
for file in files:
print(file)
salt_path = salt_dir + file
saltDF = salt_parser.salt_loader(saltpath=salt_path)
salt_parser.salt_df_parser(saltDF, salt_dir)
# generate reft file here
reft_path = './data/reft/'
import scripts.odf_reft_parser as reft_parser
for station in ssscc:
with open(reft_path + station + '.cap', 'r') as ssscc_reftemp:
# create csv files
df_part = reft_parser.parse(ssscc_reftemp, station)
df_part.to_csv(reft_path + station + '_reft.csv', index=False)
# generate oxy file here (separate out from calib)
###########################################################################
### File I/O
# Load in all bottle, time, ref_data files into DataFrame
btl_data_all = process_ctd.load_all_ctd_files(ssscc, btl_data_prefix,
btl_data_postfix, 'bottle',
cols)
time_data_all = process_ctd.load_all_ctd_files(ssscc, time_data_prefix,
time_data_postfix, 'time',
None)
##
# end of data collection, begin calibration
################################################################################
### Pressure Calibration
# Determine Pressure offset from logs
pressure_log = process_ctd.load_pressure_logs(p_log_file)
p_off = process_ctd.get_pressure_offset(
pressure_log.ondeck_start_p,
pressure_log.ondeck_end_p) # insert logic to check if good value?
btl_data_all = fit_ctd.apply_pressure_offset(
btl_data_all, p_btl_col, p_off) # not actually bottle data
time_data_all = fit_ctd.apply_pressure_offset(time_data_all, p_col,
p_off) # continuous data
# btl_data_all[p_btl_col] = fit_ctd.apply_pressure_offset(btl_data_all[p_btl_col], p_off)
# time_data_all[p_col] = fit_ctd.apply_pressure_offset(time_data_all[p_col],p_off)
###########################################################################
### Temperature Calibration
for x in range(
2): # 2 passes checks for outliers twice (i.e. before/after fit)
# Second order calibration
df_temp_good = process_ctd.prepare_fit_data(btl_data_all, 'T90')
df_ques_reft = process_ctd.quality_check(df_temp_good[t2_btl_col],
df_temp_good[t1_btl_col],
df_temp_good[p_btl_col],
df_temp_good['SSSCC'],
df_temp_good['btl_fire_num'],
'quest')
df_ques_reft['Parameter'] = 'REF_TEMP'
# sort out fit parameters, what order P, what order T
# only fit `linear' regions, i.e. not mixed/benthic layers
# built better way to handle xRange?
coef_temp_1, df_ques_t1 = process_ctd.calibrate_param(
df_temp_good[t1_btl_col],
df_temp_good[reft_col],
df_temp_good[p_btl_col],
'TP',
2,
df_temp_good.SSSCC,
df_temp_good.btl_fire_num,
xRange='800:6000')
coef_temp_2, df_ques_t2 = process_ctd.calibrate_param(
df_temp_good[t2_btl_col],
df_temp_good[reft_col],
df_temp_good[p_btl_col],
'TP',
2,
df_temp_good.SSSCC,
df_temp_good.btl_fire_num,
xRange='1500:6000')
# Apply fitting coef to data
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col], coef_temp_1)
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col], coef_temp_2)
time_data_all[t1_col] = fit_ctd.temperature_polyfit(
time_data_all[t1_col], time_data_all[p_col], coef_temp_1)
time_data_all[t2_col] = fit_ctd.temperature_polyfit(
time_data_all[t2_col], time_data_all[p_col], coef_temp_2)
# Construct Quality Flag file
qual_flag_temp = process_ctd.combine_quality_flags(
[df_ques_reft, df_ques_t1, df_ques_t2])
## First order calibtation
df_temp_good = process_ctd.prepare_fit_data(btl_data_all, 'T90')
# df_ques_reft = process_ctd.quality_check(df_temp_good[t2_btl_col], df_temp_good[t1_btl_col], df_temp_good[p_btl_col], df_temp_good['SSSCC'], df_temp_good['btl_fire_num'], 'quest')
# df_ques_reft['Parameter'] = 'REF_TEMP'
coef_temp_prim, df_ques_t1 = process_ctd.calibrate_param(
df_temp_good[t1_btl_col], df_temp_good[reft_col],
df_temp_good[p_btl_col], 'T', 1, df_temp_good.SSSCC,
df_temp_good.btl_fire_num)
coef_temp_sec, df_ques_t2 = process_ctd.calibrate_param(
df_temp_good[t2_btl_col], df_temp_good[reft_col],
df_temp_good[p_btl_col], 'T', 1, df_temp_good.SSSCC,
df_temp_good.btl_fire_num)
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col], coef_temp_prim)
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col], coef_temp_sec)
# Apply fitting coef to data
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col], coef_temp_prim)
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col], coef_temp_sec)
time_data_all[t1_col] = fit_ctd.temperature_polyfit(
time_data_all[t1_col], time_data_all[p_col], coef_temp_prim)
time_data_all[t2_col] = fit_ctd.temperature_polyfit(
time_data_all[t2_col], time_data_all[p_col], coef_temp_sec)
qual_flag_temp = process_ctd.combine_quality_flags(
[df_ques_reft, df_ques_t1, df_ques_t2])
###########################################################################
#
### Conductivity Calibration
for x in range(2):
# Update ref conductivity from salinometer
btl_data_all['BTLCOND'] = fit_ctd.CR_to_cond(btl_data_all.CRavg,
btl_data_all.BathTEMP,
btl_data_all.CTDTMP1,
btl_data_all.CTDPRS)
df_cond_good = process_ctd.prepare_fit_data(btl_data_all, 'BTLCOND')
df_ques_refc = process_ctd.quality_check(df_cond_good['CTDCOND2'],
df_temp_good['CTDCOND1'],
df_temp_good['CTDPRS'],
df_temp_good['SSSCC'],
df_temp_good['btl_fire_num'],
'quest')
df_ques_refc['Parameter'] = 'REF_COND'
# Second Order Calibration
# can probably use same xRange as temp, unless C sensor has some quirk
coef_cond_1, df_ques_c1 = process_ctd.calibrate_param(
df_cond_good.CTDCOND1,
df_cond_good.BTLCOND,
df_cond_good.CTDPRS,
'CP',
2,
df_cond_good.SSSCC,
df_cond_good.btl_fire_num,
xRange='800:6000')
coef_cond_2, df_ques_c2 = process_ctd.calibrate_param(
df_cond_good.CTDCOND2,
df_cond_good.BTLCOND,
df_cond_good.CTDPRS,
'CP',
2,
df_cond_good.SSSCC,
df_cond_good.btl_fire_num,
xRange='1500:6000')
# Apply fitting coef to data
# fit_ctd.conductivity_polyfit has salinity commented out? (fit_ctd.py, ln 460) MK
btl_data_all['CTDCOND1'] = fit_ctd.conductivity_polyfit(
btl_data_all['CTDCOND1'], btl_data_all['CTDTMP1'],
btl_data_all['CTDPRS'], coef_cond_1)
btl_data_all['CTDCOND2'] = fit_ctd.conductivity_polyfit(
btl_data_all['CTDCOND2'], btl_data_all['CTDTMP2'],
btl_data_all['CTDPRS'], coef_cond_2)
# btl_data_all['CTDCOND1'],btl_data_all['CTDSAL'] = fit_ctd.conductivity_polyfit(btl_data_all['CTDCOND1'],btl_data_all['CTDTMP1'],btl_data_all['CTDPRS'],coef_cond_1)
# btl_data_all['CTDCOND2'],sal_2 = fit_ctd.conductivity_polyfit(btl_data_all['CTDCOND2'],btl_data_all['CTDTMP2'],btl_data_all['CTDPRS'],coef_cond_2)
# fit_ctd.conductivity_polyfit has salinity commented out? (fit_ctd.py, ln 460) MK
time_data_all['CTDCOND1'] = fit_ctd.conductivity_polyfit(
time_data_all['CTDCOND1'], time_data_all['CTDTMP1'],
time_data_all['CTDPRS'], coef_cond_1)
time_data_all['CTDCOND2'] = fit_ctd.conductivity_polyfit(
time_data_all['CTDCOND2'], time_data_all['CTDTMP2'],
time_data_all['CTDPRS'], coef_cond_2)
# time_data_all['CTDCOND1'],time_data_all['CTDSAL'] = fit_ctd.conductivity_polyfit(time_data_all['CTDCOND1'], time_data_all['CTDTMP1'], time_data_all['CTDPRS'], coef_cond_1)
# time_data_all['CTDCOND2'],sal2 = fit_ctd.conductivity_polyfit(time_data_all['CTDCOND2'], time_data_all['CTDTMP2'], time_data_all['CTDPRS'], coef_cond_2)
qual_flag_cond = process_ctd.combine_quality_flags(
[df_ques_c1, df_ques_c2, df_ques_refc])
# Finally WRT to Cond
btl_data_all['BTLCOND'] = fit_ctd.CR_to_cond(btl_data_all.CRavg,
btl_data_all.BathTEMP,
btl_data_all.CTDTMP1,
btl_data_all.CTDPRS)
df_cond_good = process_ctd.prepare_fit_data(btl_data_all, 'BTLCOND')
coef_cond_prim, df_ques_c1 = process_ctd.calibrate_param(
df_cond_good.CTDCOND1, df_cond_good.BTLCOND, df_cond_good.CTDPRS,
'C', 2, df_cond_good.SSSCC, df_cond_good.btl_fire_num)
coef_cond_sec, df_ques_c2 = process_ctd.calibrate_param(
df_cond_good.CTDCOND2, df_cond_good.BTLCOND, df_cond_good.CTDPRS,
'C', 2, df_cond_good.SSSCC, df_cond_good.btl_fire_num)
# Apply fitting coef to data
# fit_ctd.conductivity_polyfit has salinity commented out? (fit_ctd.py, ln 460) MK
btl_data_all['CTDCOND1'] = fit_ctd.conductivity_polyfit(
btl_data_all['CTDCOND1'], btl_data_all['CTDTMP1'],
btl_data_all['CTDPRS'], coef_cond_prim)
btl_data_all['CTDCOND2'] = fit_ctd.conductivity_polyfit(
btl_data_all['CTDCOND2'], btl_data_all['CTDTMP2'],
btl_data_all['CTDPRS'], coef_cond_sec)
# btl_data_all['CTDCOND1'],sal_1 = fit_ctd.conductivity_polyfit(btl_data_all['CTDCOND1'],btl_data_all['CTDTMP1'],btl_data_all['CTDPRS'],coef_cond_prim)
# btl_data_all['CTDCOND2'],sal_2 = fit_ctd.conductivity_polyfit(btl_data_all['CTDCOND2'],btl_data_all['CTDTMP2'],btl_data_all['CTDPRS'],coef_cond_sec)
# fit_ctd.conductivity_polyfit has salinity commented out? (fit_ctd.py, ln 460) MK
time_data_all['CTDCOND1'] = fit_ctd.conductivity_polyfit(
time_data_all['CTDCOND1'], time_data_all['CTDTMP1'],
time_data_all['CTDPRS'], coef_cond_prim)
time_data_all['CTDCOND2'] = fit_ctd.conductivity_polyfit(
time_data_all['CTDCOND2'], time_data_all['CTDTMP2'],
time_data_all['CTDPRS'], coef_cond_sec)
# time_data_all['CTDCOND1'],time_data_all['CTDSAL'] = fit_ctd.conductivity_polyfit(time_data_all['CTDCOND1'], time_data_all['CTDTMP1'], time_data_all['CTDPRS'], coef_cond_prim)
# time_data_all['CTDCOND2'],sal2 = fit_ctd.conductivity_polyfit(time_data_all['CTDCOND2'], time_data_all['CTDTMP2'], time_data_all['CTDPRS'], coef_cond_sec)
qual_flag_cond = process_ctd.combine_quality_flags(
[df_ques_c1, df_ques_c2, df_ques_refc])
###########################################################################
#
# ## Oxygen Calibration
# btl_data_all,time_data_all = oxy_fitting.calibrate_oxygen(time_data_all,btl_data_all,ssscc)
# subprocess.run(['oxy_fit_script.py'], stdout=subprocess.PIPE)
#
#
# subprocess.run(['ctd_to_bottle.py'], stdout=subprocess.PIPE)
#
#
# ##########################################################################
#####
# Oxygen calibration (MK)
# look over this code again, maybe refence matlab scripts from PMEL
import ctdcal.oxy_fitting as oxy_fitting
import gsw
# definitions
import ctdcal.settings as settings
hex_prefix = settings.ctd_processing_dir['hex_prefix']
hex_postfix = settings.ctd_processing_dir['hex_postfix']
xml_prefix = settings.ctd_processing_dir['xml_prefix']
xml_postfix = settings.ctd_processing_dir['xml_postfix']
oxy_btl_col = settings.bottle_inputs['btl_oxy']
rinko_volts = settings.ctd_inputs['rinko_oxy']
rinko_btl_volts = settings.bottle_inputs['rinko_oxy']
dov_col = settings.ctd_inputs['dov']
# unsure about these
lat_col = config['analytical_inputs']['lat']
lon_col = config['analytical_inputs']['lon']
lat_btl_col = config['inputs']['lat']
lon_btl_col = config['inputs']['lon']
sal_btl_col = config['analytical_inputs']['btl_salt']
t_btl_col = config['inputs']['t']
t_col = config['analytical_inputs']['t']
# calculate sigma
btl_data_all['sigma_btl'] = oxy_fitting.sigma_from_CTD(
btl_data_all[sal_btl_col], btl_data_all[t_btl_col],
btl_data_all[p_btl_col], btl_data_all[lon_btl_col],
btl_data_all[lat_btl_col])
time_data_all['sigma_ctd'] = oxy_fitting.sigma_from_CTD(
time_data_all[sal_col], time_data_all[t_col], time_data_all[p_col],
time_data_all[lon_col], time_data_all[lat_col])
btl_data_all[oxy_btl_col] = oxy_fitting.calculate_bottle_oxygen(
ssscc, btl_data_all['SSSCC'], btl_data_all['TITR_VOL'],
btl_data_all['TITR_TEMP'], btl_data_all['FLASKNO'])
btl_data_all[oxy_btl_col] = oxy_fitting.oxy_ml_to_umolkg(
btl_data_all[oxy_btl_col], btl_data_all['sigma_btl'])
btl_data_all['OXYGEN_FLAG_W'] = oxy_fitting.flag_winkler_oxygen(
btl_data_all[oxy_btl_col])
# Calculate SA and PT
btl_data_all['SA'] = gsw.SA_from_SP(btl_data_all[sal_btl_col],
btl_data_all[p_btl_col],
btl_data_all[lon_btl_col],
btl_data_all[lat_btl_col])
btl_data_all['PT'] = gsw.pt0_from_t(btl_data_all['SA'],
btl_data_all[t_btl_col],
btl_data_all[p_btl_col])
time_data_all['SA'] = gsw.SA_from_SP(time_data_all[sal_col],
time_data_all[p_col],
time_data_all[lon_col],
time_data_all[lat_col])
time_data_all['PT'] = gsw.pt0_from_t(time_data_all['SA'],
time_data_all[t_col],
time_data_all[p_col])
# Calculate OS in µmol/kg
btl_data_all['OS_btl'] = oxy_fitting.os_umol_kg(btl_data_all['SA'],
btl_data_all['PT'])
time_data_all['OS_ctd'] = oxy_fitting.os_umol_kg(time_data_all['SA'],
time_data_all['PT'])
# collect data by stations
btl_data_all['oxy_stn_group'] = btl_data_all['SSSCC'].str[0:3]
time_data_all['oxy_stn_group'] = time_data_all['SSSCC'].str[0:3]
station_list = time_data_all['oxy_stn_group'].unique()
station_list = station_list.tolist()
station_list.sort()
btl_data_oxy = btl_data_all.copy() #loc[btl_data_all['OXYGEN'].notnull()]
rinko_coef0 = rinko.rinko_o2_cal_parameters()
all_rinko_df = pd.DataFrame()
all_sbe43_df = pd.DataFrame()
rinko_dict = {}
sbe43_dict = {}
for station in station_list:
#time_data = time_data_all[time_data_all['SSSCC'].str[0:3] == station].copy()
#btl_data = btl_data_oxy[btl_data_oxy['SSSCC'].str[0:3] == station].copy()
time_data = time_data_all[time_data_all['oxy_stn_group'] ==
station].copy()
btl_data = btl_data_oxy[btl_data_oxy['oxy_stn_group'] ==
station].copy()
if (btl_data['OXYGEN_FLAG_W'] == 9).all():
rinko_dict[station] = np.full(8, np.nan)
sbe43_dict[station] = np.full(7, np.nan)
print(station + ' skipped, all oxy data is NaN')
continue
rinko_coef, rinko_oxy_df = rinko.rinko_oxygen_fit(
btl_data[p_btl_col], btl_data[oxy_btl_col], btl_data['sigma_btl'],
time_data['sigma_ctd'], time_data['OS_ctd'], time_data[p_col],
time_data[t_col], time_data[rinko_volts], rinko_coef0,
btl_data['SSSCC'])
station_ssscc = time_data['SSSCC'].values[0]
hex_file = hex_prefix + station_ssscc + hex_postfix
xml_file = xml_prefix + station_ssscc + xml_postfix
sbe_coef0 = oxy_fitting.get_SB_coef(hex_file, xml_file)
sbe_coef, sbe_oxy_df = oxy_fitting.sbe43_oxy_fit(
btl_data[p_btl_col], btl_data[oxy_btl_col], btl_data['sigma_btl'],
time_data['sigma_ctd'], time_data['OS_ctd'], time_data[p_col],
time_data[t_col], time_data[dov_col], time_data['scan_datetime'],
sbe_coef0, btl_data['SSSCC'])
rinko_dict[station] = rinko_coef
sbe43_dict[station] = sbe_coef
all_rinko_df = pd.concat([all_rinko_df, rinko_oxy_df])
all_sbe43_df = pd.concat([all_sbe43_df, sbe_oxy_df])
print(station + ' Done!')
sbe43_coef_df = oxy_fitting.create_coef_df(sbe43_dict)
rinko_coef_df = oxy_fitting.create_coef_df(rinko_dict)
btl_data_all = btl_data_all.merge(
all_rinko_df,
left_on=['SSSCC', p_btl_col],
right_on=['SSSCC_rinko', 'CTDPRS_rinko_btl'],
how='left')
btl_data_all = btl_data_all.merge(
all_sbe43_df,
left_on=['SSSCC', p_btl_col],
right_on=['SSSCC_sbe43', 'CTDPRS_sbe43_btl'],
how='left')
btl_data_all.drop(list(btl_data_all.filter(regex='rinko')),
axis=1,
inplace=True)
btl_data_all.drop(list(btl_data_all.filter(regex='sbe43')),
axis=1,
inplace=True)
### Handle Missing Values
X = btl_data_all['CTDOXYVOLTS_x'], btl_data_all['CTDPRS'], btl_data_all[
t_btl_col], btl_data_all['dv_dt_x'], btl_data_all['OS_btl']
rinko_X = btl_data_all[p_btl_col], btl_data_all[t_btl_col], btl_data_all[
'CTDRINKOVOLTS'], btl_data_all['OS_btl']
for station in btl_data_all['SSSCC']:
coef_43 = sbe43_coef_df.loc[station[0:3]]
coef_rinko = rinko_coef_df.loc[station[0:3]]
btl_data_all['CTDOXY_fill'] = oxy_fitting.oxy_equation(
X, coef_43[0], coef_43[1], coef_43[2], coef_43[3], coef_43[4],
coef_43[5], coef_43[6])
btl_data_all['CTDRINKO_fill'] = rinko.rinko_curve_fit_eq(
rinko_X, coef_rinko[0], coef_rinko[1], coef_rinko[2],
coef_rinko[3], coef_rinko[4], coef_rinko[5], coef_rinko[6],
coef_rinko[7])
btl_data_all.loc[btl_data_all['CTDOXY'].isnull(), 'CTDOXY_FLAG_W'] = 2
btl_data_all.loc[btl_data_all['CTDOXY'].isnull(),
'CTDOXY'] = btl_data_all.loc[
btl_data_all['CTDOXY'].isnull(), 'CTDOXY_fill']
btl_data_all.loc[btl_data_all['CTDRINKO'].isnull(), 'CTDRINKO_FLAG_W'] = 2
btl_data_all.loc[btl_data_all['CTDRINKO'].isnull(),
'CTDRINKO'] = btl_data_all.loc[
btl_data_all['CTDRINKO'].isnull(), 'CTDRINKO_fill']
btl_data_all = oxy_fitting.flag_oxy_data(btl_data_all)
btl_data_all = oxy_fitting.flag_oxy_data(btl_data_all,
ctd_oxy_col='CTDRINKO',
flag_col='CTDRINKO_FLAG_W')
btl_data_all[
'res_rinko'] = btl_data_all['OXYGEN'] - btl_data_all['CTDRINKO']
btl_data_all['res_sbe43'] = btl_data_all['OXYGEN'] - btl_data_all['CTDOXY']
btl_data_all.loc[
np.abs(btl_data_all['res_rinko']) >= 6,
'CTDRINKO_FLAG_W'] = 3 # what's the benefit of np.abs() vs abs()?
btl_data_all.loc[np.abs(btl_data_all['res_sbe43']) >= 6,
'CTDOXY_FLAG_W'] = 3
time_data_all['CTDOXY'] = '-999'
time_data_all['CTDRINKO'] = '-999'
btl_data_all.sort_values(by='sigma_btl', inplace=True)
time_data_all.sort_values(by='sigma_ctd', inplace=True)
for station in station_list:
rinko_coef = rinko_coef_df.loc[station].values
if np.isnan(rinko_coef).all():
print(station + ' data bad, leaving -999 values and flagging as 9')
time_data_all.loc[time_data_all['SSSCC'].str[0:3] == station,
'CTDRINKO_FLAG_W'] = 9
time_data_all.loc[time_data_all['SSSCC'].str[0:3] == station,
'CTDOXY_FLAG_W'] = 9
continue
time_data = time_data_all[time_data_all['oxy_stn_group'] ==
station].copy()
time_data['CTDOXY'] = oxy_fitting.SB_oxy_eq(
sbe43_coef_df.loc[station], time_data[dov_col], time_data[p_col],
time_data[t_col], time_data['dv_dt'], time_data['OS_ctd'])
time_data['CTDRINKO'] = rinko.rinko_curve_fit_eq(
(time_data[p_col], time_data[t_col], time_data[rinko_volts],
time_data['OS_ctd']), rinko_coef[0], rinko_coef[1], rinko_coef[2],
rinko_coef[3], rinko_coef[4], rinko_coef[5], rinko_coef[6],
rinko_coef[7])
time_data_all.loc[time_data_all['SSSCC'].str[0:3] == station,
'CTDOXY'] = time_data['CTDOXY']
time_data_all.loc[time_data_all['SSSCC'].str[0:3] == station,
'CTDRINKO'] = time_data['CTDRINKO']
time_data_all.loc[time_data_all['SSSCC'].str[0:3] == station,
'CTDRINKO_FLAG_W'] = 2
time_data_all.loc[time_data_all['SSSCC'].str[0:3] == station,
'CTDOXY_FLAG_W'] = 2
print(station + ' time data done')
#####
#######################
# turn correction instantaneous CTD measurements into points @ bottle firings
# subprocess.run(['ctd_to_bottle.py'], stdout=subprocess.PIPE)
################ Clean and export data #######################
# btl_data_all = process_ctd.merge_cond_flags(btl_data_all,qual_flag_cond, c_btl_col) # make code not require specific column input
btl_data_all = process_ctd.merge_refcond_flags(btl_data_all,
qual_flag_cond)
# btl_data_all = process_ctd.merge_temp_flags(btl_data_all, qual_flag_temp, t_btl_col)
btl_data_all = process_ctd.merged_reftemp_flags(btl_data_all,
qual_flag_temp)
### Export Quality Flags
qual_flag_temp.to_csv('data/logs/qual_flag_temp_new.csv', index=False)
qual_flag_cond.to_csv('data/logs/qual_flag_cond_new.csv', index=False)
### Clean up Bottle Data by removing rows with no ctd data
btl_data_all = oxy_fitting.clean_oxygen_df(
btl_data_all) # clean up column names
btl_data_all = btl_data_all.dropna(subset=cols)
### Create CT Files(oxygen done by hand)
# needed to run when there's no data available yet (i.e. at sea)
# btl_data_all['OXYGEN'] = -999
# btl_data_all['OXYGEN'] = -999
# time_data_all['CTDOXY'] = -999
# depreciated according to process_ctd.py (MK)
# process_ctd.export_btl_data(btl_data_all,expocode,sectionID,expocode)
# process_ctd.export_time_data(time_data_all,ssscc,int(sample_rate),int(search_time),expocode,sectionID,ctd,p_column_names,p_column_units)
### MK
# create depth_log.csv
station_list = time_data_all['SSSCC'].str[0:3].unique()
depth_dict = {}
for station in station_list:
print(station)
time_data = time_data_all[time_data_all['SSSCC'].str[0:3] ==
station].copy()
max_depth = process_ctd.find_cast_depth(time_data['CTDPRS'],
time_data['GPSLAT'],
time_data['ALT'])
depth_dict[station] = max_depth
depth_df = pd.DataFrame.from_dict(depth_dict, orient='index')
depth_df.reset_index(inplace=True)
depth_df.rename(columns={0: 'DEPTH', 'index': 'STNNBR'}, inplace=True)
depth_df.to_csv('data/logs/depth_log.csv', index=False)
# needs depth_log.csv, manual_depth_log.csv
process_ctd.export_ct1(time_data_all, ssscc, expocode, sectionID, ctd,
p_column_names, p_column_units)
def derive_ctd_parameters(dba):
# Get the list of sensors parsed from the dba_file
dba_sensors = [s['sensor_name'] for s in dba['sensors']]
ctd_sensors = []
found_ctd_sensors = [s for s in SCI_CTD_SENSORS if s in dba_sensors]
if len(found_ctd_sensors) == len(SCI_CTD_SENSORS):
ctd_sensors = SCI_CTD_SENSORS
else:
found_ctd_sensors = [s for s in M_CTD_SENSORS if s in dba_sensors]
if len(found_ctd_sensors) == len(M_CTD_SENSORS):
ctd_sensors = M_CTD_SENSORS
if not ctd_sensors:
logging.warning('Required CTD sensors not found in {:s}'.format(dba['file_metadata']['source_file']))
return dba
else:
lati = dba_sensors.index(ctd_sensors[0])
loni = dba_sensors.index(ctd_sensors[1])
pi = dba_sensors.index(ctd_sensors[2])
ti = dba_sensors.index(ctd_sensors[3])
ci = dba_sensors.index(ctd_sensors[4])
lat = dba['data'][:, lati]
lon = dba['data'][:, loni]
p = dba['data'][:, pi]
t = dba['data'][:, ti]
c = dba['data'][:, ci]
# Calculate salinity
salt = calculate_practical_salinity(c, t, p)
# Calculate density
density = calculate_density(t, p, salt, lat, lon)
# Calculate potential temperature
p_temp = pt0_from_t(salt, t, p)
# Add parameters as long as they exist in ncw.nc_sensor_defs
# if 'salinity' in ncw.nc_sensor_defs:
sensor_def = {'sensor_name': 'salinity',
'attrs': {'ancillary_variables': 'conductivity,temperature,presssure',
'observation_type': 'calculated'}}
dba['data'] = np.append(dba['data'], np.expand_dims(salt, 1), axis=1)
dba['sensors'].append(sensor_def)
# if 'density' in ncw.nc_sensor_defs:
sensor_def = {'sensor_name': 'density',
'attrs': {
'ancillary_variables': 'conductivity,temperature,presssure,latitude,longitude',
'observation_type': 'calculated'}}
dba['data'] = np.append(dba['data'], np.expand_dims(density, 1), axis=1)
dba['sensors'].append(sensor_def)
# if 'potential_temperature' in ncw.nc_sensor_defs:
sensor_def = {'sensor_name': 'potential_temperature',
'attrs': {'ancillary_variables': 'salinity,temperature,presssure',
'observation_type': 'calculated'}}
dba['data'] = np.append(dba['data'], np.expand_dims(p_temp, 1), axis=1)
dba['sensors'].append(sensor_def)
# if 'sound_speed' in ncw.nc_sensor_defs:
# Calculate sound speed
svel = calculate_sound_speed(t, p, salt, lat, lon)
sensor_def = {'sensor_name': 'sound_speed',
'attrs': {
'ancillary_variables': 'conductivity,temperature,presssure,latitude,longitude',
'observation_type': 'calculated'}}
dba['data'] = np.append(dba['data'], np.expand_dims(svel, 1), axis=1)
dba['sensors'].append(sensor_def)
return dba
def add_oxsol(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
t = var_temp[:]
SP = var_psal[:]
if "PRES" in ds.variables:
var_pres = ds.variables["PRES"]
pres_var = "PRES"
comment = ""
p = var_pres[:]
elif "NOMINAL_DEPTH" in ds.variables:
var_pres = ds.variables["NOMINAL_DEPTH"]
pres_var = "NOMINAL_DEPTH"
comment = ", using nominal depth of " + str(var_pres[:])
p = var_pres[:]
else:
p = 0
pres_var = "nominal depth of 0 dbar"
comment = ", using nominal depth 0 dbar"
lat = -47
lon = 142
try:
lat = ds.variables["LATITUDE"][0]
lon = ds.variables["LONGITUDE"][0]
except:
pass
SA = gsw.SA_from_SP(SP, p, lon, lat)
pt = gsw.pt0_from_t(SA, t, p)
oxsol = gsw.O2sol_SP_pt(SP, pt)
if 'OXSOL' in ds.variables:
ncVarOut = ds.variables['OXSOL']
else:
ncVarOut = ds.createVariable(
"OXSOL", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = oxsol
ncVarOut.units = "umol/kg"
ncVarOut.long_name = "moles_of_oxygen_per_unit_mass_in_sea_water_at_saturation"
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html function gsw.O2sol_SP_pt" + comment
ncVarOut.coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr(
'history', hist + datetime.utcnow().strftime("%Y-%m-%d") +
" added oxygen solubility")
ds.close()
return netCDFfile
def eps_overturn(P, Z, T, S, lon, lat, dnoise=0.001, pdref=4000):
'''
Calculate profile of turbulent dissipation epsilon from structure of a ctd
profile.
Currently this takes only one profile and not a matrix from a whole twoyo.
'''
import numpy as np
import gsw
# avoid error due to nan's in conditional statements
np.seterr(invalid='ignore')
z0 = Z.copy()
z0 = z0.astype('float')
# Find non-NaNs
x = np.where(np.isfinite(T))
x = x[0]
# Extract variables without the NaNs
p = P[x].copy()
z = Z[x].copy()
z = z.astype('float')
t = T[x].copy()
s = S[x].copy()
# cn2 = ctdn['n2'][x].copy()
SA = gsw.SA_from_SP(s, t, lon, lat)
CT = gsw.CT_from_t(SA, t, p)
PT = gsw.pt0_from_t(SA, t, p)
sg4 = gsw.pot_rho_t_exact(SA, t, p, pdref)-1000
# Create intermediate density profile
D0 = sg4[0]
sgt = D0-sg4[0]
n = sgt/dnoise
n = np.round(n)
sgi = [D0+n*dnoise] # first element
for i in np.arange(1, np.alen(sg4), 1):
sgt = sg4[i]-sgi[i-1]
n = sgt/dnoise
n = np.round(n)
sgi.append(sgi[i-1]+n*dnoise)
sgi = np.array(sgi)
# Sort
Ds = np.sort(sgi, kind='mergesort')
Is = np.argsort(sgi, kind='mergesort')
# Calculate Thorpe length scale
TH = z[Is]-z
cumTH = np.cumsum(TH)
aa = np.where(cumTH > 1)
aa = aa[0]
# last index in overturns
aatmp = aa.copy()
aatmp = np.append(aatmp, np.nanmax(aa)+10)
aad = np.diff(aatmp)
aadi = np.where(aad > 1)
aadi = aadi[0]
LastItems = aa[aadi].copy()
# first index in overturns
aatmp = aa.copy()
aatmp = np.insert(aatmp, 0, -1)
aad = np.diff(aatmp)
aadi = np.where(aad > 1)
aadi = aadi[0]
FirstItems = aa[aadi].copy()
# % Sort temperature and salinity for calculating the buoyancy frequency
PTs = PT[Is]
SAs = SA[Is]
CTs = CT[Is]
# % Loop through detected overturns
# % and calculate Thorpe Scales, N2 and dT/dz over the overturn
# THsc = nan(size(Z));
THsc = np.zeros_like(z)*np.nan
N2 = np.zeros_like(z)*np.nan
# CN2 = np.ones_like(z)*np.nan
DTDZ = np.zeros_like(z)*np.nan
out = {}
out['idx'] = np.zeros_like(z0)
for iostart, ioend in zip(FirstItems, LastItems):
idx = np.arange(iostart, ioend+1, 1)
out['idx'][x[idx]] = 1
sc = np.sqrt(np.mean(np.square(TH[idx])))
# ctdn2 = np.nanmean(cn2[idx])
# Buoyancy frequency calculated over the overturn from sorted profiles
# Go beyond overturn (I am sure this will cause trouble with the
# indices at some point)
n2, Np = gsw.Nsquared(SAs[[iostart-1, ioend+1]],
CTs[[iostart-1, ioend+1]],
p[[iostart-1, ioend+1]], lat)
# Fill depth range of the overturn with the Thorpe scale
THsc[idx] = sc
# Fill depth range of the overturn with N^2
N2[idx] = n2
# Fill depth range of the overturn with average 10m N^2
# CN2[idx] = ctdn2
# % Calculate epsilon
THepsilon = 0.9*THsc**2.0*np.sqrt(N2)**3
THepsilon[N2 <= 0] = np.nan
THk = 0.2*THepsilon/N2
out['eps'] = np.zeros_like(z0)*np.nan
out['eps'][x] = THepsilon
out['k'] = np.zeros_like(z0)*np.nan
out['k'][x] = THk
out['n2'] = np.zeros_like(z0)*np.nan
out['n2'][x] = N2
out['Lt'] = np.zeros_like(z0)*np.nan
out['Lt'][x] = THsc
return out
def oxygen(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
var_pres = ds.variables["PRES"]
t = var_temp[:]
SP = var_psal[:]
p = var_pres[:]
SA = gsw.SA_from_SP(SP, p, ds.variables["LONGITUDE"][0], ds.variables["LATITUDE"][0])
CT = gsw.CT_from_t(SA, t, p)
pt = gsw.pt0_from_t(SA, t, p)
sigma_theta0 = gsw.sigma0(SA, CT)
oxsol = ds.variables["OXSOL"][:]
ts = np.log((298.15 - t) / (273.15 + t))
# psal correction, from Aanderra TD-218, Gordon and Garcia oxygaen solubility salinity coefficents
B0 = -6.24097e-3
C0 = -3.11680e-7
B1 = -6.93498e-3
B2 = -6.90358e-3
B3 = -4.29155e-3
psal_correction = np.exp(SP *(B0 + B1 * ts + B2 * np.power(ts, 2) + B3 * np.power(ts, 3)) + np.power(SP, 2) * C0)
# get correction slope, offset
slope = 1.0
offset = 0.0
try:
slope = ds.variables['DOX2_RAW'].calibration_slope
offset = ds.variables['DOX2_RAW'].calibration_offset
except AttributeError:
pass
# calculate disolved oxygen, umol/kg
dox2_raw = ds.variables['DOX2_RAW'] * psal_correction * slope + offset
dox2 = 1000 * dox2_raw / (sigma_theta0 + 1000)
if 'DOX2' not in ds.variables:
ncVarOut = ds.createVariable("DOX2", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOX2']
ncVarOut[:] = dox2
ncVarOut.units = "umol/kg"
ncVarOut.comment = "calculated using DOX2 = DOX2_RAW * PSAL_CORRECTION / ((sigma_theta(P=0,Theta,S) + 1000).. Sea Bird AN 64, Aanderaa TD210 Optode Manual"
ncVarOut.comment_calibration = "calibration slope " + str(slope) + " offset " + str(offset) + " umol/l"
# calculate, and write the oxygen mass/seawater mass
if 'DOX_MG' not in ds.variables:
ncVarOut = ds.createVariable("DOXY", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOXY']
ncVarOut[:] = dox2_raw / 31.24872
ncVarOut.units = "mg/l"
ncVarOut.comment = "calculated using DOXY = * DOX2_RAW * PSAL_CORRECTION / 31.24872... Aanderaa TD210 Optode Manual"
ncVarOut.comment_calibration = "calibration slope " + str(slope) + " offset " + str(offset) + " umol/l"
# calculate and write oxygen solubility, ratio of disolved oxgen / oxygen solubility
if 'DOXS' not in ds.variables:
ncVarOut = ds.createVariable("DOXS", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
else:
ncVarOut = ds.variables['DOXS']
ncVarOut[:] = dox2/oxsol
ncVarOut.units = "1"
ncVarOut.comment = "calculated using DOX2/OXSOL"
# finish off, and close file
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + " : added derived oxygen, DOX2, DOXS, DOX_MG")
ds.close()
def process_all_new():
# Directory and file information
expocode = settings.cruise['expocode']
sectionID = settings.cruise['sectionid']
raw_directory = settings.ctd_processing_dir['raw_data_directory']
time_directory = settings.ctd_processing_dir['time_data_directory']
converted_directory = settings.ctd_processing_dir['converted_directory']
pressure_directory = settings.ctd_processing_dir['pressure_data_directory']
oxygen_directory = settings.ctd_processing_dir['oxygen_directory']
btl_directory = settings.ctd_processing_dir['bottle_directory']
o2flask_file = settings.ctd_processing_dir['o2flask_file']
log_directory = settings.ctd_processing_dir['log_directory']
p_log_file = settings.ctd_processing_dir['pressure_log']
hex_prefix = settings.ctd_processing_dir['hex_prefix']
hex_postfix = settings.ctd_processing_dir['hex_postfix']
xml_prefix = settings.ctd_processing_dir['xml_prefix']
xml_postfix = settings.ctd_processing_dir['xml_postfix']
# CTD Data Inputs
p_col = settings.ctd_inputs['p']
t_col = settings.ctd_inputs['t']
t1_col = settings.ctd_inputs['t1']
t2_col = settings.ctd_inputs['t2']
c_col = settings.ctd_inputs['c']
c1_col = settings.ctd_inputs['c1']
c2_col = settings.ctd_inputs['c2']
sal_col = settings.ctd_inputs['salt']
dov_col = settings.ctd_inputs['dov']
lat_col = settings.ctd_inputs['lat']
lon_col = settings.ctd_inputs['lon']
time_col = settings.ctd_inputs['scan_datetime']
# Bottle Data Inputs
p_btl_col = settings.bottle_inputs['p']
t_btl_col = settings.bottle_inputs['t']
t1_btl_col = settings.bottle_inputs['t1']
t2_btl_col = settings.bottle_inputs['t2']
c_btl_col = settings.bottle_inputs['c']
c1_btl_col = settings.bottle_inputs['c1']
c2_btl_col = settings.bottle_inputs['c2']
reft_col = settings.bottle_inputs['reft']
cond_col = settings.bottle_inputs['btl_cond']
cr_avg = settings.bottle_inputs['cond_ratio']
bath_temp = settings.bottle_inputs['bath_temp']
sal_btl_col = settings.bottle_inputs['salt']
dov_btl_col = settings.bottle_inputs['dov']
lat_btl_col = settings.bottle_inputs['lat']
lon_btl_col = settings.bottle_inputs['lon']
oxy_btl_col = settings.bottle_inputs['btl_oxy']
time_btl_col = settings.bottle_inputs['scan_datetime']
# CTD Information
sample_rate = settings.ctd_processing_constants['sample_rate']
search_time = settings.ctd_processing_constants['roll_filter_time']
ctd = settings.ctd_processing_constants['ctd_serial']
p_column_names = settings.pressure_series_output['column_name']
p_column_units = settings.pressure_series_output['column_units']
btl_data_prefix = 'data/bottle/'
btl_data_postfix = '_btl_mean.pkl'
time_data_prefix = 'data/time/'
time_data_postfix = '_time.pkl'
p_log_file = 'data/logs/ondeck_pressure.csv'
# Columns from btl and ctd file to be read:
btl_cols = settings.btl_input_array
ctd_cols = settings.ctd_input_array
ssscc = settings.ssscc
# time_start = time.perf_counter()
cnv_dir_list = os.listdir(converted_directory)
time_dir_list = os.listdir(time_directory)
btl_dir_list = os.listdir(btl_directory)
for station in ssscc:
if '{}.pkl'.format(station) in cnv_dir_list:
continue
#convert hex to ctd
hex_file = hex_prefix + station + hex_postfix
xml_file = xml_prefix + station + xml_postfix
sbe_convert.convert_sbe(station, hex_file, xml_file,
converted_directory)
print('Converted_sbe SSSCC: ' + station + ' done')
# time_convert = time.perf_counter()
for station in ssscc:
if '{}_time.pkl'.format(station) in time_dir_list:
continue
sbe_convert.sbe_metadata(station)
print('sbe_metadata SSSCC: ' + station + ' done')
for station in ssscc:
if '{}_btl_mean.pkl'.format(station) in btl_dir_list:
continue
#process bottle file
sbe_convert.process_bottle(station)
print('process_bottle SSSCC: ' + station + ' done')
# time_bottle = time.perf_counter()
###########################################################################
### File I/O
# Load in all bottle, time, ref_data files into DataFrame
btl_data_all = process_ctd.load_all_ctd_files(ssscc, btl_data_prefix,
btl_data_postfix, 'bottle',
btl_cols)
time_data_all = process_ctd.load_all_ctd_files(ssscc, time_data_prefix,
time_data_postfix, 'time',
ctd_cols)
################################################################################
### Pressure Calibration
# Determine Pressure offset from logs
pressure_log = process_ctd.load_pressure_logs(p_log_file)
p_off = process_ctd.get_pressure_offset(pressure_log.ondeck_start_p,
pressure_log.ondeck_end_p)
btl_data_all[p_btl_col] = fit_ctd.apply_pressure_offset(
btl_data_all[p_btl_col], p_off)
time_data_all[p_col] = fit_ctd.apply_pressure_offset(
time_data_all[p_col], p_off)
###########################################################################
df_ques_t1 = pd.DataFrame()
df_ques_t2 = pd.DataFrame()
df_ques_c1 = pd.DataFrame()
df_ques_c2 = pd.DataFrame()
### Temperature Calibration
for x in range(2):
# Second order calibration
df_temp_good = process_ctd.prepare_fit_data(btl_data_all, reft_col)
df_ques_reft = process_ctd.quality_check(df_temp_good[t2_btl_col],
df_temp_good[t1_btl_col],
df_temp_good[p_btl_col],
df_temp_good['SSSCC'],
df_temp_good['btl_fire_num'],
'quest')
df_ques_reft['Parameter'] = 'REF_TEMP'
if settings.do_primary == 1:
coef_temp_1, df_ques_t1 = process_ctd.calibrate_param(
df_temp_good[t1_btl_col],
df_temp_good[reft_col],
df_temp_good[p_btl_col],
'TP',
2,
df_temp_good.SSSCC,
df_temp_good.btl_fire_num,
xRange='800:6000')
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col], coef_temp_1)
time_data_all[t1_col] = fit_ctd.temperature_polyfit(
time_data_all[t1_col], time_data_all[p_col], coef_temp_1)
elif settings.do_secondary == 1:
coef_temp_2, df_ques_t2 = process_ctd.calibrate_param(
df_temp_good[t2_btl_col],
df_temp_good[reft_col],
df_temp_good[p_btl_col],
'TP',
2,
df_temp_good.SSSCC,
df_temp_good.btl_fire_num,
xRange='1500:6000')
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col], coef_temp_2)
time_data_all[t2_col] = fit_ctd.temperature_polyfit(
time_data_all[t2_col], time_data_all[p_col], coef_temp_2)
# Apply fitting coef to data
# Construct Quality Flag file
qual_flag_temp = process_ctd.combine_quality_flags(
[df_ques_reft, df_ques_t1, df_ques_t2])
## First order calibtation
df_temp_good = process_ctd.prepare_fit_data(btl_data_all, reft_col)
# df_ques_reft = process_ctd.quality_check(df_temp_good[t2_btl_col], df_temp_good[t1_btl_col], df_temp_good[p_btl_col], df_temp_good['SSSCC'], df_temp_good['btl_fire_num'], 'quest')
# df_ques_reft['Parameter'] = 'REF_TEMP'
if settings.do_primary == 1:
coef_temp_prim, df_ques_t1 = process_ctd.calibrate_param(
df_temp_good[t1_btl_col], df_temp_good[reft_col],
df_temp_good[p_btl_col], 'T', 1, df_temp_good.SSSCC,
df_temp_good.btl_fire_num)
btl_data_all[t1_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t1_btl_col], btl_data_all[p_btl_col],
coef_temp_prim)
time_data_all[t1_col] = fit_ctd.temperature_polyfit(
time_data_all[t1_col], time_data_all[p_col], coef_temp_prim)
elif settings.do_secondary == 1:
coef_temp_sec, df_ques_t2 = process_ctd.calibrate_param(
df_temp_good[t2_btl_col], df_temp_good[reft_col],
df_temp_good[p_btl_col], 'T', 1, df_temp_good.SSSCC,
df_temp_good.btl_fire_num)
btl_data_all[t2_btl_col] = fit_ctd.temperature_polyfit(
btl_data_all[t2_btl_col], btl_data_all[p_btl_col],
coef_temp_sec)
time_data_all[t2_col] = fit_ctd.temperature_polyfit(
time_data_all[t2_col], time_data_all[p_col], coef_temp_sec)
# Apply fitting coef to data
qual_flag_temp = process_ctd.combine_quality_flags(
[df_ques_reft, df_ques_t1, df_ques_t2])
###########################################################################
#
### Conductivity Calibration
for x in range(2):
btl_data_all[cond_col] = fit_ctd.CR_to_cond(btl_data_all[cr_avg],
btl_data_all[bath_temp],
btl_data_all[t1_btl_col],
btl_data_all[p_btl_col])
df_cond_good = process_ctd.prepare_fit_data(btl_data_all, cond_col)
df_ques_refc = process_ctd.quality_check(df_cond_good[c2_btl_col],
df_temp_good[c1_btl_col],
df_temp_good[p_btl_col],
df_temp_good['SSSCC'],
df_temp_good['btl_fire_num'],
'quest')
df_ques_refc['Parameter'] = 'REF_COND'
# Second Order Calibration
if settings.do_primary == 1:
coef_cond_1, df_ques_c1 = process_ctd.calibrate_param(
df_cond_good[c1_btl_col],
df_cond_good[cond_col],
df_cond_good[p_btl_col],
'CP',
2,
df_cond_good['SSSCC'],
df_cond_good['btl_fire_num'],
xRange='800:6000')
btl_data_all[c1_btl_col], btl_data_all[
sal_btl_col] = fit_ctd.conductivity_polyfit(
btl_data_all[c1_btl_col], btl_data_all[t1_btl_col],
btl_data_all[p_btl_col], coef_cond_1)
time_data_all[c1_col], time_data_all[
sal_col] = fit_ctd.conductivity_polyfit(
time_data_all[c1_col], time_data_all[t1_col],
time_data_all[p_col], coef_cond_1)
elif settings.do_secondary == 1:
coef_cond_2, df_ques_c2 = process_ctd.calibrate_param(
df_cond_good[c2_btl_col],
df_cond_good[cond_col],
df_cond_good[p_btl_col],
'CP',
2,
df_cond_good['SSSCC'],
df_cond_good['btl_fire_num'],
xRange='1500:6000')
btl_data_all[c2_btl_col], sal_2 = fit_ctd.conductivity_polyfit(
btl_data_all[c2_btl_col], btl_data_all[t2_btl_col],
btl_data_all[p_btl_col], coef_cond_2)
time_data_all[c2_btl_col], sal2 = fit_ctd.conductivity_polyfit(
time_data_all[c2_col], time_data_all[t2_col],
time_data_all[p_col], coef_cond_2)
qual_flag_cond = process_ctd.combine_quality_flags(
[df_ques_c1, df_ques_c2, df_ques_refc])
btl_data_all[cond_col] = fit_ctd.CR_to_cond(btl_data_all[cr_avg],
btl_data_all[bath_temp],
btl_data_all[t1_btl_col],
btl_data_all[p_btl_col])
df_cond_good = process_ctd.prepare_fit_data(btl_data_all, cond_col)
if settings.do_primary == 1:
coef_cond_prim, df_ques_c1 = process_ctd.calibrate_param(
df_cond_good[c1_btl_col], df_cond_good[cond_col],
df_cond_good[p_btl_col], 'C', 2, df_cond_good['SSSCC'],
df_cond_good['btl_fire_num'])
btl_data_all[c1_btl_col], btl_data_all[
sal_btl_col] = fit_ctd.conductivity_polyfit(
btl_data_all[c1_btl_col], btl_data_all[t1_btl_col],
btl_data_all[p_btl_col], coef_cond_prim)
time_data_all[c1_col], time_data_all[
sal_col] = fit_ctd.conductivity_polyfit(
time_data_all[c1_col], time_data_all[t1_col],
time_data_all[p_col], coef_cond_prim)
elif settings.do_secondary == 1:
coef_cond_sec, df_ques_c2 = process_ctd.calibrate_param(
df_cond_good.CTDCOND2, df_cond_good.BTLCOND,
df_cond_good.CTDPRS, 'C', 2, df_cond_good.SSSCC,
df_cond_good.btl_fire_num)
btl_data_all[c2_btl_col], sal_2 = fit_ctd.conductivity_polyfit(
btl_data_all[c2_btl_col], btl_data_all[t2_btl_col],
btl_data_all[p_btl_col], coef_cond_sec)
time_data_all[c2_col], sal2 = fit_ctd.conductivity_polyfit(
time_data_all[c2_col], time_data_all[t2_col],
time_data_all[p_col], coef_cond_sec)
qual_flag_cond = process_ctd.combine_quality_flags(
[df_ques_c1, df_ques_c2, df_ques_refc])
###########################################################################
#
# ## Oxygen Calibration
# Calculate Sigma
btl_data_all['sigma_btl'] = oxy_fitting.sigma_from_CTD(
btl_data_all[sal_btl_col], btl_data_all[t_btl_col],
btl_data_all[p_btl_col], btl_data_all[lon_btl_col],
btl_data_all[lat_btl_col])
time_data_all['sigma_ctd'] = oxy_fitting.sigma_from_CTD(
time_data_all[sal_col], time_data_all[t_col], time_data_all[p_col],
time_data_all[lon_col], time_data_all[lat_col])
btl_data_all[oxy_btl_col] = oxy_fitting.calculate_bottle_oxygen(
ssscc, btl_data_all['SSSCC'], btl_data_all['TITR_VOL'],
btl_data_all['TITR_TEMP'], btl_data_all['FLASKNO'])
btl_data_all[oxy_btl_col] = oxy_fitting.oxy_ml_to_umolkg(
btl_data_all[oxy_btl_col], btl_data_all['sigma_btl'])
# Calculate SA and PT
btl_data_all['SA'] = gsw.SA_from_SP(btl_data_all[sal_btl_col],
btl_data_all[p_btl_col],
btl_data_all[lon_btl_col],
btl_data_all[lat_btl_col])
btl_data_all['PT'] = gsw.pt0_from_t(btl_data_all['SA'],
btl_data_all[t_btl_col],
btl_data_all[p_btl_col])
time_data_all['SA'] = gsw.SA_from_SP(time_data_all[sal_col],
time_data_all[p_col],
time_data_all[lon_col],
time_data_all[lat_col])
time_data_all['PT'] = gsw.pt0_from_t(time_data_all['SA'],
time_data_all[t_col],
time_data_all[p_col])
# Calculate OS in µmol/kg
btl_data_all['OS_btl'] = oxy_fitting.os_umol_kg(btl_data_all['SA'],
btl_data_all['PT'])
time_data_all['OS_ctd'] = oxy_fitting.os_umol_kg(time_data_all['SA'],
time_data_all['PT'])
oxy_df = pd.DataFrame()
coef_dict = {}
for station in ssscc:
btl_data = btl_data_all[btl_data_all['SSSCC'] == station]
time_data = time_data_all[time_data_all['SSSCC'] == station]
hex_file = hex_prefix + station + hex_postfix
xml_file = xml_prefix + station + xml_postfix
coef0 = oxy_fitting.get_SB_coef(hex_file, xml_file)
cfw_coef, df = oxy_fitting.oxy_fit(
btl_data[p_btl_col], btl_data[oxy_btl_col], btl_data['sigma_btl'],
time_data['sigma_ctd'], time_data['OS_ctd'], time_data[p_col],
time_data[t_col], time_data[dov_col], time_data[time_col], coef0)
df['SSSCC'] = station
coef_dict[station] = cfw_coef
oxy_df = pd.concat([oxy_df, df])
print(station, ' Completed')
coef_df = oxy_fitting.create_coef_df(coef_dict)
oxy_df = oxy_fitting.flag_oxy_data(oxy_df)
# Merge oxygen fitting DF to btl_data_all
btl_data_all = oxy_fitting.merge_oxy_df(btl_data_all, oxy_df)
# Apply coef to Time Data
time_data_all = oxy_fitting.apply_oxygen_coef_ctd(time_data_all, coef_df,
ssscc)
################ Clean and export data #######################
btl_data_all = process_ctd.merge_cond_flags(btl_data_all, qual_flag_cond)
btl_data_all = process_ctd.merge_refcond_flags(btl_data_all,
qual_flag_cond)
btl_data_all = process_ctd.merged_reftemp_flags(btl_data_all,
qual_flag_temp)
### Export Quality Flags
qual_flag_temp.to_csv('data/logs/qual_flag_temp_new.csv', index=False)
qual_flag_cond.to_csv('data/logs/qual_flag_cond_new.csv', index=False)
### Clean up Bottle Data by removing rows with no ctd data
btl_data_all = btl_data_all.dropna(subset=btl_cols)
### Add DATE and TIME
btl_data_all['DATE'] = ''
btl_data_all['TIME'] = ''
for station in ssscc:
df = btl_data_all.loc[btl_data_all['SSSCC'] == station].copy()
btl_data_all.loc[btl_data_all['SSSCC'] ==
station] = process_ctd.get_btl_time(
df, 'btl_fire_num', time_col)
### Create CT Files and HY files
process_ctd.export_btl_data(btl_data_all, expocode, sectionID, expocode)
process_ctd.export_time_data(time_data_all, ssscc, int(sample_rate),
int(search_time), expocode, sectionID, ctd,
p_column_names, p_column_units)
def extract_sbe43(netCDFfile):
ds = Dataset(netCDFfile, 'r')
ds.set_auto_mask(False)
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
var_pres = ds.variables["PRES"]
# the SBE43 voltage
var_v0 = ds.variables["V0"]
dep_code = ds.deployment_code
print('deployment', dep_code)
out_file = dep_code + "-SBE43.nc"
ds_out = Dataset(out_file, 'w', data_model='NETCDF4_CLASSIC')
ds_out.createDimension("TIME", len(ds.variables['TIME']))
ncVarIn = ds.variables['TIME']
ncTimesOut = ds_out.createVariable('TIME', "f8", ("TIME", ), zlib=True)
for a in ncVarIn.ncattrs():
if a != '_FillValue':
ncTimesOut.setncattr(a, ncVarIn.getncattr(a))
ncTimesOut[:] = ds.variables['TIME'][:]
# copy old variables into new file
in_vars = set([x for x in ds.variables])
z = in_vars.intersection([
'TEMP', 'PSAL', 'PRES', 'V0', 'TEMP_quality_control',
'PSAL_quality_control', 'PRES_quality_control', 'V0_quality_control',
'LATITUDE', 'LONGITUDE', 'NOMINAL_DEPTH'
])
for v in z:
print('copying', v, 'dimensions', ncVarIn.dimensions)
ncVarIn = ds.variables[v]
if '_FillValue' in ncVarIn.ncattrs():
fill = ncVarIn._FillValue
else:
fill = None
ncVarOut = ds_out.createVariable(
v, ncVarIn.dtype, ncVarIn.dimensions, fill_value=fill,
zlib=True) # fill_value=nan otherwise defaults to max
for a in ncVarIn.ncattrs():
if a != '_FillValue':
if a == 'ancillary_variables':
ncVarOut.setncattr(
a, v + '_quality_control'
) # only copying main quality control, not all individual flags
else:
ncVarOut.setncattr(a, ncVarIn.getncattr(a))
ncVarOut[:] = ncVarIn[:]
if 'DOX2' in ds.variables:
ncVarIn = ds.variables['DOX2']
ncVarOut = ds_out.createVariable(
'DOX2_SBE', "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
for a in ncVarIn.ncattrs():
if a not in ['_FillValue', 'ancillary_variables']:
ncVarOut.setncattr(a, ncVarIn.getncattr(a))
ncVarOut[:] = ncVarIn[:]
T = var_temp[:]
calibration_Soc = float(var_v0.calibration_Soc)
calibration_offset = float(var_v0.calibration_offset)
calibration_A = float(var_v0.calibration_A)
calibration_B = float(var_v0.calibration_B)
calibration_C = float(var_v0.calibration_C)
calibration_E = float(var_v0.calibration_E)
slope_correction = 1.0
if 'oxygen_correction_slope' in var_v0.ncattrs():
slope_correction = float(var_v0.oxygen_correction_slope)
lat = -47
lon = 142
try:
lat = ds.variables["LATITUDE"][0]
lon = ds.variables["LONGITUDE"][0]
except:
pass
SP = var_psal[:]
p = var_pres[:]
SA = gsw.SA_from_SP(SP, p, lon, lat)
pt = gsw.pt0_from_t(SA, T, p)
CT = gsw.CT_from_t(SA, T, p)
sigma_t0 = gsw.sigma0(SA, CT)
# calc oxygen solubility
# 0.01 % difference to sea bird calculation
#oxsol = gsw.O2sol_SP_pt(SP, pt) # umol/kg returned
# calc OXSOL in ml/l as per seabird application note 64
# this method gives a 0.2 % difference to what is calculated by sea bird (and what is calculated by TEOS-10)
A0 = 2.00907
A1 = 3.22014
A2 = 4.0501
A3 = 4.94457
A4 = -0.256847
A5 = 3.88767
B0 = -0.00624523
B1 = -0.00737614
B2 = -0.010341
B3 = -0.00817083
C0 = -0.000000488682
ts = np.log((298.15 - T) / (273.15 + T))
oxsol = np.exp(A0 + A1 * ts + A2 * (ts**2) + A3 * (ts**3) + A4 * (ts**4) +
A5 * (ts**5) +
SP * [B0 + B1 * (ts) + B2 * (ts**2) + B3 *
(ts**3)] + C0 * (SP**2))
#
# # calculate oxygen from V0
dox = slope_correction * calibration_Soc * (
var_v0[:] + calibration_offset) * oxsol * (
1 + calibration_A * T + calibration_B * T**2 +
calibration_C * T**3) * np.exp(calibration_E * p / (T + 273.15))
#
# # create SBE43 oxygen ml/l
# # ncVarOut = ds_out.createVariable("DOX", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
# #
# # ncVarOut[:] = dox
# # ncVarOut.long_name = "volume_concentration_of_dissolved_molecular_oxygen_in_sea_water"
# # ncVarOut.valid_min = 0
# # ncVarOut.valid_max = 40
# # ncVarOut.units = "ml/l"
# # ncVarOut.equation_1 = "Ox(ml/l)=Soc.(V+Voffset).(1+A.T+B.T^2+V.T^3).OxSOL(T,S).exp(E.P/K) ... SeaBird (AN64-2)"
# create SBE43 oxygen in umol/kg
ncVarOut = ds_out.createVariable(
"DOX2", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = dox * 44600 / (1000 + sigma_t0)
#ncVarOut[:] = dox * 44.6
ncVarOut.standard_name = "moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut.long_name = "moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut.coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'
ncVarOut.valid_min = 0
ncVarOut.valid_max = 400
ncVarOut.units = "umol/kg"
ncVarOut.equation_1 = "Ox[ml/l]=Soc.(V+Voffset).(1+A.T+B.T^2+C.T^3).OxSOL(T,S)[ml/l].exp(E.P/K) ... SeaBird (AN64)"
ncVarOut.equation_2 = "Ox[umol/kg]=Ox[ml/l].44660/(sigma-theta(P=0,Theta,S)+1000)"
#ncVarOut.equation_1 = "Ox[umol/kg]=Soc.(V+Voffset).(1+A.T+B.T^2+C.T^3).OxSOL(T,S)[umol/kg].exp(E.P/K) ... SeaBird (AN64)"
#ncVarOut.comment = 'OxSOL in umol/kg'
ncVarOut.ancillary_variables = "DOX2_quality_control DOX2_quality_control_in"
# quality flags
ncVarOut_qc = ds_out.createVariable(
"DOX2_quality_control", "i1", ("TIME", ), fill_value=99, zlib=True
) # fill_value=99 otherwise defaults to max, imos-toolbox uses 99
ncVarOut_qc[:] = np.zeros(ncVarOut_qc.shape)
if 'V0_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['V0_quality_control'][:]],
axis=0)
ncVarOut_qc[:] = mx
if 'TEMP_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['TEMP_quality_control'][:]],
axis=0)
print('TEMP max', mx)
ncVarOut_qc[:] = mx
if 'PSAL_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['PSAL_quality_control'][:]],
axis=0)
print('PSAL max', mx)
ncVarOut_qc[:] = mx
if 'PRES_quality_control' in ds.variables:
mx = np.max([ncVarOut_qc[:], ds.variables['PRES_quality_control'][:]],
axis=0)
print('PRES max', mx)
ncVarOut_qc[:] = mx
ncVarOut_qc.standard_name = ncVarOut.standard_name + " status_flag"
ncVarOut_qc.quality_control_conventions = "IMOS standard flags"
ncVarOut_qc.flag_values = np.array([0, 1, 2, 3, 4, 6, 7, 9], dtype=np.int8)
ncVarOut_qc.flag_meanings = 'unknown good_data probably_good_data probably_bad_data bad_data not_deployed interpolated missing_value'
ncVarOut_qc.comment = 'maximum of all flags'
# create a QC flag variable for the input data
ncVarOut_qc = ds_out.createVariable(
"DOX2_quality_control_in", "i1", ("TIME", ), fill_value=99, zlib=True
) # fill_value=99 otherwise defaults to max, imos-toolbox uses 99
ncVarOut_qc[:] = mx
ncVarOut_qc.long_name = "input data flag for moles_of_oxygen_per_unit_mass_in_sea_water"
ncVarOut_qc.units = "1"
ncVarOut_qc.comment = "data flagged from input variables TEMP, PSAL, PRES"
# save the OxSOL
if 'OXSOL' in ds_out.variables:
ncVarOut = ds_out.variables['OXSOL']
else:
ncVarOut = ds_out.createVariable(
"OXSOL", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
#ncVarOut[:] = oxsol * 44600 / (1000+sigma_t0)
ncVarOut[:] = oxsol
ncVarOut.units = "umol/kg"
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html function gsw.O2sol_SP_pt"
ncVarOut.long_name = "moles_of_oxygen_per_unit_mass_in_sea_water_at_saturation"
ncVarOut.coordinates = 'TIME LATITUDE LONGITUDE NOMINAL_DEPTH'
for v in ds.ncattrs():
if not v.startswith('sea_bird'):
ds_out.setncattr(v, ds.getncattr(v))
ds_out.instrument = 'Sea-Bird Electronics ; SBE43'
ds_out.instrument_model = 'SBE43'
ds_out.instrument_serial_number = '43' + var_v0.calibration_SerialNumber
ncTimeFormat = "%Y-%m-%dT%H:%M:%SZ"
attrs = ds.ncattrs()
for at in attrs:
if at not in [
'title', 'instrument', 'instrument_model',
'instrument_serial_number', 'history', 'date_created', 'title'
]:
#print('copy att', at)
ds_out.setncattr(at, ds.getncattr(at))
ds_out.deployment_code = ds.deployment_code
ds_out.instrument = 'SeaBird Electronics ; SBE43'
ds_out.instrument_model = 'SBE43'
ds_out.instrument_serial_number = ds.variables[
'V0'].calibration_SerialNumber
ds_out.title = 'Oceanographic mooring data deployment of {platform_code} at latitude {geospatial_lat_max:3.1f} longitude {geospatial_lon_max:3.1f} depth {geospatial_vertical_max:3.0f} (m) instrument {instrument} serial {instrument_serial_number}'
# add creating and history entry
ds_out.setncattr("date_created", datetime.utcnow().strftime(ncTimeFormat))
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
# keep the history so we know where it came from
ds_out.setncattr(
'history', hist + datetime.utcnow().strftime("%Y-%m-%d") +
" calculate DOX2 from file " + os.path.basename(netCDFfile))
ds.close()
ds_out.close()
return out_file
def add_optode_oxygen(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_psal = ds.variables["PSAL"]
var_bphase = ds.variables["BPHASE"]
var_otemp = ds.variables["OTEMP"]
#var_otemp = ds.variables["TEMP"]
t = var_temp[:]
SP = var_psal[:]
phase = var_bphase[:]
otemp = var_otemp[:]
try:
var_pres = ds.variables["PRES"]
p = var_pres[:]
except:
p = 0
lat = -47
lon = 142
try:
lat = ds.variables["LATITUDE"][0]
lon = ds.variables["LONGITUDE"][0]
except:
pass
SA = gsw.SA_from_SP(SP, p, lon, lat)
pt = gsw.pt0_from_t(SA, t, p)
sigmat = gsw.sigma0(SA, t)
oxsol = gsw.O2sol_SP_pt(SP, pt)
if 'OXSOL' in ds.variables:
out_oxsol_var = ds.variables['OXSOL']
else:
out_oxsol_var = ds.createVariable(
"OXSOL", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
out_oxsol_var[:] = oxsol
out_oxsol_var.units = "umol/kg"
out_oxsol_var.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html function gsw.O2sol_SP_pt"
C0 = var_bphase.calibration_C0
C1 = var_bphase.calibration_C1
C2 = var_bphase.calibration_C2
A0 = var_bphase.calibration_A0
A1 = var_bphase.calibration_A1
B0 = var_bphase.calibration_B0
B1 = var_bphase.calibration_B1
if 'DOX2_RAW' in ds.variables:
out_ox_var = ds.variables['DOX2_RAW']
else:
out_ox_var = ds.createVariable(
"DOX2_RAW", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
oxygen = (
(A0 + A1 * otemp) /
(B0 + B1 * phase) - 1) / (C0 + C1 * otemp + C2 * np.power(otemp, 2))
out_ox_var[:] = oxygen
out_ox_var.comment_calc1 = 'calculated using ((A0 + A1 * otemp)/(B0 + B1 * phase) - 1)/(C0 + C1 * otemp + C2 * otemp * otemp)'
out_ox_var.units = "umol"
out_ox_var.long_name = "mole_concentration_of_dissolved_molecular_oxygen_in_sea_water (not salinity or pressure corrected)"
out_ox_var.valid_max = np.float32(400)
out_ox_var.valid_min = np.float32(0)
out_ox_var.coordinates = "TIME LATITUDE LONGITUDE NOMINAL_DEPTH"
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr(
'history', hist + datetime.utcnow().strftime("%Y-%m-%d") +
" : calculated optode oxygen and oxygen solubility")
ds.close()