def sbe43(volts, p, t, c, coefs, lat=0.0, lon=0.0):
# NOTE: lat/lon = 0 is not "acceptable" for GSW, come up with something else?
"""
SBE equation for converting SBE43 engineering units to oxygen (ml/l).
SensorID: 38
Parameters
----------
volts : array-like
Raw voltage
p : array-like
Converted pressure (dbar)
t : array-like
Converted temperature (Celsius)
c : array-like
Converted conductivity (mS/cm)
coefs : dict
Dictionary of calibration coefficients (Soc, Voffset, Tau20, A, B, C, E)
lat : array-like, optional
Latitude (decimal degrees north)
lon : array-like, optional
Longitude (decimal degrees)
Returns
-------
oxy_ml_l : array-like
Converted oxygen (mL/L)
"""
# TODO: is there any reason for this to output mL/L? if oxygen eq uses o2sol
# in umol/kg, result is in umol/kg... which is what we use at the end anyway?
t_Kelvin = t + 273.15
SP = gsw.SP_from_C(c, t, p)
SA = gsw.SA_from_SP(SP, p, lon, lat)
CT = gsw.CT_from_t(SA, t, p)
sigma0 = gsw.sigma0(SA, CT)
o2sol = gsw.O2sol(SA, CT, p, lon, lat) # umol/kg
o2sol_ml_l = oxy_umolkg_to_ml(o2sol,
sigma0) # equation expects mL/L (see TODO)
# NOTE: lat/lon always required to get o2sol (and need SA/CT for sigma0 anyway)
# the above is equivalent to:
# pt = gsw.pt0_from_t(SA, t, p)
# o2sol = gsw.O2sol_SP_pt(s, pt)
oxy_ml_l = (coefs["Soc"] * (volts + coefs["offset"]) *
(1.0 + coefs["A"] * t + coefs["B"] * np.power(t, 2) +
coefs["C"] * np.power(t, 3)) * o2sol_ml_l *
np.exp(coefs["E"] * p / t_Kelvin))
return np.around(oxy_ml_l, 4)
def get_AOU_stoich(sal, temp, O2, sigma0, water_age):
import gsw
import numpy as np
zeros = np.zeros_like(sal)
osol = gsw.O2sol(sal, temp, zeros, -125, 50)
#convert osol to umol/L
osol_umolL = osol * (1000 / (1000 + sigma0))
AOU = osol_umolL - O2
#print('max AOU: '+str(np.max(AOU)) + ', min AOU: '+ str(np. min(AOU)))
AOU_stoich = np.copy(AOU)
AOU_stoich = AOU_stoich * (117 / 170)
#AOU_zeroed[AOU<0] = 0
## AOU for young waters?!
##AOU_stoich[water_age<10] = 0
return AOU_stoich
def saturation_ox(cast_xarray):
# calculate absolute salinity, conservative temperature, saturation oxygen conc.
SA = gsw.SA_from_SP(cast_xarray.salinity.values,
cast_xarray.pressure.values,
cast_xarray.longitude.values,
cast_xarray.latitude.values)
CT = gsw.CT_from_t(SA, cast_xarray.temperature.values,
cast_xarray.pressure.values)
O2sol = gsw.O2sol(SA, CT, cast_xarray.pressure.values,
cast_xarray.longitude.values,
cast_xarray.latitude.values)
# this is the only way I could figure out to add as "data variable"
cast_xarray = cast_xarray.reset_coords()
cast_xarray = cast_xarray.set_coords(['station', 'latitude', 'longitude'])
# add the calculated variables to the xarray
cast_xarray['SA'] = (['depth'], SA)
cast_xarray['CT'] = (['depth'], CT)
cast_xarray['O2sol'] = (['depth'], O2sol)
cast_xarray.O2sol.attrs['units'] = 'umol/kg'
return cast_xarray
lat_interp = lat_interp_time(new_dates_num)
lon_interp_time = interpolate.interp1d(date_reform_num, lon)
lon_interp = lon_interp_time(new_dates_num)
for l in np.arange(lat_array_pd.shape[0]):
# rows: Pressure, columns dates
lat_array_pd[l, :] = lat_interp.T
lon_array_pd[l, :] = lon_interp.T
P_PD = np.zeros(T_PD.shape)
for l in np.arange(len(new_dates_num)):
P_PD[:, l] = depth_range
SA = gsw.SA_from_SP(S_PD, P_PD, lon_array_pd, lat_array_pd)
CT = gsw.CT_from_t(SA, T_PD, P_PD)
oxy_eq = gsw.O2sol(SA, CT, P_PD, lon_array_pd, lat_array_pd)
OS_PD = O_PD / oxy_eq * 100
D_PD = np.zeros(T_PD.shape)
D_PD[:] = np.NaN
for r in np.arange(D_PD.shape[0]):
for c in np.arange(D_PD.shape[1]):
if np.isnan(SA[r, c]) == False and np.isnan(
CT[r, c]) == False:
D_PD[r, c] = gsw.sigma0(SA[r, c], CT[r, c])
plt.figure(figsize=(figx, figy))
plt.pcolormesh(X,
Y,
OS_PD,
## 3.3 --- Station Riki
## print('load riki.pkl - Make sure it is updated')
## df_riki = pd.read_pickle('riki.pkl')
## 4. Merged dataset
#df = pd.concat([df_MAR, df_NL, df_IML, df_riki], axis=0, sort=False)
df = pd.concat([df_MAR, df_NL, df_IML], axis=0, sort=False)
df = df.reset_index(drop=True)
#O2sol = gsw.O2sol(SA,CT,p,long,lat) # <--- use this!
SA = gsw.SA_from_SP(df['salinity'], df['depth'], df['longitude'],
df['latitude'])
CT = gsw.CT_from_t(SA, df['temperature'], df['depth'])
O2sol = gsw.O2sol(SA, CT, df['depth'], df['longitude'],
df['latitude']) # in umol/kg
O2sol = O2sol / 43.570 # in ml/l
#satO2 = swx.satO2(df['salinity'], df['temperature'])
df['O2sat_perc'] = df['O2'] / O2sol * 100 # calculate oxygen saturation %
df['NO3'] = df[
'NO3'] / 1.025 # convert nutrient data from uM=mmol/m3/umol/L to umol/kgSW
df['SiO'] = df['SiO'] / 1.025
df['PO4'] = df['PO4'] / 1.025
###########################################
## ------ Prepare data for CO2ys ------- ##
###########################################
# Deal with weird flags (e.g., 'I23', 'I22')
df = df.astype({
SA_sort = np.array(list(map(lambda x, y: y[x], np.argsort(sigma0, axis=1), SA))) CT_sort = np.array(list(map(lambda x, y: y[x], np.argsort(sigma0, axis=1), CT))) P_sort = np.array(list(map(lambda x, y: y[x], np.argsort(sigma0, axis=1), P))) N2, p_mid = gsw.Nsquared(SA_sort, CT_sort, P_sort, lat=lat0, axis=1) # Oxygen conversion Ofreq = sx_oxygen_frequency.values Soc = float(dpl['calibration.SBE43F.soc']) Foffset = float(dpl['calibration.SBE43F.foffset']) A = float(dpl['calibration.SBE43F.a']) B = float(dpl['calibration.SBE43F.b']) C = float(dpl['calibration.SBE43F.c']) E = float(dpl['calibration.SBE43F.e']) K = CT + 273.15 # Temp in Kelvin O2sol = gsw.O2sol(SA, CT, P, lon0, lat0); #umol/Kg O2 = Soc*(Ofreq+Foffset)*(1.0+A*CT + B*CT**2 + C*CT**3)*O2sol*np.exp(E*P/K); # MiniFluo calib (# Assume new version) TRY_calib = float(dpl['calibration.MFL.TRY_std']) NAP_calib = float(dpl['calibration.MFL.NAP_spf']) PHE_calib = float(dpl['calibration.MFL.PHE_spf']) FLU_calib = float(dpl['calibration.MFL.FLU_std']) PYR_calib = float(dpl['calibration.MFL.PYR_std']) TRY_blank = float(dpl['calibration.MFL.TRY_std_blank']) NAP_blank = float(dpl['calibration.MFL.NAP_spf_blank']) PHE_blank = float(dpl['calibration.MFL.PHE_spf_blank']) FLU_blank = float(dpl['calibration.MFL.FLU_std_blank']) PYR_blank = float(dpl['calibration.MFL.PYR_std_blank']) DARK = float(dpl['calibration.MFL.DARK']) # Get Relative units (e.g., scaled)
def calc_preind_co2_AOU_method(ncname, datestr):
import numpy as np
import netCDF4 as nc
import gsw
"""
- Opens a LiveOcean netcdf file
- Calculates potential density
- Uses a derived relationship between potential density (sigma0) and
cfc-freon11 estimated watermass age to determine what atmospheric co2
the watermass saw when it was last at surface.
- see comments below
Parameters
----------
ncname : name of path + filename of LiveOcean boundary condition file
for SalishSeaCast, containing temperature, salinity, DIC, TA
datestr : a string of form y2018m06d01 to write in resulting netcdf name.
Returns
-------
writes netcdf file of form ./preind_DIC/LO_intrusion_' + datestr +'_preind_DIC.nc
with the following variables:
sigma0, pycnal_last_at_surface, pycnal_witnessed_atm_co2
insitu_pco2, preind_pco2, preind_dic
"""
print(datestr)
#open dataset & retrieve relevant variables, calculate potential density
test_LO = nc.Dataset(ncname)
zlevels = (test_LO['deptht'][:])
sal = test_LO['vosaline'][0, :, 0, :]
temp = test_LO['votemper'][0, :, 0, :]
sigma0 = gsw.sigma0(sal, temp)
DIC = test_LO['DIC'][0, :, 0, :]
TA = test_LO['TA'][0, :, 0, :]
O2 = test_LO['OXY'][0, :, 0, :]
depth_this = np.zeros_like(TA)
#depth_this - array of depths of same shape as DIC
for i in range(0, 950):
depth_this[:, i] = zlevels
#calculate pycnal's last surfacing, according to exp function
#found using cfc ages
params0 = 0.1301889490932413
params1 = 3.8509914822057825
params2 = 8.301166081413104
pycnal_last_at_surface = 2019 - (params0 * np.exp(-params1 *
(25.15 - sigma0)) +
params2)
#find last seen atmospheric co2
pycnal_witnessed_atm_co2 = np.zeros_like(pycnal_last_at_surface)
for i in range(0, 40):
for j in range(0, 950):
ty = pycnal_last_at_surface[i, j]
tco2 = co2_from_year(ty)
pycnal_witnessed_atm_co2[i, j] = tco2
#(1) estimate AOU on 26 (assoc with water parcel with DIC_{w,2019,26,jdf})
# = f(O2_{w,2019,26,jdf},S_{w,2019,26,jdf},T_{w,2019,26,jdf}, P_{w,2019,26,jdf})
#(P is there to determine T when last at surface - I'll call it preT next)
osol = gsw.O2sol(sal, temp, depth_this, -125, 50)
AOU = osol - O2
print('max AOU: ' + str(np.max(AOU)) + ', min AOU: ' + str(np.min(AOU)))
AOU_stoich = np.copy(AOU)
AOU_stoich = AOU_stoich * (117 / 170)
#AOU_zeroed[AOU<0] = 0
#(2) estimate preformed DIC on 26 when last at surface (say 16 years ago or in 2003):
# preDIC_{w,2003,26} = DIC_{w,2019,26,jdf} - AOU_{w,2019,26,jdf}
#{AOU may be about 130 umol/kg or so in this e.g. - we are taking away the AOU because the water hasn't received the organic rain yet - here I am assuming that one
# unit of AOU means one unit of DIC and it will be close but we should check Gruber et al. 2006}
preformed_DIC = DIC - AOU_stoich
#(3) estimate preformed PCO2
#prePCO2_{w,jdf,26} = f(preDIC_{w,2003,26},TA_{w,2019,26,jdf},S_{w,2019,26,jdf},preT_{w,2019,26,jdf})
print('finding preformed_pco2 at surface')
zeros_here = np.zeros_like(depth_this)
pHr, OmAr, pco2r = oned_moxy(sal, temp, preformed_DIC, TA, 1, zeros_here)
preformed_pco2 = pco2r.reshape(40, 950)
print('max preformed_pco2: ' + str(np.max(preformed_pco2)) +
', min preformed_pco2: ' + str(np.min(preformed_pco2)))
#(4) estimate disequilibrium PCO2 when last at surface
#diseqPCO2 = prePCO2_{w,jdf,26} - PCO2_a,2003
#{expect diseqPCO2 may be about 0-30 uatm but not 300uatm or more}
diseqPCO2 = preformed_pco2 - pycnal_witnessed_atm_co2
print('max diseqPCO2: ' + str(np.max(diseqPCO2)) + ', min diseqPCO2: ' +
str(np.min(diseqPCO2)))
pref_pco2_inc_diseqpco2 = diseqPCO2 + 284
#(5) estimate preindustrial preformed DIC
#preDIC_PI = f( (PCO2_PI + diseqPCO2), TA_{w,2019,26,jdf},S_{w,2019,26,jdf},preT_{w,2019,26,jdf})
#pref_pco2_inc_diseqpco2 = diseqPCO2 + preformed_pco2
print('calculating preindustrial preformed DIC')
preind_dic = np.zeros_like(DIC)
preind_dic_r = np.ravel(preind_dic)
pref_pco2_inc_diseqpco2_r = np.ravel(pref_pco2_inc_diseqpco2)
depth_r = np.ravel(depth_this)
sal_r = np.ravel(sal)
temp_r = np.ravel(temp)
TA_r = np.ravel(TA)
zeros_r = np.zeros_like(depth_r)
#for i in range(0,len(depth_r)):
for i in range(0, len(depth_r)):
if i % 950 == 0:
print(i)
t_dic = find_DIC_corresp_to_pco2(sal_r[i], temp_r[i],
pref_pco2_inc_diseqpco2_r[i], TA_r[i],
1, zeros_r[i])
preind_dic_r[i] = t_dic
preind_pref_dic = preind_dic_r.reshape(40, 950)
deltaDIC = preformed_DIC - preind_pref_dic
print('max deltaDIC: ' + str(np.max(deltaDIC)) + ', min deltaDIC: ' +
str(np.min(deltaDIC)))
final_preind_DIC = DIC - deltaDIC
f = nc.Dataset('./preind_DIC/LO_AOUmethod_stoicCO_diseq_prefC_zeroed_' +
datestr + '_preind_DIC.nc',
'w',
format='NETCDF4') #'w' stands for write
g = f.createGroup('preindustrial_DIC')
g.createDimension('xval', 950)
g.createDimension('depth', 40)
ts = g.createVariable('sigma0', 'f4', ('depth', 'xval'))
ts[:] = sigma0
ts2 = g.createVariable('pycnal_last_at_surface', 'f4', ('depth', 'xval'))
ts2[:] = pycnal_last_at_surface
ts3 = g.createVariable('pycnal_witnessed_atm_co2', 'f4', ('depth', 'xval'))
ts3[:] = pycnal_witnessed_atm_co2
ts4 = g.createVariable('AOU', 'f4', ('depth', 'xval'))
ts4[:] = AOU
ts5 = g.createVariable('preformed_pco2', 'f4', ('depth', 'xval'))
ts5[:] = preformed_pco2
ts6 = g.createVariable('deltaDIC', 'f4', ('depth', 'xval'))
ts6[:] = deltaDIC
ts7 = g.createVariable('preind_dic', 'f4', ('depth', 'xval'))
ts7[:] = final_preind_DIC
f.close()
# 1. NL
df_NL = pd.read_excel('/home/cyrf0006/github/AZMP-NL/datasets/carbonates/AZMP_NL_plot.xlsx')
df_NL = df_NL.set_index('timestamp', drop=False)
# CO2sys
CO2dict_NL = CO2SYS(df_NL.TA, df_NL.TIC, 1, 2, df_NL.salinity, 20, df_NL.temperature, 0, df_NL.depth, 0, 0, 1, 1, 1)
co2sys_NL = pd.DataFrame.from_dict(CO2dict_NL)
co2sys_NL.index = df_NL.index
# try to by-pass Gary's calculation
df_NL['pH'] = co2sys_NL['pHoutTOTAL']
df_NL['Omega_C'] = co2sys_NL['OmegaCAout']
df_NL['Omega_A'] = co2sys_NL['OmegaARout']
df_NL['pH'] = co2sys_NL['pHoutTOTAL']
# Calculate in situ o2
SA = gsw.SA_from_SP(df_NL.salinity, df_NL.depth , df_NL.latitude, df_NL.longitude)
CT = gsw.CT_from_t(SA, df_NL.temperature, df_NL.depth)
df_NL['satO2'] = gsw.O2sol(SA, CT, df_NL.depth , df_NL.latitude, df_NL.longitude)
df_NL['O2_ml_l'] = df_NL['satO2_perc']/100*df_NL['satO2']/44.6
del SA, CT
# 2. Gulf
df_IML = pd.read_excel('/home/cyrf0006/github/AZMP-NL/datasets/carbonates/IMLJune2019(Gibb).xlsx', header=1, parse_dates = {'timestamp' : [6, 7]})
df_IML = df_IML.drop(0)
df_IML.index = pd.to_datetime(df_IML.timestamp)
df_IML.drop(columns='timestamp', inplace=True)
df_IML = df_IML.rename(columns={'Unnamed: 64' : 'pH'})
df_IML = df_IML.rename(columns={'Unnamed: 65' : 'Omega_C'})
df_IML = df_IML.rename(columns={'Unnamed: 66' : 'Omega_A'})
df_IML = df_IML.rename(columns={'Unnamed: 67' : 'strate'})
# Drop empty values
df_IML.dropna(subset=['pH labo'], inplace=True)
# CO2sys
SA = gsw.SA_from_SP(sx_salinity.values, P, lon0, lat0)
CT = gsw.CT_from_t(SA, sx_temperature.values, P)
sigma0 = gsw.sigma0(SA, CT)
N2, p_mid = gsw.Nsquared(SA, CT, P, lat=lat0)
# Oxygen conversion
Ofreq = sx_oxygen_frequency.values
Soc = float(dpl['calibration.SBE43F.soc'])
Foffset = float(dpl['calibration.SBE43F.foffset'])
A = float(dpl['calibration.SBE43F.a'])
B = float(dpl['calibration.SBE43F.b'])
C = float(dpl['calibration.SBE43F.c'])
E = float(dpl['calibration.SBE43F.e'])
K = CT + 273.15 # Temp in Kelvin
O2sol = gsw.O2sol(SA, CT, P, lon0, lat0)
#umol/Kg
O2 = Soc * (Ofreq + Foffset) * (1.0 + A * CT + B * CT**2 +
C * CT**3) * O2sol * np.exp(E * P / K)
# MiniFluo
DARK = float(dpl['calibration.MFL.DARK'])
# Get Relative units (e.g., scaled)
TRY_ru = (sx_fluorescence_270_340 -
DARK) / (sx_fluorescence_monitoring_270_340 - DARK)
NAP_ru = TRY_ru.copy() # Same as TRY
PHE_ru = (sx_fluorescence_255_360 -
DARK) / (sx_fluorescence_monitoring_255_360 - DARK)
FLU_ru = (sx_fluorescence_260_315 -
DARK) / (sx_fluorescence_monitoring_260_315 - DARK)
PYR_ru = (sx_fluorescence_270_376 -
def prepare_oxy(btl_df, time_df, ssscc_list):
"""
Calculate oxygen-related variables needed for calibration:
sigma, oxygen solubility (OS), and bottle oxygen
Parameters
----------
btl_df : DataFrame
CTD data at bottle stops
time_df : DataFrame
Continuous CTD data
ssscc_list : list of str
List of stations to process
Returns
-------
"""
# Calculate SA and CT
btl_df["SA"] = gsw.SA_from_SP(
btl_df[cfg.column["sal"]],
btl_df[cfg.column["p_btl"]],
btl_df[cfg.column["lon_btl"]],
btl_df[cfg.column["lat_btl"]],
)
btl_df["CT"] = gsw.CT_from_t(
btl_df["SA"],
btl_df[
cfg.column["t1_btl"]], # oxygen sensor is on primary line (ie t1)
btl_df[cfg.column["p_btl"]],
)
time_df["SA"] = gsw.SA_from_SP(
time_df[cfg.column["sal"]],
time_df[cfg.column["p"]],
time_df[cfg.column["lon_btl"]],
time_df[cfg.column["lat_btl"]],
)
time_df["CT"] = gsw.CT_from_t(
time_df["SA"],
time_df[cfg.column["t1"]], # oxygen sensor is on primary line (ie t1)
time_df[cfg.column["p"]],
)
# calculate sigma
btl_df["sigma_btl"] = sigma_from_CTD(
btl_df[cfg.column["sal"]],
btl_df[
cfg.column["t1_btl"]], # oxygen sensor is on primary line (ie t1)
btl_df[cfg.column["p_btl"]],
btl_df[cfg.column["lon_btl"]],
btl_df[cfg.column["lat_btl"]],
)
time_df["sigma_ctd"] = sigma_from_CTD(
time_df[cfg.column["sal"]],
time_df[cfg.column["t1"]], # oxygen sensor is on primary line (ie t1)
time_df[cfg.column["p"]],
time_df[cfg.column["lon_btl"]],
time_df[cfg.column["lat_btl"]],
)
# Calculate oxygen solubility in µmol/kg
btl_df["OS"] = gsw.O2sol(
btl_df["SA"],
btl_df["CT"],
btl_df[cfg.column["p_btl"]],
btl_df[cfg.column["lon_btl"]],
btl_df[cfg.column["lat_btl"]],
)
time_df["OS"] = gsw.O2sol(
time_df["SA"],
time_df["CT"],
time_df[cfg.column["p"]],
time_df[cfg.column["lon"]],
time_df[cfg.column["lat"]],
)
# Convert CTDOXY units
btl_df["CTDOXY"] = oxy_ml_to_umolkg(btl_df["CTDOXY1"], btl_df["sigma_btl"])
# Calculate bottle oxygen
btl_df[cfg.column["oxy_btl"]] = calculate_bottle_oxygen(
ssscc_list,
btl_df["SSSCC"],
btl_df["TITR_VOL"],
btl_df["TITR_TEMP"],
btl_df["FLASKNO"],
)
btl_df[cfg.column["oxy_btl"]] = oxy_ml_to_umolkg(
btl_df[cfg.column["oxy_btl"]], btl_df["sigma_btl"])
btl_df["OXYGEN_FLAG_W"] = flag_winkler_oxygen(
btl_df[cfg.column["oxy_btl"]])
# Load manual OXYGEN flags
if Path("data/oxygen/manual_oxy_flags.csv").exists():
manual_flags = pd.read_csv("data/oxygen/manual_oxy_flags.csv",
dtype={"SSSCC": str})
for _, flags in manual_flags.iterrows():
df_row = (btl_df["SSSCC"] == flags["SSSCC"]) & (
btl_df["btl_fire_num"] == flags["Bottle"])
btl_df.loc[df_row, "OXYGEN_FLAG_W"] = flags["Flag"]
return True
def load_clean_interpolate_and_shift_data(datafile_path, shift_value):
"""
loads a a .mat file of mss measurements
a few files are cleaned through a hardcoded routine (emb217/TR1-8.mat, emb169/TS118, emb169/TRR109)
datafile_path: absolute path of the mss measurements saved as a .mat file
return values:
number_of_profiles number of profiles/casts in the transect
lat latitude in degrees (as a float) of the casts
lon longitude in degrees (as a float) of the casts
distance distance in km from the starting point of the transect
interp_pressure equidistant pressure array between the highest and the lowest measured pressure value
pressure_grid
oxygen_grid
oxygen_sat_grid oxygen saturation in percent as a grid (number_of_profiles x len(interp_pressure))
salinity_grid salinity in g/kg as a grid (number_of_profiles x len(interp_pressure))
consv_temperature_grid conservative temperature in degrees Celsius as a grid (number_of_profiles x len(interp_pressure))
density_grid density in kg/m^3 as a grid (number_of_profiles x len(interp_pressure))
eps_pressure 1D array of pressure values to the dissipation rate values (the pressure distance between points is bigger than in interp_pressure)
eps_pressure_grid
eps_grid measured dissipation rate values (number_of_profiles x len(eps_pressure))
eps_salinity_grid salinity in g/kg as a grid (number_of_profiles x len(eps_pressure))
eps_consv_temperature_grid conservative temperature as a grid (number_of_profiles x len(eps_pressure))
eps_oxygen_grid
eps_oxygen_sat_grid oxygen saturation as a grid (number_of_profiles x len(eps_pressure))
eps_N_squared_grid N^2, the Brunt-Vaisala frequency in 1/s^2 as a grid (number_of_profiles x len(eps_pressure))
eps_density_grid density in kg/m^3 as a grid (number_of_profiles x len(eps_pressure))
0 error code, returned if distance is not monotonically increasing (ergo a loop in the data)
"""
import scipy.io as sio
import geopy.distance as geo
import numpy as np
import gsw
splitted_path = datafile_path.split("/")
cruisename = splitted_path[4][0:6]
DATAFILENAME = splitted_path[-1]
transect_name = DATAFILENAME[:-4]
print(cruisename + "_" + transect_name)
#print("Filename:",sio.whosmat(FILENAME))
data = sio.loadmat(datafile_path)
STA_substructure = data["STA"]
DATA_substructure = data["DATA"]
MIX_substructure = data["MIX"]
CTD_substructure = data["CTD"]
#print(STA_substructure.dtype)
#print(DATA_substructure.dtype)
#print(CTD_substructure.dtype)
#print(MIX_substructure.dtype)
lat = STA_substructure["LAT"][0]
lon = STA_substructure["LON"][0]
pressure = CTD_substructure["P"][0]
absolute_salinity = CTD_substructure["SA"][0] #is this unit sufficient
consv_temperature = CTD_substructure["CT"][
0] #TODO better use conservative temperature?
alpha = CTD_substructure["ALPHA"][0]
beta = CTD_substructure["BETA"][0]
if cruisename == "emb217":
oxygen_sat = CTD_substructure["O2"][0]
elif cruisename == "emb177":
try:
oxygen_data = sio.loadmat(
"/home/ole/windows/all_data/emb177/deployments/ship/mss/mss38/TODL/TODL_merged_oxy/"
+ transect_name + "_TODL_merged_oxy.mat")
except OSError:
print("##########################################")
print(cruisename, transect_name, "is skipped!")
print("No oxygen data for this file")
print("##########################################")
return 0
#the following code block is written by Peter Holtermann and Copy pasted here
emb177_oxygen = []
emb177_oxygen_pressure = []
for icast in range(len(lon)):
oxy_p_temp = oxygen_data['TODL_MSS']['P'][0, :][icast][0, :]
#mT = oxygen_data['TODL_MSS']['mT'][0,:][icast][0,:]
#C = oxygen_data['TODL_MSS']['C'][0,:][icast][0,:]
oxy = oxygen_data['TODL_MSS']['oxy'][0, :][icast][:, 4]
emb177_oxygen.append(oxy)
emb177_oxygen_pressure.append(oxy_p_temp)
elif cruisename == "emb169":
try:
oxygen_data = sio.loadmat(
"/home/ole/windows/all_data/emb169/deployments/ship/mss/TODL/merged/"
+ transect_name + "_TODL_merged_oxy.mat")
except OSError:
print("##########################################")
print(cruisename, transect_name, "is skipped!")
print("No oxygen data for this file")
print("##########################################")
return 0
#the following code block is written by Peter Holtermann and Copy pasted here
emb169_oxygen = []
emb169_oxygen_pressure = []
for icast in range(len(lon)):
oxy_p_temp = oxygen_data['TODL_MSS']['P'][0, :][icast][0, :]
#mT = oxygen_data['TODL_MSS']['mT'][0,:][icast][0,:]
#C = oxygen_data['TODL_MSS']['C'][0,:][icast][0,:]
oxy = oxygen_data['TODL_MSS']['oxy'][0, :][icast][:, 4]
emb169_oxygen.append(oxy)
emb169_oxygen_pressure.append(oxy_p_temp)
else:
raise AssertionError
eps = MIX_substructure["eps"][0]
eps_pressure = MIX_substructure["P"][0]
number_of_profiles = np.shape(pressure)[-1]
latitude = []
longitude = []
#calculates the distance in km of the position from every profile to the position of the starting profile.
distance = np.zeros(number_of_profiles)
origin = (float(lat[0][0][0]), float(lon[0][0][0])
) #lots of brackets to get a number, not an array (workaround)
for i in range(number_of_profiles):
current_point = (float(lat[i][0][0]), float(lon[i][0][0]))
latitude.append(float(lat[i][0][0]))
longitude.append(float(lon[i][0][0]))
distance[i] = geo.geodesic(
origin,
current_point).km #Distance in km, change to nautical miles?
lat = np.asarray(latitude)
lon = np.asarray(longitude)
#Data Cleaning
#-----------------------------------------------------------------------------------------------------------------
#remove data from a file, with overlapping positional points
if cruisename == "emb217" and transect_name == "TR1-8":
lat = np.delete(lat, np.s_[33:47])
lon = np.delete(lon, np.s_[33:47])
distance = np.delete(distance, np.s_[33:47])
pressure = np.delete(pressure, np.s_[33:47], axis=0)
oxygen_sat = np.delete(oxygen_sat, np.s_[33:47], axis=0)
absolute_salinity = np.delete(absolute_salinity, np.s_[33:47], axis=0)
consv_temperature = np.delete(consv_temperature, np.s_[33:47], axis=0)
alpha = np.delete(alpha, np.s_[33:47], axis=0)
beta = np.delete(beta, np.s_[33:47], axis=0)
eps = np.delete(eps, np.s_[33:47], axis=0)
eps_pressure = np.delete(eps_pressure, np.s_[33:47], axis=0)
number_of_profiles = np.shape(pressure)[-1]
if cruisename == "emb169" and DATAFILENAME[:-4] == "TS118":
lat = lat[:21]
lon = lon[:21]
distance = distance[:21]
pressure = pressure[:21]
oxygen_sat = oxygen_sat[:21]
absolute_salinity = absolute_salinity[:21]
consv_temperature = consv_temperature[:21]
alpha = alpha[:21]
beta = beta[:21]
eps = eps[:21]
eps_pressure = eps_pressure[:21]
number_of_profiles = np.shape(pressure)[-1]
#removes the last data point, that dont seem to belong to the transect
if cruisename == "emb169" and DATAFILENAME[:-4] == "TRR109":
lat = lat[:-1]
lon = lon[:-1]
distance = distance[:-1]
pressure = pressure[:-1]
oxygen_sat = oxygen_sat[:-1]
absolute_salinity = absolute_salinity[:-1]
consv_temperature = consv_temperature[:-1]
alpha = alpha[:-1]
beta = beta[:-1]
eps = eps[:-1]
eps_pressure = eps_pressure[:-1]
number_of_profiles = np.shape(pressure)[-1]
#-----------------------------------------------------------------------------------------------------------------
#test if distance is monotonically increasing
try:
assert (np.all(np.diff(distance) > 0))
except AssertionError:
print("##########################################")
print(cruisename, DATAFILENAME[:-4], "is skipped!")
print(
"Distance is not monotonically increasing. Mabye due to a loop in the transect?"
)
print("##########################################")
return 0
#continue #jump to the next datafile
#TODO point density?
#initial values
min_pressure = 10
max_pressure = 60
max_size = 1000
min_size = 3000
#select the min and max values for the equidistant pressure array (later used for the grid)
for i in range(number_of_profiles):
if np.nanmin(pressure[i]) < min_pressure:
min_pressure = np.nanmin(pressure[i])
if np.nanmax(pressure[i]) > max_pressure:
max_pressure = np.nanmax(pressure[i])
if pressure[i].size > max_size:
max_size = pressure[i].size
if pressure[i].size < min_size:
min_size = pressure[i].size
#print("High resolution",min_pressure,max_pressure,min_size,max_size)
#check if that worked correctly
assert (max_size >= min_size)
#creates a pressure axis between the maximum and minimum pressure
interp_pressure = np.linspace(min_pressure, max_pressure, min_size)
#creates pressure grid, where every column is equal to interp_pressure
pressure_grid = np.reshape(interp_pressure, (1, -1)) * np.ones(
(np.shape(pressure)[-1], min_size))
#create grids that changes in distance on x and in depth on y-axis
salinity_grid = np.zeros((np.shape(pressure)[-1], min_size))
consv_temperature_grid = np.copy(salinity_grid)
alpha_grid = np.copy(salinity_grid)
beta_grid = np.copy(salinity_grid)
if cruisename == "emb217":
oxygen_sat_grid = np.copy(salinity_grid)
elif cruisename == "emb177" or cruisename == "emb169":
oxygen_grid = np.copy(salinity_grid)
#check if the pressure points for every eps profile are the same
for i in range(number_of_profiles):
assert np.all(eps_pressure[i].flatten() == eps_pressure[0].flatten())
#if yes the pressure can be coverted to a 1D array instead of a 2D array
eps_pressure = eps_pressure[0].flatten()
#print(eps[i].flatten().size,eps_pressure.size, np.arange(1,160.5,0.5).size, np.arange(1,160.5,0.5).size - eps_pressure.size)
#print(np.shape(eps),np.shape(eps[0]))
desired_pressure_array = np.arange(
1, 160.5, 0.5
) #stems from the orginal matlab script that process the raw MSS data
last_element = eps_pressure[-1]
old_size = eps_pressure.size
if last_element < 160:
for index, element in enumerate(eps):
eps[index] = np.pad(
eps[index],
((0, desired_pressure_array.size - eps_pressure.size), (0, 0)),
'constant',
constant_values=np.nan)
eps_pressure = np.append(eps_pressure,
np.arange(last_element + 0.5, 160.5, 0.5))
assert (eps[0].flatten().size == eps_pressure.size)
#averaged of approx 5 depth bins (???)
eps_grid = np.zeros((number_of_profiles, eps_pressure.size))
#needed for the interpolation of S and T to the same grid as the eps
eps_salinity_grid = np.ones((np.shape(eps_grid)))
eps_consv_temperature_grid = np.ones(np.shape(eps_grid))
if cruisename == "emb217":
eps_oxygen_sat_grid = np.copy(eps_salinity_grid)
elif cruisename == "emb177" or cruisename == "emb169":
eps_oxygen_grid = np.copy(eps_salinity_grid)
#vector times matrix multiplication to get a 2D array, where every column is equal to eps_pressure
eps_pressure_grid = np.reshape(eps_pressure,
(1, -1)) * np.ones(np.shape(eps_grid))
#create a pressure axis where every point is shifted by half the distance to the next one
shifted_pressure = eps_pressure + np.mean(np.diff(eps_pressure)) / 2
#prepend a point at the beginning to be a point longer than the pressure axis we want from the Nsquared function
shifted_pressure = np.append(
eps_pressure[0] - np.mean(np.diff(eps_pressure)) / 2, shifted_pressure)
#from that create a grid, where we have just n-times the pressure axis with n the number of profiles
shifted_pressure_grid = np.reshape(shifted_pressure, (1, -1)) * np.ones(
(number_of_profiles, shifted_pressure.size))
shifted_salinity_grid = np.ones((np.shape(shifted_pressure_grid)))
shifted_consv_temperature_grid = np.ones((np.shape(shifted_pressure_grid)))
#Grid interpolation (loops over all profiles)
for i in range(number_of_profiles):
#interpolation to a common fine grid
salinity_grid[i] = np.interp(interp_pressure,
pressure[i].flatten(),
absolute_salinity[i].flatten(),
left=np.nan,
right=np.nan)
consv_temperature_grid[i] = np.interp(interp_pressure,
pressure[i].flatten(),
consv_temperature[i].flatten(),
left=np.nan,
right=np.nan)
alpha_grid[i] = np.interp(interp_pressure,
pressure[i].flatten(),
alpha[i].flatten(),
left=np.nan,
right=np.nan)
beta_grid[i] = np.interp(interp_pressure,
pressure[i].flatten(),
beta[i].flatten(),
left=np.nan,
right=np.nan)
#interpolation to a shifted grid for the Nsquared function
shifted_salinity_grid[i] = np.interp(shifted_pressure,
pressure[i].flatten(),
absolute_salinity[i].flatten(),
left=np.nan,
right=np.nan)
shifted_consv_temperature_grid[i] = np.interp(
shifted_pressure,
pressure[i].flatten(),
consv_temperature[i].flatten(),
left=np.nan,
right=np.nan)
#just changes the format of eps slightly
assert (eps[i].flatten().size == eps_pressure.size)
eps_grid[i] = eps[i].flatten()
#interpolate S and T to the same grid as eps
eps_salinity_grid[i] = np.interp(eps_pressure,
pressure[i].flatten(),
absolute_salinity[i].flatten(),
left=np.nan,
right=np.nan)
eps_consv_temperature_grid[i] = np.interp(
eps_pressure,
pressure[i].flatten(),
consv_temperature[i].flatten(),
left=np.nan,
right=np.nan)
#interpolation of the oxygen depends on which source it is
if cruisename == "emb217":
oxygen_sat_grid[i] = np.interp(interp_pressure,
pressure[i].flatten() + shift_value,
oxygen_sat[i].flatten(),
left=np.nan,
right=np.nan)
eps_oxygen_sat_grid[i] = np.interp(eps_pressure,
pressure[i].flatten() +
shift_value,
oxygen_sat[i].flatten(),
left=np.nan,
right=np.nan)
elif cruisename == "emb177":
oxygen_grid[i] = np.interp(interp_pressure,
emb177_oxygen_pressure[i] + shift_value,
emb177_oxygen[i],
left=np.nan,
right=np.nan)
eps_oxygen_grid[i] = np.interp(eps_pressure,
emb177_oxygen_pressure[i] +
shift_value,
emb177_oxygen[i].flatten(),
left=np.nan,
right=np.nan)
elif cruisename == "emb169":
oxygen_grid[i] = np.interp(interp_pressure,
emb169_oxygen_pressure[i] + shift_value,
emb169_oxygen[i],
left=np.nan,
right=np.nan)
eps_oxygen_grid[i] = np.interp(eps_pressure,
emb169_oxygen_pressure[i] +
shift_value,
emb169_oxygen[i].flatten(),
left=np.nan,
right=np.nan)
if cruisename == "emb217":
assert (np.shape(oxygen_sat_grid) == np.shape(salinity_grid))
assert (np.shape(eps_oxygen_sat_grid) == np.shape(eps_salinity_grid))
if cruisename == "emb177" or cruisename == "emb169":
assert (np.shape(oxygen_grid) == np.shape(salinity_grid))
assert (np.shape(eps_oxygen_grid) == np.shape(eps_salinity_grid))
#fine density grid
density_grid = gsw.rho(salinity_grid, consv_temperature_grid,
pressure_grid)
pot_density_grid = 1000 + gsw.density.sigma0(salinity_grid,
consv_temperature_grid)
#density grid on the same points as the eps grid
eps_density_grid = gsw.rho(eps_salinity_grid, eps_consv_temperature_grid,
eps_pressure_grid)
eps_pot_density_grid = 1000 + gsw.density.sigma0(
eps_salinity_grid, eps_consv_temperature_grid)
#TODO compare with density_grid?
#density_grid_check = (1 - alpha_grid * (consv_temperature_grid) + beta_grid * (salinity_grid))*rho_0
#difference = density_grid-density_grid_check
#calculate N^2 with the gsw toolbox, by using the shifted grid we should get a N^2 grid that is defined at the same points as eps_grid
eps_N_squared_grid, crosscheck_pressure_grid = gsw.Nsquared(
shifted_salinity_grid,
shifted_consv_temperature_grid,
shifted_pressure_grid,
lat=np.mean(lat),
axis=1)
crosscheck_pressure = np.mean(crosscheck_pressure_grid, axis=0)
#test if we really calculated N^2 for the same points as pressure points from the dissipation measurement
assert (np.all(crosscheck_pressure == eps_pressure))
#create grids that have the latitude/longitude values for every depth (size: number_of_profiles x len(interp_pressure))
lat_grid = np.reshape(lat, (-1, 1)) * np.ones(
(number_of_profiles, max_size))
lon_grid = np.reshape(lon, (-1, 1)) * np.ones(
(number_of_profiles, max_size))
#check if lon grid was correctly created
assert (np.all(lon_grid[:, 0] == lon))
#create grids that have the latitude/longitude values for every depth (size: number_of_profiles x len(eps_pressure))
eps_lat_grid = np.reshape(lat, (-1, 1)) * np.ones(
(number_of_profiles, eps_pressure.size))
eps_lon_grid = np.reshape(lat, (-1, 1)) * np.ones(
(number_of_profiles, eps_pressure.size))
if cruisename == "emb217":
#convert oxygen saturation to oxygen concentration (functions are selfwritten but use the gsw toolbox, for more informations see the function (also in this file))
oxygen_grid = thesis.oxygen_saturation_to_concentration(
oxygen_sat_grid, salinity_grid, consv_temperature_grid,
pressure_grid, lat_grid, lon_grid)
eps_oxygen_grid = thesis.oxygen_saturation_to_concentration(
eps_oxygen_sat_grid, eps_salinity_grid, eps_consv_temperature_grid,
eps_pressure_grid, eps_lat_grid, eps_lon_grid)
elif cruisename == "emb177":
#scale the oxygen to be at 100% at the surface
maximum_concentration_grid = gsw.O2sol(salinity_grid,
consv_temperature_grid,
pressure_grid, lat_grid,
lon_grid)
correction_factor = np.ones((number_of_profiles, 1))
for profile in range(number_of_profiles):
for i in range(100):
if np.isnan(oxygen_grid[profile, i]) or np.isnan(
maximum_concentration_grid[profile, i]):
continue
else:
correction_factor[profile] = maximum_concentration_grid[
profile, i] / oxygen_grid[profile, i]
break
oxygen_grid = oxygen_grid * correction_factor
eps_oxygen_grid = eps_oxygen_grid * correction_factor
#convert oxygen concentration to oxygen saturation (functions are selfwritten but use the gsw toolbox, for more informations see the function (also in this file))
oxygen_sat_grid = thesis.oxygen_concentration_to_saturation(
oxygen_grid, salinity_grid, consv_temperature_grid, pressure_grid,
lat_grid, lon_grid)
eps_oxygen_sat_grid = thesis.oxygen_concentration_to_saturation(
eps_oxygen_grid, eps_salinity_grid, eps_consv_temperature_grid,
eps_pressure_grid, eps_lat_grid, eps_lon_grid)
elif cruisename == "emb169":
#convert oxygen concentration to oxygen saturation (functions are selfwritten but use the gsw toolbox, for more informations see the function (also in this file))
oxygen_sat_grid = thesis.oxygen_concentration_to_saturation(
oxygen_grid, salinity_grid, consv_temperature_grid, pressure_grid,
lat_grid, lon_grid)
eps_oxygen_sat_grid = thesis.oxygen_concentration_to_saturation(
eps_oxygen_grid, eps_salinity_grid, eps_consv_temperature_grid,
eps_pressure_grid, eps_lat_grid, eps_lon_grid)
else:
raise AssertionError
return [[number_of_profiles, lat, lon, distance],
[
interp_pressure, oxygen_sat_grid, oxygen_grid, salinity_grid,
consv_temperature_grid, density_grid, pot_density_grid
],
[
eps_pressure, eps_oxygen_sat_grid, eps_oxygen_grid, eps_grid,
eps_salinity_grid, eps_consv_temperature_grid,
eps_N_squared_grid, eps_density_grid, eps_pot_density_grid
]]
for l in np.arange(len(date_reform)):
date_reform_num[l] = date_reform_sub[
l].total_seconds()
lat_array = np.zeros(pres.shape)
lat_array[:] = np.NaN
lon_array = np.zeros(pres.shape)
lon_array[:] = np.NaN
for l in np.arange(lat_array.shape[0]):
lat_array[l, :] = lat[l]
lon_array[l, :] = lon[l]
SA = gsw.SA_from_SP(sal, pres, lon_array, lat_array)
CT = gsw.CT_from_t(SA, temp, pres)
oxy_eq = gsw.O2sol(SA, CT, pres, lon_array, lat_array)
# calculate depth
geo_strf = gsw.geo_strf_dyn_height(SA=SA.T,
CT=CT.T,
p=pres.T)
geo_strf = geo_strf.T
z = gsw.z_from_p(p=pres,
lat=lat_array,
geo_strf_dyn_height=geo_strf)
z = -1 * z
oxy_sat = doxy / oxy_eq * 100
# Calculate density
density = np.zeros(temp.shape)
close_date = rounddown_date
# Unit Conversion
SA = gsw.SA_from_SP(
surf_S, P, lon[j], lat[j])
# Calculate conservative temp or temp
CT = gsw.CT_from_t(SA, surf_T, P)
# Calculate density
#dense=gsw.density.rho_t_exact(SA, T, P) # kg/m^3
surf_dense = gsw.density.sigma0(SA,
CT) + 1000
# Calculate oxygen saturation at each point
O2_sol = gsw.O2sol(SA, CT, P, lon[j],
lat[j])
oxy_sat = np.round(surf_O / O2_sol * 100,
2)
oxy_dev = surf_O - O2_sol
surf_O_units = (surf_O *
surf_dense) * (10**-6)
# Nearest neighbor
date_ind = np.where(
era5_time == close_date)
# Should add a check to make sure there is only 1 date ind
# use index to get lat-lon slices and other variable
# wind speed structure (time,lat, lon)
# load u_10 data
for i in np.arange(len(surf_density)):
# Calculate absolute salinity
SA=gsw.SA_from_SP(surf_S_interp[i], P, surf_lon_interp[i], surf_lat_interp[i])
# Calculate conservative temp or temp
T=surf_T_interp[i]
CT=gsw.CT_from_t(SA, T, P)
# Calculate density
#dense=gsw.density.rho_t_exact(SA, T, P) # kg/m^3
dense=gsw.density.sigma0(SA,CT)+1000
surf_density[i]=dense
# Calculate oxygen saturation at each point
O2_sol=gsw.O2sol(SA,CT,P,surf_lon_interp[i],surf_lat_interp[i])
oxy_sat[i]=np.round(surf_O_interp[i]/O2_sol*100,2)
oxy_dev[i]=surf_O_interp[i]-O2_sol
# Convert oxygen to appropriate units
surf_O_interp_units = np.multiply(surf_O_interp, surf_density)*(10**-6)
Fd_L13_float=np.zeros(len(date_6hr))
Fd_L13_float[:]=np.NaN
Fc_L13_float=np.zeros(len(date_6hr))
Fc_L13_float[:]=np.NaN
Fp_L13_float=np.zeros(len(date_6hr))
Fp_L13_float[:]=np.NaN
K_L13=np.zeros(len(date_6hr))
K_L13[:]=np.NaN