def read_dbds(self, fns, settings):
if settings['buoyancy_engine']=='shallow':
buoyancy_variable='m_ballast_pumped'
elif settings['buoyancy_engine']=='deep':
buoyancy_variable='m_de_oil_vol'
else:
raise ValueError('Unknown buoyancy engine. Accepted values: (shallow|deep)')
dbd = dbdreader.MultiDBD(filenames=fns, include_paired=True)
tmp = dbd.get_sync("sci_ctd41cp_timestamp",["sci_water_pressure",
"sci_water_cond",
"sci_water_temp",
"m_pitch",
"m_battpos",
buoyancy_variable])
_, tctd, P, C, T, pitch, battpos, buoyancy_change = tmp.compress(tmp[3]>0.01, axis=1)
SP = gsw.SP_from_C(C*10, T, P*10)
SA = gsw.SA_from_SP(SP, P*10, settings['longitude'], settings['latitude'])
density = gsw.rho_t_exact(SA, T, P*10)
#density = fast_gsw.rho(C*10, T, P*10, settings['longitude'], settings['latitude'])
data = dict(time=tctd, pressure=P, pitch=pitch,
buoyancy_change=buoyancy_change, battpos=battpos, density=density)
self.model.input_data = data
mask=np.zeros(tctd.shape,'int').astype(bool) # use all data by default
self.model.set_mask(mask)
self.model.OR(P*10<settings['minlimitdepth'])
self.model.OR(P*10>settings['maxlimitdepth'])
def _calcSal(temp, cond, pres, coefs):
a_offset = coefs[0]
a_slope = coefs[1]
t_offset = coefs[2]
t_slope = coefs[3]
alpha = a_offset + a_slope / flowSpeed
tau = t_offset + t_slope / np.sqrt(flowSpeed)
#tau = 11 # Parameter for Lueck and Picklo (1990)
#alpha = 0.57/tau + 0.03122 # Parameter for Lueck and Picklo (1990)
alpha[~np.isfinite(alpha)] = a_offset
tau[~np.isfinite(tau)] = t_offset
beta = 1 / tau # Parameter for Lueck and Picklo (1990)
fn = Fs / 2 # Nyquist frequency
a = 4 * fn * alpha * tau / (1 + 4 * fn * tau
) # Parameter for Lueck and Picklo (1990)
b = 1 - 2 * a / alpha # Parameter for Lueck and Picklo (1990)
# Compute temperature correction to obtained water temperature within the conductivity cell (Morison, 1994)
_internal_bias = np.full_like(temp, 0)
for sample in np.arange(1, len(_internal_bias)):
# Recursive filter from Morison (1994)
# if np.isfinite(temp[sample-1]):
_internal_bias[sample] = -b[sample] * _internal_bias[
sample - 1] + a[sample] * (temp[sample] - temp[sample - 1])
return gsw.SP_from_C(cond, temp + _internal_bias,
pres) # Practical salinity
def process_brown(df_raw, cfg: Mapping[str, Any]):
'''
Calc physical values from codes
:param df_raw:
:param cfg:
:return: pandas dataframe
# todo: use signs. For now our data haven't negative values and it is noise if signs!=0. Check: df_raw.signs[df_raw.signs!=0]
'''
Val = {}
if b'Pres' in cfg['for']['k_names'] and not 'Pres' in df_raw.columns:
df_raw = df_raw.rename(columns={'P': 'Pres'})
for nameb in np.intersect1d(np.array(df_raw.columns, 'S10'),
cfg['for']['k_names']):
name = nameb.decode('ascii')
Val[name] = np.polyval(
cfg['for']['kk'][nameb == cfg['for']['k_names']].flat,
df_raw[name])
# Practical Salinity PSS-78
Val['Sal'] = gsw.SP_from_C(Val['Cond'], Val['Temp'], Val['Pres'])
df = pd.DataFrame(Val,
columns=cfg['out']['data_columns'],
index=df_raw.index)
# import seawater as sw
# T90conv = lambda t68: t68/1.00024
# Val['sigma0'] sw.pden(s, T90conv(t), p=0) - 1000
return df
def ctd_residuals_df(df_bottle):
'''Compute residuals and add to dataframe.
Operate in place.
'''
#Salinity should really be grabbed straight from the files, but...
try:
df_bottle['BTLCOND'] = gsw.C_from_SP(df_bottle['SALNTY'],
df_bottle['CTDTMP1'],
df_bottle['CTDPRS'])
except ValueError:
print(f'WHOOPS SOMETHING WENT WRONG')
df_bottle['SALNTY'] = df_bottle['SALNTY'].replace(to_replace=-999,
value=np.nan)
df_bottle['BTLCOND'] = gsw.C_from_SP(df_bottle['SALNTY'],
df_bottle['CTDTMP1'],
df_bottle['CTDPRS'])
df_bottle['BTL_O'] = df_bottle['OXYGEN'] - df_bottle['CTDOXY']
df_bottle['BTL_C1'] = df_bottle['BTLCOND'] - df_bottle['CTDCOND1']
df_bottle['BTL_C2'] = df_bottle['BTLCOND'] - df_bottle['CTDCOND2']
df_bottle['C1_C2'] = df_bottle['CTDCOND1'] - df_bottle['CTDCOND2']
df_bottle['BTL_T1'] = df_bottle['REFTMP'] - df_bottle['CTDTMP1']
df_bottle['BTL_T2'] = df_bottle['REFTMP'] - df_bottle['CTDTMP2']
df_bottle['T1_T2'] = df_bottle['CTDTMP1'] - df_bottle['CTDTMP2']
df_bottle['BTL_SAL_UP'] = df_bottle['SALNTY'] - gsw.SP_from_C(
df_bottle['CTDCOND2'], df_bottle['CTDTMP2'], df_bottle['CTDPRS'])
df_bottle['BTL_SAL'] = df_bottle['SALNTY'] - df_bottle['CTDSAL']
return None
def ctd_residuals_df(df_bottle):
"""Compute residuals and add to dataframe.
Operate in place.
"""
# Salinity should really be grabbed straight from the files, but...
try:
df_bottle["BTLCOND"] = gsw.C_from_SP(df_bottle["SALNTY"],
df_bottle["CTDTMP1"],
df_bottle["CTDPRS"])
except ValueError:
print("WHOOPS SOMETHING WENT WRONG")
df_bottle["SALNTY"] = df_bottle["SALNTY"].replace(to_replace=-999,
value=np.nan)
df_bottle["BTLCOND"] = gsw.C_from_SP(df_bottle["SALNTY"],
df_bottle["CTDTMP1"],
df_bottle["CTDPRS"])
df_bottle["BTL_O"] = df_bottle["OXYGEN"] - df_bottle["CTDOXY"]
df_bottle["BTL_C1"] = df_bottle["BTLCOND"] - df_bottle["CTDCOND1"]
df_bottle["BTL_C2"] = df_bottle["BTLCOND"] - df_bottle["CTDCOND2"]
df_bottle["C1_C2"] = df_bottle["CTDCOND1"] - df_bottle["CTDCOND2"]
df_bottle["BTL_T1"] = df_bottle["REFTMP"] - df_bottle["CTDTMP1"]
df_bottle["BTL_T2"] = df_bottle["REFTMP"] - df_bottle["CTDTMP2"]
df_bottle["T1_T2"] = df_bottle["CTDTMP1"] - df_bottle["CTDTMP2"]
df_bottle["BTL_SAL_UP"] = df_bottle["SALNTY"] - gsw.SP_from_C(
df_bottle["CTDCOND2"], df_bottle["CTDTMP2"], df_bottle["CTDPRS"])
df_bottle["BTL_SAL"] = df_bottle["SALNTY"] - df_bottle["CTDSAL"]
return None
def gsw_rho(C, T, P, lon, lat):
SP = gsw.SP_from_C(C, T, P)
# This particular data set was collected in the Baltic.
SA = gsw.SA_from_SP_Baltic(SP, lon, lat)
# in-situ density
rho = gsw.density.rho_t_exact(SA, T, P)
return rho
def calculate_density(T, p, C, lat, long):
"""Calculates density from temp, pressure, conductivity, and lat/long.
All parameters and output are float or array-like.
:param T: temperature (deg C)
:param p: pressure (bar)
:param C: conductivity (S/m)
:param lat: latitude (decimal deg)
:param long: longitude (decimal deg)
:return: density (kg/m^3)
"""
# pressure in dbars = pressure * 10
if type(p) == float:
p_dbar = p * 10
else:
p_dbar = [pi * 10 for pi in p]
# conductivity in mS/cm = conductivity * 10
if type(C) == float:
C_mScm = C * 10
else:
C_mScm = [Ci * 10 for Ci in C]
# calculate SP from conductivity (mS/cm), in-situ temperature (deg C), and gauge pressure (dbar)
SP = gsw.SP_from_C(C_mScm, T, p_dbar)
# calculate SA from SP (unitless), gauge pressure (dbar), longitude and latitude (decimal degrees)
SA = gsw.SA_from_SP(SP, p_dbar, long, lat)
# calculate CT from SA (g/kg), in-situ temperature (deg C), and gauge pressure (dbar)
CT = gsw.CT_from_t(SA, T, p_dbar)
# calculate density
return gsw.rho(SA, CT, p_dbar)
def add_psal(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_cndc = ds.variables["CNDC"]
var_pres = ds.variables["PRES"]
t = var_temp[:]
C = var_cndc[:] * 10
p = var_pres[:]
psal = gsw.SP_from_C(C, t, p)
ncVarOut = ds.createVariable(
"PSAL", "f4", ("TIME", ), fill_value=np.nan,
zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = psal
ncVarOut.units = "1"
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html"
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr(
'history', hist + datetime.utcnow().strftime("%Y-%m-%d") +
" : added PSAL from TEMP, CNDC, PRES")
ds.close()
def calculate_n2(T, p, C, lat, long):
"""Calculates the buoyancy frequency N^2
All parameters and output are float or array-like.
:param T: temperature (deg C)
:param p: pressure (bar)
:param C: conductivity (S/m)
:param lat: latitude (decimal deg)
:param long: longitude (decimal deg)
:return: buoyancy (i.e. brunt-vaisala) frequency N^2 (s^-2)
"""
# pressure in dbars = pressure * 10
p_dbar = p * 10
# conductivity in mS/cm = conductivity * 10
C_mScm = C * 10
print(T.count())
print(p.count())
print(C.count())
print(lat.count())
print(long.count())
# calculate SP from conductivity (mS/cm), in-situ temperature (deg C), and gauge pressure (dbar)
SP = gsw.SP_from_C(C_mScm, T, p_dbar)
# calculate SA from SP (unitless), gauge pressure (dbar), longitude and latitude (decimal degrees)
SA = gsw.SA_from_SP(SP, p_dbar, long, lat)
# calculate CT from SA (g/kg), in-situ temperature (deg C), and gauge pressure (dbar)
CT = gsw.CT_from_t(SA, T, p_dbar)
# calculate N^2
nsquared, p_mid = gsw.Nsquared(SA, CT, p_dbar, lat=lat)
return nsquared
def test_SP_from_C(self):
"""Practical Salinity from Conductivity"""
output = gsw.SP_from_C(C, t, p)
check_values = np.array((20.009869599086951,
20.265511864874270,
22.981513062527689,
31.204503263727982,
34.032315787432829,
36.400308494388170))
numpy.testing.assert_array_equal(output, check_values)
def derive_cnv(self):
"""Compute SP, SA, CT, z, and GP from a cnv pre-processed cast."""
import gsw
cast = self.copy()
p = cast.index.values.astype(float)
cast['SP'] = gsw.SP_from_C(cast['c0S/m'].values * 10.,
cast['t090C'].values, p)
cast['SA'] = gsw.SA_from_SP(cast['SP'].values, p, self.lon, self.lat)
cast['SR'] = gsw.SR_from_SP(cast['SP'].values)
cast['CT'] = gsw.CT_from_t(cast['SA'].values, cast['t090C'].values, p)
cast['z'] = -gsw.z_from_p(p, self.lat)
cast['sigma0_CT'] = gsw.sigma0(cast['SA'].values, cast['CT'].values)
return cast
def derive_cnv(self):
"""Compute SP, SA, CT, z, and GP from a cnv pre-processed cast."""
cast = self.copy() # FIXME: Use MetaDataFrame to propagate lon, lat.
p = cast.index.values.astype(float)
cast['SP'] = gsw.SP_from_C(cast['c0S/m'].values * 10.,
cast['t090C'].values, p)
cast['SA'] = gsw.SA_from_SP(cast['SP'].values, p, self.lon, self.lat)
cast['SR'] = gsw.SR_from_SP(cast['SP'].values)
cast['CT'] = gsw.CT_from_t(cast['SA'].values, cast['t090C'].values, p)
cast['z'] = -gsw.z_from_p(p, self.lat)
cast['sigma0_CT'] = gsw.sigma0_CT_exact(cast['SA'].values,
cast['CT'].values)
return cast
def sbe43(volts, p, t, c, coefs, lat=0.0, lon=0.0):
# NOTE: lat/lon = 0 is not "acceptable" for GSW, come up with something else?
"""
SBE equation for converting SBE43 engineering units to oxygen (ml/l).
SensorID: 38
Parameters
----------
volts : array-like
Raw voltage
p : array-like
Converted pressure (dbar)
t : array-like
Converted temperature (Celsius)
c : array-like
Converted conductivity (mS/cm)
coefs : dict
Dictionary of calibration coefficients (Soc, Voffset, Tau20, A, B, C, E)
lat : array-like, optional
Latitude (decimal degrees north)
lon : array-like, optional
Longitude (decimal degrees)
Returns
-------
oxy_ml_l : array-like
Converted oxygen (mL/L)
"""
# TODO: is there any reason for this to output mL/L? if oxygen eq uses o2sol
# in umol/kg, result is in umol/kg... which is what we use at the end anyway?
t_Kelvin = t + 273.15
SP = gsw.SP_from_C(c, t, p)
SA = gsw.SA_from_SP(SP, p, lon, lat)
CT = gsw.CT_from_t(SA, t, p)
sigma0 = gsw.sigma0(SA, CT)
o2sol = gsw.O2sol(SA, CT, p, lon, lat) # umol/kg
o2sol_ml_l = oxy_umolkg_to_ml(o2sol,
sigma0) # equation expects mL/L (see TODO)
# NOTE: lat/lon always required to get o2sol (and need SA/CT for sigma0 anyway)
# the above is equivalent to:
# pt = gsw.pt0_from_t(SA, t, p)
# o2sol = gsw.O2sol_SP_pt(s, pt)
oxy_ml_l = (coefs["Soc"] * (volts + coefs["offset"]) *
(1.0 + coefs["A"] * t + coefs["B"] * np.power(t, 2) +
coefs["C"] * np.power(t, 3)) * o2sol_ml_l *
np.exp(coefs["E"] * p / t_Kelvin))
return np.around(oxy_ml_l, 4)
def calc_salinity(self, group):
''' If salinity is not in the list
calculate if from TSP
'''
print('calculating_salinity')
salinity = []
for n in range(len(group['Cond.'])):
s = gsw.SP_from_C(group['Cond.'].values[n],
group['Temp'].values[n],
group['Press'].values[n])
salinity.append(s)
group['Salinity'] = salinity
return group
def test_L2_params(self):
self.contexts = _get_pc_dict('tempwat_l1', 'condwat_l1', 'preswat_l1',
'pracsal', 'density')
self.value_classes = {}
dom_set = SimpleDomainSet((10, ))
# Add the callback for retrieving values
for n, p in self.contexts.iteritems():
if hasattr(p, '_pval_callback'):
p._pval_callback = self._get_param_vals
p._ctxt_callback = self._ctxt_callback
self.value_classes[n] = get_value_class(p.param_type, dom_set)
# Get the L2 data
psval = get_value_class(self.contexts['pracsal'].param_type, dom_set)
rhoval = get_value_class(self.contexts['density'].param_type, dom_set)
# Perform assertions - involves "manual" calculation of values
# Get the L0 data needed for validating output
latvals = self._get_param_vals('lat', slice(None))
lonvals = self._get_param_vals('lon', slice(None))
# Get the L1 data needed for validating output
t1val = get_value_class(self.contexts['tempwat_l1'].param_type,
dom_set)
c1val = get_value_class(self.contexts['condwat_l1'].param_type,
dom_set)
p1val = get_value_class(self.contexts['preswat_l1'].param_type,
dom_set)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# pracsal = gsw.SP_from_C((condwat_l1 * 10), tempwat_l1, preswat_l1)
import gsw
ps = gsw.SP_from_C((c1val[:] * 10.), t1val[:], p1val[:])
np.testing.assert_allclose(psval[:], ps)
# absolute_salinity = gsw.SA_from_SP(pracsal, preswat_l1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, tempwat_l1, preswat_l1)
# density = gsw.rho(absolute_salinity, conservative_temperature, preswat_l1)
abs_sal = gsw.SA_from_SP(psval[:], p1val[:], lonvals, latvals)
cons_temp = gsw.CT_from_t(abs_sal, t1val[:], p1val[:])
rho = gsw.rho(abs_sal, cons_temp, p1val[:])
np.testing.assert_allclose(rhoval[:], rho)
def _compute_data(self,data, units, names, p_ref = 0, baltic = False, lon=0, lat=0, isen = '0'):
""" Computes convservative temperature, absolute salinity and potential density from input data, expects a recarray with the following entries data['C']: conductivity in mS/cm, data['T']: in Situ temperature in degree Celsius (ITS-90), data['p']: in situ sea pressure in dbar
Arguments:
p_ref: Reference pressure for potential density
baltic: if True use the Baltic Sea density equation instead of open ocean
lon: Longitude of ctd cast default=0
lat: Latitude of ctd cast default=0
Returns:
list [cdata,cunits,cnames] with cdata: recarray with entries 'SP', 'SA', 'pot_rho', etc., cunits: dictionary with units, cnames: dictionary with names
"""
sen = isen + isen
# Check for units and convert them if neccessary
if(units['C' + isen] == 'S/m'):
logger.info('Converting conductivity units from S/m to mS/cm')
Cfac = 10
if(('68' in units['T' + isen]) or ('68' in names['T' + isen]) ):
logger.info('Converting IPTS-68 to T90')
T = gsw.t90_from_t68(data['T' + isen])
else:
T = data['T' + isen]
SP = gsw.SP_from_C(data['C' + isen], T, data['p'])
SA = gsw.SA_from_SP(SP,data['p'],lon = lon, lat = lat)
if(baltic == True):
SA = gsw.SA_from_SP_Baltic(SA,lon = lon, lat = lat)
PT = gsw.pt0_from_t(SA, T, data['p'])
CT = gsw.CT_from_t(SA, T, data['p'])
pot_rho = gsw.pot_rho_t_exact(SA, T, data['p'], p_ref)
names = ['SP' + sen,'SA' + sen,'pot_rho' + sen,'pt0' + sen,'CT' + sen]
formats = ['float','float','float','float','float']
cdata = {}
cdata['SP' + sen] = SP
cdata['SA' + sen] = SA
cdata['pot_rho' + sen] = pot_rho
cdata['pt' + sen] = PT
cdata['CT' + sen] = CT
cnames = {'SA' + sen:'Absolute salinity','SP' + sen: 'Practical Salinity on the PSS-78 scale',
'pot_rho' + sen: 'Potential density',
'pt' + sen:'potential temperature with reference sea pressure (p_ref) = 0 dbar',
'CT' + sen:'Conservative Temperature (ITS-90)'}
cunits = {'SA' + sen:'g/kg','SP' + sen:'PSU','pot_rho' + sen:'kg/m^3' ,'CT' + sen:'deg C','pt' + sen:'deg C'}
return [cdata,cunits,cnames]
def calc_teos10_columns(self, lat, lng):
# Practical Salinity
SP = gsw.SP_from_C(self.data['Cond'], self.data['Temp'], self.data['Pres'])
# Absolute Salinity
SA = gsw.SA_from_SP_Baltic(SP, lng, lat)
# Conservative Temperature
CT = gsw.CT_from_t(SA, self.data['Temp'], self.data['Pres'])
# Sigma(density) with reference pressure of 0 dbar
sigma = gsw.sigma0(SA, CT)
# Depth
depth = list(map(abs, gsw.z_from_p(self.data['Pres'], lat)))
return {'PracticalSalinity' : SP,
'AbsoluteSalinity' : SA,
'ConservativeTemperature' : CT,
'Sigma(density)' : sigma,
'Depth' : depth}
def main(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_cndc = ds.variables["CNDC"]
var_pres = ds.variables["PRES"]
t = var_temp[:]
C = var_cndc[:] * 10
p = var_pres[:]
psal = gsw.SP_from_C(C, t, p)
ncVarOut = ds.createVariable("PSAL", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = psal
ncVarOut.units = "1"
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html"
ds.close()
def read_dbds(self, fns, settings):
if settings['buoyancy_engine'] == 'shallow':
buoyancy_variable = 'm_ballast_pumped'
elif settings['buoyancy_engine'] == 'deep':
buoyancy_variable = 'm_de_oil_vol'
else:
raise ValueError(
'Unknown buoyancy engine. Accepted values: (shallow|deep)')
dbd = dbdreader.MultiDBD(filenames=fns, complement_files=True)
try:
tmp = dbd.get_CTD_sync("m_pitch", "m_battpos", buoyancy_variable)
except dbdreader.DbdError:
tmp = dbd.get_sync("sci_water_cond", "sci_water_temp",
"sci_water_pressure", "m_pitch", "m_battpos",
buoyancy_variable)
try:
(_, lat), (_, lon) = dbd.get("m_gps_lat", "m_gps_lon")
except dbdreader.DbdError:
latitude = float(input("Provide decimal latitude:"))
longitude = float(input("Provide decimal longitude:"))
else:
latitude = np.median(lat)
longitude = np.median(lon)
condition = np.logical_and(tmp[2] > 0.01,
np.isfinite(np.sum(tmp, axis=0)))
tctd, C, T, P, pitch, battpos, buoyancy_change = np.compress(condition,
tmp,
axis=1)
SP = gsw.SP_from_C(C * 10, T, P * 10)
SA = gsw.SA_from_SP(SP, P * 10, longitude, latitude)
density = gsw.rho_t_exact(SA, T, P * 10)
#density = fast_gsw.rho(C*10, T, P*10, longitude, latitude)
data = dict(time=tctd,
pressure=P,
pitch=pitch,
buoyancy_change=buoyancy_change,
battpos=battpos,
density=density)
self.model.input_data = data
mask = np.zeros(tctd.shape,
'int').astype(bool) # use all data by default
self.model.set_mask(mask)
self.model.OR(P * 10 < settings['minlimitdepth'])
self.model.OR(P * 10 > settings['maxlimitdepth'])
def sp_dict(c, t, p):
"""Wrapper of SP_from_C from gsw library.
Take in non-numpy data, format to numpy array, then run through.
Goal to eventually deprecate this.
Inputs:
c: array, Conductivity in mS/cm
t: array, in-situ temp in Celcius
p: array, sea pressure
Output:
SP: array, practical salinity (PSS-78)
"""
c = np.array(c)
t = np.array(t)
p = np.array(p)
SP = gsw.SP_from_C(c, t, p)
return SP
def rbr_data_correction(fname):
"""Remove some strange data from the time series in RBR data.
This function is inteded to be run after the dataset was generated. The
TEOS10 equations are used here to correct the RBR conductivity readings.
"""
import warnings
warnings.filterwarnings("ignore", category=RuntimeWarning)
# open dataset as append mode
dataset = nc.Dataset(fname, "a")
# extract data into numpy arrays
Cw, Tw, Sw, p = (dataset[v][:].filled()
for v in ["Cw", "Tw", "Sw", "depth"])
# remove salinity data when the gradient is greater than 0.05
ix = np.append(False, np.abs(np.diff(Sw)) >= 0.1)
Sw[ix] = np.nan
# compute conductivity from teos
Sw_clean = Sw.copy()
Sw_clean[np.isnan(Sw)] = np.nanmean(Sw)
Cw_from_teos = gsw.C_from_SP(Sw_clean, Tw, p)
Sw_from_teos = gsw.SP_from_C(Cw_from_teos, Tw, p)
# compute the clean water density
rhow = gsw.rho(Sw_clean, Tw, p)
# save data into the dataset
dataset["rhow"][:] = rhow
dataset["Cw"][:] = 0.1 * Cw + 0.9 * Cw_from_teos
dataset["Sw"][:] = Sw
#
dens_rel = np.sqrt(dataset["rhoa"][:] / dataset["rhow"][:])
dataset["wstar"][:] = dens_rel * dataset["ustar"][:]
# close dataset
dataset.close()
def add_psal(netCDFfile):
ds = Dataset(netCDFfile, 'a')
var_temp = ds.variables["TEMP"]
var_cndc = ds.variables["CNDC"]
comment = ""
try:
var_pres = ds.variables["PRES"]
except KeyError:
var_pres = ds.variables["NOMINAL_DEPTH"]
comment = ", using nominal depth " + str(var_pres[:])
t = var_temp[:]
C = var_cndc[:] * 10
p = var_pres[:]
psal = gsw.SP_from_C(C, t, p)
if "PSAL" in ds.variables:
ncVarOut = ds.variables["PSAL"]
else:
ncVarOut = ds.createVariable("PSAL", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max
ncVarOut[:] = psal
ncVarOut.units = "1"
ncVarOut.standard_name = "sea_water_practical_salinity"
ncVarOut.long_name = "sea_water_practical_salinity"
ncVarOut.valid_max = np.float32(40)
ncVarOut.valid_min = np.float32(-1)
ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html" + comment
# update the history attribute
try:
hist = ds.history + "\n"
except AttributeError:
hist = ""
ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + " : added PSAL from TEMP, CNDC, PRES")
ds.close()
def createNewFile(filename, station, CTDConfig):
infile = open(filename, 'r')
lines = infile.readlines()
l = lines[0].split(";")
options = None
if l[2] == "Cond." and len(l) == 13:
CTDConfig.conductivity_to_salinity = True
CTDConfig.conductivityMissing = False
elif l[2] == "Sal." and 'Cond.' not in l:
CTDConfig.conductivity_to_salinity = False
CTDConfig.conductivityMissing = True
elif l[2] == "Sal." and 'Cond.' in l:
CTDConfig.conductivity_to_salinity = False
CTDConfig.conductivityMissing = False
if ('S. vel.' in l and 'Cond.' in l):
CTDConfig.conductivityMissing = False
CTDConfig.conductivity_to_salinity = False
l[-4] = l[-3]
l[-3] = l[-2]
l[-2] = l[-1]
options = ['S. vel.']
if l[8] == "Press":
CTDConfig.calculate_depth_from_pressure = True
okokyst_metadata.addStationMeadata(station, CTDConfig, options)
# Create a new file by adding a header to the existing CTD file.
newfilename = filename[0:-4] + "_edited.txt"
if os.path.exists(newfilename):
os.remove(newfilename)
if CTDConfig.debug:
print("=> Creating new output file: %s" % newfilename)
outfile = open(newfilename, 'a')
outfile.writelines(station.header + station.mainHeader)
counter = 0
for line in lines[1:]:
l = line.split(";")
# Calculate salinity from conductivty and replace with conductivity in data array
if CTDConfig.conductivity_to_salinity and counter > 0:
C = float(l[2])
T = float(l[3])
P = float(l[8])
salinity = gsw.SP_from_C(C, T, P)
l[2] = salinity
# If only pressure is given, convert to depth in meters
if CTDConfig.calculate_depth_from_pressure:
P = float(l[8])
l[8] = pressure_to_depth(P, station.latitude)
line = ';'.join(map(str, l))
if counter > 0:
outfile.writelines(line)
counter += 1
outfile.close()
infile.close()
return newfilename
def data_correction(var_attrs, var_names, metadata_file, metadata_file_info, input_data_file, output_data_file, config_settings):
# Declare dictionaries where to store corrections attributes
cond_corr_atts = {}
salt_corr_atts = {}
with netCDF4.Dataset(metadata_file) as nc:
# List variables contained in metadata file
nc_vars = nc.variables.keys()
#load metadata from conductivity and salinity to be corrected from sensor1
cond_01_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][0])
salt_01_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][1])
# in case there exists sensor 2
if metadata_file_info['platform_subtype'] == 'ctd':
if len(var_names['var_to_corr']) >= 2:
if var_names['var_to_corr'][2] in nc_vars:
cond_02_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][2])
salt_02_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][3])
with netCDF4.Dataset(input_data_file) as nc:
#load metadata and data from sensor 1
cond_01_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][0])
salt_01_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][1])
cond_01_data = nc.variables[var_names['var_original'][0]][:]
temp_01_data = nc.variables[var_names['var_aux'][0]][:]
cond_corr_atts['cond_01_corr_atts'] = cond_01_corr_atts
salt_corr_atts['salt_01_corr_atts']= salt_01_corr_atts
if metadata_file_info['platform_subtype'] == 'glider':
prof_dir_data = nc.variables[var_names['var_aux'][2]][:]
# in case there exists sensor 2
if metadata_file_info['platform_subtype'] == 'ctd':
if len(var_names['var_to_corr']) >= 2:
if var_names['var_to_corr'][2] in nc_vars:
cond_02_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][2])
salt_02_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][3])
cond_02_data = nc.variables[var_names['var_original'][2]][:]
temp_02_data = nc.variables[var_names['var_aux'][2]][:]
cond_corr_atts['cond_02_corr_atts'] = cond_02_corr_atts
salt_corr_atts['salt_02_corr_atts'] = salt_02_corr_atts
#load data from pressure sensor
pres_data = nc.variables[var_names['var_aux'][1]][:]
#APPLY CORRECTIONS
logging.info('Exporting corrected variables to ' + output_data_file + '\n')
#correct conductivity values
if config_settings['multi_coeff_A'] == False:
if 'correction_coefficient_A' in cond_01_corr_atts:
coefficient_A = np.float64(cond_01_corr_atts['correction_coefficient_A'])
elif 'CorrectionCoefficient_A' in cond_01_corr_atts:
coefficient_A = np.float64(cond_01_corr_atts['CorrectionCoefficient_A'])
cond_01_corr_data = cond_01_data * coefficient_A
if metadata_file_info['platform_subtype'] == 'glider':
cond_01_corr_data = cond_01_corr_data * 10 # convert Sm-1 to mScm-1
cond_01_corr_atts, cond_01_corr_data = VariableSettings.fill_empty_value_with_nan(cond_01_corr_atts, cond_01_corr_data)
elif config_settings['multi_coeff_A'] == True:
coeff_A_list = config_settings['coeff_A_list']
idx_list = config_settings['idx_list']
cond_01_corr_data = cond_01_data.copy()
if config_settings['prof_dir_sections'] == False:
range_idx = len(idx_list)-1
elif config_settings['prof_dir_sections'] == True:
range_idx = len(idx_list)
for i in range(0, range_idx):
coefficient_A = coeff_A_list[i]
if config_settings['prof_dir_sections'] == False:
cond_01_corr_data[idx_list[i]:idx_list[i+1]] = cond_01_data[idx_list[i]:idx_list[i+1]] * coefficient_A
elif config_settings['prof_dir_sections'] == True:
cond_01_corr_data[prof_dir_data == idx_list[i]] = cond_01_data[prof_dir_data == idx_list[i]] * coefficient_A
if metadata_file_info['platform_subtype'] == 'glider':
cond_01_corr_data = cond_01_corr_data * 10 # convert Sm-1 to mScm-1
cond_01_corr_atts, cond_01_corr_data = VariableSettings.fill_empty_value_with_nan(cond_01_corr_atts, cond_01_corr_data)
#derive corrected salinities using gsw package
salt_01_corr_data = gsw.SP_from_C(cond_01_corr_data,temp_01_data,pres_data)
if metadata_file_info['platform_subtype'] == 'glider':
salt_01_corr_atts, salt_01_corr_data = VariableSettings.fill_empty_value_with_nan(salt_01_corr_atts, salt_01_corr_data)
# in case there exists sensor 2
if metadata_file_info['platform_subtype'] == 'ctd':
if len(var_names['var_to_corr']) >= 2:
if var_names['var_to_corr'][2] in nc_vars:
if 'correction_coefficient_B' in cond_02_corr_atts:
coefficient_B = cond_02_corr_atts['correction_coefficient_B']
elif 'CorrectionCoefficient_B' in cond_02_corr_atts:
coefficient_B = cond_02_corr_atts['CorrectionCoefficient_B']
cond_02_corr_data = cond_02_data * coefficient_B
#cond_02_corr_atts, cond_02_corr_data = VariableSettings.fill_empty_value_with_nan(cond_02_corr_atts, cond_02_corr_data)
#derive corrected salinities using gsw package
salt_02_corr_data = gsw.SP_from_C(cond_02_corr_data,temp_02_data,pres_data)
#salt_02_corr_atts, salt_02_corr_data = VariableSettings.fill_empty_value_with_nan(salt_02_corr_atts, salt_02_corr_data)
#WRITE CORRECTED VARIABLES
#write conductivity and salinity corrected variables corrected L1 file
with netCDF4.Dataset(output_data_file, 'r+') as nc:
#generate corrected variables to fill data&metadata
VariableSettings.create_corrected_variable(metadata_file_info, nc, var_names['var_to_corr'][0],cond_01_corr_data,cond_01_atts,cond_01_corr_atts)
VariableSettings.create_corrected_variable(metadata_file_info, nc, var_names['var_to_corr'][1],salt_01_corr_data,salt_01_atts,salt_01_corr_atts)
# in case there exists sensor 2
if metadata_file_info['platform_subtype'] == 'ctd':
if len(var_names['var_to_corr']) >= 2:
if var_names['var_to_corr'][2] in nc_vars:
VariableSettings.create_corrected_variable(metadata_file_info,nc, var_names['var_to_corr'][2],cond_02_corr_data,cond_02_atts,cond_02_corr_atts)
VariableSettings.create_corrected_variable(metadata_file_info,nc, var_names['var_to_corr'][3],salt_02_corr_data,salt_02_atts,salt_02_corr_atts)
# Edit global attributes
with netCDF4.Dataset(output_data_file, 'r+') as nc:
nc.setncattr('data_mode', 'DM')
nc.setncattr('processing_level', 'L1_corr processed data with salinity and conductivity corrections')
nc.setncattr('history', 'Product generated by the glider toolbox version 1.3.0 (https://github.com/socib/glider_toolbox). Salinity and conductivity corrections generated by the SOCIB Salinity Correction Toolbox version 0.1.0')
return cond_corr_atts, salt_corr_atts
def derive(self,property):
"""
Derives seawater properties as salinity, buoyancy frequency squared etc.
Args:
ST:
N2:
"""
try:
gsw
except:
logger.warning('GSW toolbox missing, derive will not work, doing nothing')
return
if(property == 'ST'):
# Poor mans check if the variables exist
sensor_pair0 = False
sensor_pair1 = False
try:
tmp1 = self.data['cond0']
tmp1 = self.data['temp0']
tmp1 = self.data['p']
sensor_pair0 = True
except:
sensor_pair0 = False
try:
tmp1 = self.data['cond1']
tmp1 = self.data['temp1']
tmp1 = self.data['p']
sensor_pair1 = True
except:
sensor_pair1 = False
if(sensor_pair0):
logger.debug('Calculating PSU0/SA11/CT11/rho11')
SP = gsw.SP_from_C(self.data['cond0'],self.data['temp0'],self.data['p'])
self.derived['SP00'] = SP
SA = gsw.SA_from_SP(SP,self.data['p'],self.header['lon'],self.header['lat'])
self.derived['SA00'] = SA
CT = gsw.CT_from_t(SA,self.data['temp0'],self.data['p'])
self.derived['CT00'] = CT
rho = gsw.rho_CT_exact(SA,CT,self.data['p'])
self.derived['rho00'] = rho
if(sensor_pair1):
logger.debug('Calculating PSU1/SA11/CT11/rho11')
SP = gsw.SP_from_C(self.data['cond1'],self.data['temp1'],self.data['p'])
self.derived['SP11'] = SP
SA = gsw.SA_from_SP(SP,self.data['p'],self.header['lon'],self.header['lat'])
self.derived['SA11'] = SA
CT = gsw.CT_from_t(SA,self.data['temp1'],self.data['p'])
self.derived['CT11'] = CT
rho = gsw.rho_CT_exact(SA,CT,self.data['p'])
self.derived['rho11'] = rho
if(property == 'N2'):
# Poor mans check if the variables exist
sensor_pair0 = False
sensor_pair1 = False
try:
tmp1 = self.derived['SA00']
tmp1 = self.derived['CT00']
tmp1 = self.data['p']
sensor_pair0 = True
except:
logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")')
sensor_pair0 = False
try:
tmp1 = self.derived['SA11']
tmp1 = self.derived['CT11']
tmp1 = self.data['p']
sensor_pair1 = True
except:
logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")')
sensor_pair1 = False
if(sensor_pair0):
logger.debug('Calculating Nsquared00')
[N2,p_mid] = gsw.Nsquared(self.derived['SA00'],self.derived['CT00'],self.data['p'])
self.derived['Nsquared00'] = interp(self.data['p'],p_mid,N2)
if(sensor_pair1):
logger.debug('Calculating Nsquared11')
[N2,p_mid] = gsw.Nsquared(self.derived['SA11'],self.derived['CT11'],self.data['p'])
self.derived['Nsquared11'] = interp(self.data['p'],p_mid,N2)
if(property == 'alphabeta'):
# Poor mans check if the variables exist
sensor_pair0 = False
sensor_pair1 = False
try:
tmp1 = self.derived['SA00']
tmp1 = self.derived['CT00']
tmp1 = self.data['p']
sensor_pair0 = True
except:
logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")')
sensor_pair0 = False
try:
tmp1 = self.derived['SA11']
tmp1 = self.derived['CT11']
tmp1 = self.data['p']
sensor_pair1 = True
except:
logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")')
sensor_pair1 = False
if(sensor_pair0):
logger.debug('Calculating Nsquared00')
alpha = gsw.alpha(self.derived['SA00'],self.derived['CT00'],self.data['p'])
beta = gsw.beta(self.derived['SA00'],self.derived['CT00'],self.data['p'])
self.derived['alpha00'] = alpha
self.derived['beta00'] = beta
if(sensor_pair1):
logger.debug('Calculating Nsquared11')
alpha = gsw.alpha(self.derived['SA11'],self.derived['CT11'],self.data['p'])
beta = gsw.beta(self.derived['SA11'],self.derived['CT11'],self.data['p'])
self.derived['alpha11'] = alpha
self.derived['beta11'] = beta
# pres = data['P_slow'][5000:15000].reshape(500,20).mean(1)
cond = data['JAC_C'][141000:155000].reshape(700, 20).mean(1)
temp = data['JAC_T'][141000:155000].reshape(700, 20).mean(1)
pres = data['P_slow'][141000:155000].reshape(700, 20).mean(1)
plt.plot(data['P_slow'])
plt.show()
# cond = data['JAC_C'][5000:20000]
# temp = data['JAC_T'][5000:20000]
# pres = data['P_slow'][5000:20000]
# salt = seawater.salt(cond,temp,pres)
# # plt.plot(salt)
# # plt.show()
sal = gsw.SP_from_C(cond, temp, pres)
# # plt.plot(cond)
# # plt.show()
# # plt.plot(temp)
# # plt.show()
# plt.plot(pres)
# plt.show()
# # plt.plot(sal)
# # plt.show()
# plt.plot(sal)
# plt.show()
CT = gsw.CT_from_t(sal, temp, pres)
SA = gsw.SA_from_SP(sal, pres, 174, -43)
dens = gsw.sigma0(SA, CT)
depth = gsw.z_from_p(pres, -43)
def main(argv):
parser = argparse.ArgumentParser(
description=
'Convert SBE raw data to a converted, csv-formatted text file')
parser.add_argument('timeFile',
metavar='time_file',
help='the .csv data file to fit by bottle data')
# debug messages
parser.add_argument('-d',
'--debug',
action='store_true',
help='display debug messages')
# raw output
parser.add_argument('-pres',
'--pressure',
action='store_true',
help='Fit pressure data')
# raw output
parser.add_argument('-temp',
'--temperature',
action='store_true',
help='Fit temperture data')
# raw output
parser.add_argument('-cond',
'--conductivity',
action='store_true',
help='Fit conductivity data')
# raw output
parser.add_argument('-oxy',
'--oxygen',
type=argparse.FileType('r'),
help='return the oxygen data file')
# Process Command-line args
args = parser.parse_args()
if args.debug:
global DEBUG
DEBUG = True
debugPrint("Running in debug mode")
# Verify hex file exists
if not os.path.isfile(args.timeFile):
errPrint('ERROR: Input time dependent .csv file:', args.timeFile,
'not found\n')
sys.exit(1)
# Used later for building output file names
filename_ext = os.path.basename(
args.timeFile) # original filename with ext
filename_base = os.path.splitext(filename_ext)[
0] # original filename w/o ext
if '_' in filename_base:
filename_base = filename_base.split('_')[0]
#Import Cruise Configuration File
iniFile = 'data/ini-files/configuration.ini'
config = configparser.RawConfigParser()
config.read(iniFile)
#Initialise Configuration Parameters
expocode = config['cruise']['expocode']
sectionID = config['cruise']['sectionid']
raw_directory = config['ctd_processing']['raw_data_directory']
time_directory = config['ctd_processing']['time_data_directory']
pressure_directory = config['ctd_processing']['pressure_data_directory']
oxygen_directory = config['ctd_processing']['oxygen_directory']
btl_directory = config['ctd_processing']['bottle_directory']
o2flask_file = config['ctd_processing']['o2flask_file']
log_directory = config['ctd_processing']['log_directory']
sample_rate = config['ctd_processing']['sample_rate']
search_time = config['ctd_processing']['roll_filter_time']
ctd = config['ctd_processing']['ctd_serial']
time_zone = config['inputs']['time_zone']
p_col = config['analytical_inputs']['p']
p_btl_col = config['inputs']['p']
t_col = config['analytical_inputs']['t']
t_btl_col = config['inputs']['t']
t1_col = config['analytical_inputs']['t1']
t1_btl_col = config['inputs']['t1']
t2_col = config['analytical_inputs']['t2']
t2_btl_col = config['inputs']['t2']
c_col = config['analytical_inputs']['c']
c1_col = config['analytical_inputs']['c1']
c1_btl_col = config['inputs']['c1']
c2_col = config['analytical_inputs']['c2']
c2_btl_col = config['inputs']['c2']
sal_col = config['analytical_inputs']['salt']
sal_btl_col = config['inputs']['salt']
btl_sal_col = config['analytical_inputs']['btl_salt']
dov_col = config['analytical_inputs']['dov']
dov_btl_col = config['inputs']['dov']
dopl_col = config['analytical_inputs']['dopl']
dopl_btl_col = config['inputs']['dopl']
dopkg_col = config['analytical_inputs']['dopkg']
btl_oxy_col = config['analytical_inputs']['btl_oxy']
xmis_col = config['analytical_inputs']['xmis']
fluor_col = config['analytical_inputs']['fluor']
timedate = config['analytical_inputs']['datetime']
lat_col = config['analytical_inputs']['lat']
lat_btl_col = config['inputs']['lat']
lon_col = config['analytical_inputs']['lon']
lon_btl_col = config['inputs']['lon']
reft_col = config['inputs']['reft']
btl_num_col = config['inputs']['btl_num']
#time_column_data = config['time_series_output']['data_names'].split(',')
time_column_data = config['time_series_output']['data_output']
time_column_names = config['time_series_output']['column_name'].split(',')
time_column_units = config['time_series_output']['column_units'].split(',')
time_column_format = config['time_series_output']['format']
#pressure_column_data = config['time_series_output']['data_names'].split(',')
p_column_data = config['pressure_series_output']['data'].split(',')
p_column_names = config['pressure_series_output']['column_name'].split(',')
p_column_units = config['pressure_series_output']['column_units'].split(
',')
p_column_format = config['pressure_series_output']['format']
p_column_qual = config['pressure_series_output']['qual_columns'].split(',')
p_column_one = list(
config['pressure_series_output']['q1_columns'].split(','))
#bottle_data outputs
btl_dtype = config['bottle_series_output']['dtype']
hexfileName = str(filename_base + '.' + HEX_EXT)
hexfilePath = os.path.join(raw_directory, hexfileName)
xmlfileName = str(filename_base + '.' + XML_EXT)
xmlfilePath = os.path.join(raw_directory, xmlfileName)
outtimefileName = str(filename_base + TIME_SUFFIX + '.' + FILE_EXT)
outtimefilePath = os.path.join(time_directory, outtimefileName)
pressfileName = str(filename_base + FIT_SUFFIX + '.' + FILE_EXT)
pressfilePath = os.path.join(pressure_directory, pressfileName)
btlfileName = str(filename_base + BTL_SUFFIX + MEAN_SUFFIX + '.' +
FILE_EXT)
btlfilePath = os.path.join(btl_directory, btlfileName)
# Get bottle data
btl_data = process_ctd.dataToNDarray(btlfilePath, float, True, ',', 0)
btl_data = btl_data[:][1:]
# Get procesed time data
time_data = process_ctd.dataToNDarray(args.timeFile, float, True, ',', 1)
btm = np.argmax(time_data[p_col][1:])
time_data = time_data[:][1:btm]
if args.pressure:
print('In -pres flag fit condition')
print(filename_base)
pfileName = str('poffset' + '.' + FILE_EXT)
pfilePath = os.path.join(log_directory, pfileName)
poff_data = process_ctd.dataToNDarray(pfilePath, str, None, ',', None)
for line in poff_data:
if filename_base in line[0]:
for val in line:
if 'offset' in val:
offset = float(str.split(val, ':')[1])
continue
# Pressure offset
btl_data[p_btl_col] = fit_ctd.offset(offset, btl_data[p_btl_col])
time_data[p_col] = fit_ctd.offset(offset, time_data[p_col])
# End pressure if condition
if args.temperature:
print('In -temp flag fit condition')
print(filename_base)
coef1 = []
coef2 = []
# Get descrete ref temp data
t1fileName = str('fitting_t1' + '.' + FILE_EXT)
t1filePath = os.path.join(log_directory, t1fileName)
t1_coef = process_ctd.dataToNDarray(t1filePath, str, None, ',', None)
for line in t1_coef:
if filename_base in line[0]:
val = line[1:]
for i in range(0, len(val)):
if 'coef' in val[i]:
coef1.append(float(str.split(val[i], ':')[1]))
else:
coef1.append(float(val[i]))
continue
btl_data[t1_btl_col] = fit_ctd.temperature_polyfit(
coef1, btl_data[p_btl_col], btl_data[t1_btl_col], 'all')
time_data[t1_col] = fit_ctd.temperature_polyfit(
coef1, time_data[p_col], time_data[t1_col], 'all')
t2fileName = str('fitting_t2' + '.' + FILE_EXT)
t2filePath = os.path.join(log_directory, t2fileName)
t2_coef = process_ctd.dataToNDarray(t2filePath, str, None, ',', None)
for line in t2_coef:
if filename_base in line[0]:
val = line[1:]
for i in range(0, len(val)):
if 'coef' in val[i]:
coef2.append(float(str.split(val[i], ':')[1]))
else:
coef2.append(float(val[i]))
continue
btl_data[t2_btl_col] = fit_ctd.temperature_polyfit(
coef2, btl_data[p_btl_col], btl_data[t2_btl_col], 'all')
time_data[t2_col] = fit_ctd.temperature_polyfit(
coef2, time_data[p_col], time_data[t2_col], 'all')
if args.conductivity:
print('In -cond flag fit condition')
print(filename_base)
coef1 = []
coef2 = []
# Get descrete ref temp data
c1fileName = str('fitting_c1' + '.' + FILE_EXT)
c1filePath = os.path.join(log_directory, c1fileName)
c1_coef = process_ctd.dataToNDarray(c1filePath, str, None, ',', None)
for line in c1_coef:
if filename_base in line[0]:
val = line[1:]
for i in range(0, len(val)):
if 'coef' in val[i]:
coef1.append(float(str.split(val[i], ':')[1]))
else:
coef1.append(float(val[i]))
continue
btl_data[c1_btl_col] = fit_ctd.conductivity_polyfit(
coef1, btl_data[p_btl_col], btl_data[t1_btl_col],
btl_data[c1_btl_col])
time_data[c1_col] = fit_ctd.conductivity_polyfit(
coef1, time_data[p_col], time_data[c1_col], time_data[c1_col])
c2fileName = str('fitting_c2' + '.' + FILE_EXT)
c2filePath = os.path.join(log_directory, c2fileName)
c2_coef = process_ctd.dataToNDarray(c2filePath, str, None, ',', None)
for line in c2_coef:
if filename_base in line[0]:
val = line[1:]
for i in range(0, len(val)):
if 'coef' in val[i]:
coef2.append(float(str.split(val[i], ':')[1]))
else:
coef2.append(float(val[i]))
continue
btl_data[c2_btl_col] = fit_ctd.conductivity_polyfit(
coef2, btl_data[p_btl_col], btl_data[t2_btl_col],
btl_data[c2_btl_col])
time_data[c2_col] = fit_ctd.conductivity_polyfit(
coef2, time_data[p_col], time_data[c2_col], time_data[c2_col])
time_data[sal_col] = gsw.SP_from_C(time_data[c_col], time_data[t_col],
time_data[p_col])
if args.oxygen:
print('In -oxy flag fit condition')
print(filename_base)
# Get Analytical Oxygen data
o2pkg_btl, o2pl_btl = fit_ctd.o2_calc(o2flask_file, args.oxygen.name,
btl_data[btl_num_col],
btl_data[sal_btl_col])
kelvin = []
for i in range(0, len(time_data[t_col])):
kelvin.append(time_data[t_col][i] + 273.15)
# Find New Oxygen Coef
oxy_coef = fit_ctd.find_oxy_coef(o2pl_btl['OXYGEN'],
btl_data[p_btl_col],
btl_data[t_btl_col],
btl_data[sal_btl_col],
btl_data[dov_btl_col], hexfilePath,
xmlfilePath)
# Convert CTD Oxygen Voltage Data with New DO Coef
time_data[dopl_col] = fit_ctd.oxy_dict(oxy_coef, time_data[p_col],
kelvin, time_data[t_col],
time_data[sal_col],
time_data[dov_col])
# End oxygen flag fitting if condition
# Find Isopycnal Down Trace Bottle Trip Equivalent
# Write bottle data to file
report_ctd.report_btl_data(btlfilePath, btl_dtype, btl_data)
# Write time data to file
report_ctd.report_time_series_data(filename_base, time_directory, expocode,
time_column_names, time_column_units,
time_column_names, time_column_format,
time_data)
# Pressure Sequence
pressure_seq_data = process_ctd.pressure_sequence(filename_base, p_col,
timedate, 2.0, -1.0, 0.0,
'down', int(sample_rate),
int(search_time),
time_data)
# Convert dissolved oxygen from ml/l to umol/kg
dopkg = process_ctd.o2pl2pkg(p_col, t1_col, sal_col, dopl_col, dopkg_col,
lat_col, lon_col, pressure_seq_data)
# Add quality codes to data
qual_pseq_data = process_ctd.ctd_quality_codes(dopkg_col, None, None, True,
p_column_qual, p_column_one,
pressure_seq_data)
# Collect Cast Details from Log
logfileName = str('cast_details' + '.' + FILE_EXT)
logfilePath = os.path.join(log_directory, logfileName)
cast_details = process_ctd.dataToNDarray(logfilePath, str, None, ',', 0)
for line in cast_details:
if filename_base in line[0]:
for val in line:
if 'at_depth' in val: btime = float(str.split(val, ':')[1])
if 'latitude' in val: btm_lat = float(str.split(val, ':')[1])
if 'longitude' in val: btm_lon = float(str.split(val, ':')[1])
if 'altimeter_bottom' in val:
btm_alt = float(str.split(val, ':')[1])
break
# Write time data to file
depth = -999
report_ctd.report_pressure_series_data(filename_base, expocode, sectionID,
btime, btm_lat, btm_lon, depth,
btm_alt, ctd, pressure_directory,
p_column_names, p_column_units,
p_column_data, qual_pseq_data,
dopkg, pressure_seq_data)
#plt.plot(o2pl_btl['OXYGEN'], btl_data[p_btl_col], color='b', marker='o')
#plt.plot(tmpo2, time_data[p_col], color='g', label='raw')
#plt.plot(time_data[dopl_col], time_data[p_col], color='b', label='raw')
#plt.plot(pressure_seq_data[dopl_col], pressure_seq_data[p_col], color='r', label='raw')
#plt.plot(pressure_seq_data[t2_col], pressure_seq_data[p_col], color='r', label='raw')
#plt.plot(btl_data[t_btl_col], time_data[p_col], color='r', label='raw')
#plt.plot(o2pl_btl['OXYGEN'], btl_data[p_btl_col], color='g', marker='o', label='raw')
#plt.plot(pressure_seq_data[do_col], pressure_seq_data[p_col], color='b', label='raw')
#plt.gca().invert_yaxis()
#plt.axis()
#plt.show()
debugPrint('Done!')
def ctdproc(lista,
temp_name='t068C',
lathint='Latitude =',
lonhint='Longitude =',
cond_name='c0S/m',
press_name='prDM',
down_cast=True,
looped=True,
hann_f=False,
hann_block=20,
hann_times=2,
latline=[],
lonline=[]):
'''
This function do the basic proccess to all .cnv CTD data from
given list.
'''
for fname in lista:
lon, lat, data = ctdread(fname,
press_name=press_name,
down_cast=down_cast,
lathint=lathint,
lonhint=lonhint,
lonline=lonline,
latline=latline)
if looped:
data = loopedit(data)
dataname = basename(fname)[1]
if (data.shape[0] < 101) & (
data.shape[0] >
10): # se o tamanho do perfil for com menos de 101 medidas
if (data.shape[0] /
2) % 2 == 0: # caso a metade dos dados seja par
blk = (data.shape[0] / 2) + 1 # bloco = a metade +1
else:
blk = data.shape[0] / 2 # se for impar o bloco e a metade
# remove spikes dos perfis de temperatura e condutividade
data = despike(data, propname=temp_name, block=blk, wgth=2)
data = despike(data, propname=cond_name, block=blk, wgth=2)
elif data.shape[0] >= 101:
# para perfis com mais de 101 medidas, utiliza-se blocos de 101
data = despike(data, propname=temp_name, block=101, wgth=2)
data = despike(data, propname=cond_name, block=101, wgth=2)
else:
print('radial muito rasa')
# realiza média em caixa de 1 metro
data = binning(data, delta=1.)
if temp_name == 't068C':
data['t090C'] = gsw.t90_from_t68(data['t068C'])
data['sp'] = gsw.SP_from_C(data[cond_name] * 10, data['t090C'],
data.index.values)
if hann_f:
times = 0
while times < hann_times:
data = hann_filter(data, 't090C', hann_block)
data = hann_filter(data, 'sp', hann_block)
times += 1
data['pt'] = sw.ptmp(data['sp'], data['t090C'], data.index.values)
#data['ct'] = gsw.CT_from_pt(data['sa'],data['pt'])
data['psigma0'] = sw.pden(
data['sp'], data['t090C'], data.index.values, pr=0) - 1000
data['psigma1'] = sw.pden(
data['sp'], data['t090C'], data.index.values, pr=1000) - 1000
data['psigma2'] = sw.pden(
data['sp'], data['t090C'], data.index.values, pr=2000) - 1000
data['gpan'] = sw.gpan(data['sp'], data['t090C'], data.index.values)
data['lat'] = lat
data['lon'] = lon
data.to_pickle(
os.path.split(fname)[0] + '/' +
os.path.splitext(os.path.split(fname)[1])[0])
print(dataname)
def read_nos(buoy, dstart, dend, usemodel=False):
'''Set up urls and then read from them to get TCOON and NOS data.
Most stations have several data sources, so they are aggregated here.
This calls to read_nos_df() to do the reading and rearranging.
dstart and dend are datetime objects.
'''
if not usemodel:
# tide, met, and phys data
prefixes = [
'https://tidesandcurrents.noaa.gov/api/datagetter?product=water_level&application=NOS.COOPS.TAC.WL&station=',
'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=air_pressure&application=NOS.COOPS.TAC.MET&station=',
'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=air_temperature&application=NOS.COOPS.TAC.MET&station=',
'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=wind&application=NOS.COOPS.TAC.MET&station=',
'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=water_temperature&application=NOS.COOPS.TAC.PHYSOCEAN&station=',
'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=conductivity&application=NOS.COOPS.TAC.PHYSOCEAN&station='
]
# dstart and dend need to be in format YYYYMMDD
dstarts = dstart.strftime('%Y%m%d')
dends = dend.strftime('%Y%m%d')
suffixes = [
'&begin_date=' + dstarts + '&end_date=' + dends +
'&datum=MSL&units=metric&time_zone=GMT&format=csv',
'&begin_date=' + dstarts + '&end_date=' + dends +
'&time_zone=GMT&units=english&interval=6&format=csv',
'&begin_date=' + dstarts + '&end_date=' + dends +
'&time_zone=GMT&units=english&interval=6&format=csv',
'&begin_date=' + dstarts + '&end_date=' + dends +
'&time_zone=GMT&units=english&interval=6&format=csv',
'&begin_date=' + dstarts + '&end_date=' + dends +
'&time_zone=GMT&units=english&interval=6&format=csv',
'&begin_date=' + dstarts + '&end_date=' + dends +
'&time_zone=GMT&units=english&interval=6&format=csv'
]
dfs = []
for prefix, suffix in zip(prefixes, suffixes):
url = prefix + buoy + suffix
dft = read_nos_df(url)
if dft is not None:
dft = dft[~dft.index.duplicated(
keep='first')] # remove any duplicated indices
dfs.append(dft)
# combine the dataframes together
# don't append if all df are empty
if [df.empty for df in dfs].count(True) != len(dfs):
df = pd.concat([df for df in dfs if not df.empty],
axis=1,
sort=False)
else:
df = pd.concat([df for df in dfs], axis=1, sort=False)
# calculate salinity from conductivity, if available
if 'Conductivity [mS/cm]' in df.keys():
if not 'AtmPr [mb]' in df.keys():
pr = np.zeros(len(df))
else:
pr = df['AtmPr [mb]'] / 100. - 10.1325
df['Salinity'] = gsw.SP_from_C(df['Conductivity [mS/cm]'],
df['WaterT [deg C]'], pr)
# dictionary for rounding decimal places
rdict = {'Salinity': 2}
df = df.round(rdict)
else: # use model
prefix = 'https://tidesandcurrents.noaa.gov/api/datagetter?product=predictions&application=NOS.COOPS.TAC.WL&station='
dstarts = dstart.strftime('%Y%m%d')
dends = dend.strftime('%Y%m%d')
suffix = '&begin_date=' + dstarts + '&end_date=' + dends + '&datum=MSL&time_zone=GMT&units=metric&interval=h&format=csv'
url = prefix + buoy + suffix
df = read_nos_df(url)
return df
ctd_data.append(df)
ctd_data = pd.concat(ctd_data, axis=0, sort=False)
# load bottle trip file
file_list = sorted(glob.glob("../data/bottle/*.pkl"))
ssscc_list = [ssscc.strip("../data/bottle/")[:5] for ssscc in file_list]
upcast_data = []
for f in file_list:
with open(f, "rb") as x:
df = pickle.load(x)
df["SSSCC"] = f.strip("../data/bottle/")[:5]
# change to secondary if that is what's used
upcast_data.append(df[["SSSCC", "CTDCOND1", "CTDTMP1", "CTDPRS"]])
upcast_data = pd.concat(upcast_data, axis=0, sort=False)
upcast_data["CTDSAL"] = gsw.SP_from_C(
upcast_data["CTDCOND1"], upcast_data["CTDTMP1"], upcast_data["CTDPRS"]
)
# load salt file (adapted from compare_salinities.ipynb)
file_list = sorted(glob.glob("../data/salt/*.csv"))
ssscc_list = [ssscc.strip("../data/salt/")[:5] for ssscc in file_list]
salt_data = []
for f in file_list:
df = pd.read_csv(f, usecols=["STNNBR", "CASTNO", "SAMPNO", "SALNTY"])
df["SSSCC"] = f.strip("../data/salt/")[:5]
salt_data.append(df)
salt_data = pd.concat(salt_data, axis=0, sort=False)
salt_data["SALNTY"] = salt_data["SALNTY"].round(4)
if "SALNTY_FLAG_W" not in salt_data.columns:
salt_data["SALNTY_FLAG_W"] = 2
