def read_dbds(self, fns, settings): if settings['buoyancy_engine']=='shallow': buoyancy_variable='m_ballast_pumped' elif settings['buoyancy_engine']=='deep': buoyancy_variable='m_de_oil_vol' else: raise ValueError('Unknown buoyancy engine. Accepted values: (shallow|deep)') dbd = dbdreader.MultiDBD(filenames=fns, include_paired=True) tmp = dbd.get_sync("sci_ctd41cp_timestamp",["sci_water_pressure", "sci_water_cond", "sci_water_temp", "m_pitch", "m_battpos", buoyancy_variable]) _, tctd, P, C, T, pitch, battpos, buoyancy_change = tmp.compress(tmp[3]>0.01, axis=1) SP = gsw.SP_from_C(C*10, T, P*10) SA = gsw.SA_from_SP(SP, P*10, settings['longitude'], settings['latitude']) density = gsw.rho_t_exact(SA, T, P*10) #density = fast_gsw.rho(C*10, T, P*10, settings['longitude'], settings['latitude']) data = dict(time=tctd, pressure=P, pitch=pitch, buoyancy_change=buoyancy_change, battpos=battpos, density=density) self.model.input_data = data mask=np.zeros(tctd.shape,'int').astype(bool) # use all data by default self.model.set_mask(mask) self.model.OR(P*10<settings['minlimitdepth']) self.model.OR(P*10>settings['maxlimitdepth'])
def _calcSal(temp, cond, pres, coefs): a_offset = coefs[0] a_slope = coefs[1] t_offset = coefs[2] t_slope = coefs[3] alpha = a_offset + a_slope / flowSpeed tau = t_offset + t_slope / np.sqrt(flowSpeed) #tau = 11 # Parameter for Lueck and Picklo (1990) #alpha = 0.57/tau + 0.03122 # Parameter for Lueck and Picklo (1990) alpha[~np.isfinite(alpha)] = a_offset tau[~np.isfinite(tau)] = t_offset beta = 1 / tau # Parameter for Lueck and Picklo (1990) fn = Fs / 2 # Nyquist frequency a = 4 * fn * alpha * tau / (1 + 4 * fn * tau ) # Parameter for Lueck and Picklo (1990) b = 1 - 2 * a / alpha # Parameter for Lueck and Picklo (1990) # Compute temperature correction to obtained water temperature within the conductivity cell (Morison, 1994) _internal_bias = np.full_like(temp, 0) for sample in np.arange(1, len(_internal_bias)): # Recursive filter from Morison (1994) # if np.isfinite(temp[sample-1]): _internal_bias[sample] = -b[sample] * _internal_bias[ sample - 1] + a[sample] * (temp[sample] - temp[sample - 1]) return gsw.SP_from_C(cond, temp + _internal_bias, pres) # Practical salinity
def process_brown(df_raw, cfg: Mapping[str, Any]): ''' Calc physical values from codes :param df_raw: :param cfg: :return: pandas dataframe # todo: use signs. For now our data haven't negative values and it is noise if signs!=0. Check: df_raw.signs[df_raw.signs!=0] ''' Val = {} if b'Pres' in cfg['for']['k_names'] and not 'Pres' in df_raw.columns: df_raw = df_raw.rename(columns={'P': 'Pres'}) for nameb in np.intersect1d(np.array(df_raw.columns, 'S10'), cfg['for']['k_names']): name = nameb.decode('ascii') Val[name] = np.polyval( cfg['for']['kk'][nameb == cfg['for']['k_names']].flat, df_raw[name]) # Practical Salinity PSS-78 Val['Sal'] = gsw.SP_from_C(Val['Cond'], Val['Temp'], Val['Pres']) df = pd.DataFrame(Val, columns=cfg['out']['data_columns'], index=df_raw.index) # import seawater as sw # T90conv = lambda t68: t68/1.00024 # Val['sigma0'] sw.pden(s, T90conv(t), p=0) - 1000 return df
def ctd_residuals_df(df_bottle): '''Compute residuals and add to dataframe. Operate in place. ''' #Salinity should really be grabbed straight from the files, but... try: df_bottle['BTLCOND'] = gsw.C_from_SP(df_bottle['SALNTY'], df_bottle['CTDTMP1'], df_bottle['CTDPRS']) except ValueError: print(f'WHOOPS SOMETHING WENT WRONG') df_bottle['SALNTY'] = df_bottle['SALNTY'].replace(to_replace=-999, value=np.nan) df_bottle['BTLCOND'] = gsw.C_from_SP(df_bottle['SALNTY'], df_bottle['CTDTMP1'], df_bottle['CTDPRS']) df_bottle['BTL_O'] = df_bottle['OXYGEN'] - df_bottle['CTDOXY'] df_bottle['BTL_C1'] = df_bottle['BTLCOND'] - df_bottle['CTDCOND1'] df_bottle['BTL_C2'] = df_bottle['BTLCOND'] - df_bottle['CTDCOND2'] df_bottle['C1_C2'] = df_bottle['CTDCOND1'] - df_bottle['CTDCOND2'] df_bottle['BTL_T1'] = df_bottle['REFTMP'] - df_bottle['CTDTMP1'] df_bottle['BTL_T2'] = df_bottle['REFTMP'] - df_bottle['CTDTMP2'] df_bottle['T1_T2'] = df_bottle['CTDTMP1'] - df_bottle['CTDTMP2'] df_bottle['BTL_SAL_UP'] = df_bottle['SALNTY'] - gsw.SP_from_C( df_bottle['CTDCOND2'], df_bottle['CTDTMP2'], df_bottle['CTDPRS']) df_bottle['BTL_SAL'] = df_bottle['SALNTY'] - df_bottle['CTDSAL'] return None
def ctd_residuals_df(df_bottle): """Compute residuals and add to dataframe. Operate in place. """ # Salinity should really be grabbed straight from the files, but... try: df_bottle["BTLCOND"] = gsw.C_from_SP(df_bottle["SALNTY"], df_bottle["CTDTMP1"], df_bottle["CTDPRS"]) except ValueError: print("WHOOPS SOMETHING WENT WRONG") df_bottle["SALNTY"] = df_bottle["SALNTY"].replace(to_replace=-999, value=np.nan) df_bottle["BTLCOND"] = gsw.C_from_SP(df_bottle["SALNTY"], df_bottle["CTDTMP1"], df_bottle["CTDPRS"]) df_bottle["BTL_O"] = df_bottle["OXYGEN"] - df_bottle["CTDOXY"] df_bottle["BTL_C1"] = df_bottle["BTLCOND"] - df_bottle["CTDCOND1"] df_bottle["BTL_C2"] = df_bottle["BTLCOND"] - df_bottle["CTDCOND2"] df_bottle["C1_C2"] = df_bottle["CTDCOND1"] - df_bottle["CTDCOND2"] df_bottle["BTL_T1"] = df_bottle["REFTMP"] - df_bottle["CTDTMP1"] df_bottle["BTL_T2"] = df_bottle["REFTMP"] - df_bottle["CTDTMP2"] df_bottle["T1_T2"] = df_bottle["CTDTMP1"] - df_bottle["CTDTMP2"] df_bottle["BTL_SAL_UP"] = df_bottle["SALNTY"] - gsw.SP_from_C( df_bottle["CTDCOND2"], df_bottle["CTDTMP2"], df_bottle["CTDPRS"]) df_bottle["BTL_SAL"] = df_bottle["SALNTY"] - df_bottle["CTDSAL"] return None
def gsw_rho(C, T, P, lon, lat): SP = gsw.SP_from_C(C, T, P) # This particular data set was collected in the Baltic. SA = gsw.SA_from_SP_Baltic(SP, lon, lat) # in-situ density rho = gsw.density.rho_t_exact(SA, T, P) return rho
def calculate_density(T, p, C, lat, long): """Calculates density from temp, pressure, conductivity, and lat/long. All parameters and output are float or array-like. :param T: temperature (deg C) :param p: pressure (bar) :param C: conductivity (S/m) :param lat: latitude (decimal deg) :param long: longitude (decimal deg) :return: density (kg/m^3) """ # pressure in dbars = pressure * 10 if type(p) == float: p_dbar = p * 10 else: p_dbar = [pi * 10 for pi in p] # conductivity in mS/cm = conductivity * 10 if type(C) == float: C_mScm = C * 10 else: C_mScm = [Ci * 10 for Ci in C] # calculate SP from conductivity (mS/cm), in-situ temperature (deg C), and gauge pressure (dbar) SP = gsw.SP_from_C(C_mScm, T, p_dbar) # calculate SA from SP (unitless), gauge pressure (dbar), longitude and latitude (decimal degrees) SA = gsw.SA_from_SP(SP, p_dbar, long, lat) # calculate CT from SA (g/kg), in-situ temperature (deg C), and gauge pressure (dbar) CT = gsw.CT_from_t(SA, T, p_dbar) # calculate density return gsw.rho(SA, CT, p_dbar)
def add_psal(netCDFfile): ds = Dataset(netCDFfile, 'a') var_temp = ds.variables["TEMP"] var_cndc = ds.variables["CNDC"] var_pres = ds.variables["PRES"] t = var_temp[:] C = var_cndc[:] * 10 p = var_pres[:] psal = gsw.SP_from_C(C, t, p) ncVarOut = ds.createVariable( "PSAL", "f4", ("TIME", ), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max ncVarOut[:] = psal ncVarOut.units = "1" ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html" # update the history attribute try: hist = ds.history + "\n" except AttributeError: hist = "" ds.setncattr( 'history', hist + datetime.utcnow().strftime("%Y-%m-%d") + " : added PSAL from TEMP, CNDC, PRES") ds.close()
def calculate_n2(T, p, C, lat, long): """Calculates the buoyancy frequency N^2 All parameters and output are float or array-like. :param T: temperature (deg C) :param p: pressure (bar) :param C: conductivity (S/m) :param lat: latitude (decimal deg) :param long: longitude (decimal deg) :return: buoyancy (i.e. brunt-vaisala) frequency N^2 (s^-2) """ # pressure in dbars = pressure * 10 p_dbar = p * 10 # conductivity in mS/cm = conductivity * 10 C_mScm = C * 10 print(T.count()) print(p.count()) print(C.count()) print(lat.count()) print(long.count()) # calculate SP from conductivity (mS/cm), in-situ temperature (deg C), and gauge pressure (dbar) SP = gsw.SP_from_C(C_mScm, T, p_dbar) # calculate SA from SP (unitless), gauge pressure (dbar), longitude and latitude (decimal degrees) SA = gsw.SA_from_SP(SP, p_dbar, long, lat) # calculate CT from SA (g/kg), in-situ temperature (deg C), and gauge pressure (dbar) CT = gsw.CT_from_t(SA, T, p_dbar) # calculate N^2 nsquared, p_mid = gsw.Nsquared(SA, CT, p_dbar, lat=lat) return nsquared
def test_SP_from_C(self): """Practical Salinity from Conductivity""" output = gsw.SP_from_C(C, t, p) check_values = np.array((20.009869599086951, 20.265511864874270, 22.981513062527689, 31.204503263727982, 34.032315787432829, 36.400308494388170)) numpy.testing.assert_array_equal(output, check_values)
def derive_cnv(self): """Compute SP, SA, CT, z, and GP from a cnv pre-processed cast.""" import gsw cast = self.copy() p = cast.index.values.astype(float) cast['SP'] = gsw.SP_from_C(cast['c0S/m'].values * 10., cast['t090C'].values, p) cast['SA'] = gsw.SA_from_SP(cast['SP'].values, p, self.lon, self.lat) cast['SR'] = gsw.SR_from_SP(cast['SP'].values) cast['CT'] = gsw.CT_from_t(cast['SA'].values, cast['t090C'].values, p) cast['z'] = -gsw.z_from_p(p, self.lat) cast['sigma0_CT'] = gsw.sigma0(cast['SA'].values, cast['CT'].values) return cast
def derive_cnv(self): """Compute SP, SA, CT, z, and GP from a cnv pre-processed cast.""" cast = self.copy() # FIXME: Use MetaDataFrame to propagate lon, lat. p = cast.index.values.astype(float) cast['SP'] = gsw.SP_from_C(cast['c0S/m'].values * 10., cast['t090C'].values, p) cast['SA'] = gsw.SA_from_SP(cast['SP'].values, p, self.lon, self.lat) cast['SR'] = gsw.SR_from_SP(cast['SP'].values) cast['CT'] = gsw.CT_from_t(cast['SA'].values, cast['t090C'].values, p) cast['z'] = -gsw.z_from_p(p, self.lat) cast['sigma0_CT'] = gsw.sigma0_CT_exact(cast['SA'].values, cast['CT'].values) return cast
def sbe43(volts, p, t, c, coefs, lat=0.0, lon=0.0): # NOTE: lat/lon = 0 is not "acceptable" for GSW, come up with something else? """ SBE equation for converting SBE43 engineering units to oxygen (ml/l). SensorID: 38 Parameters ---------- volts : array-like Raw voltage p : array-like Converted pressure (dbar) t : array-like Converted temperature (Celsius) c : array-like Converted conductivity (mS/cm) coefs : dict Dictionary of calibration coefficients (Soc, Voffset, Tau20, A, B, C, E) lat : array-like, optional Latitude (decimal degrees north) lon : array-like, optional Longitude (decimal degrees) Returns ------- oxy_ml_l : array-like Converted oxygen (mL/L) """ # TODO: is there any reason for this to output mL/L? if oxygen eq uses o2sol # in umol/kg, result is in umol/kg... which is what we use at the end anyway? t_Kelvin = t + 273.15 SP = gsw.SP_from_C(c, t, p) SA = gsw.SA_from_SP(SP, p, lon, lat) CT = gsw.CT_from_t(SA, t, p) sigma0 = gsw.sigma0(SA, CT) o2sol = gsw.O2sol(SA, CT, p, lon, lat) # umol/kg o2sol_ml_l = oxy_umolkg_to_ml(o2sol, sigma0) # equation expects mL/L (see TODO) # NOTE: lat/lon always required to get o2sol (and need SA/CT for sigma0 anyway) # the above is equivalent to: # pt = gsw.pt0_from_t(SA, t, p) # o2sol = gsw.O2sol_SP_pt(s, pt) oxy_ml_l = (coefs["Soc"] * (volts + coefs["offset"]) * (1.0 + coefs["A"] * t + coefs["B"] * np.power(t, 2) + coefs["C"] * np.power(t, 3)) * o2sol_ml_l * np.exp(coefs["E"] * p / t_Kelvin)) return np.around(oxy_ml_l, 4)
def calc_salinity(self, group): ''' If salinity is not in the list calculate if from TSP ''' print('calculating_salinity') salinity = [] for n in range(len(group['Cond.'])): s = gsw.SP_from_C(group['Cond.'].values[n], group['Temp'].values[n], group['Press'].values[n]) salinity.append(s) group['Salinity'] = salinity return group
def test_L2_params(self): self.contexts = _get_pc_dict('tempwat_l1', 'condwat_l1', 'preswat_l1', 'pracsal', 'density') self.value_classes = {} dom_set = SimpleDomainSet((10, )) # Add the callback for retrieving values for n, p in self.contexts.iteritems(): if hasattr(p, '_pval_callback'): p._pval_callback = self._get_param_vals p._ctxt_callback = self._ctxt_callback self.value_classes[n] = get_value_class(p.param_type, dom_set) # Get the L2 data psval = get_value_class(self.contexts['pracsal'].param_type, dom_set) rhoval = get_value_class(self.contexts['density'].param_type, dom_set) # Perform assertions - involves "manual" calculation of values # Get the L0 data needed for validating output latvals = self._get_param_vals('lat', slice(None)) lonvals = self._get_param_vals('lon', slice(None)) # Get the L1 data needed for validating output t1val = get_value_class(self.contexts['tempwat_l1'].param_type, dom_set) c1val = get_value_class(self.contexts['condwat_l1'].param_type, dom_set) p1val = get_value_class(self.contexts['preswat_l1'].param_type, dom_set) # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # pracsal = gsw.SP_from_C((condwat_l1 * 10), tempwat_l1, preswat_l1) import gsw ps = gsw.SP_from_C((c1val[:] * 10.), t1val[:], p1val[:]) np.testing.assert_allclose(psval[:], ps) # absolute_salinity = gsw.SA_from_SP(pracsal, preswat_l1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, tempwat_l1, preswat_l1) # density = gsw.rho(absolute_salinity, conservative_temperature, preswat_l1) abs_sal = gsw.SA_from_SP(psval[:], p1val[:], lonvals, latvals) cons_temp = gsw.CT_from_t(abs_sal, t1val[:], p1val[:]) rho = gsw.rho(abs_sal, cons_temp, p1val[:]) np.testing.assert_allclose(rhoval[:], rho)
def _compute_data(self,data, units, names, p_ref = 0, baltic = False, lon=0, lat=0, isen = '0'): """ Computes convservative temperature, absolute salinity and potential density from input data, expects a recarray with the following entries data['C']: conductivity in mS/cm, data['T']: in Situ temperature in degree Celsius (ITS-90), data['p']: in situ sea pressure in dbar Arguments: p_ref: Reference pressure for potential density baltic: if True use the Baltic Sea density equation instead of open ocean lon: Longitude of ctd cast default=0 lat: Latitude of ctd cast default=0 Returns: list [cdata,cunits,cnames] with cdata: recarray with entries 'SP', 'SA', 'pot_rho', etc., cunits: dictionary with units, cnames: dictionary with names """ sen = isen + isen # Check for units and convert them if neccessary if(units['C' + isen] == 'S/m'): logger.info('Converting conductivity units from S/m to mS/cm') Cfac = 10 if(('68' in units['T' + isen]) or ('68' in names['T' + isen]) ): logger.info('Converting IPTS-68 to T90') T = gsw.t90_from_t68(data['T' + isen]) else: T = data['T' + isen] SP = gsw.SP_from_C(data['C' + isen], T, data['p']) SA = gsw.SA_from_SP(SP,data['p'],lon = lon, lat = lat) if(baltic == True): SA = gsw.SA_from_SP_Baltic(SA,lon = lon, lat = lat) PT = gsw.pt0_from_t(SA, T, data['p']) CT = gsw.CT_from_t(SA, T, data['p']) pot_rho = gsw.pot_rho_t_exact(SA, T, data['p'], p_ref) names = ['SP' + sen,'SA' + sen,'pot_rho' + sen,'pt0' + sen,'CT' + sen] formats = ['float','float','float','float','float'] cdata = {} cdata['SP' + sen] = SP cdata['SA' + sen] = SA cdata['pot_rho' + sen] = pot_rho cdata['pt' + sen] = PT cdata['CT' + sen] = CT cnames = {'SA' + sen:'Absolute salinity','SP' + sen: 'Practical Salinity on the PSS-78 scale', 'pot_rho' + sen: 'Potential density', 'pt' + sen:'potential temperature with reference sea pressure (p_ref) = 0 dbar', 'CT' + sen:'Conservative Temperature (ITS-90)'} cunits = {'SA' + sen:'g/kg','SP' + sen:'PSU','pot_rho' + sen:'kg/m^3' ,'CT' + sen:'deg C','pt' + sen:'deg C'} return [cdata,cunits,cnames]
def calc_teos10_columns(self, lat, lng): # Practical Salinity SP = gsw.SP_from_C(self.data['Cond'], self.data['Temp'], self.data['Pres']) # Absolute Salinity SA = gsw.SA_from_SP_Baltic(SP, lng, lat) # Conservative Temperature CT = gsw.CT_from_t(SA, self.data['Temp'], self.data['Pres']) # Sigma(density) with reference pressure of 0 dbar sigma = gsw.sigma0(SA, CT) # Depth depth = list(map(abs, gsw.z_from_p(self.data['Pres'], lat))) return {'PracticalSalinity' : SP, 'AbsoluteSalinity' : SA, 'ConservativeTemperature' : CT, 'Sigma(density)' : sigma, 'Depth' : depth}
def main(netCDFfile): ds = Dataset(netCDFfile, 'a') var_temp = ds.variables["TEMP"] var_cndc = ds.variables["CNDC"] var_pres = ds.variables["PRES"] t = var_temp[:] C = var_cndc[:] * 10 p = var_pres[:] psal = gsw.SP_from_C(C, t, p) ncVarOut = ds.createVariable("PSAL", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max ncVarOut[:] = psal ncVarOut.units = "1" ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html" ds.close()
def read_dbds(self, fns, settings): if settings['buoyancy_engine'] == 'shallow': buoyancy_variable = 'm_ballast_pumped' elif settings['buoyancy_engine'] == 'deep': buoyancy_variable = 'm_de_oil_vol' else: raise ValueError( 'Unknown buoyancy engine. Accepted values: (shallow|deep)') dbd = dbdreader.MultiDBD(filenames=fns, complement_files=True) try: tmp = dbd.get_CTD_sync("m_pitch", "m_battpos", buoyancy_variable) except dbdreader.DbdError: tmp = dbd.get_sync("sci_water_cond", "sci_water_temp", "sci_water_pressure", "m_pitch", "m_battpos", buoyancy_variable) try: (_, lat), (_, lon) = dbd.get("m_gps_lat", "m_gps_lon") except dbdreader.DbdError: latitude = float(input("Provide decimal latitude:")) longitude = float(input("Provide decimal longitude:")) else: latitude = np.median(lat) longitude = np.median(lon) condition = np.logical_and(tmp[2] > 0.01, np.isfinite(np.sum(tmp, axis=0))) tctd, C, T, P, pitch, battpos, buoyancy_change = np.compress(condition, tmp, axis=1) SP = gsw.SP_from_C(C * 10, T, P * 10) SA = gsw.SA_from_SP(SP, P * 10, longitude, latitude) density = gsw.rho_t_exact(SA, T, P * 10) #density = fast_gsw.rho(C*10, T, P*10, longitude, latitude) data = dict(time=tctd, pressure=P, pitch=pitch, buoyancy_change=buoyancy_change, battpos=battpos, density=density) self.model.input_data = data mask = np.zeros(tctd.shape, 'int').astype(bool) # use all data by default self.model.set_mask(mask) self.model.OR(P * 10 < settings['minlimitdepth']) self.model.OR(P * 10 > settings['maxlimitdepth'])
def sp_dict(c, t, p): """Wrapper of SP_from_C from gsw library. Take in non-numpy data, format to numpy array, then run through. Goal to eventually deprecate this. Inputs: c: array, Conductivity in mS/cm t: array, in-situ temp in Celcius p: array, sea pressure Output: SP: array, practical salinity (PSS-78) """ c = np.array(c) t = np.array(t) p = np.array(p) SP = gsw.SP_from_C(c, t, p) return SP
def rbr_data_correction(fname): """Remove some strange data from the time series in RBR data. This function is inteded to be run after the dataset was generated. The TEOS10 equations are used here to correct the RBR conductivity readings. """ import warnings warnings.filterwarnings("ignore", category=RuntimeWarning) # open dataset as append mode dataset = nc.Dataset(fname, "a") # extract data into numpy arrays Cw, Tw, Sw, p = (dataset[v][:].filled() for v in ["Cw", "Tw", "Sw", "depth"]) # remove salinity data when the gradient is greater than 0.05 ix = np.append(False, np.abs(np.diff(Sw)) >= 0.1) Sw[ix] = np.nan # compute conductivity from teos Sw_clean = Sw.copy() Sw_clean[np.isnan(Sw)] = np.nanmean(Sw) Cw_from_teos = gsw.C_from_SP(Sw_clean, Tw, p) Sw_from_teos = gsw.SP_from_C(Cw_from_teos, Tw, p) # compute the clean water density rhow = gsw.rho(Sw_clean, Tw, p) # save data into the dataset dataset["rhow"][:] = rhow dataset["Cw"][:] = 0.1 * Cw + 0.9 * Cw_from_teos dataset["Sw"][:] = Sw # dens_rel = np.sqrt(dataset["rhoa"][:] / dataset["rhow"][:]) dataset["wstar"][:] = dens_rel * dataset["ustar"][:] # close dataset dataset.close()
def add_psal(netCDFfile): ds = Dataset(netCDFfile, 'a') var_temp = ds.variables["TEMP"] var_cndc = ds.variables["CNDC"] comment = "" try: var_pres = ds.variables["PRES"] except KeyError: var_pres = ds.variables["NOMINAL_DEPTH"] comment = ", using nominal depth " + str(var_pres[:]) t = var_temp[:] C = var_cndc[:] * 10 p = var_pres[:] psal = gsw.SP_from_C(C, t, p) if "PSAL" in ds.variables: ncVarOut = ds.variables["PSAL"] else: ncVarOut = ds.createVariable("PSAL", "f4", ("TIME",), fill_value=np.nan, zlib=True) # fill_value=nan otherwise defaults to max ncVarOut[:] = psal ncVarOut.units = "1" ncVarOut.standard_name = "sea_water_practical_salinity" ncVarOut.long_name = "sea_water_practical_salinity" ncVarOut.valid_max = np.float32(40) ncVarOut.valid_min = np.float32(-1) ncVarOut.comment = "calculated using gsw-python https://teos-10.github.io/GSW-Python/index.html" + comment # update the history attribute try: hist = ds.history + "\n" except AttributeError: hist = "" ds.setncattr('history', hist + datetime.utcnow().strftime("%Y-%m-%d") + " : added PSAL from TEMP, CNDC, PRES") ds.close()
def createNewFile(filename, station, CTDConfig): infile = open(filename, 'r') lines = infile.readlines() l = lines[0].split(";") options = None if l[2] == "Cond." and len(l) == 13: CTDConfig.conductivity_to_salinity = True CTDConfig.conductivityMissing = False elif l[2] == "Sal." and 'Cond.' not in l: CTDConfig.conductivity_to_salinity = False CTDConfig.conductivityMissing = True elif l[2] == "Sal." and 'Cond.' in l: CTDConfig.conductivity_to_salinity = False CTDConfig.conductivityMissing = False if ('S. vel.' in l and 'Cond.' in l): CTDConfig.conductivityMissing = False CTDConfig.conductivity_to_salinity = False l[-4] = l[-3] l[-3] = l[-2] l[-2] = l[-1] options = ['S. vel.'] if l[8] == "Press": CTDConfig.calculate_depth_from_pressure = True okokyst_metadata.addStationMeadata(station, CTDConfig, options) # Create a new file by adding a header to the existing CTD file. newfilename = filename[0:-4] + "_edited.txt" if os.path.exists(newfilename): os.remove(newfilename) if CTDConfig.debug: print("=> Creating new output file: %s" % newfilename) outfile = open(newfilename, 'a') outfile.writelines(station.header + station.mainHeader) counter = 0 for line in lines[1:]: l = line.split(";") # Calculate salinity from conductivty and replace with conductivity in data array if CTDConfig.conductivity_to_salinity and counter > 0: C = float(l[2]) T = float(l[3]) P = float(l[8]) salinity = gsw.SP_from_C(C, T, P) l[2] = salinity # If only pressure is given, convert to depth in meters if CTDConfig.calculate_depth_from_pressure: P = float(l[8]) l[8] = pressure_to_depth(P, station.latitude) line = ';'.join(map(str, l)) if counter > 0: outfile.writelines(line) counter += 1 outfile.close() infile.close() return newfilename
def data_correction(var_attrs, var_names, metadata_file, metadata_file_info, input_data_file, output_data_file, config_settings): # Declare dictionaries where to store corrections attributes cond_corr_atts = {} salt_corr_atts = {} with netCDF4.Dataset(metadata_file) as nc: # List variables contained in metadata file nc_vars = nc.variables.keys() #load metadata from conductivity and salinity to be corrected from sensor1 cond_01_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][0]) salt_01_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][1]) # in case there exists sensor 2 if metadata_file_info['platform_subtype'] == 'ctd': if len(var_names['var_to_corr']) >= 2: if var_names['var_to_corr'][2] in nc_vars: cond_02_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][2]) salt_02_corr_atts = VariableSettings.import_corrected_variable_attributes(var_attrs, nc, var_names['var_to_corr'][3]) with netCDF4.Dataset(input_data_file) as nc: #load metadata and data from sensor 1 cond_01_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][0]) salt_01_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][1]) cond_01_data = nc.variables[var_names['var_original'][0]][:] temp_01_data = nc.variables[var_names['var_aux'][0]][:] cond_corr_atts['cond_01_corr_atts'] = cond_01_corr_atts salt_corr_atts['salt_01_corr_atts']= salt_01_corr_atts if metadata_file_info['platform_subtype'] == 'glider': prof_dir_data = nc.variables[var_names['var_aux'][2]][:] # in case there exists sensor 2 if metadata_file_info['platform_subtype'] == 'ctd': if len(var_names['var_to_corr']) >= 2: if var_names['var_to_corr'][2] in nc_vars: cond_02_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][2]) salt_02_atts = VariableSettings.import_original_variable_attributes(nc, var_names['var_original'][3]) cond_02_data = nc.variables[var_names['var_original'][2]][:] temp_02_data = nc.variables[var_names['var_aux'][2]][:] cond_corr_atts['cond_02_corr_atts'] = cond_02_corr_atts salt_corr_atts['salt_02_corr_atts'] = salt_02_corr_atts #load data from pressure sensor pres_data = nc.variables[var_names['var_aux'][1]][:] #APPLY CORRECTIONS logging.info('Exporting corrected variables to ' + output_data_file + '\n') #correct conductivity values if config_settings['multi_coeff_A'] == False: if 'correction_coefficient_A' in cond_01_corr_atts: coefficient_A = np.float64(cond_01_corr_atts['correction_coefficient_A']) elif 'CorrectionCoefficient_A' in cond_01_corr_atts: coefficient_A = np.float64(cond_01_corr_atts['CorrectionCoefficient_A']) cond_01_corr_data = cond_01_data * coefficient_A if metadata_file_info['platform_subtype'] == 'glider': cond_01_corr_data = cond_01_corr_data * 10 # convert Sm-1 to mScm-1 cond_01_corr_atts, cond_01_corr_data = VariableSettings.fill_empty_value_with_nan(cond_01_corr_atts, cond_01_corr_data) elif config_settings['multi_coeff_A'] == True: coeff_A_list = config_settings['coeff_A_list'] idx_list = config_settings['idx_list'] cond_01_corr_data = cond_01_data.copy() if config_settings['prof_dir_sections'] == False: range_idx = len(idx_list)-1 elif config_settings['prof_dir_sections'] == True: range_idx = len(idx_list) for i in range(0, range_idx): coefficient_A = coeff_A_list[i] if config_settings['prof_dir_sections'] == False: cond_01_corr_data[idx_list[i]:idx_list[i+1]] = cond_01_data[idx_list[i]:idx_list[i+1]] * coefficient_A elif config_settings['prof_dir_sections'] == True: cond_01_corr_data[prof_dir_data == idx_list[i]] = cond_01_data[prof_dir_data == idx_list[i]] * coefficient_A if metadata_file_info['platform_subtype'] == 'glider': cond_01_corr_data = cond_01_corr_data * 10 # convert Sm-1 to mScm-1 cond_01_corr_atts, cond_01_corr_data = VariableSettings.fill_empty_value_with_nan(cond_01_corr_atts, cond_01_corr_data) #derive corrected salinities using gsw package salt_01_corr_data = gsw.SP_from_C(cond_01_corr_data,temp_01_data,pres_data) if metadata_file_info['platform_subtype'] == 'glider': salt_01_corr_atts, salt_01_corr_data = VariableSettings.fill_empty_value_with_nan(salt_01_corr_atts, salt_01_corr_data) # in case there exists sensor 2 if metadata_file_info['platform_subtype'] == 'ctd': if len(var_names['var_to_corr']) >= 2: if var_names['var_to_corr'][2] in nc_vars: if 'correction_coefficient_B' in cond_02_corr_atts: coefficient_B = cond_02_corr_atts['correction_coefficient_B'] elif 'CorrectionCoefficient_B' in cond_02_corr_atts: coefficient_B = cond_02_corr_atts['CorrectionCoefficient_B'] cond_02_corr_data = cond_02_data * coefficient_B #cond_02_corr_atts, cond_02_corr_data = VariableSettings.fill_empty_value_with_nan(cond_02_corr_atts, cond_02_corr_data) #derive corrected salinities using gsw package salt_02_corr_data = gsw.SP_from_C(cond_02_corr_data,temp_02_data,pres_data) #salt_02_corr_atts, salt_02_corr_data = VariableSettings.fill_empty_value_with_nan(salt_02_corr_atts, salt_02_corr_data) #WRITE CORRECTED VARIABLES #write conductivity and salinity corrected variables corrected L1 file with netCDF4.Dataset(output_data_file, 'r+') as nc: #generate corrected variables to fill data&metadata VariableSettings.create_corrected_variable(metadata_file_info, nc, var_names['var_to_corr'][0],cond_01_corr_data,cond_01_atts,cond_01_corr_atts) VariableSettings.create_corrected_variable(metadata_file_info, nc, var_names['var_to_corr'][1],salt_01_corr_data,salt_01_atts,salt_01_corr_atts) # in case there exists sensor 2 if metadata_file_info['platform_subtype'] == 'ctd': if len(var_names['var_to_corr']) >= 2: if var_names['var_to_corr'][2] in nc_vars: VariableSettings.create_corrected_variable(metadata_file_info,nc, var_names['var_to_corr'][2],cond_02_corr_data,cond_02_atts,cond_02_corr_atts) VariableSettings.create_corrected_variable(metadata_file_info,nc, var_names['var_to_corr'][3],salt_02_corr_data,salt_02_atts,salt_02_corr_atts) # Edit global attributes with netCDF4.Dataset(output_data_file, 'r+') as nc: nc.setncattr('data_mode', 'DM') nc.setncattr('processing_level', 'L1_corr processed data with salinity and conductivity corrections') nc.setncattr('history', 'Product generated by the glider toolbox version 1.3.0 (https://github.com/socib/glider_toolbox). Salinity and conductivity corrections generated by the SOCIB Salinity Correction Toolbox version 0.1.0') return cond_corr_atts, salt_corr_atts
def derive(self,property): """ Derives seawater properties as salinity, buoyancy frequency squared etc. Args: ST: N2: """ try: gsw except: logger.warning('GSW toolbox missing, derive will not work, doing nothing') return if(property == 'ST'): # Poor mans check if the variables exist sensor_pair0 = False sensor_pair1 = False try: tmp1 = self.data['cond0'] tmp1 = self.data['temp0'] tmp1 = self.data['p'] sensor_pair0 = True except: sensor_pair0 = False try: tmp1 = self.data['cond1'] tmp1 = self.data['temp1'] tmp1 = self.data['p'] sensor_pair1 = True except: sensor_pair1 = False if(sensor_pair0): logger.debug('Calculating PSU0/SA11/CT11/rho11') SP = gsw.SP_from_C(self.data['cond0'],self.data['temp0'],self.data['p']) self.derived['SP00'] = SP SA = gsw.SA_from_SP(SP,self.data['p'],self.header['lon'],self.header['lat']) self.derived['SA00'] = SA CT = gsw.CT_from_t(SA,self.data['temp0'],self.data['p']) self.derived['CT00'] = CT rho = gsw.rho_CT_exact(SA,CT,self.data['p']) self.derived['rho00'] = rho if(sensor_pair1): logger.debug('Calculating PSU1/SA11/CT11/rho11') SP = gsw.SP_from_C(self.data['cond1'],self.data['temp1'],self.data['p']) self.derived['SP11'] = SP SA = gsw.SA_from_SP(SP,self.data['p'],self.header['lon'],self.header['lat']) self.derived['SA11'] = SA CT = gsw.CT_from_t(SA,self.data['temp1'],self.data['p']) self.derived['CT11'] = CT rho = gsw.rho_CT_exact(SA,CT,self.data['p']) self.derived['rho11'] = rho if(property == 'N2'): # Poor mans check if the variables exist sensor_pair0 = False sensor_pair1 = False try: tmp1 = self.derived['SA00'] tmp1 = self.derived['CT00'] tmp1 = self.data['p'] sensor_pair0 = True except: logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")') sensor_pair0 = False try: tmp1 = self.derived['SA11'] tmp1 = self.derived['CT11'] tmp1 = self.data['p'] sensor_pair1 = True except: logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")') sensor_pair1 = False if(sensor_pair0): logger.debug('Calculating Nsquared00') [N2,p_mid] = gsw.Nsquared(self.derived['SA00'],self.derived['CT00'],self.data['p']) self.derived['Nsquared00'] = interp(self.data['p'],p_mid,N2) if(sensor_pair1): logger.debug('Calculating Nsquared11') [N2,p_mid] = gsw.Nsquared(self.derived['SA11'],self.derived['CT11'],self.data['p']) self.derived['Nsquared11'] = interp(self.data['p'],p_mid,N2) if(property == 'alphabeta'): # Poor mans check if the variables exist sensor_pair0 = False sensor_pair1 = False try: tmp1 = self.derived['SA00'] tmp1 = self.derived['CT00'] tmp1 = self.data['p'] sensor_pair0 = True except: logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")') sensor_pair0 = False try: tmp1 = self.derived['SA11'] tmp1 = self.derived['CT11'] tmp1 = self.data['p'] sensor_pair1 = True except: logger.info('Did not find absolute salinities and temperature, do first a .derive("ST")') sensor_pair1 = False if(sensor_pair0): logger.debug('Calculating Nsquared00') alpha = gsw.alpha(self.derived['SA00'],self.derived['CT00'],self.data['p']) beta = gsw.beta(self.derived['SA00'],self.derived['CT00'],self.data['p']) self.derived['alpha00'] = alpha self.derived['beta00'] = beta if(sensor_pair1): logger.debug('Calculating Nsquared11') alpha = gsw.alpha(self.derived['SA11'],self.derived['CT11'],self.data['p']) beta = gsw.beta(self.derived['SA11'],self.derived['CT11'],self.data['p']) self.derived['alpha11'] = alpha self.derived['beta11'] = beta
# pres = data['P_slow'][5000:15000].reshape(500,20).mean(1) cond = data['JAC_C'][141000:155000].reshape(700, 20).mean(1) temp = data['JAC_T'][141000:155000].reshape(700, 20).mean(1) pres = data['P_slow'][141000:155000].reshape(700, 20).mean(1) plt.plot(data['P_slow']) plt.show() # cond = data['JAC_C'][5000:20000] # temp = data['JAC_T'][5000:20000] # pres = data['P_slow'][5000:20000] # salt = seawater.salt(cond,temp,pres) # # plt.plot(salt) # # plt.show() sal = gsw.SP_from_C(cond, temp, pres) # # plt.plot(cond) # # plt.show() # # plt.plot(temp) # # plt.show() # plt.plot(pres) # plt.show() # # plt.plot(sal) # # plt.show() # plt.plot(sal) # plt.show() CT = gsw.CT_from_t(sal, temp, pres) SA = gsw.SA_from_SP(sal, pres, 174, -43) dens = gsw.sigma0(SA, CT) depth = gsw.z_from_p(pres, -43)
def main(argv): parser = argparse.ArgumentParser( description= 'Convert SBE raw data to a converted, csv-formatted text file') parser.add_argument('timeFile', metavar='time_file', help='the .csv data file to fit by bottle data') # debug messages parser.add_argument('-d', '--debug', action='store_true', help='display debug messages') # raw output parser.add_argument('-pres', '--pressure', action='store_true', help='Fit pressure data') # raw output parser.add_argument('-temp', '--temperature', action='store_true', help='Fit temperture data') # raw output parser.add_argument('-cond', '--conductivity', action='store_true', help='Fit conductivity data') # raw output parser.add_argument('-oxy', '--oxygen', type=argparse.FileType('r'), help='return the oxygen data file') # Process Command-line args args = parser.parse_args() if args.debug: global DEBUG DEBUG = True debugPrint("Running in debug mode") # Verify hex file exists if not os.path.isfile(args.timeFile): errPrint('ERROR: Input time dependent .csv file:', args.timeFile, 'not found\n') sys.exit(1) # Used later for building output file names filename_ext = os.path.basename( args.timeFile) # original filename with ext filename_base = os.path.splitext(filename_ext)[ 0] # original filename w/o ext if '_' in filename_base: filename_base = filename_base.split('_')[0] #Import Cruise Configuration File iniFile = 'data/ini-files/configuration.ini' config = configparser.RawConfigParser() config.read(iniFile) #Initialise Configuration Parameters expocode = config['cruise']['expocode'] sectionID = config['cruise']['sectionid'] raw_directory = config['ctd_processing']['raw_data_directory'] time_directory = config['ctd_processing']['time_data_directory'] pressure_directory = config['ctd_processing']['pressure_data_directory'] oxygen_directory = config['ctd_processing']['oxygen_directory'] btl_directory = config['ctd_processing']['bottle_directory'] o2flask_file = config['ctd_processing']['o2flask_file'] log_directory = config['ctd_processing']['log_directory'] sample_rate = config['ctd_processing']['sample_rate'] search_time = config['ctd_processing']['roll_filter_time'] ctd = config['ctd_processing']['ctd_serial'] time_zone = config['inputs']['time_zone'] p_col = config['analytical_inputs']['p'] p_btl_col = config['inputs']['p'] t_col = config['analytical_inputs']['t'] t_btl_col = config['inputs']['t'] t1_col = config['analytical_inputs']['t1'] t1_btl_col = config['inputs']['t1'] t2_col = config['analytical_inputs']['t2'] t2_btl_col = config['inputs']['t2'] c_col = config['analytical_inputs']['c'] c1_col = config['analytical_inputs']['c1'] c1_btl_col = config['inputs']['c1'] c2_col = config['analytical_inputs']['c2'] c2_btl_col = config['inputs']['c2'] sal_col = config['analytical_inputs']['salt'] sal_btl_col = config['inputs']['salt'] btl_sal_col = config['analytical_inputs']['btl_salt'] dov_col = config['analytical_inputs']['dov'] dov_btl_col = config['inputs']['dov'] dopl_col = config['analytical_inputs']['dopl'] dopl_btl_col = config['inputs']['dopl'] dopkg_col = config['analytical_inputs']['dopkg'] btl_oxy_col = config['analytical_inputs']['btl_oxy'] xmis_col = config['analytical_inputs']['xmis'] fluor_col = config['analytical_inputs']['fluor'] timedate = config['analytical_inputs']['datetime'] lat_col = config['analytical_inputs']['lat'] lat_btl_col = config['inputs']['lat'] lon_col = config['analytical_inputs']['lon'] lon_btl_col = config['inputs']['lon'] reft_col = config['inputs']['reft'] btl_num_col = config['inputs']['btl_num'] #time_column_data = config['time_series_output']['data_names'].split(',') time_column_data = config['time_series_output']['data_output'] time_column_names = config['time_series_output']['column_name'].split(',') time_column_units = config['time_series_output']['column_units'].split(',') time_column_format = config['time_series_output']['format'] #pressure_column_data = config['time_series_output']['data_names'].split(',') p_column_data = config['pressure_series_output']['data'].split(',') p_column_names = config['pressure_series_output']['column_name'].split(',') p_column_units = config['pressure_series_output']['column_units'].split( ',') p_column_format = config['pressure_series_output']['format'] p_column_qual = config['pressure_series_output']['qual_columns'].split(',') p_column_one = list( config['pressure_series_output']['q1_columns'].split(',')) #bottle_data outputs btl_dtype = config['bottle_series_output']['dtype'] hexfileName = str(filename_base + '.' + HEX_EXT) hexfilePath = os.path.join(raw_directory, hexfileName) xmlfileName = str(filename_base + '.' + XML_EXT) xmlfilePath = os.path.join(raw_directory, xmlfileName) outtimefileName = str(filename_base + TIME_SUFFIX + '.' + FILE_EXT) outtimefilePath = os.path.join(time_directory, outtimefileName) pressfileName = str(filename_base + FIT_SUFFIX + '.' + FILE_EXT) pressfilePath = os.path.join(pressure_directory, pressfileName) btlfileName = str(filename_base + BTL_SUFFIX + MEAN_SUFFIX + '.' + FILE_EXT) btlfilePath = os.path.join(btl_directory, btlfileName) # Get bottle data btl_data = process_ctd.dataToNDarray(btlfilePath, float, True, ',', 0) btl_data = btl_data[:][1:] # Get procesed time data time_data = process_ctd.dataToNDarray(args.timeFile, float, True, ',', 1) btm = np.argmax(time_data[p_col][1:]) time_data = time_data[:][1:btm] if args.pressure: print('In -pres flag fit condition') print(filename_base) pfileName = str('poffset' + '.' + FILE_EXT) pfilePath = os.path.join(log_directory, pfileName) poff_data = process_ctd.dataToNDarray(pfilePath, str, None, ',', None) for line in poff_data: if filename_base in line[0]: for val in line: if 'offset' in val: offset = float(str.split(val, ':')[1]) continue # Pressure offset btl_data[p_btl_col] = fit_ctd.offset(offset, btl_data[p_btl_col]) time_data[p_col] = fit_ctd.offset(offset, time_data[p_col]) # End pressure if condition if args.temperature: print('In -temp flag fit condition') print(filename_base) coef1 = [] coef2 = [] # Get descrete ref temp data t1fileName = str('fitting_t1' + '.' + FILE_EXT) t1filePath = os.path.join(log_directory, t1fileName) t1_coef = process_ctd.dataToNDarray(t1filePath, str, None, ',', None) for line in t1_coef: if filename_base in line[0]: val = line[1:] for i in range(0, len(val)): if 'coef' in val[i]: coef1.append(float(str.split(val[i], ':')[1])) else: coef1.append(float(val[i])) continue btl_data[t1_btl_col] = fit_ctd.temperature_polyfit( coef1, btl_data[p_btl_col], btl_data[t1_btl_col], 'all') time_data[t1_col] = fit_ctd.temperature_polyfit( coef1, time_data[p_col], time_data[t1_col], 'all') t2fileName = str('fitting_t2' + '.' + FILE_EXT) t2filePath = os.path.join(log_directory, t2fileName) t2_coef = process_ctd.dataToNDarray(t2filePath, str, None, ',', None) for line in t2_coef: if filename_base in line[0]: val = line[1:] for i in range(0, len(val)): if 'coef' in val[i]: coef2.append(float(str.split(val[i], ':')[1])) else: coef2.append(float(val[i])) continue btl_data[t2_btl_col] = fit_ctd.temperature_polyfit( coef2, btl_data[p_btl_col], btl_data[t2_btl_col], 'all') time_data[t2_col] = fit_ctd.temperature_polyfit( coef2, time_data[p_col], time_data[t2_col], 'all') if args.conductivity: print('In -cond flag fit condition') print(filename_base) coef1 = [] coef2 = [] # Get descrete ref temp data c1fileName = str('fitting_c1' + '.' + FILE_EXT) c1filePath = os.path.join(log_directory, c1fileName) c1_coef = process_ctd.dataToNDarray(c1filePath, str, None, ',', None) for line in c1_coef: if filename_base in line[0]: val = line[1:] for i in range(0, len(val)): if 'coef' in val[i]: coef1.append(float(str.split(val[i], ':')[1])) else: coef1.append(float(val[i])) continue btl_data[c1_btl_col] = fit_ctd.conductivity_polyfit( coef1, btl_data[p_btl_col], btl_data[t1_btl_col], btl_data[c1_btl_col]) time_data[c1_col] = fit_ctd.conductivity_polyfit( coef1, time_data[p_col], time_data[c1_col], time_data[c1_col]) c2fileName = str('fitting_c2' + '.' + FILE_EXT) c2filePath = os.path.join(log_directory, c2fileName) c2_coef = process_ctd.dataToNDarray(c2filePath, str, None, ',', None) for line in c2_coef: if filename_base in line[0]: val = line[1:] for i in range(0, len(val)): if 'coef' in val[i]: coef2.append(float(str.split(val[i], ':')[1])) else: coef2.append(float(val[i])) continue btl_data[c2_btl_col] = fit_ctd.conductivity_polyfit( coef2, btl_data[p_btl_col], btl_data[t2_btl_col], btl_data[c2_btl_col]) time_data[c2_col] = fit_ctd.conductivity_polyfit( coef2, time_data[p_col], time_data[c2_col], time_data[c2_col]) time_data[sal_col] = gsw.SP_from_C(time_data[c_col], time_data[t_col], time_data[p_col]) if args.oxygen: print('In -oxy flag fit condition') print(filename_base) # Get Analytical Oxygen data o2pkg_btl, o2pl_btl = fit_ctd.o2_calc(o2flask_file, args.oxygen.name, btl_data[btl_num_col], btl_data[sal_btl_col]) kelvin = [] for i in range(0, len(time_data[t_col])): kelvin.append(time_data[t_col][i] + 273.15) # Find New Oxygen Coef oxy_coef = fit_ctd.find_oxy_coef(o2pl_btl['OXYGEN'], btl_data[p_btl_col], btl_data[t_btl_col], btl_data[sal_btl_col], btl_data[dov_btl_col], hexfilePath, xmlfilePath) # Convert CTD Oxygen Voltage Data with New DO Coef time_data[dopl_col] = fit_ctd.oxy_dict(oxy_coef, time_data[p_col], kelvin, time_data[t_col], time_data[sal_col], time_data[dov_col]) # End oxygen flag fitting if condition # Find Isopycnal Down Trace Bottle Trip Equivalent # Write bottle data to file report_ctd.report_btl_data(btlfilePath, btl_dtype, btl_data) # Write time data to file report_ctd.report_time_series_data(filename_base, time_directory, expocode, time_column_names, time_column_units, time_column_names, time_column_format, time_data) # Pressure Sequence pressure_seq_data = process_ctd.pressure_sequence(filename_base, p_col, timedate, 2.0, -1.0, 0.0, 'down', int(sample_rate), int(search_time), time_data) # Convert dissolved oxygen from ml/l to umol/kg dopkg = process_ctd.o2pl2pkg(p_col, t1_col, sal_col, dopl_col, dopkg_col, lat_col, lon_col, pressure_seq_data) # Add quality codes to data qual_pseq_data = process_ctd.ctd_quality_codes(dopkg_col, None, None, True, p_column_qual, p_column_one, pressure_seq_data) # Collect Cast Details from Log logfileName = str('cast_details' + '.' + FILE_EXT) logfilePath = os.path.join(log_directory, logfileName) cast_details = process_ctd.dataToNDarray(logfilePath, str, None, ',', 0) for line in cast_details: if filename_base in line[0]: for val in line: if 'at_depth' in val: btime = float(str.split(val, ':')[1]) if 'latitude' in val: btm_lat = float(str.split(val, ':')[1]) if 'longitude' in val: btm_lon = float(str.split(val, ':')[1]) if 'altimeter_bottom' in val: btm_alt = float(str.split(val, ':')[1]) break # Write time data to file depth = -999 report_ctd.report_pressure_series_data(filename_base, expocode, sectionID, btime, btm_lat, btm_lon, depth, btm_alt, ctd, pressure_directory, p_column_names, p_column_units, p_column_data, qual_pseq_data, dopkg, pressure_seq_data) #plt.plot(o2pl_btl['OXYGEN'], btl_data[p_btl_col], color='b', marker='o') #plt.plot(tmpo2, time_data[p_col], color='g', label='raw') #plt.plot(time_data[dopl_col], time_data[p_col], color='b', label='raw') #plt.plot(pressure_seq_data[dopl_col], pressure_seq_data[p_col], color='r', label='raw') #plt.plot(pressure_seq_data[t2_col], pressure_seq_data[p_col], color='r', label='raw') #plt.plot(btl_data[t_btl_col], time_data[p_col], color='r', label='raw') #plt.plot(o2pl_btl['OXYGEN'], btl_data[p_btl_col], color='g', marker='o', label='raw') #plt.plot(pressure_seq_data[do_col], pressure_seq_data[p_col], color='b', label='raw') #plt.gca().invert_yaxis() #plt.axis() #plt.show() debugPrint('Done!')
def ctdproc(lista, temp_name='t068C', lathint='Latitude =', lonhint='Longitude =', cond_name='c0S/m', press_name='prDM', down_cast=True, looped=True, hann_f=False, hann_block=20, hann_times=2, latline=[], lonline=[]): ''' This function do the basic proccess to all .cnv CTD data from given list. ''' for fname in lista: lon, lat, data = ctdread(fname, press_name=press_name, down_cast=down_cast, lathint=lathint, lonhint=lonhint, lonline=lonline, latline=latline) if looped: data = loopedit(data) dataname = basename(fname)[1] if (data.shape[0] < 101) & ( data.shape[0] > 10): # se o tamanho do perfil for com menos de 101 medidas if (data.shape[0] / 2) % 2 == 0: # caso a metade dos dados seja par blk = (data.shape[0] / 2) + 1 # bloco = a metade +1 else: blk = data.shape[0] / 2 # se for impar o bloco e a metade # remove spikes dos perfis de temperatura e condutividade data = despike(data, propname=temp_name, block=blk, wgth=2) data = despike(data, propname=cond_name, block=blk, wgth=2) elif data.shape[0] >= 101: # para perfis com mais de 101 medidas, utiliza-se blocos de 101 data = despike(data, propname=temp_name, block=101, wgth=2) data = despike(data, propname=cond_name, block=101, wgth=2) else: print('radial muito rasa') # realiza média em caixa de 1 metro data = binning(data, delta=1.) if temp_name == 't068C': data['t090C'] = gsw.t90_from_t68(data['t068C']) data['sp'] = gsw.SP_from_C(data[cond_name] * 10, data['t090C'], data.index.values) if hann_f: times = 0 while times < hann_times: data = hann_filter(data, 't090C', hann_block) data = hann_filter(data, 'sp', hann_block) times += 1 data['pt'] = sw.ptmp(data['sp'], data['t090C'], data.index.values) #data['ct'] = gsw.CT_from_pt(data['sa'],data['pt']) data['psigma0'] = sw.pden( data['sp'], data['t090C'], data.index.values, pr=0) - 1000 data['psigma1'] = sw.pden( data['sp'], data['t090C'], data.index.values, pr=1000) - 1000 data['psigma2'] = sw.pden( data['sp'], data['t090C'], data.index.values, pr=2000) - 1000 data['gpan'] = sw.gpan(data['sp'], data['t090C'], data.index.values) data['lat'] = lat data['lon'] = lon data.to_pickle( os.path.split(fname)[0] + '/' + os.path.splitext(os.path.split(fname)[1])[0]) print(dataname)
def read_nos(buoy, dstart, dend, usemodel=False): '''Set up urls and then read from them to get TCOON and NOS data. Most stations have several data sources, so they are aggregated here. This calls to read_nos_df() to do the reading and rearranging. dstart and dend are datetime objects. ''' if not usemodel: # tide, met, and phys data prefixes = [ 'https://tidesandcurrents.noaa.gov/api/datagetter?product=water_level&application=NOS.COOPS.TAC.WL&station=', 'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=air_pressure&application=NOS.COOPS.TAC.MET&station=', 'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=air_temperature&application=NOS.COOPS.TAC.MET&station=', 'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=wind&application=NOS.COOPS.TAC.MET&station=', 'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=water_temperature&application=NOS.COOPS.TAC.PHYSOCEAN&station=', 'https://api.tidesandcurrents.noaa.gov/api/prod/datagetter?product=conductivity&application=NOS.COOPS.TAC.PHYSOCEAN&station=' ] # dstart and dend need to be in format YYYYMMDD dstarts = dstart.strftime('%Y%m%d') dends = dend.strftime('%Y%m%d') suffixes = [ '&begin_date=' + dstarts + '&end_date=' + dends + '&datum=MSL&units=metric&time_zone=GMT&format=csv', '&begin_date=' + dstarts + '&end_date=' + dends + '&time_zone=GMT&units=english&interval=6&format=csv', '&begin_date=' + dstarts + '&end_date=' + dends + '&time_zone=GMT&units=english&interval=6&format=csv', '&begin_date=' + dstarts + '&end_date=' + dends + '&time_zone=GMT&units=english&interval=6&format=csv', '&begin_date=' + dstarts + '&end_date=' + dends + '&time_zone=GMT&units=english&interval=6&format=csv', '&begin_date=' + dstarts + '&end_date=' + dends + '&time_zone=GMT&units=english&interval=6&format=csv' ] dfs = [] for prefix, suffix in zip(prefixes, suffixes): url = prefix + buoy + suffix dft = read_nos_df(url) if dft is not None: dft = dft[~dft.index.duplicated( keep='first')] # remove any duplicated indices dfs.append(dft) # combine the dataframes together # don't append if all df are empty if [df.empty for df in dfs].count(True) != len(dfs): df = pd.concat([df for df in dfs if not df.empty], axis=1, sort=False) else: df = pd.concat([df for df in dfs], axis=1, sort=False) # calculate salinity from conductivity, if available if 'Conductivity [mS/cm]' in df.keys(): if not 'AtmPr [mb]' in df.keys(): pr = np.zeros(len(df)) else: pr = df['AtmPr [mb]'] / 100. - 10.1325 df['Salinity'] = gsw.SP_from_C(df['Conductivity [mS/cm]'], df['WaterT [deg C]'], pr) # dictionary for rounding decimal places rdict = {'Salinity': 2} df = df.round(rdict) else: # use model prefix = 'https://tidesandcurrents.noaa.gov/api/datagetter?product=predictions&application=NOS.COOPS.TAC.WL&station=' dstarts = dstart.strftime('%Y%m%d') dends = dend.strftime('%Y%m%d') suffix = '&begin_date=' + dstarts + '&end_date=' + dends + '&datum=MSL&time_zone=GMT&units=metric&interval=h&format=csv' url = prefix + buoy + suffix df = read_nos_df(url) return df
ctd_data.append(df) ctd_data = pd.concat(ctd_data, axis=0, sort=False) # load bottle trip file file_list = sorted(glob.glob("../data/bottle/*.pkl")) ssscc_list = [ssscc.strip("../data/bottle/")[:5] for ssscc in file_list] upcast_data = [] for f in file_list: with open(f, "rb") as x: df = pickle.load(x) df["SSSCC"] = f.strip("../data/bottle/")[:5] # change to secondary if that is what's used upcast_data.append(df[["SSSCC", "CTDCOND1", "CTDTMP1", "CTDPRS"]]) upcast_data = pd.concat(upcast_data, axis=0, sort=False) upcast_data["CTDSAL"] = gsw.SP_from_C( upcast_data["CTDCOND1"], upcast_data["CTDTMP1"], upcast_data["CTDPRS"] ) # load salt file (adapted from compare_salinities.ipynb) file_list = sorted(glob.glob("../data/salt/*.csv")) ssscc_list = [ssscc.strip("../data/salt/")[:5] for ssscc in file_list] salt_data = [] for f in file_list: df = pd.read_csv(f, usecols=["STNNBR", "CASTNO", "SAMPNO", "SALNTY"]) df["SSSCC"] = f.strip("../data/salt/")[:5] salt_data.append(df) salt_data = pd.concat(salt_data, axis=0, sort=False) salt_data["SALNTY"] = salt_data["SALNTY"].round(4) if "SALNTY_FLAG_W" not in salt_data.columns: salt_data["SALNTY_FLAG_W"] = 2