def get_hydro(my_file): f = Dataset(my_file, mode='r') eps = f.variables['EPSILON'][:] eps = remove_bad_eps(eps) lat = f.variables['LATITUDE'][:] lon = f.variables['LONGITUDE'][:] p = f.variables['PRESSURE'][:] SP = f.variables['PSAL'][:] T = f.variables['TEMPERATURE'][:] z = gsw.z_from_p(p, lat) # m SA = gsw.SA_from_SP(SP, p, lon, lat) # g/kg, absolute salinity CT = gsw.CT_from_t(SA, T, p) # C, conservative temperature SA = remove_bad_SA(SA) CT = remove_bad_CT(CT) [N2_mid, p_mid] = gsw.Nsquared(SA, CT, p, lat) z_mid = gsw.z_from_p(p_mid, lat) N2 = interp_to_edges(N2_mid, z, z_mid, 4) N2 = np.append(np.append(N2, [np.nan]), [np.nan]) N2 = remove_bad_N2(N2) return N2, SA, CT, eps, z
def test_xarray_with_coords(): pytest.importorskip('dask') SA_chunk = SA.chunk(chunks={'y': 1, 't': 1}) CT_chunk = CT.chunk(chunks={'y': 1, 't': 1}) lat_chunk = lat.chunk(chunks={'y': 1}) # Dimensions and cordinates match: expected = gsw.sigma0(SA_vals, CT_vals) xarray = gsw.sigma0(SA, CT) chunked = gsw.sigma0(SA_chunk, CT_chunk) assert_allclose(xarray, expected) assert_allclose(chunked, expected) # Broadcasting along dimension required (dimensions known) expected = gsw.alpha(SA_vals, CT_vals, p_vals[np.newaxis, :, np.newaxis]) xarray = gsw.alpha(SA, CT, p) chunked = gsw.alpha(SA_chunk, CT_chunk, p) assert_allclose(xarray, expected) assert_allclose(chunked, expected) # Broadcasting along dimension required (dimensions unknown/exclusive) expected = gsw.z_from_p(p_vals[:, np.newaxis], lat_vals[np.newaxis, :]) xarray = gsw.z_from_p(p, lat) chunked = gsw.z_from_p(p, lat_chunk) assert_allclose(xarray, expected) assert_allclose(chunked, expected)
def _load_from_pts(self, pressure, temperature, salinity, lon, lat, name): self._pts = True # if all([ isinstance(i, float) for i in [pressure, temperature, salinity, lon, lat] ]): #pressure = np.array([-1, 1e4]) #temperature = np.array([temperature, temperature]) #salinity = np.array([salinity, salinity]) #lon = np.array([lon,lon]) #lat = np.array([lat,lat]) pressure = [-1, 1e4] temperature = [temperature, temperature] salinity = [salinity, salinity] lon = [lon, lon] lat = [lat, lat] # self.lon, self.lat = lon, lat # self.p = pressure self.z = gsw.z_from_p(self.p, self.lat) # self.temp, self.s = temperature, salinity # derive absolute salinity and conservative temperature self.SA = gsw.SA_from_SP(self.s, self.p, self.lon, self.lat) self.CT = gsw.CT_from_t(self.SA, self.temp, self.p) # isopycnal displacement and velocity self.eta = 0. self.detadt = 0. # if name is None: self.name = 'Provided water profile at lon=%.0f, lat=%.0f' % ( self.lon, self.lat)
def make_depth_log(time_df, threshold=80): # TODO: get column names from config file """ Create depth log file from maximum depth of each station/cast in time DataFrame. If rosette does not get within the threshold distance of the bottom, returns NaN. Parameters ---------- time_df : DataFrame DataFrame containing continuous CTD data threshold : int, optional Maximum altimeter reading to consider cast "at the bottom" (defaults to 80) """ # TODO: make inputs be arraylike rather than dataframe df = time_df[["SSSCC", "CTDPRS", "GPSLAT", "ALT"]].copy().reset_index() df_group = df.groupby("SSSCC", sort=False) idx_p_max = df_group["CTDPRS"].idxmax() bottom_df = pd.DataFrame( data={ "SSSCC": df["SSSCC"].unique(), "max_p": df.loc[idx_p_max, "CTDPRS"], "lat": df.loc[idx_p_max, "GPSLAT"], "alt": df.loc[idx_p_max, "ALT"], }) bottom_df.loc[bottom_df["alt"] > threshold, "alt"] = np.nan # pandas 1.2.1 ufunc issue workaround with pd.to_numpy() bottom_df["DEPTH"] = ((bottom_df["alt"] + np.abs( gsw.z_from_p(bottom_df["max_p"], bottom_df["lat"].to_numpy()))).fillna( value=-999).round().astype(int)) bottom_df[["SSSCC", "DEPTH"]].to_csv(cfg.directory["logs"] + "depth_log.csv", index=False) return True
def get_glider_depth(ds): good = np.where(~np.isnan(ds.pressure))[0] ds['depth'] = ds.pressure * 0. ds['depth'].values = -gsw.z_from_p(ds.pressure.values, ds.latitude.values) # now we really want to know where it is, so interpolate: if len(good) > 0: ds['depth'].values = np.interp(np.arange(len(ds.depth)), good, ds['depth'].values[good]) attr = { 'source': 'pressure', 'long_name': 'glider depth', 'standard_name': 'depth', 'units': 'm', 'comment': 'from science pressure and interpolated', 'observation_type': 'calulated', 'accuracy': '1', 'precision': '2', 'resolution': '0.02', 'valid_min': '0', 'valid_max': '2000', 'reference_datum': 'surface', 'positive': 'down' } ds['depth'].attrs = attr return ds
def _load_from_pts(self, pressure, temperature, salinity, lon, lat, name): self._pts = True # if all([ isinstance(i, float) for i in [pressure, temperature, salinity, lon, lat] ]): #pressure = np.array([-1, 1e4]) #temperature = np.array([temperature, temperature]) #salinity = np.array([salinity, salinity]) #lon = np.array([lon,lon]) #lat = np.array([lat,lat]) pressure = [-1, 1e4] temperature = [temperature, temperature] salinity = [salinity, salinity] lon = [lon, lon] lat = [lat, lat] # self.lon, self.lat = lon, lat # self.p = pressure self.z = gsw.z_from_p(self.p, self.lat) # self.temp, self.s = temperature, salinity # self._update_eos() # if name is None: self.name = 'Provided water profile at lon=%.0f, lat=%.0f' % ( self.lon, self.lat)
def add_ctd_params(df_in: MutableMapping[str, Sequence], cfg: Mapping[str, Any], lon=16.7, lat=55.2): """ Calculate all parameters from 'sigma0', 'depth', 'soundV', 'SA' that is specified in cfg['out']['data_columns'] :param df_in: DataFrame with columns: 'Pres', 'Temp90' or 'Temp', and may be others: 'Lat', 'Lon': to use instead cfg['in']['lat'] and lat and -//- lon :param cfg: dict with fields: ['out']['data_columns'] - list of columns in output dataframe ['in'].['b_temp_on_its90'] - optional :param lon: # 54.8707 # least priority values :param lon: # 19.3212 :return: DataFrame with only columns specified in cfg['out']['data_columns'] """ ctd = df_in params_to_calc = set(cfg['out']['data_columns']).difference(ctd.columns) params_coord_needed_for = params_to_calc.intersection( ('depth', 'sigma0', 'SA', 'soundV')) # need for all cols? if any(params_coord_needed_for): # todo: load from nav: if np.any(ctd.get('Lat')): lat = ctd['Lat'] lon = ctd['Lon'] else: if 'lat' in cfg['in']: lat = cfg['in']['lat'] lon = cfg['in']['lon'] else: print( 'Calc', '/'.join(params_coord_needed_for), f'using MANUAL INPUTTED coordinates: lat={lat}, lon={lon}') if 'Temp90' not in ctd.columns: if cfg['in'].get('b_temp_on_its90'): ctd['Temp90'] = ctd['Temp'] else: ctd['Temp90'] = gsw.conversions.t90_from_t68(df_in['Temp']) ctd['SA'] = gsw.SA_from_SP( ctd['Sal'], ctd['Pres'], lat=lat, lon=lon) # or Sstar_from_SP() for Baltic where SA=S* # Val['Sal'] = gsw.SP_from_C(Val['Cond'], Val['Temp'], Val['P']) if 'soundV' in params_to_calc: ctd['soundV'] = gsw.sound_speed_t_exact(ctd['SA'], ctd['Temp90'], ctd['Pres']) if 'depth' in params_to_calc: ctd['depth'] = np.abs(gsw.z_from_p(np.abs(ctd['Pres']), lat)) if 'sigma0' in params_to_calc: CT = gsw.CT_from_t(ctd['SA'], ctd['Temp90'], ctd['Pres']) ctd['sigma0'] = gsw.sigma0(ctd['SA'], CT) # ctd = pd.DataFrame(ctd, columns=cfg['out']['data_columns'], index=df_in.index) if 'Lat' in params_to_calc and not 'Lat' in ctd.columns: ctd['Lat'] = lat ctd['Lon'] = lon return ctd[cfg['out']['data_columns']]
def isopycnal_displacements(rho, rho_ref, p, lat, diff_window=400, axis=0): """ Compute vertical isopycnal displacements (eta) as (rho - rho_ref)/(drho_ref/dz) with drho_ref/dz computed over a vertical window (diff_window) Parameters ---------- rho: density (nd-array) rho_ref: reference density (see adiabatic leveling for computing ref rho) p: pressure array diff_window: vertical window for computing drho_ref/dz axis: axis of vertical direction Returns ------- eta: vertical isopycnal displacements """ win = diff_window flip = False # makes the vertical be along dimension 0 if not already if axis == 1: rho = rho.T rho_ref = rho_ref.T p = p.T flip = True elif axis > 1: raise ValueError('Only handles 2-D Arrays') z = -1 * gsw.z_from_p(p[:, 0], lat[:, 0]) dz = np.nanmean(np.diff(z)) step = int(np.floor(.5 * win / dz)) eta = np.full_like(rho, np.nan) for i in range(rho.shape[0]): # If in the TOP half of the profile the window needs to be adjusted if i - step < 0: lower = 0 upper = int(2 * step) # If in the BOTTOM half of the profile the window needs to be adjusted elif i + step > (rho.shape[0] - 1): lower = int(rho.shape[0] - 2 * step) upper = -1 else: upper = i + step lower = i - step drefdz = (rho_ref[upper] - rho_ref[lower]) / win eta[i, :] = (rho[i, :] - rho_ref[i]) / drefdz return eta
def derive_csv(data): density = gsw.z_from_p(cast[6].values,lat) absolute_sal=gsw.SA_from_SP(cast[10].values,density,lon,lat) reference_sal=gsw.SR_from_SP(absolute_sal) conservative_temp=gsw.CT_from_t(absolute_sal,cast[8],density) potential_density=gsw.sigma0(absolute_sal,conservative_temp) return density,absolute_sal,reference_sal,conservative_temp,potential_density
def gen_topomask(h, lon, lat, dx=1., kind='linear', plot=False): """ Generates a topography mask from an oceanographic transect taking the deepest CTD scan as the depth of each station. Inputs ------ h : array Pressure of the deepest CTD scan for each station [dbar]. lons : array Longitude of each station [decimal degrees east]. lat : Latitude of each station. [decimal degrees north]. dx : float Horizontal resolution of the output arrays [km]. kind : string, optional Type of the interpolation to be performed. See scipy.interpolate.interp1d documentation for details. plot : bool Whether to plot mask for visualization. Outputs ------- xm : array Horizontal distances [km]. hm : array Local depth [m]. Examples -------- >>> import gsw >>> import df # FIXME: Add a dataset. >>> h = df.get_maxdepth() >>> # TODO: method to output distance. >>> x = np.append(0, np.cumsum(gsw.distance(df.lon, df.lat)[0] / 1e3)) >>> xm, hm = gen_topomask(h, df.lon, df.lat, dx=1., kind='linear') >>> fig, ax = plt.subplots() >>> ax.plot(xm, hm, 'k', linewidth=1.5) >>> ax.plot(x, h, 'ro') >>> ax.set_xlabel('Distance [km]') >>> ax.set_ylabel('Depth [m]') >>> ax.grid(True) >>> plt.show() Author ------ André Palóczy Filho ([email protected]) -- October/2012 """ h, lon, lat = map(np.asanyarray, (h, lon, lat)) # Distance in km. x = np.append(0, np.cumsum(gsw.distance(lon, lat)[0] / 1e3)) h = -gsw.z_from_p(h, lat.mean()) Ih = interp1d(x, h, kind=kind, bounds_error=False, fill_value=h[-1]) xm = np.arange(0, x.max() + dx, dx) hm = Ih(xm) return xm, hm
def calculate_depth(p, lat): """Calculates depth from pressure and latitude. :param p: pressure (bar) :param lat: latitude (decimal degrees) :return: depth (m) """ p_dbar = p * 10 return abs(gsw.z_from_p(p_dbar, lat))
def insitu_to_absolute(Tis, SP, p, lon, lat, zref): """Transform in situ variables to TEOS10 variables :rtype: float, float, float""" # SP is in p.s.u. SA = gsw.SA_from_SP(SP, p, lon, lat) CT = gsw.CT_from_t(SA, Tis, p) z = -gsw.z_from_p(p, lat) return (CT, SA, z)
def _update_eos(self): # derive absolute salinity and conservative temperature self.SA = gsw.SA_from_SP(self.s, self.p, self.lon, self.lat) self.CT = gsw.CT_from_t(self.SA, self.temp, self.p) # derive N2 self.N2, self.p_mid = gsw.Nsquared(self.SA, self.CT, self.p, lat=self.lat) self.z_mid = gsw.z_from_p(self.p_mid, self.lat)
def derive_cnv(self): """Compute SP, SA, CT, z, and GP from a cnv pre-processed cast.""" cast = self.copy() # FIXME: Use MetaDataFrame to propagate lon, lat. p = cast.index.values.astype(float) cast["SP"] = gsw.SP_from_C(cast["c0S/m"] * 10.0, cast["t090C"], p) cast["SA"] = gsw.SA_from_SP(cast["SP"], p, self.lon, self.lat) cast["SR"] = gsw.SR_from_SP(cast["SP"]) cast["CT"] = gsw.CT_from_t(cast["SA"], cast["t090C"], p) cast["z"] = -gsw.z_from_p(p, self.lat) cast["sigma0_CT"] = gsw.sigma0_CT_exact(cast["SA"], cast["CT"]) return cast
def test_pz_roundtrip(): """ The p_z conversion functions have Matlab-based checks that use only the first two arguments. Here we verify that the functions are also inverses when the optional arguments are used. """ z = np.array([-10, -100, -1000, -5000], dtype=float) p = gsw.p_from_z(z, 30, 0.5, 0.25) zz = gsw.z_from_p(p, 30, 0.5, 0.25) assert_almost_equal(z, zz)
def derive_cnv(self): """Compute SP, SA, CT, z, and GP from a cnv pre-processed cast.""" cast = self.copy() # FIXME: Use MetaDataFrame to propagate lon, lat. p = cast.index.values.astype(float) cast['SP'] = gsw.SP_from_C(cast['c0S/m'].values * 10., cast['t090C'].values, p) cast['SA'] = gsw.SA_from_SP(cast['SP'].values, p, self.lon, self.lat) cast['SR'] = gsw.SR_from_SP(cast['SP'].values) cast['CT'] = gsw.CT_from_t(cast['SA'].values, cast['t090C'].values, p) cast['z'] = -gsw.z_from_p(p, self.lat) cast['sigma0_CT'] = gsw.sigma0_CT_exact(cast['SA'].values, cast['CT'].values) return cast
def calculate_depth(pressure, latitude): """Calculates depth from pressure (dbar) and latitude. By default, gsw returns depths as negative. This routine returns the absolute values for positive depths. Paramters: pressure (decibars) latitude (decimal degrees) Returns: depth (meters) """ return abs(z_from_p(pressure, latitude))
def derive_cnv(self): """Compute SP, SA, CT, z, and GP from a cnv pre-processed cast.""" import gsw cast = self.copy() p = cast.index.values.astype(float) cast['SP'] = gsw.SP_from_C(cast['c0S/m'].values * 10., cast['t090C'].values, p) cast['SA'] = gsw.SA_from_SP(cast['SP'].values, p, self.lon, self.lat) cast['SR'] = gsw.SR_from_SP(cast['SP'].values) cast['CT'] = gsw.CT_from_t(cast['SA'].values, cast['t090C'].values, p) cast['z'] = -gsw.z_from_p(p, self.lat) cast['sigma0_CT'] = gsw.sigma0(cast['SA'].values, cast['CT'].values) return cast
def calc_teos10_columns(self, lat, lng): # Practical Salinity SP = gsw.SP_from_C(self.data['Cond'], self.data['Temp'], self.data['Pres']) # Absolute Salinity SA = gsw.SA_from_SP_Baltic(SP, lng, lat) # Conservative Temperature CT = gsw.CT_from_t(SA, self.data['Temp'], self.data['Pres']) # Sigma(density) with reference pressure of 0 dbar sigma = gsw.sigma0(SA, CT) # Depth depth = list(map(abs, gsw.z_from_p(self.data['Pres'], lat))) return {'PracticalSalinity' : SP, 'AbsoluteSalinity' : SA, 'ConservativeTemperature' : CT, 'Sigma(density)' : sigma, 'Depth' : depth}
def gen_topomask(h, lon, lat, dx=1.0, kind="linear", plot=False): """ Generates a topography mask from an oceanographic transect taking the deepest CTD scan as the depth of each station. Inputs ------ h : array Pressure of the deepest CTD scan for each station [dbar]. lons : array Longitude of each station [decimal degrees east]. lat : Latitude of each station. [decimal degrees north]. dx : float Horizontal resolution of the output arrays [km]. kind : string, optional Type of the interpolation to be performed. See scipy.interpolate.interp1d documentation for details. plot : bool Whether to plot mask for visualization. Outputs ------- xm : array Horizontal distances [km]. hm : array Local depth [m]. Author ------ André Palóczy Filho ([email protected]) -- October/2012 """ import gsw from scipy.interpolate import interp1d h, lon, lat = list(map(np.asanyarray, (h, lon, lat))) # Distance in km. x = np.append(0, np.cumsum(gsw.distance(lon, lat)[0] / 1e3)) h = -gsw.z_from_p(h, lat.mean()) Ih = interp1d(x, h, kind=kind, bounds_error=False, fill_value=h[-1]) xm = np.arange(0, x.max() + dx, dx) hm = Ih(xm) return xm, hm
def calc_depth_date(transect): '''Cacluate the depth from pressure and convert matlab date to python Args ---- transect: dict Transect dictionary with corresponding CTD data Returns ------- transect: dict Input transect dictionary with depths and updated timestamps ''' def matlab2datetime(matlab_datenum): '''Convert Matlab datenum to Python datetime''' import datetime day = datetime.datetime.fromordinal(int(matlab_datenum)) dayfrac = datetime.timedelta(days=matlab_datenum%1) - \ datetime.timedelta(days=366) return day + dayfrac import gsw import numpy n_depths, n_stations = transect['date_up'].shape transect['depth'] = numpy.zeros((n_depths, n_stations), dtype=float) transect['date_up'] = transect['date_up'].astype(object) lats = numpy.meshgrid(transect['lat'], numpy.ones(n_depths))[0] for i in range(n_depths): for j in range(n_stations): transect['depth'][i][j] = gsw.z_from_p(transect['press_up'][i][j], lats[i][j]) try: transect['date_up'][i][j] = matlab2datetime( transect['date_up'][i][j]) except: pass return transect
def calc_vertical_weights_2D(pressure_coord, latitude_coord, coord_names, data_shape): """Calculate vertical weights for a 2D depth axis (i.e. pressure, latitude) Args: pressure_coord (iris.coords.DimCoord): One-dimensional pressure coordinate latitude_coord (iris.coords.DimCoord): One-dimensional latitude coordinate coord_names (list): Names of each data coordinate data_shape (tuple): Shape of data Returns: iris.cube: Array of weights with shape matching data_shape """ if coord_names == ['time', 'depth', 'latitude', 'longitude']: ntime, nlev, nlat, nlon = data_shape elif coord_names == ['depth', 'latitude', 'longitude']: nlev, nlat, nlon = data_shape ntime = None depth_diffs = numpy.zeros([nlev, nlat]) for lat_index in range(0, nlat): height_vals = gsw.z_from_p(pressure_coord.points, latitude_coord.points[lat_index]) depth_vals = gsw.depth_from_z(height_vals) depth_bounds = guess_depth_bounds(depth_vals) depth_diffs[:, lat_index] = numpy.apply_along_axis( lambda x: x[1] - x[0], 1, depth_bounds) # Braodcast if ntime: depth_diffs = depth_diffs[numpy.newaxis, ...] depth_diffs = numpy.repeat(depth_diffs, ntime, axis=0) depth_diffs = depth_diffs[..., numpy.newaxis] depth_diffs = numpy.repeat(depth_diffs, nlon, axis=-1) assert depth_diffs.min() > 0.0 return depth_diffs
def save_as_xr(input, output): '''Read float files, compose xarray dataset, convert variables, and save as netcdf last updated: june 11, 2019 ''' a = load_matfile(str(input)) output = str(output) # u = 0.5 * (a['A']['u1'].flatten()[0] + a['A']['u2'].flatten()[0]) # v = 0.5 * (a['A']['v1'].flatten()[0] + a['A']['v2'].flatten()[0]) # dudz = 0.5 * (a['A']['du1dz'].flatten()[0] + a['A']['du2dz'].flatten()[0]) # dvdz = 0.5 * (a['A']['dv1dz'].flatten()[0] + a['A']['dv2dz'].flatten()[0]) # eps = np.nanmedian(np.dstack((a['A']['eps1'].flatten()[0], # a['A']['eps2'].flatten()[0])), axis=2) # chi = np.nanmedian(np.dstack((a['A']['chi1'].flatten()[0], # a['A']['chi2'].flatten()[0])), axis=2) # compose dataset object ds = xr.Dataset( { # define wanted variables here! 'sigma': (['z', 'time'], a['A']['Sigma'].flatten()[0]), 'u1': (['z', 'time'], a['A']['u1'].flatten()[0]), 'u2': (['z', 'time'], a['A']['u2'].flatten()[0]), 'v1': (['z', 'time'], a['A']['v1'].flatten()[0]), 'v2': (['z', 'time'], a['A']['v2'].flatten()[0]), 'du1dz': (['z', 'time'], a['A']['du1dz'].flatten()[0]), 'du2dz': (['z', 'time'], a['A']['du2dz'].flatten()[0]), 'dv1dz': (['z', 'time'], a['A']['dv1dz'].flatten()[0]), 'dv2dz': (['z', 'time'], a['A']['dv2dz'].flatten()[0]), 'eps1': (['z', 'time'], a['A']['eps1'].flatten()[0]), 'eps2': (['z', 'time'], a['A']['eps2'].flatten()[0]), 'chi1': (['z', 'time'], a['A']['chi1'].flatten()[0]), 'chi2': (['z', 'time'], a['A']['chi2'].flatten()[0]), # variables needed for QC 'RotP': (['z', 'time'], a['A']['RotP'].flatten()[0]), 'W': (['z', 'time'], a['A']['W'].flatten()[0]), 'verr1': (['z', 'time'], a['A']['verr1'].flatten()[0]), 'verr2': (['z', 'time'], a['A']['verr2'].flatten()[0]), 'kT1': (['z', 'time'], a['A']['kT1'].flatten()[0]), 'kT2': (['z', 'time'], a['A']['kT2'].flatten()[0]), 'T': (['z', 'time'], a['A']['T'].flatten()[0]), 'S': (['z', 'time'], a['A']['S'].flatten()[0]), 'n2': (['z', 'time'], a['A']['N2'].flatten()[0]) }, coords={ 'pressure': (['z'], a['A']['Pr'].flatten()[0].astype(float)), 'z': a['A']['Pr'].flatten()[0].astype(float), 'lat': (['time'], a['A']['lat'].flatten()[0]), 'lon': (['time'], a['A']['lon'].flatten()[0]), 'time': a['A']['Jday_gmt'].flatten()[0].astype(float) }, attrs={'floatid': output.split('_')[1].split('.')[0]}) # remove nans ds = ds.dropna(dim='time', how='all') # convert to datetime ds = ds.assign_coords(time=(dn2dt_vec(ds.time))) # comvert pressure to depth ds = ds.assign_coords(z=(gsw.z_from_p(ds.z, ds.lat.mean()).astype(float))) # ds = ds.assign_coords(z=-ds.z) _, index = np.unique(ds.time, return_index=True) ds = ds.isel(time=index) # convert variables p, lon, lat = xr.broadcast(ds.pressure, ds.lon, ds.lat) ds['S'] = xr.DataArray(gsw.SA_from_SP(ds.S, p, lon, lat), dims=['z', 'time']) ds['T'] = xr.DataArray(gsw.CT_from_t(ds.S, ds.T, p), dims=['z', 'time']) ds['rho0'] = xr.DataArray(gsw.sigma0(ds.S, ds.T) + 1000, dims=['z', 'time']) # save as netcdf ds.to_netcdf(str(output))
print('\n Making figures...') date_reform = np.array(date_reform) date_reform_sub = date_reform - refdate date_reform_num = np.zeros(len(date_reform)) date_reform_num[:] = np.NaN for l in np.arange(len(date_reform)): date_reform_num[l] = date_reform_sub[l].total_seconds() # Convert from pressure to depth z_values = np.zeros(pres.shape) z_values[:] = np.NaN for pp in np.arange(pres.shape[0]): z_values[pp, :] = gsw.z_from_p(pres[pp, :], lat[pp]) z_values = z_values * -1 # Only make sections if there is oxygen data # Interpolate data T_P = np.zeros((temp.shape[0], len(depth_range))) T_P[:] = np.NaN S_P = np.zeros((temp.shape[0], len(depth_range))) S_P[:] = np.NaN O_P = np.zeros((temp.shape[0], len(depth_range))) O_P[:] = np.NaN T_P = RF.PresInterpolation1m(OriginalData=temp,
def standardize(self, gps_prefix=None): df = self.data.copy() # Convert NMEA coordinates to decimal degrees for col in df.columns: # Ignore if the m_gps_lat and/or m_gps_lon value is the default masterdata value if col.endswith('_lat'): df[col] = df[col].map(lambda x: get_decimal_degrees(x) if x <= 9000 else np.nan) elif col.endswith('_lon'): df[col] = df[col].map(lambda x: get_decimal_degrees(x) if x < 18000 else np.nan) # Standardize 'time' to the 't' column for t in self.TIMESTAMP_SENSORS: if t in df.columns: df['t'] = pd.to_datetime(df[t], unit='s') break # Interpolate GPS coordinates if 'm_gps_lat' in df.columns and 'm_gps_lon' in df.columns: df['drv_m_gps_lat'] = df.m_gps_lat.copy() df['drv_m_gps_lon'] = df.m_gps_lon.copy() # Fill in data will nulls where value is the default masterdata value masterdatas = (df.drv_m_gps_lon >= 18000) | (df.drv_m_gps_lat > 9000) df.loc[masterdatas, 'drv_m_gps_lat'] = np.nan df.loc[masterdatas, 'drv_m_gps_lon'] = np.nan try: # Interpolate the filled in 'x' and 'y' y_interp, x_interp = interpolate_gps(masked_epoch(df.t), df.drv_m_gps_lat, df.drv_m_gps_lon) except (ValueError, IndexError): L.warning("Raw GPS values not found!") y_interp = np.empty(df.drv_m_gps_lat.size) * np.nan x_interp = np.empty(df.drv_m_gps_lon.size) * np.nan df['y'] = y_interp df['x'] = x_interp """ ---- Option 1: Always calculate Z from pressure ---- It's really a matter of data provider preference and varies from one provider to another. That being said, typically the sci_water_pressure or m_water_pressure variables, if present in the raw data files, will typically have more non-NaN values than m_depth. For example, all MARACOOS gliders typically have both m_depth and sci_water_pressure contained in them. However, m_depth is typically heavily decimated while sci_water_pressure contains a more complete pressure record. So, while we transmit both m_depth and sci_water_pressure, I calculate depth from pressure & (interpolated) latitude and use that as my NetCDF depth variable. - Kerfoot """ # Search for a 'pressure' column for p in self.PRESSURE_SENSORS: if p in df.columns: # Convert bar to dbar here df['pressure'] = df[p].copy() * 10 # Calculate depth from pressure and latitude # Negate the results so that increasing values note increasing depths df['z'] = -z_from_p(df.pressure, df.y) break if 'z' not in df and 'pressure' not in df: # Search for a 'z' column for p in self.DEPTH_SENSORS: if p in df.columns: df['z'] = df[p].copy() # Calculate pressure from depth and latitude # Negate the results so that increasing values note increasing depth df['pressure'] = -p_from_z(df.z, df.y) break # End Option 1 """ ---- Option 2: Use raw pressure/depth data that was sent across ---- # Standardize to the 'pressure' column for p in self.PRESSURE_SENSORS: if p in df.columns: # Convert bar to dbar here df['pressure'] = df[p].copy() * 10 break # Standardize to the 'z' column for p in self.DEPTH_SENSORS: if p in df.columns: df['z'] = df[p].copy() break # Don't calculate Z from pressure if a metered depth column exists already if 'pressure' in df and 'z' not in df: # Calculate depth from pressure and latitude # Negate the results so that increasing values note increasing depths df['z'] = -z_from_p(df.pressure, df.y) if 'z' in df and 'pressure' not in df: # Calculate pressure from depth and latitude # Negate the results so that increasing values note increasing depth df['pressure'] = -p_from_z(df.z, df.y) # End Option 2 """ rename_columns = { 'm_water_vx': 'u_orig', 'm_water_vy': 'v_orig', } # These need to be standardize so we can compute salinity and density! for vname in self.TEMPERATURE_SENSORS: if vname in df.columns: rename_columns[vname] = 'temperature' break for vname in self.CONDUCTIVITY_SENSORS: if vname in df.columns: rename_columns[vname] = 'conductivity' break # Standardize columns df = df.rename(columns=rename_columns) # Compute additional columns df = self.compute(df) return df
# %% gbu = upqc.groupby_bins("time", timebins) upa = gbu.mean(skipna=True, keep_attrs=True) # Use mid time as dimension, rather than Interval. upa["time_bins"] = interval_to_mid(upa.time_bins.values).astype("datetime64[s]") upa = upa.rename({"time_bins": "time"}) # Mean of heading should be performed using circular mean. (Technically, so should pitch and roll, but for small angles the noncircular mean is ok) upa["heading"] = (["time"], upa.heading.groupby_bins("time", timebins).reduce(stats.circmean, high=360.).values) # %% [markdown] # Finally, thermodynamics. # %% doa["z"] = (doa.p.dims, gsw.z_from_p(doa.p, doa.lat), {"units": "m", "long_name": "height"}) doa["depth"] = (doa.p.dims, -doa.z, {"units": "m", "long_name": "depth"}) # %% [markdown] # ## Old cut off data above surface and below bottom # # Use a simple echo intensity threshold to find the maximum. # %% # dmin = 50. # Minimum distance above which to look for the maximum # nroll = 180 # Number of points in rolling mode window # fcut = 0.1 # Extra distance to remove (1 - fcut)*dcut # %% # fig, ax = plt.subplots() # upa.a1.isel(time=10000).plot.line(ax=ax, marker='.')
def compute_thermodynamics(prof): """ Perform a pre-processing on the glider data """ # 1) Make a nice time record for the profile max_depth = prof['P'].max(dim='NT') bottom = prof['P'].argmax(dim='NT') record = prof['time_since_start_of_dive'] deltat_bottom = record.data[bottom].astype('f8') deltat_total = record.data[-1].astype('f8') alpha = deltat_bottom / deltat_total t_start = prof.GPS_time[0].data t_stop = prof.GPS_time[1].data t_bottom = t_start + pd.to_timedelta( alpha * (t_stop - t_start).astype('f8')) time_profile = t_bottom + pd.to_timedelta( prof['time_since_start_of_dive'] - deltat_bottom, unit='s') prof = prof.rename({'NT': 'time'}).assign_coords( time=('time', time_profile)) # 2) Get the coordinates of the profile lat_start = prof.GPS_latitude[0].data lat_stop = prof.GPS_latitude[1].data lat_bottom = lat_start + alpha * (lat_stop - lat_start) lon_start = prof.GPS_longitude[0].data lon_stop = prof.GPS_longitude[1].data lon_bottom = lon_start + alpha * (lon_stop - lon_start) distance_start_to_bottom = 1e-3 * gsw.distance([lon_start, lon_bottom], [lat_start, lat_bottom], p=[0, max_depth]).squeeze() distance_bottom_to_stop = 1e-3 * gsw.distance([lon_bottom, lon_stop], [lat_bottom, lat_stop], p=[max_depth, 0]).squeeze() # 3) Clean up unvalid data niceT = prof['T'] niceS = prof['S'] # Do not forget to correct the offset due to surface pressure niceP = (prof['P'] - prof['Psurf']) niceDive = prof['dive'] # 4) Compute thermodynamic quantities from GSW toolbox # - Absolute Salinity SA = gsw.SA_from_SP(niceS, niceP, lat_start, lon_start) # - Conservative Temperature CT = gsw.CT_from_t(SA, niceT, niceP) # - In situ density rho = gsw.rho(SA, CT, niceP) # - Potential density referenced to surface pressure sigma0 = gsw.sigma0(SA, CT) # - Buoyancy b = 9.81 * (1 - rho / 1025) N2 = xr.DataArray(gsw.Nsquared(SA, CT, niceP)[0], name='Buoyancy frequency', dims='time', coords = {'time': ('time', prof['time'].isel(time=slice(1, None))) } ) # - Depth depth = - gsw.z_from_p(niceP, lat_start) # 5) Split the dive into one descending and one ascending path bottom = niceP.argmax(dim='time') ones = xr.ones_like(niceP) newdive = xr.concat([2 * niceDive[:bottom] - 1, 2 * niceDive[bottom:]], dim='time') lat = xr.concat([0.5 * (lat_start + lat_bottom) * ones[:bottom], 0.5 * (lat_stop + lat_bottom) * ones[bottom:]], dim='time') lon = xr.concat([0.5 * (lon_start + lon_bottom) * ones[:bottom], 0.5 * (lon_stop + lon_bottom) * ones[bottom:]], dim='time') Pmax = xr.concat([max_depth * ones[:bottom], max_depth * ones[bottom:]], dim='time') distance = xr.concat([distance_start_to_bottom * ones[:bottom], distance_bottom_to_stop * ones[bottom:]], dim='time') distance.name = 'distance between profiles in km' return xr.Dataset({'Temperature': niceT, 'Salinity': niceS, 'Pressure': niceP, 'Rho': ('time', rho), 'CT': ('time', CT), 'SA': ('time', SA), 'Sigma0': ('time', sigma0), 'b': ('time', b), 'Pmax': ('time', Pmax), 'N2': N2}, coords={'profile': ('time', newdive.data), 'depth': ('time', depth), 'lat': ('time', lat), 'lon': ('time', lon), 'distance': distance})
# %% fig, ax = plt.subplots() ds.p_SBE37.plot(ax=ax) ds.p.plot(ax=ax, yincrease=False) # %% [markdown] # Estimate some more thermodynamic variables. # %% import gsw # %% ds["SA_SBE37"] = (ds.p.dims, gsw.SA_from_SP(ds.SP_SBE37, ds.p_SBE37, ds.lon, ds.lat), {"units": "g/kg", "long_name": "Absolute_salinity"}) ds["CT_SBE37"] = (ds.p.dims, gsw.CT_from_t(ds.SA_SBE37, ds.t_SBE37, ds.p_SBE37), {"units": "deg C", "long_name": "Conservative_temperature"}) ds["z_SBE37"] = (ds.p.dims, gsw.z_from_p(ds.p_SBE37, ds.lat), {"units": "m", "long_name": "height"}) ds["depth_SBE37"] = (ds.p.dims, -ds.z_SBE37, {"units": "m", "long_name": "depth"}) ds["z_ADCP"] = (ds.p.dims, gsw.z_from_p(ds.p, ds.lat), {"units": "m", "long_name": "height"}) ds["depth_ADCP"] = (ds.p.dims, -ds.z_ADCP, {"units": "m", "long_name": "depth"}) ds["z"] = (ds.distance.dims, ds.distance + ds.z_ADCP.mean(dim="time"), {"units": "m", "long_name": "height"}) ds["depth"] = (ds.distance.dims, -ds.z, {"units": "m", "long_name": "depth"}) ds = ds.set_coords(["z", "depth"]) # %% [markdown] # Save dataset to netcdf. # %% ds.to_netcdf("../proc/ABLE_sentinel_mooring_2018.nc") # %% [markdown] # ## Examine a short segment of the dataset
# pressure p = argo.pres[0] # practical salinity psal = argo.psal[0] # in-situ temperature t = argo.temp[0] print"3.4 Absolute salinity (g/lg)" Sa=sw.SA_from_SP(psal,p,lon,lat) print Sa print"3.5 Conservative temperature (C)" Tc = sw.CT_from_t(Sa,t,p) print Tc print"3.6 Water column height (m)" z=sw.z_from_p(p,lat) print z # Then make plots of SA and Tc vs z. Include axis labels and titles. lblSize = 10 fig,ax = plt.subplots(ncols=2,figsize=(8,6)) ax[0].set_xlabel('Absolute Salinity [g/kg]',size=lblSize ) ax[0].set_ylabel('Water column height [m]',size=lblSize ) #ax[0].set_ylim((-1850,0)) #ax.set_ylim(-20,1) ax[0].scatter(Sa,z ,marker='o',s=12,\ color='g', alpha=.7,zorder=10) ax[0].plot(Sa,z,'g') ax[0].set_title(r'$S_A$',size=16)
def loadDataFRP_raw( exp='all', sel='narrow', meshPath='/ocean/eolson/MEOPAR/NEMO-forcing/grid/mesh_mask201702.nc'): import gsw # use to convert p to z if exp not in {'exp1', 'exp2', 'exp3', 'all'}: print('option exp=' + exp + ' is not defined.') raise if sel not in {'narrow', 'wide'}: print('option sel=' + sel + ' is not defined.') raise df0, clist, tcor, cast19, cast25 = loadDataFRP_init(exp=exp) zCasts = dict() for nn in clist: zCasts[nn] = rawCast() ip = np.argmax(cast25[nn].df['prSM'].values) ilag = df0.loc[df0.Station == nn, 'ishift_sub19'].values[0] pS_pr = df0.loc[df0.Station == nn, 'pS_pr'].values[0] pE_pr = df0.loc[df0.Station == nn, 'pE_pr'].values[0] pS_tur = df0.loc[df0.Station == nn, 'pStart25'].values[0] pE_tur = df0.loc[df0.Station == nn, 'pEnd25'].values[0] if sel == 'narrow': pS = pS_tur pE = pE_tur elif sel == 'wide': pS = pS_pr pE = pE_pr parDZ = .78 xmisDZ = .36 turbDZ = .67 zshiftdict = { 'gsw_ctA0': 0.0, 'gsw_srA0': 0.0, 'xmiss': xmisDZ, 'seaTurbMtr': turbDZ, 'par': parDZ, 'wetStar': 0.0, 'sbeox0ML_L': 0.0 } for var in ('gsw_ctA0', 'gsw_srA0', 'xmiss', 'par', 'wetStar', 'sbeox0ML_L'): if not nn == 14.2: #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - zshiftdict[ var] # down z inV = cast25[nn].df.loc[pS:ip][var].values # down var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].dCast[var] = dataPair(inP, inV) else: # special case where there is no downcast zCasts[nn].dCast[var] = dataPair(np.nan, np.nan) if not nn == 14.1: #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - zshiftdict[ var] # down z inV = cast25[nn].df.loc[ip:pE][var].values # down var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].uCast[var] = dataPair(inP, inV) else: # special case where there is no upcast zCasts[nn].uCast[var] = dataPair(np.nan, np.nan) if not nn == 14.2: #turbidity downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - turbDZ # down z inV0 = cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['seaTurbMtr'].values # down var if sel == 'wide': # additional QC for broader data selection ii1 = amp(rolling_window_padded(inV0, 5), -1) > .5 * np.nanmax(inV0) # get rid of near-zero turbidity values; seem to be dropped signal ii2 = np.nanmin(rolling_window_padded(inV0, 5), -1) < .3 inV0[np.logical_or(ii1, ii2)] = np.nan inV = ssig.medfilt(inV0, 3) # down var if sel == 'wide': # exclude above surface data with np.errstate(invalid='ignore'): inV[inP < .1] = np.nan zCasts[nn].dCast['turb_uncor'] = dataPair(inP, inV) zCasts[nn].dCast['turb'] = dataPair(inP, inV * 1.0 / tcor) else: # special case where there is no downcast zCasts[nn].dCast['turb_uncor'] = dataPair(np.nan, np.nan) zCasts[nn].dCast['turb'] = dataPair(np.nan, np.nan) if not nn == 14.1: #turbidity upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - turbDZ # up z inV0 = cast19[nn].df.loc[(ip + ilag):( pE + ilag)]['seaTurbMtr'].values # up var if sel == 'wide': # additional QC for broader data selection ii1 = amp(rolling_window_padded(inV0, 5), -1) > .5 * np.nanmax(inV0) # get rid of near-zero turbidity values; seem to be dropped signal ii2 = np.nanmin(rolling_window_padded(inV0, 5), -1) < .3 inV0[np.logical_or(ii1, ii2)] = np.nan inV = ssig.medfilt(inV0, 3) # down var if sel == 'wide': # exclude above surface data with np.errstate(invalid='ignore'): inV[inP < .1] = np.nan zCasts[nn].uCast['turb_uncor'] = dataPair(inP, inV) zCasts[nn].uCast['turb'] = dataPair(inP, inV * 1.0 / tcor) else: # special case where there is no upcasts zCasts[nn].uCast['turb_uncor'] = dataPair(np.nan, np.nan) zCasts[nn].uCast['turb'] = dataPair(np.nan, np.nan) # fix first 2 casts for which sb25 pump did not turn on. use sb19 if (exp == 'exp1' or exp == 'all'): for nn in range(1, 3): ip = np.argmax(cast25[nn].df['prSM'].values) ilag = df0.loc[df0.Station == nn, 'ishift_sub19'].values[0] pS_pr = df0.loc[df0.Station == nn, 'pS_pr'].values[0] pE_pr = df0.loc[df0.Station == nn, 'pE_pr'].values[0] pS_tur = df0.loc[df0.Station == nn, 'pStart25'].values[0] pE_tur = df0.loc[df0.Station == nn, 'pEnd25'].values[0] if sel == 'narrow': pS = pS_tur pE = pE_tur elif sel == 'wide': pS = pS_pr pE = pE_pr ##temperature #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # down z inV = cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['gsw_ctA0'].values # down var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].dCast['gsw_ctA0'] = dataPair(inP, inV) #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # up z inV = cast19[nn].df.loc[(ip + ilag):(pE + ilag)]['gsw_ctA0'].values # up var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].uCast['gsw_ctA0'] = dataPair(inP, inV) ##sal #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # down z inV = cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['gsw_srA0'].values # down var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].dCast['gsw_srA0'] = dataPair(inP, inV) #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # up z inV = cast19[nn].df.loc[(ip + ilag):(pE + ilag)]['gsw_srA0'].values # up var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].uCast['gsw_srA0'] = dataPair(inP, inV) ##xmiss: xmis25=1.14099414691*xmis19+-1.6910134322 #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - xmisDZ # down z inV = 1.14099414691 * cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['CStarTr0'].values - 1.6910134322 # down var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].dCast['xmiss'] = dataPair(inP, inV) #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - xmisDZ # up p inV = 1.14099414691 * cast19[nn].df.loc[(ip + ilag):( pE + ilag)]['CStarTr0'].values - 1.6910134322 # up var if sel == 'wide': inV[inP < .1] = np.nan zCasts[nn].dCast['wetStar'] = dataPair(np.nan, np.nan) zCasts[nn].uCast['wetStar'] = dataPair(np.nan, np.nan) zCasts[nn].dCast['sbeox0ML_L'] = dataPair(np.nan, np.nan) zCasts[nn].uCast['sbeox0ML_L'] = dataPair(np.nan, np.nan) return df0, zCasts
def post_process(self, verbose=True): print("\nPost processing") print("---------------\n") # Very basic self.ascent = self.hpid % 2 == 0 self.ascent_ctd = self.ascent*np.ones_like(self.UTC, dtype=int) self.ascent_ef = self.ascent*np.ones_like(self.UTCef, dtype=int) # Estimate number of observations. self.nobs_ctd = np.sum(~np.isnan(self.UTC), axis=0) self.nobs_ef = np.sum(~np.isnan(self.UTCef), axis=0) # Figure out some useful times. self.UTC_start = self.UTC[0, :] self.UTC_end = np.nanmax(self.UTC, axis=0) if verbose: print("Creating time variable dUTC with units of seconds.") self.dUTC = (self.UTC - self.UTC_start)*86400 self.dUTCef = (self.UTCef - self.UTC_start)*86400 if verbose: print("Interpolated GPS positions to starts and ends of profiles.") # GPS interpolation to the start and end time of each half profile. idxs = ~np.isnan(self.lon_gps) & ~np.isnan(self.lat_gps) self.lon_start = np.interp(self.UTC_start, self.utc_gps[idxs], self.lon_gps[idxs]) self.lat_start = np.interp(self.UTC_start, self.utc_gps[idxs], self.lat_gps[idxs]) self.lon_end = np.interp(self.UTC_end, self.utc_gps[idxs], self.lon_gps[idxs]) self.lat_end = np.interp(self.UTC_end, self.utc_gps[idxs], self.lat_gps[idxs]) if verbose: print("Calculating heights.") # Depth. self.z = gsw.z_from_p(self.P, self.lat_start) # self.z_ca = gsw.z_from_p(self.P_ca, self.lat_start) self.zef = gsw.z_from_p(self.Pef, self.lat_start) if verbose: print("Calculating distance along trajectory.") # Distance along track from first half profile. self.__ddist = utils.lldist(self.lon_start, self.lat_start) self.dist = np.hstack((0., np.cumsum(self.__ddist))) if verbose: print("Interpolating distance to measurements.") # Distances, velocities and speeds of each half profile. self.profile_ddist = np.zeros_like(self.lon_start) self.profile_dt = np.zeros_like(self.lon_start) self.profile_bearing = np.zeros_like(self.lon_start) lons = np.zeros((len(self.lon_start), 2)) lats = lons.copy() times = lons.copy() lons[:, 0], lons[:, 1] = self.lon_start, self.lon_end lats[:, 0], lats[:, 1] = self.lat_start, self.lat_end times[:, 0], times[:, 1] = self.UTC_start, self.UTC_end self.dist_ctd = self.UTC.copy() nans = np.isnan(self.dist_ctd) for i, (lon, lat, time) in enumerate(zip(lons, lats, times)): self.profile_ddist[i] = utils.lldist(lon, lat) # Convert time from days to seconds. self.profile_dt[i] = np.diff(time)*86400. d = np.array([self.dist[i], self.dist[i] + self.profile_ddist[i]]) idxs = ~nans[:, i] self.dist_ctd[idxs, i] = np.interp(self.UTC[idxs, i], time, d) self.dist_ef = self.__regrid('ctd', 'ef', self.dist_ctd) if verbose: print("Estimating bearings.") # Pythagorian approximation (?) of bearing. self.profile_bearing = np.arctan2(self.lon_end - self.lon_start, self.lat_end - self.lat_start) if verbose: print("Calculating sub-surface velocity.") # Convert to m s-1 calculate meridional and zonal velocities. self.sub_surf_speed = self.profile_ddist*1000./self.profile_dt self.sub_surf_u = self.sub_surf_speed*np.sin(self.profile_bearing) self.sub_surf_v = self.sub_surf_speed*np.cos(self.profile_bearing) if verbose: print("Interpolating missing velocity values.") # Fill missing U, V values using linear interpolation otherwise we # run into difficulties using cumtrapz next. self.U = self.__fill_missing(self.U) self.V = self.__fill_missing(self.V) # Absolute velocity self.calculate_absolute_velocity(verbose=verbose) if verbose: print("Calculating thermodynamic variables.") # Derive some important thermodynamics variables. # Absolute salinity. self.SA = gsw.SA_from_SP(self.S, self.P, self.lon_start, self.lat_start) # Conservative temperature. self.CT = gsw.CT_from_t(self.SA, self.T, self.P) # Potential temperature with respect to 0 dbar. self.PT = gsw.pt_from_CT(self.SA, self.CT) # In-situ density. self.rho = gsw.rho(self.SA, self.CT, self.P) # Potential density with respect to 1000 dbar. self.rho_1 = gsw.pot_rho_t_exact(self.SA, self.T, self.P, p_ref=1000.) # Buoyancy frequency regridded onto ctd grid. N2_ca, __ = gsw.Nsquared(self.SA, self.CT, self.P, self.lat_start) self.N2 = self.__regrid('ctd_ca', 'ctd', N2_ca) if verbose: print("Calculating float vertical velocity.") # Vertical velocity regridded onto ctd grid. dt = 86400.*np.diff(self.UTC, axis=0) # [s] Wz_ca = np.diff(self.z, axis=0)/dt self.Wz = self.__regrid('ctd_ca', 'ctd', Wz_ca) if verbose: print("Renaming Wp to Wpef.") # Vertical water velocity. self.Wpef = self.Wp.copy() del self.Wp if verbose: print("Calculating shear.") # Shear calculations. dUdz_ca = np.diff(self.U, axis=0)/np.diff(self.zef, axis=0) dVdz_ca = np.diff(self.V, axis=0)/np.diff(self.zef, axis=0) self.dUdz = self.__regrid('ef_ca', 'ef', dUdz_ca) self.dVdz = self.__regrid('ef_ca', 'ef', dVdz_ca) if verbose: print("Calculating Richardson number.") N2ef = self.__regrid('ctd', 'ef', self.N2) self.Ri = N2ef/(self.dUdz**2 + self.dVdz**2) if verbose: print("Regridding piston position to ctd.\n") # Regrid piston position. self.ppos = self.__regrid('ctd_ca', 'ctd', self.ppos_ca) self.update_profiles()
def create_llat_dba_reader(dba_file, timesensor=None, pressuresensor=None, depthsensor=None): if not os.path.isfile(dba_file): logging.error('dba file does not exist: {:s}'.format(dba_file)) return # Parse the dba file dba = parse_dba(dba_file) if not dba: return # List of available dba sensors dba_sensors = [s['sensor_name'] for s in dba['sensors']] # Select the time sensor time_sensor = select_time_sensor(dba, timesensor=timesensor) if not time_sensor: return # Select the pressure sensor pressure_sensor = select_pressure_sensor(dba) # Select the depth sensor depth_sensor = select_depth_sensor(dba) # We must have either a pressure_sensor or depth_sensor to continue if not pressure_sensor and not depth_sensor: logger.warning( 'No pressure sensor and no depth sensor found: {:s}'.format( dba_file)) return # Must have m_gps_lat and m_gps_lon to convert to decimal degrees if 'm_gps_lat' not in dba_sensors or 'm_gps_lon' not in dba_sensors: logger.warning( 'Missing m_gps_lat and/or m_gps_lon: {:s}'.format(dba_file)) else: # Convert m_gps_lat to decimal degrees and create the new sensor definition c = dba_sensors.index('m_gps_lat') lat_sensor = deepcopy(dba['sensors'][c]) lat_sensor['sensor_name'] = 'llat_latitude' lat_sensor['attrs']['source_sensor'] = u'm_gps_lat' lat_sensor['attrs'][ 'comment'] = u'm_gps_lat converted to decimal degrees and interpolated' lat_sensor['data'] = np.empty((len(dba['data']), 1)) * np.nan for x in range(len(dba['data'])): # Skip default values (69696969) if abs(dba['data'][x, c]) > 9000: continue lat_sensor['data'][x] = get_decimal_degrees(dba['data'][x, c]) # Convert m_gps_lon to decimal degrees and create the new sensor definition c = dba_sensors.index('m_gps_lon') lon_sensor = deepcopy(dba['sensors'][c]) lon_sensor['sensor_name'] = 'llat_longitude' lon_sensor['attrs']['source_sensor'] = u'm_gps_lon' lon_sensor['attrs'][ 'comment'] = u'm_gps_lon converted to decimal degrees and interpolated' lon_sensor['data'] = np.empty((len(dba['data']), 1)) * np.nan for x in range(len(dba['data'])): # Skip default values (69696969) if abs(dba['data'][x, c]) > 18000: continue lon_sensor['data'][x] = get_decimal_degrees(dba['data'][x, c]) # Interpolate llat_latitude and llat_longitude lat_sensor['data'], lon_sensor['data'] = interpolate_gps( time_sensor['data'], lat_sensor['data'], lon_sensor['data']) # If no depth_sensor was selected, use llat_latitude, llat_longitude and llat_pressure to calculate if not depth_sensor: logger.info( 'Calculating depth from select pressure sensor: {:s}'.format( pressure_sensor['attrs']['source_sensor'])) depth_sensor = {'sensor_name': 'llat_depth', 'attrs': {}} depth_sensor['attrs']['source_sensor'] = 'llat_pressure,llat_latitude' depth_sensor['attrs'][ 'comment'] = u'Calculated from llat_pressure and llat_latitude using gsw.z_from_p' depth_sensor['data'] = -z_from_p(pressure_sensor['data'], lat_sensor['data']) # Append the llat variables dba['data'] = np.append(dba['data'], time_sensor['data'], axis=1) del (time_sensor['data']) dba['sensors'].append(time_sensor) if pressure_sensor: dba['data'] = np.append(dba['data'], pressure_sensor['data'], axis=1) del (pressure_sensor['data']) dba['sensors'].append(pressure_sensor) dba['data'] = np.append(dba['data'], depth_sensor['data'], axis=1) del (depth_sensor['data']) dba['sensors'].append(depth_sensor) dba['data'] = np.append(dba['data'], lat_sensor['data'], axis=1) del (lat_sensor['data']) dba['sensors'].append(lat_sensor) dba['data'] = np.append(dba['data'], lon_sensor['data'], axis=1) del (lon_sensor['data']) dba['sensors'].append(lon_sensor) return dba
def create_llat_dba_reader(dba_file, timesensor=None, pressuresensor=None): if not os.path.isfile(dba_file): logger.error('dba file does not exist: {:s}'.format(dba_file)) return dba = create_dba_reader(dba_file) if len(dba['data']) == 0: logger.warning('No data records parsed: {:s}'.format(dba_file)) return timestamp_sensor = None pressure_sensor = None dba_sensors = [s['sensor'] for s in dba['sensors']] if timesensor and timesensor not in dba_sensors: logger.warning('Specified timesensor {:s} not found in dba: {:s}'.format(timesensor, dba_file)) return dba else: for t in VALID_DBA_TIMESTAMP_SENSORS: if t in dba_sensors: timestamp_sensor = t sensor_def = {'sensor' : 'drv_timestamp', 'attrs' : {}} sensor_def['attrs']['dba_sensor'] = timestamp_sensor sensor_def['attrs']['comment'] = 'Alias for {:s}'.format(timestamp_sensor) dba['sensors'].append(sensor_def) break if pressuresensor and pressuresensor not in dba_sensors: logger.warning('Specified pressuresensor {:s} not found in dba: {:s}'.format(pressuresensor, dba_file)) else: for p in VALID_DBA_PRESSURE_SENSORS: if p in dba_sensors: pressure_sensor = p sensor_def = {'sensor' : 'drv_pressure', 'attrs' : {}} sensor_def['attrs']['dba_sensor'] = pressure_sensor sensor_def['attrs']['comment'] = 'Alias for {:s}'.format(pressure_sensor) dba['sensors'].append(sensor_def) break # Create the aliases counter = 0 raw_gps = np.empty((len(dba['data']),4)) * np.nan for r in dba['data']: row_sensors = r.keys() if timestamp_sensor in row_sensors: r['drv_timestamp'] = r[timestamp_sensor] # Fill in the timestamps raw_gps[counter,0] = r['drv_timestamp'] # Fill in raw_gps if 'm_gps_lat' in row_sensors and 'm_gps_lon' in row_sensors: # Ignore if the m_gps_lat and/or m_gps_lon value is the default # masterdata value if r['m_gps_lat'] <= 9000 or r['m_gps_lon'] < 18000: r['drv_m_gps_lat'] = get_decimal_degrees(r['m_gps_lat']) r['drv_m_gps_lon'] = get_decimal_degrees(r['m_gps_lon']) raw_gps[counter,1] = r['drv_m_gps_lat'] raw_gps[counter,2] = r['drv_m_gps_lon'] else: logger.warning('Ignoring m_gps_lat/m_gps_lon default masterdata values: {:s}'.format(dba_file)) if pressure_sensor and pressure_sensor in row_sensors: r['drv_pressure'] = r[pressure_sensor] raw_gps[counter,3] = r[pressure_sensor] counter += 1 # Interpolate m_gps_lat/lon and add records to dataset['data'] #try: interp_lat,interp_lon = interpolate_gps(raw_gps[:,0], raw_gps[:,1], raw_gps[:,2]) #except IndexError as e: # logger.error('{:s}: {:s}'.format(dba_file, e)) #return # Calculate depth from pressure (multiplied by 10 to get to decibars) and latitude # Negate the results so that increasing values note increasing depths depths = -z_from_p(raw_gps[:,3]*10, interp_lat) depth_def = {'sensor' : 'drv_depth', 'attrs' : {}} depth_def['attrs']['comment'] = 'Depth calculated from pressure*10 (decibars) and raw latitude. Negative values denote increased depths' depth_def['attrs']['ancillary_variables'] = '{:s},drv_m_gps_lat'.format(pressure_sensor) dba['sensors'].append(depth_def) counter = 0 for r in dba['data']: if not np.isnan(interp_lat[counter]): r['drv_interp_m_gps_lat'] = interp_lat[counter] if not np.isnan(interp_lon[counter]): r['drv_interp_m_gps_lon'] = interp_lon[counter] if not np.isnan(depths[counter]): r['drv_depth'] = depths[counter] counter += 1 # Create and add the converted m_gps_lat/lon sensor defs lat_dd_def = {'sensor' : 'drv_m_gps_lat', 'attrs' : {}} lat_dd_def['attrs']['comment'] = 'm_gps_lat converted to decimal degrees' lat_dd_def['attrs']['dba_sensor'] = 'm_gps_lat' dba['sensors'].append(lat_dd_def) lon_dd_def = {'sensor' : 'drv_m_gps_lon', 'attrs' : {}} lon_dd_def['attrs']['comment'] = 'm_gps_lon converted to decimal degrees' lon_dd_def['attrs']['dba_sensor'] = 'm_gps_lon' dba['sensors'].append(lon_dd_def) # Create and add the converted & interpolated m_gps_lat/lon sensor defs ilat_dd_def = {'sensor' : 'drv_interp_m_gps_lat', 'attrs' : {}} ilat_dd_def['attrs']['comment'] = 'm_gps_lat converted to decimal degrees and interpolated' ilat_dd_def['attrs']['dba_sensor'] = 'm_gps_lat' dba['sensors'].append(ilat_dd_def) ilon_dd_def = {'sensor' : 'drv_interp_m_gps_lon', 'attrs' : {}} ilon_dd_def['attrs']['comment'] = 'm_gps_lon converted to decimal degrees and interpolated' ilon_dd_def['attrs']['dba_sensor'] = 'm_gps_lon' dba['sensors'].append(ilon_dd_def) return dba
def loadDataFRP_SSGrid( exp='all', sel='narrow', meshPath='/ocean/eolson/MEOPAR/NEMO-forcing/grid/mesh_mask201702.nc'): import gsw # only in this function; use to convert p to z if exp not in {'exp1', 'exp2', 'all'}: print('option exp=' + exp + ' is not defined.') raise if sel not in {'narrow', 'wide'}: print('option sel=' + sel + ' is not defined.') raise df0, clist, tcor, cast19, cast25 = loadDataFRP_init(exp=exp) # load mesh mesh = nc.Dataset(meshPath, 'r') tmask = mesh.variables['tmask'][0, :, :, :] gdept = mesh.variables['gdept_0'][0, :, :, :] gdepw = mesh.variables['gdepw_0'][0, :, :, :] nav_lat = mesh.variables['nav_lat'][:, :] nav_lon = mesh.variables['nav_lon'][:, :] mesh.close() zCasts = dict() for nn in clist: ip = np.argmax(cast25[nn].df['prSM'].values) ilag = df0.loc[df0.Station == nn, 'ishift_sub19'].values[0] pS_pr = df0.loc[df0.Station == nn, 'pS_pr'].values[0] pE_pr = df0.loc[df0.Station == nn, 'pE_pr'].values[0] pS_tur = df0.loc[df0.Station == nn, 'pStart25'].values[0] pE_tur = df0.loc[df0.Station == nn, 'pEnd25'].values[0] if sel == 'narrow': pS = pS_tur pE = pE_tur prebin = False elif sel == 'wide': pS = pS_pr pE = pE_pr prebin = True jj, ii = geo_tools.find_closest_model_point( df0.loc[df0.Station == nn]['LonDecDeg'].values[0], df0.loc[df0.Station == nn]['LatDecDeg'].values[0], nav_lon, nav_lat) zmax = -1 * gsw.z_from_p(cast25[nn].df.loc[ip, 'prSM'], df0.loc[df0.Station == nn]['LatDecDeg']) edges = gdepw[:, jj, ii] targets = gdept[:, jj, ii] edges = edges[edges < zmax] targets = targets[:(len(edges) - 1)] parDZ = .78 xmisDZ = .36 turbDZ = .67 zshiftdict = { 'gsw_ctA0': 0.0, 'gsw_srA0': 0.0, 'xmiss': xmisDZ, 'seaTurbMtr': turbDZ, 'par': parDZ, 'wetStar': 0.0, 'sbeox0ML_L': 0.0 } dCast = pd.DataFrame() uCast = pd.DataFrame() for var in ('gsw_ctA0', 'gsw_srA0', 'xmiss', 'par', 'wetStar', 'sbeox0ML_L'): if not nn == 14.2: #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - zshiftdict[ var] # down z inV = cast25[nn].df.loc[pS:ip][var].values # down var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) if var == 'gsw_ctA0': dCast = pd.DataFrame(p, columns=['depth_m']) dCast['indk'] = np.arange(0, len(p)) dCast[var] = out else: # special case where there is no downcast if var == 'gsw_ctA0': dCast = pd.DataFrame(np.nan * np.ones(10), columns=['depth_m']) dCast['indk'] = np.nan * np.ones(10) dCast[var] = np.nan * np.ones(10) if not nn == 14.1: #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - zshiftdict[ var] # down z inV = cast25[nn].df.loc[ip:pE][var].values # down var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) if var == 'gsw_ctA0': uCast = pd.DataFrame(p, columns=['depth_m']) uCast['indk'] = np.arange(0, len(p)) uCast[var] = out else: # special case where there is no upcast if var == 'gsw_ctA0': uCast = pd.DataFrame(np.nan * np.ones(10), columns=['depth_m']) uCast['indk'] = np.nan * np.ones(10) uCast[var] = np.nan * np.ones(10) if not nn == 14.2: #turbidity downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - turbDZ # down z inV0 = cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['seaTurbMtr'].values # down var if sel == 'wide': # additional QC for broader data selection ii1 = amp(rolling_window_padded(inV0, 5), -1) > .5 * np.nanmax(inV0) # get rid of near-zero turbidity values; seem to be dropped signal ii2 = np.nanmin(rolling_window_padded(inV0, 5), -1) < .3 inV0[np.logical_or(ii1, ii2)] = np.nan inV = ssig.medfilt(inV0, 3) # down var if sel == 'wide': # exclude above surface data with np.errstate(invalid='ignore'): inV[inP < .1] = np.nan p, tur = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) dCast['turb'] = tur * 1.0 / tcor else: # special case where there is no downcast dCast['turb'] = np.nan * np.ones(10) if not nn == 14.1: #turbidity upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - turbDZ # up z inV0 = cast19[nn].df.loc[(ip + ilag):( pE + ilag)]['seaTurbMtr'].values # up var if sel == 'wide': # additional QC for broader data selection ii1 = amp(rolling_window_padded(inV0, 5), -1) > .5 * np.nanmax(inV0) # get rid of near-zero turbidity values; seem to be dropped signal ii2 = np.nanmin(rolling_window_padded(inV0, 5), -1) < .3 inV0[np.logical_or(ii1, ii2)] = np.nan inV = ssig.medfilt(inV0, 3) # down var if sel == 'wide': # exclude above surface data with np.errstate(invalid='ignore'): inV[inP < .1] = np.nan p, tur = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) uCast['turb'] = tur * 1.0 / tcor else: # special case where there is no upcasts uCast['turb'] = np.nan * np.ones(10) zCasts[nn] = zCast(uCast, dCast) # fix first 2 casts for which sb25 pump did not turn on. use sb19 if (exp == 'exp1' or exp == 'all'): for nn in range(1, 3): uCast = zCasts[nn].uCast dCast = zCasts[nn].dCast ip = np.argmax(cast25[nn].df['prSM'].values) ilag = df0.loc[df0.Station == nn, 'ishift_sub19'].values[0] pS_pr = df0.loc[df0.Station == nn, 'pS_pr'].values[0] pE_pr = df0.loc[df0.Station == nn, 'pE_pr'].values[0] pS_tur = df0.loc[df0.Station == nn, 'pStart25'].values[0] pE_tur = df0.loc[df0.Station == nn, 'pEnd25'].values[0] if sel == 'narrow': pS = pS_tur pE = pE_tur elif sel == 'wide': pS = pS_pr pE = pE_pr jj, ii = geo_tools.find_closest_model_point( df0.loc[df0.Station == nn]['LonDecDeg'].values[0], df0.loc[df0.Station == nn]['LatDecDeg'].values[0], nav_lon, nav_lat) zmax = -1 * gsw.z_from_p(cast25[nn].df.loc[ip, 'prSM'], df0.loc[df0.Station == nn]['LatDecDeg']) edges = gdepw[:, jj, ii] targets = gdept[:, jj, ii] edges = edges[edges < zmax] targets = targets[:(len(edges) - 1)] ##temperature #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # down z inV = cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['gsw_ctA0'].values # down var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) dCast['gsw_ctA0'] = out #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # up z inV = cast19[nn].df.loc[(ip + ilag):(pE + ilag)]['gsw_ctA0'].values # up var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) uCast['gsw_ctA0'] = out ##sal #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # down z inV = cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['gsw_srA0'].values # down var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) dCast['gsw_srA0'] = out #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) # up z inV = cast19[nn].df.loc[(ip + ilag):(pE + ilag)]['gsw_srA0'].values # up var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges, prebin=prebin) uCast['gsw_srA0'] = out ##xmiss: xmis25=1.14099414691*xmis19+-1.6910134322 #downcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[pS:ip]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - xmisDZ # down z inV = 1.14099414691 * cast19[nn].df.loc[(pS + ilag):( ip + ilag)]['CStarTr0'].values - 1.6910134322 # down var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) dCast['xmiss'] = out #upcast inP = -1 * gsw.z_from_p( cast25[nn].df.loc[ip:pE]['prSM'].values, df0.loc[df0.Station == nn]['LatDecDeg']) - xmisDZ # up p inV = 1.14099414691 * cast19[nn].df.loc[(ip + ilag):( pE + ilag)]['CStarTr0'].values - 1.6910134322 # up var if sel == 'wide': inV[inP < .1] = np.nan p, out = bindepth(inP, inV, edges=edges, targets=targets, prebin=prebin) uCast['xmiss'] = out uCast['wetStar'] = np.nan dCast['wetStar'] = np.nan uCast['sbeox0ML_L'] = np.nan dCast['sbeox0ML_L'] = np.nan zCasts[nn] = zCast(uCast, dCast) return df0, zCasts
def pressure_to_depth(p, lat): """Wrapper function to convert from ocean pressure to depth. """ return -gsw.z_from_p(p, lat)