def kernel_matrix(X, kernel, n1, n2): (n, d) = X.shape assert n == n1 + n2 K = mat.zeros((n,n)) for i in xrange(n): for j in xrange(i+1): K[i,j] = kernel(X[i,:], X[j,:]) K[j,i] = K[i,j] U1 = mat.sum(K[0:n1,:],0) / n1 U2 = mat.sum(K[n1:n,:],0) / n2 U1m = mat.tile(U1, (n1,1)) U2m = mat.tile(U2, (n2,1)) U = mat.bmat('U1m; U2m') m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1*n1) m1m2 = mat.sum(K[0:n1, n1:n]) / (n1*n2) m2m2 = mat.sum(K[n1:n, n1:n]) / (n2*n2) mumu = mat.zeros((n,n)) mumu[0:n1, 0:n1] = m1m1 mumu[0:n1, n1:n] = m1m2 mumu[n1:n, 0:n1] = m1m2 mumu[n1:n, n1:n] = m2m2 Kcu = K - U Kuc = Kcu.T N = mat.ones((n,n))/n Kc = K - U - U.T + mumu return (K, Kuc, Kc)
def kernel_matrix(X, kernel, n1, n2): (n, d) = X.shape assert n == n1 + n2 K = mat.zeros((n, n)) for i in xrange(n): for j in xrange(i + 1): K[i, j] = kernel(X[i, :], X[j, :]) K[j, i] = K[i, j] U1 = mat.sum(K[0:n1, :], 0) / n1 U2 = mat.sum(K[n1:n, :], 0) / n2 U1m = mat.tile(U1, (n1, 1)) U2m = mat.tile(U2, (n2, 1)) U = mat.bmat('U1m; U2m') m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1 * n1) m1m2 = mat.sum(K[0:n1, n1:n]) / (n1 * n2) m2m2 = mat.sum(K[n1:n, n1:n]) / (n2 * n2) mumu = mat.zeros((n, n)) mumu[0:n1, 0:n1] = m1m1 mumu[0:n1, n1:n] = m1m2 mumu[n1:n, 0:n1] = m1m2 mumu[n1:n, n1:n] = m2m2 Kcu = K - U Kuc = Kcu.T N = mat.ones((n, n)) / n Kc = K - U - U.T + mumu return (K, Kuc, Kc)
def nk_bhatta(X1, X2, eta): # Returns the non-kernelized Bhattacharrya #I.e. fits normal distributions in input space and calculates Bhattacharrya overlap between them (n1, d1) = X1.shape (n2, d ) = X2.shape assert d1 == d mu1 = mat.sum(X1,0) / n1 mu2 = mat.sum(X2,0) / n2 X1c = X1 - mat.tile(mu1, (n1,1)) X2c = X2 - mat.tile(mu2, (n2,1)) Eta = mat.eye(d) * eta S1 = X1c.T * X1c / n1 + Eta S2 = X2c.T * X2c / n2 + Eta mu3 = .5 * (S1.I * mu1.T + S2.I * mu2.T).T S3 = 2 * (S1.I + S2.I).I d1 = la.det(S1) ** -.25 d2 = la.det(S2) ** -.25 d3 = la.det(S3) ** .5 dterm = d1 * d2 * d3 e1 = -.25 * mu1 * S1.I * mu1.T e2 = -.25 * mu2 * S2.I * mu2.T e3 = .5 * mu3 * S3 * mu3.T eterm = math.exp(e1 + e2 + e3) return float(dterm * eterm)
def nk_bhatta(X1, X2, eta): # Returns the non-kernelized Bhattacharrya #I.e. fits normal distributions in input space and calculates Bhattacharrya overlap between them (n1, d1) = X1.shape (n2, d) = X2.shape assert d1 == d mu1 = mat.sum(X1, 0) / n1 mu2 = mat.sum(X2, 0) / n2 X1c = X1 - mat.tile(mu1, (n1, 1)) X2c = X2 - mat.tile(mu2, (n2, 1)) Eta = mat.eye(d) * eta S1 = X1c.T * X1c / n1 + Eta S2 = X2c.T * X2c / n2 + Eta mu3 = .5 * (S1.I * mu1.T + S2.I * mu2.T).T S3 = 2 * (S1.I + S2.I).I d1 = la.det(S1)**-.25 d2 = la.det(S2)**-.25 d3 = la.det(S3)**.5 dterm = d1 * d2 * d3 e1 = -.25 * mu1 * S1.I * mu1.T e2 = -.25 * mu2 * S2.I * mu2.T e3 = .5 * mu3 * S3 * mu3.T eterm = math.exp(e1 + e2 + e3) return float(dterm * eterm)
def verify_kernel_matrix(): n1 = 10 n2 = 10 n = n1+n2 d = 5 degree = 3 X = randn(n,d) Phi = poly.phi(X, degree) (K, Kuc, Kc) = kernel_matrix(X, polyk(degree), n1, n2) P1 = Phi[0:n1,:] P2 = Phi[n1:n,:] mu1 = mat.sum(P1,0) / n1 mu2 = mat.sum(P2,0) / n2 P1c = P1 - mat.tile(mu1, (n1,1)) P2c = P2 - mat.tile(mu2, (n2,1)) Pc = bmat('P1c; P2c') KP = mat.zeros((n,n)) for i in xrange(n): for j in xrange(i+1): KP[i,j] = dotp(Phi[i,:], Phi[j,:]) KP[j,i] = KP[i,j] KucP = mat.zeros((n,n)) for i in xrange(n): for j in xrange(n): KucP[i,j] = dotp(Phi[i,:], Pc[j,:]) KcP = mat.zeros((n,n)) for i in xrange(n): for j in xrange(n): KcP[i,j] = dotp(Pc[i,:], Pc[j,:]) #KcP[j,i] = KcP[i,j] #debug() print "Div1: " + str(sum(abs(K-KP))) print "Div2: " + str(sum(abs(Kuc-KucP))) print "Div3: " + str(sum(abs(Kc-KcP)))
def kernel_supermatrix(self, i, j): kernel = self.kernel D1 = self.datasets[i] D2 = self.datasets[j] X1 = D1.X X2 = D2.X (n1, d) = X1.shape (n2, d) = X2.shape n = n1 + n2 X = mat.bmat('X1; X2') K1 = D1.K K2 = D2.K K = mat.zeros((n,n)) K[0:n1, 0:n1] = K1 K[n1:n, n1:n] = K2 for i in xrange(n1): for j in xrange(n1, n): K[i,j] = kernel(X[i,:], X[j,:]) K[j,i] = K[i,j] # Inelegant - improve later U1 = mat.sum(K[0:n1,:],0) / n1 U2 = mat.sum(K[n1:n,:],0) / n2 U1m = mat.tile(U1, (n1,1)) U2m = mat.tile(U2, (n2,1)) U = mat.bmat('U1m; U2m') m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1*n1) m1m2 = mat.sum(K[0:n1, n1:n]) / (n1*n2) m2m2 = mat.sum(K[n1:n, n1:n]) / (n2*n2) mumu = mat.zeros((n,n)) mumu[0:n1, 0:n1] = m1m1 mumu[0:n1, n1:n] = m1m2 mumu[n1:n, 0:n1] = m1m2 mumu[n1:n, n1:n] = m2m2 Kcu = K - U Kuc = Kcu.T N = mat.ones((n,n))/n Kc = K - U - U.T + mumu return (K, Kuc, Kc)
def kernel_supermatrix(self, i, j): kernel = self.kernel D1 = self.datasets[i] D2 = self.datasets[j] X1 = D1.X X2 = D2.X (n1, d) = X1.shape (n2, d) = X2.shape n = n1 + n2 X = mat.bmat('X1; X2') K1 = D1.K K2 = D2.K K = mat.zeros((n, n)) K[0:n1, 0:n1] = K1 K[n1:n, n1:n] = K2 for i in xrange(n1): for j in xrange(n1, n): K[i, j] = kernel(X[i, :], X[j, :]) K[j, i] = K[i, j] # Inelegant - improve later U1 = mat.sum(K[0:n1, :], 0) / n1 U2 = mat.sum(K[n1:n, :], 0) / n2 U1m = mat.tile(U1, (n1, 1)) U2m = mat.tile(U2, (n2, 1)) U = mat.bmat('U1m; U2m') m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1 * n1) m1m2 = mat.sum(K[0:n1, n1:n]) / (n1 * n2) m2m2 = mat.sum(K[n1:n, n1:n]) / (n2 * n2) mumu = mat.zeros((n, n)) mumu[0:n1, 0:n1] = m1m1 mumu[0:n1, n1:n] = m1m2 mumu[n1:n, 0:n1] = m1m2 mumu[n1:n, n1:n] = m2m2 Kcu = K - U Kuc = Kcu.T N = mat.ones((n, n)) / n Kc = K - U - U.T + mumu return (K, Kuc, Kc)
def classify0(in_x, data_set, labels, k): data_set_size = data_set.shape[0] diff_mat = tile(in_x, (data_set_size, 1)) - data_set sq_diff_mat = diff_mat**2 sq_distances = sq_diff_mat.sum(axis=1) distances = sq_distances**0.5 sorted_dist_indicies = distances.argsort() class_count = {} for i in range(k): vote_ilabel = labels[sorted_dist_indicies[i]] class_count[vote_ilabel] = class_count.get(vote_ilabel, 0) + 1 sorted_class_count = sorted(class_count.items(), key=operator.itemgetter(1), reverse=True) return sorted_class_count[0][0]
def demean(x, dim=0): # DEMEAN(X) # Removes the Average or mean value. # # DEMEAN(X,DIM) # Removes the mean along the dimension DIM of X. # if (dim == -1): # dim = 0 # if (x.shape[0] > 1): # dim = 0; # elif (x.shape[1] > 1): # dim = 1; dims = x.size # dimsize = x.shape[dim] # dimrep = np.ones(1,len(dims)); # dimrep[dim] = dimsize; return x - mtlib.tile(np.mean(x, dim), dims) # repmat(np.mean(x,dim),dimrep)
def test_bsp_light_dark(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--no-plotting',action='store_true',default=False) parser.add_argument('--profile',action='store_true',default=False) parser.add_argument('--gradient_free',action='store_true',default=False) args = parser.parse_args() plotting = not args.no_plotting profile = args.profile gradient_free = args.gradient_free model = LightDarkModel() X1 = ml.matrix([[-3.5,2],[-3.5,-2],[-4,0],[2,2],[-4,2]]).T G = ml.matrix([[-3.5,-2],[-3.5,2],[-1,0],[2,-2],[-1,-2]]).T # [Reference] final belief trajectory costs for verification # Allow for numerical inaccuracy verify_cost = [45.181701, 45.181643, 49.430339, 27.687003, 56.720314] for i_problem in xrange(0,5): # Setup initial conditions for problem x1 = X1[:,i_problem] # start (mean of initial belief) SqrtSigma1 = ml.eye(2) # initial covariance goal = G[:,i_problem] # goal model.setStartState(x1) model.setGoalState(goal) # setup initial control vector -- straight line initialization from start to goal U = ml.tile(((model.goal - model.start)/(model.T-1)), (1,model.T-1)) B = ml.zeros([model.bDim,model.T]) B[:,0] = belief.compose_belief(x1, SqrtSigma1, model) for t in xrange(0,model.T-1): B[:,t+1] = belief.belief_dynamics(B[:,t], U[:,t], None, model,None,None) # display initialization if plotting: plot.plot_belief_trajectory(B, U, model) """ if gradient_free : [Bopt, Uopt] = belief_grad_free.STOMP_BSP(B,model,plotting,profile) else: [Bopt, Uopt] = belief_opt.belief_opt_penalty_sqp(B, U, model, plotting, profile) """ if plotting: plot.plot_belief_trajectory(Bopt, Uopt, model); # Forward simulated cost cost = belief.compute_forward_simulated_cost(B[:,0], Uopt, model) print('Total cost of optimized trajectory: %f' % cost) # save trajectory to png file # saveas(gcf, sprintf('bsp-light-dark-plan-%i.png',i_problem)); print('For verification (allow for numerical inaccuracy):') print('(Reference) Total cost of optimized trajectory: %2.6f' % verify_cost[i_problem]) # Simulate execution of trajectory (test to see why the maximum likelihood observation assumption does not hold) #simulate_bsp_trajectory(B(:,1), Uopt, model); print('press enter to continue to the next problem') raw_input()
def main(grid, mat_file, save_dir, user_attributes, flags=None, domain=[], method='oi'): """ Convert MAT files created using the hfrProgs MATLAB toolbox into CF-1.6/NCEI Grid 2.0 compliant netCDF4 files :param grid: CSV file containing lon,lat grid information :param mat_file: Filepath to MAT file containing HFRProgs :param save_dir: Directory to save netCDF files to :param user_attributes: User defined dataset attributes for netCDF global attribute. Required for CF/NCEI compliance :param flags: Dictionary of thresholds at which we should filter data above :param method: 'oi' or 'lsq'. OI is optimal interpolation. LSQ is unweighted least squares """ fname = os.path.basename(mat_file) try: # load .mat file data = loadmat(mat_file, squeeze_me=True, struct_as_record=False) logging.debug('{} - MAT file successfully loaded '.format(fname)) except Exception as err: logging.error('{} - {}. MAT file could not be loaded.'.format( fname, err)) return if not domain: domain = data['TUV'].DomainName if not domain: domain = 'MARA' else: domain = 'MARA' time = timestamp_from_lluv_filename(mat_file) # convert matlab time to python datetime # time = dt.datetime.strptime(mat_time, '%Y_%m_%d_%H00') time_index = pd.date_range( time.strftime('%Y-%m-%d %H:%M:%S'), periods=1) # create pandas datetimeindex from time time_string = time.strftime( '%Y%m%dT%H%M%SZ') # create timestring from time file_name = 'RU_{}_{}.nc'.format(domain, time_string) file_and_path = os.path.join(save_dir, file_name) try: logging.debug('{} - Saving file data to variables'.format(fname)) # load longitude and latitude data associated with variables lonlat = data['TUV'].LonLat.astype(np.float32) # create variables for eastward and northward velocities u = data['TUV'].U.astype(np.float32) v = data['TUV'].V.astype(np.float32) u_units = data['TUV'].UUnits v_units = data['TUV'].VUnits maxspd = data['TUV'].OtherMetadata.cleanTotals.maxspd if method == 'oi': # create variables for associated error values u_err = data['TUV'].ErrorEstimates.Uerr.astype(np.float32) v_err = data['TUV'].ErrorEstimates.Verr.astype(np.float32) uv_covariance = data['TUV'].ErrorEstimates.UVCovariance # Data Processing Information num_rads = data[ 'TUV'].OtherMatrixVars.makeTotalsOI_TotalsNumRads.astype(int) min_rads = data[ 'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumRads min_sites = data[ 'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumSites mdlvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.mdlvar errvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.errvar sx = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sx sy = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sy temporal_threshold = data[ 'TUV'].OtherMetadata.makeTotalsOI.parameters.tempthresh processing_parameters = [ maxspd, min_sites, min_rads, temporal_threshold, sx, sy, mdlvar, errvar ] processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n' processing_parameters_info += '2) Minimum number of radial sites\n' processing_parameters_info += '3) Minimum number of radial vectors\n' processing_parameters_info += '4) Temporal search window for radial solutions (Fraction of a day)\n' processing_parameters_info += '5) Decorrelation scales in the north direction\n' processing_parameters_info += '6) Decorrelation scales in the east direction\n' processing_parameters_info += '7) Signal variance of the surface current fields (cm2 s-2)\n' processing_parameters_info += '8) Data error variance of the input radial velocities (cm2 s-2)\n' elif method == 'lsq': # create variables for associated error values u_err = data['TUV'].ErrorEstimates[1].Uerr.astype(np.float32) v_err = data['TUV'].ErrorEstimates[1].Verr.astype(np.float32) uv_covariance = data['TUV'].ErrorEstimates[1].UVCovariance.astype( np.float32) # Data Processing Information num_rads = data[ 'TUV'].OtherMatrixVars.makeTotals_TotalsNumRads.astype(int) min_rads = data[ 'TUV'].OtherMetadata.makeTotals.parameters.MinNumRads min_sites = data[ 'TUV'].OtherMetadata.makeTotals.parameters.MinNumSites spatial_threshold = data[ 'TUV'].OtherMetadata.makeTotals.parameters.spatthresh temporal_threshold = data[ 'TUV'].OtherMetadata.makeTotals.parameters.tempthresh processing_parameters = [ maxspd, min_sites, min_rads, temporal_threshold, spatial_threshold ] processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n' processing_parameters_info += '2) Minimum number of radial sites.\n' processing_parameters_info += '3) Minimum number of radial vectors.\n' processing_parameters_info += '4) Temporal search window for radial solutions (Fractions of a day)\n' processing_parameters_info += '5) Spatial search radius for radial solutions (km)\n' except AttributeError as err: logging.error( '{} - {}. MAT file missing variable needed to create netCDF4 file'. format(fname, err)) return # Create a grid to shape 1d data lon = np.unique(grid['lon'].values.astype(np.float32)) lat = np.unique(grid['lat'].values.astype(np.float32)) [x, y] = np.meshgrid(lon, lat) # Create a dictionary of variables that we want to grid data_dict = dict( u=u, v=v, u_err=u_err, v_err=v_err, uv_covariance=uv_covariance, num_radials=num_rads, ) logging.debug('{} - Gridding data to 2d grid'.format(fname)) # convert 1d data into 2d gridded form. data_dict must be a dictionary. x_ind, y_ind = gridded_index(x, y, lonlat[:, 0], lonlat[:, 1]) for key in data_dict.keys(): temp_data = mb.tile(np.nan, x.shape) temp_data[(y_ind, x_ind)] = data_dict[key] # expand dimensions for time and depth count = 0 while count < 2: # add two dimensions to from of array for time and z (depth) temp_data = np.expand_dims(temp_data, axis=0) count = count + 1 data_dict[key] = temp_data logging.debug('{} - Loading data into xarray dataset'.format(fname)) # initialize xarray dataset. Add variables. Add coordinates ds = xr.Dataset() coords = ('time', 'z', 'lat', 'lon') ds['u'] = (coords, np.float32(data_dict['u'])) ds['v'] = (coords, np.float32(data_dict['v'])) ds['u_err'] = (coords, np.float32(data_dict['u_err'])) ds['v_err'] = (coords, np.float32(data_dict['v_err'])) ds['uv_covariance'] = (coords, np.float32(data_dict['uv_covariance'])) ds['num_radials'] = (coords, data_dict['num_radials']) ds.coords['lon'] = lon ds.coords['lat'] = lat ds.coords['z'] = np.array([np.float32(0)]) ds.coords['time'] = time_index if flags: for k, v in flags.items(): ds = ds.where(ds[k] <= v) ds['processing_parameters'] = (('parameters'), processing_parameters) # Grab min and max time in dataset for entry into global attributes for cf compliance time_start = ds['time'].min().data time_end = ds['time'].max().data global_attributes = configs.netcdf_global_attributes( user_attributes, time_start, time_end) global_attributes['geospatial_lat_min'] = lat.min() global_attributes['geospatial_lat_max'] = lat.max() global_attributes['geospatial_lon_min'] = lon.min() global_attributes['geospatial_lon_max'] = lon.max() if method == 'oi': global_attributes['method'] = 'Optimal Interpolation' elif method == 'lsq': global_attributes['method'] = 'Unweighted Least Squares' logging.debug('{} - Assigning global attributes to dataset'.format(fname)) ds = ds.assign_attrs(global_attributes) logging.debug( '{} - Assigning local attributes to each variable in dataset'.format( fname)) # set time attribute ds['time'].attrs['standard_name'] = 'time' # Set lon attributes ds['lon'].attrs['long_name'] = 'Longitude' ds['lon'].attrs['standard_name'] = 'longitude' ds['lon'].attrs['short_name'] = 'lon' ds['lon'].attrs['units'] = 'degrees_east' ds['lon'].attrs['axis'] = 'X' ds['lon'].attrs['valid_min'] = np.float32(-180.0) ds['lon'].attrs['valid_max'] = np.float32(180.0) # Set lat attributes ds['lat'].attrs['long_name'] = 'Latitude' ds['lat'].attrs['standard_name'] = 'latitude' ds['lat'].attrs['short_name'] = 'lat' ds['lat'].attrs['units'] = 'degrees_north' ds['lat'].attrs['axis'] = 'Y' ds['lat'].attrs['valid_min'] = np.float32(-90.0) ds['lat'].attrs['valid_max'] = np.float32(90.0) # Set depth attributes ds['z'].attrs['long_name'] = 'Average Depth of Sensor' ds['z'].attrs['standard_name'] = 'depth' ds['z'].attrs['comment'] = 'Derived from mean value of depth variable' ds['z'].attrs['units'] = 'm' ds['z'].attrs['axis'] = 'Z' ds['z'].attrs['positive'] = 'down' # Set u attributes ds['u'].attrs['long_name'] = 'Eastward Surface Current (cm/s)' ds['u'].attrs['standard_name'] = 'surface_eastward_sea_water_velocity' ds['u'].attrs['short_name'] = 'u' ds['u'].attrs['units'] = u_units ds['u'].attrs['valid_min'] = np.float32(-300) ds['u'].attrs['valid_max'] = np.float32(300) ds['u'].attrs['coordinates'] = 'lon lat' ds['u'].attrs['grid_mapping'] = 'crs' # Set v attributes ds['v'].attrs['long_name'] = 'Northward Surface Current (cm/s)' ds['v'].attrs['standard_name'] = 'surface_northward_sea_water_velocity' ds['v'].attrs['short_name'] = 'v' ds['v'].attrs['units'] = v_units ds['v'].attrs['valid_min'] = np.float32(-300) ds['v'].attrs['valid_max'] = np.float32(300) ds['v'].attrs['coordinates'] = 'lon lat' ds['v'].attrs['grid_mapping'] = 'crs' # Set u_err attributes ds['u_err'].attrs['units'] = '1' ds['u_err'].attrs['valid_min'] = np.float32(0) ds['u_err'].attrs['valid_max'] = np.float32(1) ds['u_err'].attrs['coordinates'] = 'lon lat' ds['u_err'].attrs['grid_mapping'] = 'crs' # Set v_err attributes ds['v_err'].attrs['units'] = '1' ds['v_err'].attrs['valid_min'] = np.float32(0) ds['v_err'].attrs['valid_max'] = np.float32(1) ds['v_err'].attrs['coordinates'] = 'lon lat' ds['v_err'].attrs['grid_mapping'] = 'crs' if method == 'lsq': ds['u_err'].attrs[ 'long_name'] = 'Associated GDOP mapping error value associated with eastward velocity component' ds['v_err'].attrs[ 'long_name'] = 'Associated GDOP mapping error value associated with northward velocity component' ds['u_err'].attrs[ 'comment'] = 'velocity measurements with error values over 1.5 are of questionable quality' ds['v_err'].attrs[ 'comment'] = 'velocity measurements with error values over 1.5 are of questionable quality' elif method == 'oi': ds['u_err'].attrs[ 'long_name'] = 'Normalized uncertainty error associated with eastward velocity component' ds['v_err'].attrs[ 'long_name'] = 'Normalized uncertainty error associated with northward velocity component' ds['u_err'].attrs[ 'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality' ds['v_err'].attrs[ 'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality' # Set uv_covariance attributes ds['uv_covariance'].attrs[ 'long_name'] = 'Eastward and Northward covariance directional information of u and v' ds['uv_covariance'].attrs['units'] = '1' ds['uv_covariance'].attrs['comment'] = 'directional information of u and v' ds['uv_covariance'].attrs['coordinates'] = 'lon lat' ds['uv_covariance'].attrs['grid_mapping'] = 'crs' # Set num_radials attributes ds['num_radials'].attrs[ 'long_name'] = 'Number of radial measurements used to calculate each totals velocity' ds['num_radials'].attrs[ 'comment'] = 'totals are not calculated with fewer than 3 contributing radial measurements from 2 sites' ds['num_radials'].attrs['coordinates'] = 'lon lat' ds['num_radials'].attrs['grid_mapping'] = 'crs' # Set num_radials attributes ds['processing_parameters'].attrs[ 'long_name'] = 'General and method specific processing parameter information' ds['processing_parameters'].attrs['comment'] = processing_parameters_info # ds['processing_parameters'].attrs['coordinates'] = 'parameters' # encoded_sites = data['TUV'].OtherMatrixVars.makeTotalsOI_TotalsSiteCode # # load site ids that are set in our mysqldb # query_obj = Session.query(tables.Sites) # site_encoding = pd.read_sql(query_obj.statement, query_obj.session.bind) # # # convert site codes into binary numbers # binary_positions_mat = data['conf'].Radials.Sites.shape[0] # # decoded_sites_mat = np.tile(0, (encoded_sites.shape[0], binary_positions_mat)) # # for i, v in enumerate(encoded_sites): # decoded_sites_mat[i] = np.array(map(int, np.binary_repr(v, width=binary_positions_mat))) # # decoded_sites_mat = np.fliplr(decoded_sites_mat) # # decoded_sites_new = np.tile(0, (encoded_sites.shape[0], np.max(site_encoding['id']))) # # for site in data['RTUV']: # print site.SiteName + ' ' + str(np.log2(site.SiteCode)) # ind_mat = np.log2(site.SiteCode) # ind_real = site_encoding['id'].loc[site_encoding['site'] == site.SiteName].values[0] # decoded_sites_new[:, ind_real] = decoded_sites_mat[:, int(ind_mat)] # # decoded_sites_new = np.fliplr(decoded_sites_new) # new_encoded_sites = [bool2int(x[::-1]) for x in decoded_sites_new] # flag_masks = [2 ** int(x) for x in site_encoding['id'].tolist()] # flag_meanings = ' '.join(site_encoding['site'].tolist()) # ds['site_code_flags'].attrs['long_name'] = 'Bitwise AND representation of site contributions to a radial point' # # ds['site_code_flags'].attrs['_FillValue'] = int(0) # ds['site_code_flags'].attrs['flag_masks'] = 'b '.join(map(str, flag_masks)) # ds['site_code_flags'].attrs['flag_meanings'] = flag_meanings # ds['site_code_flags'].attrs['comment'] = 'Values are binary sums. Must be converted to binary representation to interpret flag_masks and flag_meanings' logging.debug( '{} - Setting variable encoding and fill values for netCDF4 output'. format(fname)) # encode variables for export to netcdf encoding = make_encoding(ds) encoding['lon'] = dict(zlib=False, _FillValue=False) encoding['lat'] = dict(zlib=False, _FillValue=False) encoding['z'] = dict(zlib=False, _FillValue=False) # add container variables that contain no data kwargs = dict(crs=None, instrument=None) ds = ds.assign(**kwargs) # Set crs attributes ds['crs'].attrs['grid_mapping_name'] = 'latitude_longitude' ds['crs'].attrs['inverse_flattening'] = 298.257223563 ds['crs'].attrs['long_name'] = 'Coordinate Reference System' ds['crs'].attrs['semi_major_axis'] = '6378137.0' ds['crs'].attrs['epsg_code'] = 'EPSG:4326' ds['crs'].attrs['comment'] = 'http://www.opengis.net/def/crs/EPSG/0/4326' ds['instrument'].attrs['long_name'] = 'CODAR SeaSonde High Frequency Radar' ds['instrument'].attrs[ 'sensor_type'] = 'Direction-finding high frequency radar antenna' ds['instrument'].attrs['make_model'] = 'CODAR SeaSonde' ds['instrument'].attrs['serial_number'] = 1 # Create save directory if it doesn't exist. create_dir(save_dir) logging.debug('{} - Saving dataset to netCDF4 file: {}'.format( fname, file_and_path)) ds.to_netcdf(file_and_path, encoding=encoding, format='netCDF4', engine='netcdf4', unlimited_dims=['time']) logging.info('{} - netCDF4 file successfully created: {}'.format( fname, file_and_path))
def test_bsp_light_dark(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--no-plotting',action='store_true',default=False) parser.add_argument('--profile',action='store_true',default=False) parser.add_argument('--gradient_free',action='store_true',default=False) args = parser.parse_args() plotting = not args.no_plotting profile = args.profile gradient_free = args.gradient_free model = LightDarkModel() X1 = ml.matrix([[-3.5,2],[-3.5,-2],[-4,0],[2,2],[-4,2]]).T G = ml.matrix([[-3.5,-2],[-3.5,2],[-1,0],[2,-2],[-1,-2]]).T # [Reference] final belief trajectory costs for verification # Allow for numerical inaccuracy verify_cost = [45.181701, 45.181643, 49.430339, 27.687003, 56.720314] for i_problem in xrange(0,5): # Setup initial conditions for problem x1 = X1[:,i_problem] # start (mean of initial belief) SqrtSigma1 = ml.eye(2) # initial covariance goal = G[:,i_problem] # goal model.setStartState(x1) model.setGoalState(goal) # setup initial control vector -- straight line initialization from start to goal U = ml.tile(((model.goal - model.start)/(model.T-1)), (1,model.T-1)) B = ml.zeros([model.bDim,model.T]) B[:,0] = belief.compose_belief(x1, SqrtSigma1, model) for t in xrange(0,model.T-1): B[:,t+1] = belief.belief_dynamics(B[:,t], U[:,t], None, model,None,None) # display initialization if plotting: plot.plot_belief_trajectory(B, U, model) if gradient_free : [Bopt, Uopt] = belief_grad_free.STOMP_BSP(B,model,plotting,profile) else: [Bopt, Uopt] = belief_opt.belief_opt_penalty_sqp(B, U, model, plotting, profile) if plotting: plot.plot_belief_trajectory(Bopt, Uopt, model); # Forward simulated cost cost = belief.compute_forward_simulated_cost(B[:,0], Uopt, model) print('Total cost of optimized trajectory: %f' % cost) # save trajectory to png file # saveas(gcf, sprintf('bsp-light-dark-plan-%i.png',i_problem)); print('For verification (allow for numerical inaccuracy):') print('(Reference) Total cost of optimized trajectory: %2.6f' % verify_cost[i_problem]) # Simulate execution of trajectory (test to see why the maximum likelihood observation assumption does not hold) #simulate_bsp_trajectory(B(:,1), Uopt, model); print('press enter to continue to the next problem') raw_input()
def main(grid, mat_file, save_dir, user_attributes, flags=None, domain=[], method='oi'): """ Convert MAT files created using the hfrProgs MATLAB toolbox into CF-1.6/NCEI Grid 2.0 compliant netCDF4 files :param grid: CSV file containing lon,lat grid information :param mat_file: Filepath to MAT file containing HFRProgs :param save_dir: Directory to save netCDF files to :param user_attributes: User defined dataset attributes for netCDF global attribute. Required for CF/NCEI compliance :param flags: Dictionary of thresholds at which we should filter data above :param method: 'oi' or 'lsq'. OI is optimal interpolation. LSQ is unweighted least squares """ fname = os.path.basename(mat_file) try: # load .mat file data = loadmat(mat_file, squeeze_me=True, struct_as_record=False) logging.debug('{} - MAT file successfully loaded '.format(fname)) except Exception as err: logging.error('{} - {}. MAT file could not be loaded.'.format( fname, err)) #return if not domain: domain = data['TUV'].DomainName if not domain: domain = 'MARA' else: domain = 'MARA' time = timestamp_from_lluv_filename(mat_file) # convert matlab time to python datetime # time = dt.datetime.strptime(mat_time, '%Y_%m_%d_%H00') time_index = pd.date_range( time.strftime('%Y-%m-%d %H:%M:%S'), periods=1) # create pandas datetimeindex from time time_string = time.strftime( '%Y%m%dT%H%M%SZ') # create timestring from time file_name = 'RU_{}_{}.nc'.format(domain, time_string) file_and_path = os.path.join(save_dir, file_name) try: logging.debug('{} - Saving file data to variables'.format(fname)) # load longitude and latitude data associated with variables lonlat = data['TUV'].LonLat.astype(np.float32) # create variables for eastward and northward velocities u = data['TUV'].U.astype(np.float32) v = data['TUV'].V.astype(np.float32) u_units = data['TUV'].UUnits v_units = data['TUV'].VUnits #maxspd = data['TUV'].OtherMetadata.cleanTotals.maxspd maxspd = data['conf'].Totals.MaxTotSpeed if method == 'oi': # create variables for associated error values u_err = data['TUV'].ErrorEstimates.Uerr.astype(np.float32) v_err = data['TUV'].ErrorEstimates.Verr.astype(np.float32) uv_covariance = data['TUV'].ErrorEstimates.UVCovariance.astype( np.float32) total_errors = data['TUV'].ErrorEstimates[1].TotalErrors.astype( np.float32) # Data Processing Information num_rads = data[ 'TUV'].OtherMatrixVars.makeTotalsOI_TotalsNumRads.astype(int) min_rads = data[ 'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumRads min_sites = data[ 'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumSites mdlvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.mdlvar errvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.errvar sx = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sx sy = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sy temporal_threshold = data[ 'TUV'].OtherMetadata.makeTotalsOI.parameters.tempthresh #processing_parameters = [maxspd, min_sites, min_rads, temporal_threshold, sx, sy, mdlvar, errvar] processing_parameters = [ min_sites, min_rads, temporal_threshold, sx, sy, mdlvar, errvar ] #processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n' processing_parameters_info = '1) Minimum number of radial sites\n' processing_parameters_info += '2) Minimum number of radial vectors\n' processing_parameters_info += '3) Temporal search window for radial solutions (Fraction of a day)\n' processing_parameters_info += '4) Decorrelation scales in the north direction\n' processing_parameters_info += '5) Decorrelation scales in the east direction\n' processing_parameters_info += '6) Signal variance of the surface current fields (cm2 s-2)\n' processing_parameters_info += '7) Data error variance of the input radial velocities (cm2 s-2)\n' #QC Information uerr_testname = data['conf'].Totals.OI.cleanTotalsVarargin[0][ 0] + ' ' + data['conf'].Totals.OI.cleanTotalsVarargin[0][1] uerr_threshold = data['conf'].Totals.OI.cleanTotalsVarargin[0][2] verr_testname = data['conf'].Totals.OI.cleanTotalsVarargin[1][ 0] + ' ' + data['conf'].Totals.OI.cleanTotalsVarargin[1][1] verr_threshold = data['conf'].Totals.OI.cleanTotalsVarargin[1][2] qc_info = 'Quality control reference: IOOS QARTOD HF Radar ver 1.0 May 2016\n' qc_info += 'QCFlagDefinitions: 1 = pass, 2 = not_evaluated, 3 = suspect, 4 = fail, 9 = missing_data\n' qc_primary_flag_info = 'QCPrimaryFlagDefinition: Highest flag value of QC16, QC18, QC19, QC20\n' qc_primary_flag_info += 'This flag will be set to not_evaluated only if ALL individual tests were not_evaluated.' qc_operator_mask_info = qc_info + 'The qc_operator_mask follows QCFlagDefinitions and is set at discretion of the operator or data manager.' qc16_info = qc_info + 'QC16 Max Speed Threshold [max_vel = ' + str( maxspd) + ' (cm/s)]' qc18_info = qc_info + 'QC18 Valid Location [landmask file = ' + data[ 'conf'].Totals.MaskFile + ']' qc19_info = qc_info + 'QC19 OI Uncertainty Threshold [' + uerr_testname + ' ' + str( uerr_threshold) + ']' qc20_info = qc_info + 'QC20 OI Uncertainty Threshold [' + verr_testname + ' ' + str( verr_threshold) + ']' qc16 = data['TUVqc'].QC16.astype(np.int32) qc18 = data['TUVqc'].QC18.astype(np.int32) qc19 = data['TUVqc'].QC19.astype(np.int32) qc20 = data['TUVqc'].QC20.astype(np.int32) qc_primary_flag = data['TUVqc'].PRIM.astype(np.int32) qc_operator_mask = data['TUVqc'].qc_operator_mask.astype(np.int32) elif method == 'lsq': # create variables for associated error values u_err = data['TUV'].ErrorEstimates[1].Uerr.astype(np.float32) v_err = data['TUV'].ErrorEstimates[1].Verr.astype(np.float32) uv_covariance = data['TUV'].ErrorEstimates[1].UVCovariance.astype( np.float32) total_errors = data['TUV'].ErrorEstimates[1].TotalErrors.astype( np.float32) # Data Processing Information num_rads = data[ 'TUV'].OtherMatrixVars.makeTotals_TotalsNumRads.astype(int) min_rads = data[ 'TUV'].OtherMetadata.makeTotals.parameters.MinNumRads min_sites = data[ 'TUV'].OtherMetadata.makeTotals.parameters.MinNumSites spatial_threshold = data[ 'TUV'].OtherMetadata.makeTotals.parameters.spatthresh temporal_threshold = data[ 'TUV'].OtherMetadata.makeTotals.parameters.tempthresh #processing_parameters = [maxspd, min_sites, min_rads, temporal_threshold, spatial_threshold] processing_parameters = [ min_sites, min_rads, temporal_threshold, spatial_threshold ] #processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n' processing_parameters_info = '1) Minimum number of radial sites.\n' processing_parameters_info += '2) Minimum number of radial vectors.\n' processing_parameters_info += '3) Temporal search window for radial solutions (Fractions of a day)\n' processing_parameters_info += '4) Spatial search radius for radial solutions (km)\n' #QC Information gdoptestname = data['conf'].Totals.cleanTotalsVarargin[ 0] + ' ' + data['conf'].Totals.cleanTotalsVarargin[1] gdopthreshold = data['conf'].Totals.cleanTotalsVarargin[2] qc_info = 'Quality control reference: IOOS QARTOD HF Radar ver 1.0 May 2016\n' qc_info += 'QCFlagDefinitions: 1 = pass, 2 = not_evaluated, 3 = suspect, 4 = fail, 9 = missing_data\n' qc_primary_flag_info = 'QCPrimaryFlagDefinition: Highest flag value of QC16, QC18, QC19, QC20\n' qc_primary_flag_info += 'This flag will be set to not_evaluated only if ALL tests were not_evaluated.' qc_operator_mask_info = qc_info + 'The qc_operator_mask follows QCFlagDefinitions and is set at discretion of the operator or data manager.' qc16_info = qc_info + 'QC16 Max Speed Threshold [max_vel = ' + str( maxspd) + ' (cm/s)]' qc18_info = qc_info + 'QC18 Valid Location [landmask file = ' + data[ 'conf'].Totals.MaskFile + ']' qc15_info = qc_info + 'QC15 GDOP Threshold [' + gdoptestname + ' ' + str( gdopthreshold) + ']' qc15 = data['TUVqc'].QC15.astype(np.int) qc16 = data['TUVqc'].QC16.astype(np.int) qc18 = data['TUVqc'].QC18.astype(np.int) qc_primary_flag = data['TUVqc'].PRIM.astype(np.int) qc_operator_mask = data['TUVqc'].qc_operator_mask.astype(np.int) except AttributeError as err: logging.error( '{} - {}. MAT file missing variable needed to create netCDF4 file'. format(fname, err)) #return # Create a grid to shape 1d data lon = np.unique(grid['lon'].values.astype(np.float32)) lat = np.unique(grid['lat'].values.astype(np.float32)) [x, y] = np.meshgrid(lon, lat) # Create a dictionary of variables that we want to grid if method == 'oi': data_dict = dict( u=u, v=v, u_err=u_err, v_err=v_err, uv_covariance=uv_covariance, total_errors=total_errors, num_radials=num_rads, qc16_maxspeed=qc16, qc18_validlocation=qc18, qc19_uerr=qc19, qc20_verr=qc20, qc_primary_flag=qc_primary_flag, qc_operator_mask=qc_operator_mask, ) elif method == 'lsq': data_dict = dict( u=u, v=v, u_err=u_err, v_err=v_err, uv_covariance=uv_covariance, total_errors=total_errors, num_radials=num_rads, qc15_total_errors=qc15, qc16_maxspeed=qc16, qc18_validlocation=qc18, qc_primary_flag=qc_primary_flag, qc_operator_mask=qc_operator_mask, ) logging.debug('{} - Gridding data to 2d grid'.format(fname)) # convert 1d data into 2d gridded form. data_dict must be a dictionary. x_ind, y_ind = gridded_index(x, y, lonlat[:, 0], lonlat[:, 1]) for key in data_dict.keys(): temp_data = mb.tile(np.nan, x.shape) temp_data[(y_ind, x_ind)] = data_dict[key] # expand dimensions for time and depth count = 0 while count < 2: # add two dimensions to from of array for time and z (depth) temp_data = np.expand_dims(temp_data, axis=0) count = count + 1 data_dict[key] = temp_data logging.debug('{} - Loading data into xarray dataset'.format(fname)) # initialize xarray dataset. Add variables. Add coordinates ds = xr.Dataset() coords = ('time', 'z', 'lat', 'lon') ds['u'] = (coords, np.float32(data_dict['u'])) ds['v'] = (coords, np.float32(data_dict['v'])) ds['u_err'] = (coords, np.float32(data_dict['u_err'])) ds['v_err'] = (coords, np.float32(data_dict['v_err'])) ds['uv_covariance'] = (coords, np.float32(data_dict['uv_covariance'])) ds['num_radials'] = (coords, data_dict['num_radials']) ds['qc16_maxspeed'] = (coords, np.int32(data_dict['qc16_maxspeed'])) ds['qc18_validlocation'] = (coords, np.int32(data_dict['qc18_validlocation'])) if method == 'oi': ds['qc19_uerr'] = (coords, np.int32(data_dict['qc19_uerr'])) ds['qc20_verr'] = (coords, np.int32(data_dict['qc20_verr'])) elif method == 'lsq': ds['qc15_gdop'] = (coords, np.int32(data_dict['qc15_gdop'])) ds['qc_primary_flag'] = (coords, np.int32(data_dict['qc_primary_flag'])) ds['qc_operator_mask'] = (coords, np.int32(data_dict['qc_operator_mask'])) ds.coords['lon'] = lon ds.coords['lat'] = lat ds.coords['z'] = np.array([np.float32(0)]) ds.coords['time'] = time_index if flags: for k, v in flags.items(): ds = ds.where(ds[k] <= v) ds['processing_parameters'] = (('parameters'), processing_parameters) # Grab min and max time in dataset for entry into global attributes for cf compliance time_start = ds['time'].min().data time_end = ds['time'].max().data global_attributes = configs.netcdf_global_attributes( user_attributes, time_start, time_end) global_attributes['geospatial_lat_min'] = lat.min() global_attributes['geospatial_lat_max'] = lat.max() global_attributes['geospatial_lon_min'] = lon.min() global_attributes['geospatial_lon_max'] = lon.max() if method == 'oi': global_attributes['method'] = 'Optimal Interpolation' elif method == 'lsq': global_attributes['method'] = 'Unweighted Least Squares' logging.debug('{} - Assigning global attributes to dataset'.format(fname)) ds = ds.assign_attrs(global_attributes) logging.debug( '{} - Assigning local attributes to each variable in dataset'.format( fname)) # set time attribute ds['time'].attrs['standard_name'] = 'time' # Set lon attributes ds['lon'].attrs['long_name'] = 'Longitude' ds['lon'].attrs['standard_name'] = 'longitude' ds['lon'].attrs['short_name'] = 'lon' ds['lon'].attrs['units'] = 'degrees_east' ds['lon'].attrs['axis'] = 'X' ds['lon'].attrs['valid_min'] = np.float32(-180.0) ds['lon'].attrs['valid_max'] = np.float32(180.0) # Set lat attributes ds['lat'].attrs['long_name'] = 'Latitude' ds['lat'].attrs['standard_name'] = 'latitude' ds['lat'].attrs['short_name'] = 'lat' ds['lat'].attrs['units'] = 'degrees_north' ds['lat'].attrs['axis'] = 'Y' ds['lat'].attrs['valid_min'] = np.float32(-90.0) ds['lat'].attrs['valid_max'] = np.float32(90.0) # Set depth attributes ds['z'].attrs['long_name'] = 'Average Depth of Sensor' ds['z'].attrs['standard_name'] = 'depth' ds['z'].attrs['comment'] = 'Derived from mean value of depth variable' ds['z'].attrs['units'] = 'm' ds['z'].attrs['axis'] = 'Z' ds['z'].attrs['positive'] = 'down' # Set u attributes ds['u'].attrs['long_name'] = 'Eastward Surface Current (cm/s)' ds['u'].attrs['standard_name'] = 'surface_eastward_sea_water_velocity' ds['u'].attrs['short_name'] = 'u' ds['u'].attrs['units'] = u_units ds['u'].attrs['valid_min'] = np.float32(-300) ds['u'].attrs['valid_max'] = np.float32(300) ds['u'].attrs['coordinates'] = 'lon lat' ds['u'].attrs['grid_mapping'] = 'crs' if method == 'oi': ds['u'].attrs[ 'ancillary_variables'] = 'qc16_maxspeed qc18_validlocation qc19_uerr qc20_verr qc_primary_flag qc_operator_mask' elif method == 'lsq': ds['u'].attrs[ 'ancillary_variables'] = 'qc15_gdop qc16_maxspeed qc18_validlocation qc_primary_flag qc_operator_mask' # Set v attributes ds['v'].attrs['long_name'] = 'Northward Surface Current (cm/s)' ds['v'].attrs['standard_name'] = 'surface_northward_sea_water_velocity' ds['v'].attrs['short_name'] = 'v' ds['v'].attrs['units'] = v_units ds['v'].attrs['valid_min'] = np.float32(-300) ds['v'].attrs['valid_max'] = np.float32(300) ds['v'].attrs['coordinates'] = 'lon lat' ds['v'].attrs['grid_mapping'] = 'crs' if method == 'oi': ds['v'].attrs[ 'ancillary_variables'] = 'qc16_maxspeed qc18_validlocation qc19_uerr qc20_verr qc_primary_flag qc_operator_mask' elif method == 'lsq': ds['v'].attrs[ 'ancillary_variables'] = 'qc15_gdop qc16_maxspeed qc18_validlocation qc_primary_flag qc_operator_mask' # Set u_err attributes ds['u_err'].attrs['units'] = '1' ds['u_err'].attrs['valid_min'] = np.float32(0) ds['u_err'].attrs['valid_max'] = np.float32(1) ds['u_err'].attrs['coordinates'] = 'lon lat' ds['u_err'].attrs['grid_mapping'] = 'crs' # Set v_err attributes ds['v_err'].attrs['units'] = '1' ds['v_err'].attrs['valid_min'] = np.float32(0) ds['v_err'].attrs['valid_max'] = np.float32(1) ds['v_err'].attrs['coordinates'] = 'lon lat' ds['v_err'].attrs['grid_mapping'] = 'crs' if method == 'lsq': ds['u_err'].attrs[ 'long_name'] = 'Associated GDOP mapping error value associated with eastward velocity component' ds['v_err'].attrs[ 'long_name'] = 'Associated GDOP mapping error value associated with northward velocity component' ds['total_errors'].attrs[ 'long_name'] = 'Associated GDOP mapping error value, TotalErrors are the square-root of the norm covariance matrix (or the square root of the maximum eigenvalue of the 2x2 UV covariance matrix)' ds['u_err'].attrs[ 'comment'] = 'velocity measurements with error values over 1.25 are of questionable quality' ds['v_err'].attrs[ 'comment'] = 'velocity measurements with error values over 1.25 are of questionable quality' ds['total_errors'].attrs[ 'comment'] = 'velocity measurements with error values over 1.25 are of questionable quality' elif method == 'oi': ds['u_err'].attrs[ 'long_name'] = 'Normalized uncertainty error associated with eastward velocity component' ds['v_err'].attrs[ 'long_name'] = 'Normalized uncertainty error associated with northward velocity component' ds['total_errors'].attrs[ 'long_name'] = 'Associated mapping error value, TotalErrors are the square-root of the sum of the squares of U_err and Verr' ds['u_err'].attrs[ 'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality' ds['v_err'].attrs[ 'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality' ds['total_errors'].attrs[ 'comment'] = 'velocity measurements with error values over _ are of questionable quality' # Set uv_covariance attributes ds['uv_covariance'].attrs[ 'long_name'] = 'Eastward and Northward covariance directional information of u and v' ds['uv_covariance'].attrs['units'] = '1' ds['uv_covariance'].attrs['comment'] = 'directional information of u and v' ds['uv_covariance'].attrs['coordinates'] = 'lon lat' ds['uv_covariance'].attrs['grid_mapping'] = 'crs' # Set num_radials attributes ds['num_radials'].attrs[ 'long_name'] = 'Number of radial measurements used to calculate each totals velocity' ds['num_radials'].attrs[ 'comment'] = 'totals are not calculated with fewer than 3 contributing radial measurements from 2 sites' ds['num_radials'].attrs['coordinates'] = 'lon lat' ds['num_radials'].attrs['grid_mapping'] = 'crs' # Set information attributes ds['processing_parameters'].attrs[ 'long_name'] = 'General and method specific processing parameter information' ds['processing_parameters'].attrs['comment'] = processing_parameters_info # ds['processing_parameters'].attrs['coordinates'] = 'parameters' # Set qc flag attributes ds['qc16_maxspeed'].attrs[ 'standard_name'] = 'sea_water_velocity quality_flag' ds['qc16_maxspeed'].attrs['units'] = '1' ds['qc16_maxspeed'].attrs['valid_min'] = 1 ds['qc16_maxspeed'].attrs['valid_max'] = 9 ds['qc16_maxspeed'].attrs['coordinates'] = 'lon lat' ds['qc16_maxspeed'].attrs['grid_mapping'] = 'crs' ds['qc16_maxspeed'].attrs['flag_values'] = '1 2 3 4 9' ds['qc16_maxspeed'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail missing_data' ds['qc16_maxspeed'].attrs['comment'] = qc16_info ds['qc18_validlocation'].attrs[ 'standard_name'] = 'sea_water_velocity location_test_quality_flag' ds['qc18_validlocation'].attrs['units'] = '1' ds['qc18_validlocation'].attrs['valid_min'] = 1 ds['qc18_validlocation'].attrs['valid_max'] = 9 ds['qc18_validlocation'].attrs['coordinates'] = 'lon lat' ds['qc18_validlocation'].attrs['grid_mapping'] = 'crs' ds['qc18_validlocation'].attrs['flag_values'] = '1 2 3 4 9' ds['qc18_validlocation'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail missing_data' ds['qc18_validlocation'].attrs['comment'] = qc18_info if method == 'oi': ds['qc19_uerr'].attrs[ 'standard_name'] = 'sea_water_velocity quality_flag' ds['qc19_uerr'].attrs['units'] = '1' ds['qc19_uerr'].attrs['valid_min'] = 1 ds['qc19_uerr'].attrs['valid_max'] = 9 ds['qc19_uerr'].attrs['coordinates'] = 'lon lat' ds['qc19_uerr'].attrs['grid_mapping'] = 'crs' ds['qc19_uerr'].attrs['flag_values'] = '1 2 3 4 9' ds['qc19_uerr'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail missing_data' ds['qc19_uerr'].attrs['comment'] = qc19_info ds['qc20_verr'].attrs[ 'standard_name'] = 'sea_water_velocity quality_flag' ds['qc20_verr'].attrs['units'] = '1' ds['qc20_verr'].attrs['valid_min'] = 1 ds['qc20_verr'].attrs['valid_max'] = 9 ds['qc20_verr'].attrs['coordinates'] = 'lon lat' ds['qc20_verr'].attrs['grid_mapping'] = 'crs' ds['qc20_verr'].attrs['flag_values'] = '1 2 3 4 9' ds['qc20_verr'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail missing_data' ds['qc20_verr'].attrs['comment'] = qc20_info elif method == 'lsq': ds['qc15_gdop'].attrs[ 'standard_name'] = 'sea_water_velocity quality_flag' ds['qc15_gdop'].attrs['units'] = '1' ds['qc15_gdop'].attrs['valid_min'] = 1 ds['qc15_gdop'].attrs['valid_max'] = 9 ds['qc15_gdop'].attrs['coordinates'] = 'lon lat' ds['qc15_gdop'].attrs['grid_mapping'] = 'crs' ds['qc15_gdop'].attrs['flag_values'] = '1 2 3 4 9' ds['qc15_gdop'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail missing_data' ds['qc15_gdop'].attrs['comment'] = qc15_info ds['qc_primary_flag'].attrs[ 'standard_name'] = 'sea_water_velocity aggregate_quality_flag' ds['qc_primary_flag'].attrs['units'] = '1' ds['qc_primary_flag'].attrs['valid_min'] = 1 ds['qc_primary_flag'].attrs['valid_max'] = 4 ds['qc_primary_flag'].attrs['coordinates'] = 'lon lat' ds['qc_primary_flag'].attrs['grid_mapping'] = 'crs' ds['qc_primary_flag'].attrs['flag_values'] = '1 2 3 4' ds['qc_primary_flag'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail' ds['qc_primary_flag'].attrs['comment'] = qc_primary_flag_info if method == 'oi': ds['qc_primary_flag'].attrs[ 'ancillary_variables'] = 'qc16_maxspeed qc18_validlocation qc19_uerr qc20_verr' elif method == 'lsq': ds['qc_primary_flag'].attrs[ 'ancillary_variables'] = 'qc15_gdop qc16_maxspeed qc18_validlocation' ds['qc_operator_mask'].attrs['units'] = '1' ds['qc_operator_mask'].attrs['valid_min'] = 1 ds['qc_operator_mask'].attrs['valid_max'] = 9 ds['qc_operator_mask'].attrs['coordinates'] = 'lon lat' ds['qc_operator_mask'].attrs['grid_mapping'] = 'crs' ds['qc_operator_mask'].attrs['flag_values'] = '1 2 3 4 9' ds['qc_operator_mask'].attrs[ 'flag_meanings'] = 'pass not_evaluated suspect fail missing_data' ds['qc_operator_mask'].attrs['comment'] = qc_operator_mask_info logging.debug( '{} - Setting variable encoding and fill values for netCDF4 output'. format(fname)) # encode variables for export to netcdf encoding = make_encoding(ds) encoding['lon'] = dict(zlib=False, _FillValue=False) encoding['lat'] = dict(zlib=False, _FillValue=False) encoding['z'] = dict(zlib=False, _FillValue=False) # add container variables that contain no data kwargs = dict(crs=None, instrument=None) ds = ds.assign(**kwargs) # Set crs attributes ds['crs'].attrs['grid_mapping_name'] = 'latitude_longitude' ds['crs'].attrs['inverse_flattening'] = 298.257223563 ds['crs'].attrs['long_name'] = 'Coordinate Reference System' ds['crs'].attrs['semi_major_axis'] = '6378137.0' ds['crs'].attrs['epsg_code'] = 'EPSG:4326' ds['crs'].attrs['comment'] = 'http://www.opengis.net/def/crs/EPSG/0/4326' ds['instrument'].attrs['long_name'] = 'CODAR SeaSonde High Frequency Radar' ds['instrument'].attrs[ 'sensor_type'] = 'Direction-finding high frequency radar antenna' ds['instrument'].attrs['make_model'] = 'CODAR SeaSonde' ds['instrument'].attrs['serial_number'] = 1 # Create save directory if it doesn't exist. create_dir(save_dir) logging.debug('{} - Saving dataset to netCDF4 file: {}'.format( fname, file_and_path)) ds.to_netcdf(file_and_path, encoding=encoding, format='netCDF4', engine='netcdf4', unlimited_dims=['time']) logging.info('{} - netCDF4 file successfully created: {}'.format( fname, file_and_path))