def kernel_matrix(X, kernel, n1, n2):
(n, d) = X.shape
assert n == n1 + n2
K = mat.zeros((n,n))
for i in xrange(n):
for j in xrange(i+1):
K[i,j] = kernel(X[i,:], X[j,:])
K[j,i] = K[i,j]
U1 = mat.sum(K[0:n1,:],0) / n1
U2 = mat.sum(K[n1:n,:],0) / n2
U1m = mat.tile(U1, (n1,1))
U2m = mat.tile(U2, (n2,1))
U = mat.bmat('U1m; U2m')
m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1*n1)
m1m2 = mat.sum(K[0:n1, n1:n]) / (n1*n2)
m2m2 = mat.sum(K[n1:n, n1:n]) / (n2*n2)
mumu = mat.zeros((n,n))
mumu[0:n1, 0:n1] = m1m1
mumu[0:n1, n1:n] = m1m2
mumu[n1:n, 0:n1] = m1m2
mumu[n1:n, n1:n] = m2m2
Kcu = K - U
Kuc = Kcu.T
N = mat.ones((n,n))/n
Kc = K - U - U.T + mumu
return (K, Kuc, Kc)
def kernel_matrix(X, kernel, n1, n2):
(n, d) = X.shape
assert n == n1 + n2
K = mat.zeros((n, n))
for i in xrange(n):
for j in xrange(i + 1):
K[i, j] = kernel(X[i, :], X[j, :])
K[j, i] = K[i, j]
U1 = mat.sum(K[0:n1, :], 0) / n1
U2 = mat.sum(K[n1:n, :], 0) / n2
U1m = mat.tile(U1, (n1, 1))
U2m = mat.tile(U2, (n2, 1))
U = mat.bmat('U1m; U2m')
m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1 * n1)
m1m2 = mat.sum(K[0:n1, n1:n]) / (n1 * n2)
m2m2 = mat.sum(K[n1:n, n1:n]) / (n2 * n2)
mumu = mat.zeros((n, n))
mumu[0:n1, 0:n1] = m1m1
mumu[0:n1, n1:n] = m1m2
mumu[n1:n, 0:n1] = m1m2
mumu[n1:n, n1:n] = m2m2
Kcu = K - U
Kuc = Kcu.T
N = mat.ones((n, n)) / n
Kc = K - U - U.T + mumu
return (K, Kuc, Kc)
def nk_bhatta(X1, X2, eta):
# Returns the non-kernelized Bhattacharrya
#I.e. fits normal distributions in input space and calculates Bhattacharrya overlap between them
(n1, d1) = X1.shape
(n2, d ) = X2.shape
assert d1 == d
mu1 = mat.sum(X1,0) / n1
mu2 = mat.sum(X2,0) / n2
X1c = X1 - mat.tile(mu1, (n1,1))
X2c = X2 - mat.tile(mu2, (n2,1))
Eta = mat.eye(d) * eta
S1 = X1c.T * X1c / n1 + Eta
S2 = X2c.T * X2c / n2 + Eta
mu3 = .5 * (S1.I * mu1.T + S2.I * mu2.T).T
S3 = 2 * (S1.I + S2.I).I
d1 = la.det(S1) ** -.25
d2 = la.det(S2) ** -.25
d3 = la.det(S3) ** .5
dterm = d1 * d2 * d3
e1 = -.25 * mu1 * S1.I * mu1.T
e2 = -.25 * mu2 * S2.I * mu2.T
e3 = .5 * mu3 * S3 * mu3.T
eterm = math.exp(e1 + e2 + e3)
return float(dterm * eterm)
def nk_bhatta(X1, X2, eta):
# Returns the non-kernelized Bhattacharrya
#I.e. fits normal distributions in input space and calculates Bhattacharrya overlap between them
(n1, d1) = X1.shape
(n2, d) = X2.shape
assert d1 == d
mu1 = mat.sum(X1, 0) / n1
mu2 = mat.sum(X2, 0) / n2
X1c = X1 - mat.tile(mu1, (n1, 1))
X2c = X2 - mat.tile(mu2, (n2, 1))
Eta = mat.eye(d) * eta
S1 = X1c.T * X1c / n1 + Eta
S2 = X2c.T * X2c / n2 + Eta
mu3 = .5 * (S1.I * mu1.T + S2.I * mu2.T).T
S3 = 2 * (S1.I + S2.I).I
d1 = la.det(S1)**-.25
d2 = la.det(S2)**-.25
d3 = la.det(S3)**.5
dterm = d1 * d2 * d3
e1 = -.25 * mu1 * S1.I * mu1.T
e2 = -.25 * mu2 * S2.I * mu2.T
e3 = .5 * mu3 * S3 * mu3.T
eterm = math.exp(e1 + e2 + e3)
return float(dterm * eterm)
def verify_kernel_matrix():
n1 = 10
n2 = 10
n = n1+n2
d = 5
degree = 3
X = randn(n,d)
Phi = poly.phi(X, degree)
(K, Kuc, Kc) = kernel_matrix(X, polyk(degree), n1, n2)
P1 = Phi[0:n1,:]
P2 = Phi[n1:n,:]
mu1 = mat.sum(P1,0) / n1
mu2 = mat.sum(P2,0) / n2
P1c = P1 - mat.tile(mu1, (n1,1))
P2c = P2 - mat.tile(mu2, (n2,1))
Pc = bmat('P1c; P2c')
KP = mat.zeros((n,n))
for i in xrange(n):
for j in xrange(i+1):
KP[i,j] = dotp(Phi[i,:], Phi[j,:])
KP[j,i] = KP[i,j]
KucP = mat.zeros((n,n))
for i in xrange(n):
for j in xrange(n):
KucP[i,j] = dotp(Phi[i,:], Pc[j,:])
KcP = mat.zeros((n,n))
for i in xrange(n):
for j in xrange(n):
KcP[i,j] = dotp(Pc[i,:], Pc[j,:])
#KcP[j,i] = KcP[i,j]
#debug()
print "Div1: " + str(sum(abs(K-KP)))
print "Div2: " + str(sum(abs(Kuc-KucP)))
print "Div3: " + str(sum(abs(Kc-KcP)))
def kernel_supermatrix(self, i, j):
kernel = self.kernel
D1 = self.datasets[i]
D2 = self.datasets[j]
X1 = D1.X
X2 = D2.X
(n1, d) = X1.shape
(n2, d) = X2.shape
n = n1 + n2
X = mat.bmat('X1; X2')
K1 = D1.K
K2 = D2.K
K = mat.zeros((n,n))
K[0:n1, 0:n1] = K1
K[n1:n, n1:n] = K2
for i in xrange(n1):
for j in xrange(n1, n):
K[i,j] = kernel(X[i,:], X[j,:])
K[j,i] = K[i,j]
# Inelegant - improve later
U1 = mat.sum(K[0:n1,:],0) / n1
U2 = mat.sum(K[n1:n,:],0) / n2
U1m = mat.tile(U1, (n1,1))
U2m = mat.tile(U2, (n2,1))
U = mat.bmat('U1m; U2m')
m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1*n1)
m1m2 = mat.sum(K[0:n1, n1:n]) / (n1*n2)
m2m2 = mat.sum(K[n1:n, n1:n]) / (n2*n2)
mumu = mat.zeros((n,n))
mumu[0:n1, 0:n1] = m1m1
mumu[0:n1, n1:n] = m1m2
mumu[n1:n, 0:n1] = m1m2
mumu[n1:n, n1:n] = m2m2
Kcu = K - U
Kuc = Kcu.T
N = mat.ones((n,n))/n
Kc = K - U - U.T + mumu
return (K, Kuc, Kc)
def kernel_supermatrix(self, i, j):
kernel = self.kernel
D1 = self.datasets[i]
D2 = self.datasets[j]
X1 = D1.X
X2 = D2.X
(n1, d) = X1.shape
(n2, d) = X2.shape
n = n1 + n2
X = mat.bmat('X1; X2')
K1 = D1.K
K2 = D2.K
K = mat.zeros((n, n))
K[0:n1, 0:n1] = K1
K[n1:n, n1:n] = K2
for i in xrange(n1):
for j in xrange(n1, n):
K[i, j] = kernel(X[i, :], X[j, :])
K[j, i] = K[i, j]
# Inelegant - improve later
U1 = mat.sum(K[0:n1, :], 0) / n1
U2 = mat.sum(K[n1:n, :], 0) / n2
U1m = mat.tile(U1, (n1, 1))
U2m = mat.tile(U2, (n2, 1))
U = mat.bmat('U1m; U2m')
m1m1 = mat.sum(K[0:n1, 0:n1]) / (n1 * n1)
m1m2 = mat.sum(K[0:n1, n1:n]) / (n1 * n2)
m2m2 = mat.sum(K[n1:n, n1:n]) / (n2 * n2)
mumu = mat.zeros((n, n))
mumu[0:n1, 0:n1] = m1m1
mumu[0:n1, n1:n] = m1m2
mumu[n1:n, 0:n1] = m1m2
mumu[n1:n, n1:n] = m2m2
Kcu = K - U
Kuc = Kcu.T
N = mat.ones((n, n)) / n
Kc = K - U - U.T + mumu
return (K, Kuc, Kc)
def classify0(in_x, data_set, labels, k):
data_set_size = data_set.shape[0]
diff_mat = tile(in_x, (data_set_size, 1)) - data_set
sq_diff_mat = diff_mat**2
sq_distances = sq_diff_mat.sum(axis=1)
distances = sq_distances**0.5
sorted_dist_indicies = distances.argsort()
class_count = {}
for i in range(k):
vote_ilabel = labels[sorted_dist_indicies[i]]
class_count[vote_ilabel] = class_count.get(vote_ilabel, 0) + 1
sorted_class_count = sorted(class_count.items(),
key=operator.itemgetter(1),
reverse=True)
return sorted_class_count[0][0]
def demean(x, dim=0):
# DEMEAN(X)
# Removes the Average or mean value.
#
# DEMEAN(X,DIM)
# Removes the mean along the dimension DIM of X.
# if (dim == -1):
# dim = 0
# if (x.shape[0] > 1):
# dim = 0;
# elif (x.shape[1] > 1):
# dim = 1;
dims = x.size
# dimsize = x.shape[dim]
# dimrep = np.ones(1,len(dims));
# dimrep[dim] = dimsize;
return x - mtlib.tile(np.mean(x, dim),
dims) # repmat(np.mean(x,dim),dimrep)
def test_bsp_light_dark():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--no-plotting',action='store_true',default=False)
parser.add_argument('--profile',action='store_true',default=False)
parser.add_argument('--gradient_free',action='store_true',default=False)
args = parser.parse_args()
plotting = not args.no_plotting
profile = args.profile
gradient_free = args.gradient_free
model = LightDarkModel()
X1 = ml.matrix([[-3.5,2],[-3.5,-2],[-4,0],[2,2],[-4,2]]).T
G = ml.matrix([[-3.5,-2],[-3.5,2],[-1,0],[2,-2],[-1,-2]]).T
# [Reference] final belief trajectory costs for verification
# Allow for numerical inaccuracy
verify_cost = [45.181701, 45.181643, 49.430339, 27.687003, 56.720314]
for i_problem in xrange(0,5):
# Setup initial conditions for problem
x1 = X1[:,i_problem] # start (mean of initial belief)
SqrtSigma1 = ml.eye(2) # initial covariance
goal = G[:,i_problem] # goal
model.setStartState(x1)
model.setGoalState(goal)
# setup initial control vector -- straight line initialization from start to goal
U = ml.tile(((model.goal - model.start)/(model.T-1)), (1,model.T-1))
B = ml.zeros([model.bDim,model.T])
B[:,0] = belief.compose_belief(x1, SqrtSigma1, model)
for t in xrange(0,model.T-1):
B[:,t+1] = belief.belief_dynamics(B[:,t], U[:,t], None, model,None,None)
# display initialization
if plotting:
plot.plot_belief_trajectory(B, U, model)
"""
if gradient_free :
[Bopt, Uopt] = belief_grad_free.STOMP_BSP(B,model,plotting,profile)
else:
[Bopt, Uopt] = belief_opt.belief_opt_penalty_sqp(B, U, model, plotting, profile)
"""
if plotting:
plot.plot_belief_trajectory(Bopt, Uopt, model);
# Forward simulated cost
cost = belief.compute_forward_simulated_cost(B[:,0], Uopt, model)
print('Total cost of optimized trajectory: %f' % cost)
# save trajectory to png file
# saveas(gcf, sprintf('bsp-light-dark-plan-%i.png',i_problem));
print('For verification (allow for numerical inaccuracy):')
print('(Reference) Total cost of optimized trajectory: %2.6f' % verify_cost[i_problem])
# Simulate execution of trajectory (test to see why the maximum likelihood observation assumption does not hold)
#simulate_bsp_trajectory(B(:,1), Uopt, model);
print('press enter to continue to the next problem')
raw_input()
def main(grid,
mat_file,
save_dir,
user_attributes,
flags=None,
domain=[],
method='oi'):
"""
Convert MAT files created using the hfrProgs MATLAB toolbox into CF-1.6/NCEI Grid 2.0 compliant netCDF4 files
:param grid: CSV file containing lon,lat grid information
:param mat_file: Filepath to MAT file containing HFRProgs
:param save_dir: Directory to save netCDF files to
:param user_attributes: User defined dataset attributes for netCDF global attribute. Required for CF/NCEI compliance
:param flags: Dictionary of thresholds at which we should filter data above
:param method: 'oi' or 'lsq'. OI is optimal interpolation. LSQ is unweighted least squares
"""
fname = os.path.basename(mat_file)
try:
# load .mat file
data = loadmat(mat_file, squeeze_me=True, struct_as_record=False)
logging.debug('{} - MAT file successfully loaded '.format(fname))
except Exception as err:
logging.error('{} - {}. MAT file could not be loaded.'.format(
fname, err))
return
if not domain:
domain = data['TUV'].DomainName
if not domain:
domain = 'MARA'
else:
domain = 'MARA'
time = timestamp_from_lluv_filename(mat_file)
# convert matlab time to python datetime
# time = dt.datetime.strptime(mat_time, '%Y_%m_%d_%H00')
time_index = pd.date_range(
time.strftime('%Y-%m-%d %H:%M:%S'),
periods=1) # create pandas datetimeindex from time
time_string = time.strftime(
'%Y%m%dT%H%M%SZ') # create timestring from time
file_name = 'RU_{}_{}.nc'.format(domain, time_string)
file_and_path = os.path.join(save_dir, file_name)
try:
logging.debug('{} - Saving file data to variables'.format(fname))
# load longitude and latitude data associated with variables
lonlat = data['TUV'].LonLat.astype(np.float32)
# create variables for eastward and northward velocities
u = data['TUV'].U.astype(np.float32)
v = data['TUV'].V.astype(np.float32)
u_units = data['TUV'].UUnits
v_units = data['TUV'].VUnits
maxspd = data['TUV'].OtherMetadata.cleanTotals.maxspd
if method == 'oi':
# create variables for associated error values
u_err = data['TUV'].ErrorEstimates.Uerr.astype(np.float32)
v_err = data['TUV'].ErrorEstimates.Verr.astype(np.float32)
uv_covariance = data['TUV'].ErrorEstimates.UVCovariance
# Data Processing Information
num_rads = data[
'TUV'].OtherMatrixVars.makeTotalsOI_TotalsNumRads.astype(int)
min_rads = data[
'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumRads
min_sites = data[
'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumSites
mdlvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.mdlvar
errvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.errvar
sx = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sx
sy = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sy
temporal_threshold = data[
'TUV'].OtherMetadata.makeTotalsOI.parameters.tempthresh
processing_parameters = [
maxspd, min_sites, min_rads, temporal_threshold, sx, sy,
mdlvar, errvar
]
processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n'
processing_parameters_info += '2) Minimum number of radial sites\n'
processing_parameters_info += '3) Minimum number of radial vectors\n'
processing_parameters_info += '4) Temporal search window for radial solutions (Fraction of a day)\n'
processing_parameters_info += '5) Decorrelation scales in the north direction\n'
processing_parameters_info += '6) Decorrelation scales in the east direction\n'
processing_parameters_info += '7) Signal variance of the surface current fields (cm2 s-2)\n'
processing_parameters_info += '8) Data error variance of the input radial velocities (cm2 s-2)\n'
elif method == 'lsq':
# create variables for associated error values
u_err = data['TUV'].ErrorEstimates[1].Uerr.astype(np.float32)
v_err = data['TUV'].ErrorEstimates[1].Verr.astype(np.float32)
uv_covariance = data['TUV'].ErrorEstimates[1].UVCovariance.astype(
np.float32)
# Data Processing Information
num_rads = data[
'TUV'].OtherMatrixVars.makeTotals_TotalsNumRads.astype(int)
min_rads = data[
'TUV'].OtherMetadata.makeTotals.parameters.MinNumRads
min_sites = data[
'TUV'].OtherMetadata.makeTotals.parameters.MinNumSites
spatial_threshold = data[
'TUV'].OtherMetadata.makeTotals.parameters.spatthresh
temporal_threshold = data[
'TUV'].OtherMetadata.makeTotals.parameters.tempthresh
processing_parameters = [
maxspd, min_sites, min_rads, temporal_threshold,
spatial_threshold
]
processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n'
processing_parameters_info += '2) Minimum number of radial sites.\n'
processing_parameters_info += '3) Minimum number of radial vectors.\n'
processing_parameters_info += '4) Temporal search window for radial solutions (Fractions of a day)\n'
processing_parameters_info += '5) Spatial search radius for radial solutions (km)\n'
except AttributeError as err:
logging.error(
'{} - {}. MAT file missing variable needed to create netCDF4 file'.
format(fname, err))
return
# Create a grid to shape 1d data
lon = np.unique(grid['lon'].values.astype(np.float32))
lat = np.unique(grid['lat'].values.astype(np.float32))
[x, y] = np.meshgrid(lon, lat)
# Create a dictionary of variables that we want to grid
data_dict = dict(
u=u,
v=v,
u_err=u_err,
v_err=v_err,
uv_covariance=uv_covariance,
num_radials=num_rads,
)
logging.debug('{} - Gridding data to 2d grid'.format(fname))
# convert 1d data into 2d gridded form. data_dict must be a dictionary.
x_ind, y_ind = gridded_index(x, y, lonlat[:, 0], lonlat[:, 1])
for key in data_dict.keys():
temp_data = mb.tile(np.nan, x.shape)
temp_data[(y_ind, x_ind)] = data_dict[key]
# expand dimensions for time and depth
count = 0
while count < 2: # add two dimensions to from of array for time and z (depth)
temp_data = np.expand_dims(temp_data, axis=0)
count = count + 1
data_dict[key] = temp_data
logging.debug('{} - Loading data into xarray dataset'.format(fname))
# initialize xarray dataset. Add variables. Add coordinates
ds = xr.Dataset()
coords = ('time', 'z', 'lat', 'lon')
ds['u'] = (coords, np.float32(data_dict['u']))
ds['v'] = (coords, np.float32(data_dict['v']))
ds['u_err'] = (coords, np.float32(data_dict['u_err']))
ds['v_err'] = (coords, np.float32(data_dict['v_err']))
ds['uv_covariance'] = (coords, np.float32(data_dict['uv_covariance']))
ds['num_radials'] = (coords, data_dict['num_radials'])
ds.coords['lon'] = lon
ds.coords['lat'] = lat
ds.coords['z'] = np.array([np.float32(0)])
ds.coords['time'] = time_index
if flags:
for k, v in flags.items():
ds = ds.where(ds[k] <= v)
ds['processing_parameters'] = (('parameters'), processing_parameters)
# Grab min and max time in dataset for entry into global attributes for cf compliance
time_start = ds['time'].min().data
time_end = ds['time'].max().data
global_attributes = configs.netcdf_global_attributes(
user_attributes, time_start, time_end)
global_attributes['geospatial_lat_min'] = lat.min()
global_attributes['geospatial_lat_max'] = lat.max()
global_attributes['geospatial_lon_min'] = lon.min()
global_attributes['geospatial_lon_max'] = lon.max()
if method == 'oi':
global_attributes['method'] = 'Optimal Interpolation'
elif method == 'lsq':
global_attributes['method'] = 'Unweighted Least Squares'
logging.debug('{} - Assigning global attributes to dataset'.format(fname))
ds = ds.assign_attrs(global_attributes)
logging.debug(
'{} - Assigning local attributes to each variable in dataset'.format(
fname))
# set time attribute
ds['time'].attrs['standard_name'] = 'time'
# Set lon attributes
ds['lon'].attrs['long_name'] = 'Longitude'
ds['lon'].attrs['standard_name'] = 'longitude'
ds['lon'].attrs['short_name'] = 'lon'
ds['lon'].attrs['units'] = 'degrees_east'
ds['lon'].attrs['axis'] = 'X'
ds['lon'].attrs['valid_min'] = np.float32(-180.0)
ds['lon'].attrs['valid_max'] = np.float32(180.0)
# Set lat attributes
ds['lat'].attrs['long_name'] = 'Latitude'
ds['lat'].attrs['standard_name'] = 'latitude'
ds['lat'].attrs['short_name'] = 'lat'
ds['lat'].attrs['units'] = 'degrees_north'
ds['lat'].attrs['axis'] = 'Y'
ds['lat'].attrs['valid_min'] = np.float32(-90.0)
ds['lat'].attrs['valid_max'] = np.float32(90.0)
# Set depth attributes
ds['z'].attrs['long_name'] = 'Average Depth of Sensor'
ds['z'].attrs['standard_name'] = 'depth'
ds['z'].attrs['comment'] = 'Derived from mean value of depth variable'
ds['z'].attrs['units'] = 'm'
ds['z'].attrs['axis'] = 'Z'
ds['z'].attrs['positive'] = 'down'
# Set u attributes
ds['u'].attrs['long_name'] = 'Eastward Surface Current (cm/s)'
ds['u'].attrs['standard_name'] = 'surface_eastward_sea_water_velocity'
ds['u'].attrs['short_name'] = 'u'
ds['u'].attrs['units'] = u_units
ds['u'].attrs['valid_min'] = np.float32(-300)
ds['u'].attrs['valid_max'] = np.float32(300)
ds['u'].attrs['coordinates'] = 'lon lat'
ds['u'].attrs['grid_mapping'] = 'crs'
# Set v attributes
ds['v'].attrs['long_name'] = 'Northward Surface Current (cm/s)'
ds['v'].attrs['standard_name'] = 'surface_northward_sea_water_velocity'
ds['v'].attrs['short_name'] = 'v'
ds['v'].attrs['units'] = v_units
ds['v'].attrs['valid_min'] = np.float32(-300)
ds['v'].attrs['valid_max'] = np.float32(300)
ds['v'].attrs['coordinates'] = 'lon lat'
ds['v'].attrs['grid_mapping'] = 'crs'
# Set u_err attributes
ds['u_err'].attrs['units'] = '1'
ds['u_err'].attrs['valid_min'] = np.float32(0)
ds['u_err'].attrs['valid_max'] = np.float32(1)
ds['u_err'].attrs['coordinates'] = 'lon lat'
ds['u_err'].attrs['grid_mapping'] = 'crs'
# Set v_err attributes
ds['v_err'].attrs['units'] = '1'
ds['v_err'].attrs['valid_min'] = np.float32(0)
ds['v_err'].attrs['valid_max'] = np.float32(1)
ds['v_err'].attrs['coordinates'] = 'lon lat'
ds['v_err'].attrs['grid_mapping'] = 'crs'
if method == 'lsq':
ds['u_err'].attrs[
'long_name'] = 'Associated GDOP mapping error value associated with eastward velocity component'
ds['v_err'].attrs[
'long_name'] = 'Associated GDOP mapping error value associated with northward velocity component'
ds['u_err'].attrs[
'comment'] = 'velocity measurements with error values over 1.5 are of questionable quality'
ds['v_err'].attrs[
'comment'] = 'velocity measurements with error values over 1.5 are of questionable quality'
elif method == 'oi':
ds['u_err'].attrs[
'long_name'] = 'Normalized uncertainty error associated with eastward velocity component'
ds['v_err'].attrs[
'long_name'] = 'Normalized uncertainty error associated with northward velocity component'
ds['u_err'].attrs[
'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality'
ds['v_err'].attrs[
'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality'
# Set uv_covariance attributes
ds['uv_covariance'].attrs[
'long_name'] = 'Eastward and Northward covariance directional information of u and v'
ds['uv_covariance'].attrs['units'] = '1'
ds['uv_covariance'].attrs['comment'] = 'directional information of u and v'
ds['uv_covariance'].attrs['coordinates'] = 'lon lat'
ds['uv_covariance'].attrs['grid_mapping'] = 'crs'
# Set num_radials attributes
ds['num_radials'].attrs[
'long_name'] = 'Number of radial measurements used to calculate each totals velocity'
ds['num_radials'].attrs[
'comment'] = 'totals are not calculated with fewer than 3 contributing radial measurements from 2 sites'
ds['num_radials'].attrs['coordinates'] = 'lon lat'
ds['num_radials'].attrs['grid_mapping'] = 'crs'
# Set num_radials attributes
ds['processing_parameters'].attrs[
'long_name'] = 'General and method specific processing parameter information'
ds['processing_parameters'].attrs['comment'] = processing_parameters_info
# ds['processing_parameters'].attrs['coordinates'] = 'parameters'
# encoded_sites = data['TUV'].OtherMatrixVars.makeTotalsOI_TotalsSiteCode
# # load site ids that are set in our mysqldb
# query_obj = Session.query(tables.Sites)
# site_encoding = pd.read_sql(query_obj.statement, query_obj.session.bind)
#
# # convert site codes into binary numbers
# binary_positions_mat = data['conf'].Radials.Sites.shape[0]
#
# decoded_sites_mat = np.tile(0, (encoded_sites.shape[0], binary_positions_mat))
#
# for i, v in enumerate(encoded_sites):
# decoded_sites_mat[i] = np.array(map(int, np.binary_repr(v, width=binary_positions_mat)))
#
# decoded_sites_mat = np.fliplr(decoded_sites_mat)
#
# decoded_sites_new = np.tile(0, (encoded_sites.shape[0], np.max(site_encoding['id'])))
#
# for site in data['RTUV']:
# print site.SiteName + ' ' + str(np.log2(site.SiteCode))
# ind_mat = np.log2(site.SiteCode)
# ind_real = site_encoding['id'].loc[site_encoding['site'] == site.SiteName].values[0]
# decoded_sites_new[:, ind_real] = decoded_sites_mat[:, int(ind_mat)]
#
# decoded_sites_new = np.fliplr(decoded_sites_new)
# new_encoded_sites = [bool2int(x[::-1]) for x in decoded_sites_new]
# flag_masks = [2 ** int(x) for x in site_encoding['id'].tolist()]
# flag_meanings = ' '.join(site_encoding['site'].tolist())
# ds['site_code_flags'].attrs['long_name'] = 'Bitwise AND representation of site contributions to a radial point'
# # ds['site_code_flags'].attrs['_FillValue'] = int(0)
# ds['site_code_flags'].attrs['flag_masks'] = 'b '.join(map(str, flag_masks))
# ds['site_code_flags'].attrs['flag_meanings'] = flag_meanings
# ds['site_code_flags'].attrs['comment'] = 'Values are binary sums. Must be converted to binary representation to interpret flag_masks and flag_meanings'
logging.debug(
'{} - Setting variable encoding and fill values for netCDF4 output'.
format(fname))
# encode variables for export to netcdf
encoding = make_encoding(ds)
encoding['lon'] = dict(zlib=False, _FillValue=False)
encoding['lat'] = dict(zlib=False, _FillValue=False)
encoding['z'] = dict(zlib=False, _FillValue=False)
# add container variables that contain no data
kwargs = dict(crs=None, instrument=None)
ds = ds.assign(**kwargs)
# Set crs attributes
ds['crs'].attrs['grid_mapping_name'] = 'latitude_longitude'
ds['crs'].attrs['inverse_flattening'] = 298.257223563
ds['crs'].attrs['long_name'] = 'Coordinate Reference System'
ds['crs'].attrs['semi_major_axis'] = '6378137.0'
ds['crs'].attrs['epsg_code'] = 'EPSG:4326'
ds['crs'].attrs['comment'] = 'http://www.opengis.net/def/crs/EPSG/0/4326'
ds['instrument'].attrs['long_name'] = 'CODAR SeaSonde High Frequency Radar'
ds['instrument'].attrs[
'sensor_type'] = 'Direction-finding high frequency radar antenna'
ds['instrument'].attrs['make_model'] = 'CODAR SeaSonde'
ds['instrument'].attrs['serial_number'] = 1
# Create save directory if it doesn't exist.
create_dir(save_dir)
logging.debug('{} - Saving dataset to netCDF4 file: {}'.format(
fname, file_and_path))
ds.to_netcdf(file_and_path,
encoding=encoding,
format='netCDF4',
engine='netcdf4',
unlimited_dims=['time'])
logging.info('{} - netCDF4 file successfully created: {}'.format(
fname, file_and_path))
def test_bsp_light_dark():
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--no-plotting',action='store_true',default=False)
parser.add_argument('--profile',action='store_true',default=False)
parser.add_argument('--gradient_free',action='store_true',default=False)
args = parser.parse_args()
plotting = not args.no_plotting
profile = args.profile
gradient_free = args.gradient_free
model = LightDarkModel()
X1 = ml.matrix([[-3.5,2],[-3.5,-2],[-4,0],[2,2],[-4,2]]).T
G = ml.matrix([[-3.5,-2],[-3.5,2],[-1,0],[2,-2],[-1,-2]]).T
# [Reference] final belief trajectory costs for verification
# Allow for numerical inaccuracy
verify_cost = [45.181701, 45.181643, 49.430339, 27.687003, 56.720314]
for i_problem in xrange(0,5):
# Setup initial conditions for problem
x1 = X1[:,i_problem] # start (mean of initial belief)
SqrtSigma1 = ml.eye(2) # initial covariance
goal = G[:,i_problem] # goal
model.setStartState(x1)
model.setGoalState(goal)
# setup initial control vector -- straight line initialization from start to goal
U = ml.tile(((model.goal - model.start)/(model.T-1)), (1,model.T-1))
B = ml.zeros([model.bDim,model.T])
B[:,0] = belief.compose_belief(x1, SqrtSigma1, model)
for t in xrange(0,model.T-1):
B[:,t+1] = belief.belief_dynamics(B[:,t], U[:,t], None, model,None,None)
# display initialization
if plotting:
plot.plot_belief_trajectory(B, U, model)
if gradient_free :
[Bopt, Uopt] = belief_grad_free.STOMP_BSP(B,model,plotting,profile)
else:
[Bopt, Uopt] = belief_opt.belief_opt_penalty_sqp(B, U, model, plotting, profile)
if plotting:
plot.plot_belief_trajectory(Bopt, Uopt, model);
# Forward simulated cost
cost = belief.compute_forward_simulated_cost(B[:,0], Uopt, model)
print('Total cost of optimized trajectory: %f' % cost)
# save trajectory to png file
# saveas(gcf, sprintf('bsp-light-dark-plan-%i.png',i_problem));
print('For verification (allow for numerical inaccuracy):')
print('(Reference) Total cost of optimized trajectory: %2.6f' % verify_cost[i_problem])
# Simulate execution of trajectory (test to see why the maximum likelihood observation assumption does not hold)
#simulate_bsp_trajectory(B(:,1), Uopt, model);
print('press enter to continue to the next problem')
raw_input()
def main(grid,
mat_file,
save_dir,
user_attributes,
flags=None,
domain=[],
method='oi'):
"""
Convert MAT files created using the hfrProgs MATLAB toolbox into CF-1.6/NCEI Grid 2.0 compliant netCDF4 files
:param grid: CSV file containing lon,lat grid information
:param mat_file: Filepath to MAT file containing HFRProgs
:param save_dir: Directory to save netCDF files to
:param user_attributes: User defined dataset attributes for netCDF global attribute. Required for CF/NCEI compliance
:param flags: Dictionary of thresholds at which we should filter data above
:param method: 'oi' or 'lsq'. OI is optimal interpolation. LSQ is unweighted least squares
"""
fname = os.path.basename(mat_file)
try:
# load .mat file
data = loadmat(mat_file, squeeze_me=True, struct_as_record=False)
logging.debug('{} - MAT file successfully loaded '.format(fname))
except Exception as err:
logging.error('{} - {}. MAT file could not be loaded.'.format(
fname, err))
#return
if not domain:
domain = data['TUV'].DomainName
if not domain:
domain = 'MARA'
else:
domain = 'MARA'
time = timestamp_from_lluv_filename(mat_file)
# convert matlab time to python datetime
# time = dt.datetime.strptime(mat_time, '%Y_%m_%d_%H00')
time_index = pd.date_range(
time.strftime('%Y-%m-%d %H:%M:%S'),
periods=1) # create pandas datetimeindex from time
time_string = time.strftime(
'%Y%m%dT%H%M%SZ') # create timestring from time
file_name = 'RU_{}_{}.nc'.format(domain, time_string)
file_and_path = os.path.join(save_dir, file_name)
try:
logging.debug('{} - Saving file data to variables'.format(fname))
# load longitude and latitude data associated with variables
lonlat = data['TUV'].LonLat.astype(np.float32)
# create variables for eastward and northward velocities
u = data['TUV'].U.astype(np.float32)
v = data['TUV'].V.astype(np.float32)
u_units = data['TUV'].UUnits
v_units = data['TUV'].VUnits
#maxspd = data['TUV'].OtherMetadata.cleanTotals.maxspd
maxspd = data['conf'].Totals.MaxTotSpeed
if method == 'oi':
# create variables for associated error values
u_err = data['TUV'].ErrorEstimates.Uerr.astype(np.float32)
v_err = data['TUV'].ErrorEstimates.Verr.astype(np.float32)
uv_covariance = data['TUV'].ErrorEstimates.UVCovariance.astype(
np.float32)
total_errors = data['TUV'].ErrorEstimates[1].TotalErrors.astype(
np.float32)
# Data Processing Information
num_rads = data[
'TUV'].OtherMatrixVars.makeTotalsOI_TotalsNumRads.astype(int)
min_rads = data[
'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumRads
min_sites = data[
'TUV'].OtherMetadata.makeTotalsOI.parameters.MinNumSites
mdlvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.mdlvar
errvar = data['TUV'].OtherMetadata.makeTotalsOI.parameters.errvar
sx = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sx
sy = data['TUV'].OtherMetadata.makeTotalsOI.parameters.sy
temporal_threshold = data[
'TUV'].OtherMetadata.makeTotalsOI.parameters.tempthresh
#processing_parameters = [maxspd, min_sites, min_rads, temporal_threshold, sx, sy, mdlvar, errvar]
processing_parameters = [
min_sites, min_rads, temporal_threshold, sx, sy, mdlvar, errvar
]
#processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n'
processing_parameters_info = '1) Minimum number of radial sites\n'
processing_parameters_info += '2) Minimum number of radial vectors\n'
processing_parameters_info += '3) Temporal search window for radial solutions (Fraction of a day)\n'
processing_parameters_info += '4) Decorrelation scales in the north direction\n'
processing_parameters_info += '5) Decorrelation scales in the east direction\n'
processing_parameters_info += '6) Signal variance of the surface current fields (cm2 s-2)\n'
processing_parameters_info += '7) Data error variance of the input radial velocities (cm2 s-2)\n'
#QC Information
uerr_testname = data['conf'].Totals.OI.cleanTotalsVarargin[0][
0] + ' ' + data['conf'].Totals.OI.cleanTotalsVarargin[0][1]
uerr_threshold = data['conf'].Totals.OI.cleanTotalsVarargin[0][2]
verr_testname = data['conf'].Totals.OI.cleanTotalsVarargin[1][
0] + ' ' + data['conf'].Totals.OI.cleanTotalsVarargin[1][1]
verr_threshold = data['conf'].Totals.OI.cleanTotalsVarargin[1][2]
qc_info = 'Quality control reference: IOOS QARTOD HF Radar ver 1.0 May 2016\n'
qc_info += 'QCFlagDefinitions: 1 = pass, 2 = not_evaluated, 3 = suspect, 4 = fail, 9 = missing_data\n'
qc_primary_flag_info = 'QCPrimaryFlagDefinition: Highest flag value of QC16, QC18, QC19, QC20\n'
qc_primary_flag_info += 'This flag will be set to not_evaluated only if ALL individual tests were not_evaluated.'
qc_operator_mask_info = qc_info + 'The qc_operator_mask follows QCFlagDefinitions and is set at discretion of the operator or data manager.'
qc16_info = qc_info + 'QC16 Max Speed Threshold [max_vel = ' + str(
maxspd) + ' (cm/s)]'
qc18_info = qc_info + 'QC18 Valid Location [landmask file = ' + data[
'conf'].Totals.MaskFile + ']'
qc19_info = qc_info + 'QC19 OI Uncertainty Threshold [' + uerr_testname + ' ' + str(
uerr_threshold) + ']'
qc20_info = qc_info + 'QC20 OI Uncertainty Threshold [' + verr_testname + ' ' + str(
verr_threshold) + ']'
qc16 = data['TUVqc'].QC16.astype(np.int32)
qc18 = data['TUVqc'].QC18.astype(np.int32)
qc19 = data['TUVqc'].QC19.astype(np.int32)
qc20 = data['TUVqc'].QC20.astype(np.int32)
qc_primary_flag = data['TUVqc'].PRIM.astype(np.int32)
qc_operator_mask = data['TUVqc'].qc_operator_mask.astype(np.int32)
elif method == 'lsq':
# create variables for associated error values
u_err = data['TUV'].ErrorEstimates[1].Uerr.astype(np.float32)
v_err = data['TUV'].ErrorEstimates[1].Verr.astype(np.float32)
uv_covariance = data['TUV'].ErrorEstimates[1].UVCovariance.astype(
np.float32)
total_errors = data['TUV'].ErrorEstimates[1].TotalErrors.astype(
np.float32)
# Data Processing Information
num_rads = data[
'TUV'].OtherMatrixVars.makeTotals_TotalsNumRads.astype(int)
min_rads = data[
'TUV'].OtherMetadata.makeTotals.parameters.MinNumRads
min_sites = data[
'TUV'].OtherMetadata.makeTotals.parameters.MinNumSites
spatial_threshold = data[
'TUV'].OtherMetadata.makeTotals.parameters.spatthresh
temporal_threshold = data[
'TUV'].OtherMetadata.makeTotals.parameters.tempthresh
#processing_parameters = [maxspd, min_sites, min_rads, temporal_threshold, spatial_threshold]
processing_parameters = [
min_sites, min_rads, temporal_threshold, spatial_threshold
]
#processing_parameters_info = '1) Maximum Total Speed Threshold (cm s-1)\n'
processing_parameters_info = '1) Minimum number of radial sites.\n'
processing_parameters_info += '2) Minimum number of radial vectors.\n'
processing_parameters_info += '3) Temporal search window for radial solutions (Fractions of a day)\n'
processing_parameters_info += '4) Spatial search radius for radial solutions (km)\n'
#QC Information
gdoptestname = data['conf'].Totals.cleanTotalsVarargin[
0] + ' ' + data['conf'].Totals.cleanTotalsVarargin[1]
gdopthreshold = data['conf'].Totals.cleanTotalsVarargin[2]
qc_info = 'Quality control reference: IOOS QARTOD HF Radar ver 1.0 May 2016\n'
qc_info += 'QCFlagDefinitions: 1 = pass, 2 = not_evaluated, 3 = suspect, 4 = fail, 9 = missing_data\n'
qc_primary_flag_info = 'QCPrimaryFlagDefinition: Highest flag value of QC16, QC18, QC19, QC20\n'
qc_primary_flag_info += 'This flag will be set to not_evaluated only if ALL tests were not_evaluated.'
qc_operator_mask_info = qc_info + 'The qc_operator_mask follows QCFlagDefinitions and is set at discretion of the operator or data manager.'
qc16_info = qc_info + 'QC16 Max Speed Threshold [max_vel = ' + str(
maxspd) + ' (cm/s)]'
qc18_info = qc_info + 'QC18 Valid Location [landmask file = ' + data[
'conf'].Totals.MaskFile + ']'
qc15_info = qc_info + 'QC15 GDOP Threshold [' + gdoptestname + ' ' + str(
gdopthreshold) + ']'
qc15 = data['TUVqc'].QC15.astype(np.int)
qc16 = data['TUVqc'].QC16.astype(np.int)
qc18 = data['TUVqc'].QC18.astype(np.int)
qc_primary_flag = data['TUVqc'].PRIM.astype(np.int)
qc_operator_mask = data['TUVqc'].qc_operator_mask.astype(np.int)
except AttributeError as err:
logging.error(
'{} - {}. MAT file missing variable needed to create netCDF4 file'.
format(fname, err))
#return
# Create a grid to shape 1d data
lon = np.unique(grid['lon'].values.astype(np.float32))
lat = np.unique(grid['lat'].values.astype(np.float32))
[x, y] = np.meshgrid(lon, lat)
# Create a dictionary of variables that we want to grid
if method == 'oi':
data_dict = dict(
u=u,
v=v,
u_err=u_err,
v_err=v_err,
uv_covariance=uv_covariance,
total_errors=total_errors,
num_radials=num_rads,
qc16_maxspeed=qc16,
qc18_validlocation=qc18,
qc19_uerr=qc19,
qc20_verr=qc20,
qc_primary_flag=qc_primary_flag,
qc_operator_mask=qc_operator_mask,
)
elif method == 'lsq':
data_dict = dict(
u=u,
v=v,
u_err=u_err,
v_err=v_err,
uv_covariance=uv_covariance,
total_errors=total_errors,
num_radials=num_rads,
qc15_total_errors=qc15,
qc16_maxspeed=qc16,
qc18_validlocation=qc18,
qc_primary_flag=qc_primary_flag,
qc_operator_mask=qc_operator_mask,
)
logging.debug('{} - Gridding data to 2d grid'.format(fname))
# convert 1d data into 2d gridded form. data_dict must be a dictionary.
x_ind, y_ind = gridded_index(x, y, lonlat[:, 0], lonlat[:, 1])
for key in data_dict.keys():
temp_data = mb.tile(np.nan, x.shape)
temp_data[(y_ind, x_ind)] = data_dict[key]
# expand dimensions for time and depth
count = 0
while count < 2: # add two dimensions to from of array for time and z (depth)
temp_data = np.expand_dims(temp_data, axis=0)
count = count + 1
data_dict[key] = temp_data
logging.debug('{} - Loading data into xarray dataset'.format(fname))
# initialize xarray dataset. Add variables. Add coordinates
ds = xr.Dataset()
coords = ('time', 'z', 'lat', 'lon')
ds['u'] = (coords, np.float32(data_dict['u']))
ds['v'] = (coords, np.float32(data_dict['v']))
ds['u_err'] = (coords, np.float32(data_dict['u_err']))
ds['v_err'] = (coords, np.float32(data_dict['v_err']))
ds['uv_covariance'] = (coords, np.float32(data_dict['uv_covariance']))
ds['num_radials'] = (coords, data_dict['num_radials'])
ds['qc16_maxspeed'] = (coords, np.int32(data_dict['qc16_maxspeed']))
ds['qc18_validlocation'] = (coords,
np.int32(data_dict['qc18_validlocation']))
if method == 'oi':
ds['qc19_uerr'] = (coords, np.int32(data_dict['qc19_uerr']))
ds['qc20_verr'] = (coords, np.int32(data_dict['qc20_verr']))
elif method == 'lsq':
ds['qc15_gdop'] = (coords, np.int32(data_dict['qc15_gdop']))
ds['qc_primary_flag'] = (coords, np.int32(data_dict['qc_primary_flag']))
ds['qc_operator_mask'] = (coords, np.int32(data_dict['qc_operator_mask']))
ds.coords['lon'] = lon
ds.coords['lat'] = lat
ds.coords['z'] = np.array([np.float32(0)])
ds.coords['time'] = time_index
if flags:
for k, v in flags.items():
ds = ds.where(ds[k] <= v)
ds['processing_parameters'] = (('parameters'), processing_parameters)
# Grab min and max time in dataset for entry into global attributes for cf compliance
time_start = ds['time'].min().data
time_end = ds['time'].max().data
global_attributes = configs.netcdf_global_attributes(
user_attributes, time_start, time_end)
global_attributes['geospatial_lat_min'] = lat.min()
global_attributes['geospatial_lat_max'] = lat.max()
global_attributes['geospatial_lon_min'] = lon.min()
global_attributes['geospatial_lon_max'] = lon.max()
if method == 'oi':
global_attributes['method'] = 'Optimal Interpolation'
elif method == 'lsq':
global_attributes['method'] = 'Unweighted Least Squares'
logging.debug('{} - Assigning global attributes to dataset'.format(fname))
ds = ds.assign_attrs(global_attributes)
logging.debug(
'{} - Assigning local attributes to each variable in dataset'.format(
fname))
# set time attribute
ds['time'].attrs['standard_name'] = 'time'
# Set lon attributes
ds['lon'].attrs['long_name'] = 'Longitude'
ds['lon'].attrs['standard_name'] = 'longitude'
ds['lon'].attrs['short_name'] = 'lon'
ds['lon'].attrs['units'] = 'degrees_east'
ds['lon'].attrs['axis'] = 'X'
ds['lon'].attrs['valid_min'] = np.float32(-180.0)
ds['lon'].attrs['valid_max'] = np.float32(180.0)
# Set lat attributes
ds['lat'].attrs['long_name'] = 'Latitude'
ds['lat'].attrs['standard_name'] = 'latitude'
ds['lat'].attrs['short_name'] = 'lat'
ds['lat'].attrs['units'] = 'degrees_north'
ds['lat'].attrs['axis'] = 'Y'
ds['lat'].attrs['valid_min'] = np.float32(-90.0)
ds['lat'].attrs['valid_max'] = np.float32(90.0)
# Set depth attributes
ds['z'].attrs['long_name'] = 'Average Depth of Sensor'
ds['z'].attrs['standard_name'] = 'depth'
ds['z'].attrs['comment'] = 'Derived from mean value of depth variable'
ds['z'].attrs['units'] = 'm'
ds['z'].attrs['axis'] = 'Z'
ds['z'].attrs['positive'] = 'down'
# Set u attributes
ds['u'].attrs['long_name'] = 'Eastward Surface Current (cm/s)'
ds['u'].attrs['standard_name'] = 'surface_eastward_sea_water_velocity'
ds['u'].attrs['short_name'] = 'u'
ds['u'].attrs['units'] = u_units
ds['u'].attrs['valid_min'] = np.float32(-300)
ds['u'].attrs['valid_max'] = np.float32(300)
ds['u'].attrs['coordinates'] = 'lon lat'
ds['u'].attrs['grid_mapping'] = 'crs'
if method == 'oi':
ds['u'].attrs[
'ancillary_variables'] = 'qc16_maxspeed qc18_validlocation qc19_uerr qc20_verr qc_primary_flag qc_operator_mask'
elif method == 'lsq':
ds['u'].attrs[
'ancillary_variables'] = 'qc15_gdop qc16_maxspeed qc18_validlocation qc_primary_flag qc_operator_mask'
# Set v attributes
ds['v'].attrs['long_name'] = 'Northward Surface Current (cm/s)'
ds['v'].attrs['standard_name'] = 'surface_northward_sea_water_velocity'
ds['v'].attrs['short_name'] = 'v'
ds['v'].attrs['units'] = v_units
ds['v'].attrs['valid_min'] = np.float32(-300)
ds['v'].attrs['valid_max'] = np.float32(300)
ds['v'].attrs['coordinates'] = 'lon lat'
ds['v'].attrs['grid_mapping'] = 'crs'
if method == 'oi':
ds['v'].attrs[
'ancillary_variables'] = 'qc16_maxspeed qc18_validlocation qc19_uerr qc20_verr qc_primary_flag qc_operator_mask'
elif method == 'lsq':
ds['v'].attrs[
'ancillary_variables'] = 'qc15_gdop qc16_maxspeed qc18_validlocation qc_primary_flag qc_operator_mask'
# Set u_err attributes
ds['u_err'].attrs['units'] = '1'
ds['u_err'].attrs['valid_min'] = np.float32(0)
ds['u_err'].attrs['valid_max'] = np.float32(1)
ds['u_err'].attrs['coordinates'] = 'lon lat'
ds['u_err'].attrs['grid_mapping'] = 'crs'
# Set v_err attributes
ds['v_err'].attrs['units'] = '1'
ds['v_err'].attrs['valid_min'] = np.float32(0)
ds['v_err'].attrs['valid_max'] = np.float32(1)
ds['v_err'].attrs['coordinates'] = 'lon lat'
ds['v_err'].attrs['grid_mapping'] = 'crs'
if method == 'lsq':
ds['u_err'].attrs[
'long_name'] = 'Associated GDOP mapping error value associated with eastward velocity component'
ds['v_err'].attrs[
'long_name'] = 'Associated GDOP mapping error value associated with northward velocity component'
ds['total_errors'].attrs[
'long_name'] = 'Associated GDOP mapping error value, TotalErrors are the square-root of the norm covariance matrix (or the square root of the maximum eigenvalue of the 2x2 UV covariance matrix)'
ds['u_err'].attrs[
'comment'] = 'velocity measurements with error values over 1.25 are of questionable quality'
ds['v_err'].attrs[
'comment'] = 'velocity measurements with error values over 1.25 are of questionable quality'
ds['total_errors'].attrs[
'comment'] = 'velocity measurements with error values over 1.25 are of questionable quality'
elif method == 'oi':
ds['u_err'].attrs[
'long_name'] = 'Normalized uncertainty error associated with eastward velocity component'
ds['v_err'].attrs[
'long_name'] = 'Normalized uncertainty error associated with northward velocity component'
ds['total_errors'].attrs[
'long_name'] = 'Associated mapping error value, TotalErrors are the square-root of the sum of the squares of U_err and Verr'
ds['u_err'].attrs[
'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality'
ds['v_err'].attrs[
'comment'] = 'velocity measurements with error values over 0.6 are of questionable quality'
ds['total_errors'].attrs[
'comment'] = 'velocity measurements with error values over _ are of questionable quality'
# Set uv_covariance attributes
ds['uv_covariance'].attrs[
'long_name'] = 'Eastward and Northward covariance directional information of u and v'
ds['uv_covariance'].attrs['units'] = '1'
ds['uv_covariance'].attrs['comment'] = 'directional information of u and v'
ds['uv_covariance'].attrs['coordinates'] = 'lon lat'
ds['uv_covariance'].attrs['grid_mapping'] = 'crs'
# Set num_radials attributes
ds['num_radials'].attrs[
'long_name'] = 'Number of radial measurements used to calculate each totals velocity'
ds['num_radials'].attrs[
'comment'] = 'totals are not calculated with fewer than 3 contributing radial measurements from 2 sites'
ds['num_radials'].attrs['coordinates'] = 'lon lat'
ds['num_radials'].attrs['grid_mapping'] = 'crs'
# Set information attributes
ds['processing_parameters'].attrs[
'long_name'] = 'General and method specific processing parameter information'
ds['processing_parameters'].attrs['comment'] = processing_parameters_info
# ds['processing_parameters'].attrs['coordinates'] = 'parameters'
# Set qc flag attributes
ds['qc16_maxspeed'].attrs[
'standard_name'] = 'sea_water_velocity quality_flag'
ds['qc16_maxspeed'].attrs['units'] = '1'
ds['qc16_maxspeed'].attrs['valid_min'] = 1
ds['qc16_maxspeed'].attrs['valid_max'] = 9
ds['qc16_maxspeed'].attrs['coordinates'] = 'lon lat'
ds['qc16_maxspeed'].attrs['grid_mapping'] = 'crs'
ds['qc16_maxspeed'].attrs['flag_values'] = '1 2 3 4 9'
ds['qc16_maxspeed'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail missing_data'
ds['qc16_maxspeed'].attrs['comment'] = qc16_info
ds['qc18_validlocation'].attrs[
'standard_name'] = 'sea_water_velocity location_test_quality_flag'
ds['qc18_validlocation'].attrs['units'] = '1'
ds['qc18_validlocation'].attrs['valid_min'] = 1
ds['qc18_validlocation'].attrs['valid_max'] = 9
ds['qc18_validlocation'].attrs['coordinates'] = 'lon lat'
ds['qc18_validlocation'].attrs['grid_mapping'] = 'crs'
ds['qc18_validlocation'].attrs['flag_values'] = '1 2 3 4 9'
ds['qc18_validlocation'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail missing_data'
ds['qc18_validlocation'].attrs['comment'] = qc18_info
if method == 'oi':
ds['qc19_uerr'].attrs[
'standard_name'] = 'sea_water_velocity quality_flag'
ds['qc19_uerr'].attrs['units'] = '1'
ds['qc19_uerr'].attrs['valid_min'] = 1
ds['qc19_uerr'].attrs['valid_max'] = 9
ds['qc19_uerr'].attrs['coordinates'] = 'lon lat'
ds['qc19_uerr'].attrs['grid_mapping'] = 'crs'
ds['qc19_uerr'].attrs['flag_values'] = '1 2 3 4 9'
ds['qc19_uerr'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail missing_data'
ds['qc19_uerr'].attrs['comment'] = qc19_info
ds['qc20_verr'].attrs[
'standard_name'] = 'sea_water_velocity quality_flag'
ds['qc20_verr'].attrs['units'] = '1'
ds['qc20_verr'].attrs['valid_min'] = 1
ds['qc20_verr'].attrs['valid_max'] = 9
ds['qc20_verr'].attrs['coordinates'] = 'lon lat'
ds['qc20_verr'].attrs['grid_mapping'] = 'crs'
ds['qc20_verr'].attrs['flag_values'] = '1 2 3 4 9'
ds['qc20_verr'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail missing_data'
ds['qc20_verr'].attrs['comment'] = qc20_info
elif method == 'lsq':
ds['qc15_gdop'].attrs[
'standard_name'] = 'sea_water_velocity quality_flag'
ds['qc15_gdop'].attrs['units'] = '1'
ds['qc15_gdop'].attrs['valid_min'] = 1
ds['qc15_gdop'].attrs['valid_max'] = 9
ds['qc15_gdop'].attrs['coordinates'] = 'lon lat'
ds['qc15_gdop'].attrs['grid_mapping'] = 'crs'
ds['qc15_gdop'].attrs['flag_values'] = '1 2 3 4 9'
ds['qc15_gdop'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail missing_data'
ds['qc15_gdop'].attrs['comment'] = qc15_info
ds['qc_primary_flag'].attrs[
'standard_name'] = 'sea_water_velocity aggregate_quality_flag'
ds['qc_primary_flag'].attrs['units'] = '1'
ds['qc_primary_flag'].attrs['valid_min'] = 1
ds['qc_primary_flag'].attrs['valid_max'] = 4
ds['qc_primary_flag'].attrs['coordinates'] = 'lon lat'
ds['qc_primary_flag'].attrs['grid_mapping'] = 'crs'
ds['qc_primary_flag'].attrs['flag_values'] = '1 2 3 4'
ds['qc_primary_flag'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail'
ds['qc_primary_flag'].attrs['comment'] = qc_primary_flag_info
if method == 'oi':
ds['qc_primary_flag'].attrs[
'ancillary_variables'] = 'qc16_maxspeed qc18_validlocation qc19_uerr qc20_verr'
elif method == 'lsq':
ds['qc_primary_flag'].attrs[
'ancillary_variables'] = 'qc15_gdop qc16_maxspeed qc18_validlocation'
ds['qc_operator_mask'].attrs['units'] = '1'
ds['qc_operator_mask'].attrs['valid_min'] = 1
ds['qc_operator_mask'].attrs['valid_max'] = 9
ds['qc_operator_mask'].attrs['coordinates'] = 'lon lat'
ds['qc_operator_mask'].attrs['grid_mapping'] = 'crs'
ds['qc_operator_mask'].attrs['flag_values'] = '1 2 3 4 9'
ds['qc_operator_mask'].attrs[
'flag_meanings'] = 'pass not_evaluated suspect fail missing_data'
ds['qc_operator_mask'].attrs['comment'] = qc_operator_mask_info
logging.debug(
'{} - Setting variable encoding and fill values for netCDF4 output'.
format(fname))
# encode variables for export to netcdf
encoding = make_encoding(ds)
encoding['lon'] = dict(zlib=False, _FillValue=False)
encoding['lat'] = dict(zlib=False, _FillValue=False)
encoding['z'] = dict(zlib=False, _FillValue=False)
# add container variables that contain no data
kwargs = dict(crs=None, instrument=None)
ds = ds.assign(**kwargs)
# Set crs attributes
ds['crs'].attrs['grid_mapping_name'] = 'latitude_longitude'
ds['crs'].attrs['inverse_flattening'] = 298.257223563
ds['crs'].attrs['long_name'] = 'Coordinate Reference System'
ds['crs'].attrs['semi_major_axis'] = '6378137.0'
ds['crs'].attrs['epsg_code'] = 'EPSG:4326'
ds['crs'].attrs['comment'] = 'http://www.opengis.net/def/crs/EPSG/0/4326'
ds['instrument'].attrs['long_name'] = 'CODAR SeaSonde High Frequency Radar'
ds['instrument'].attrs[
'sensor_type'] = 'Direction-finding high frequency radar antenna'
ds['instrument'].attrs['make_model'] = 'CODAR SeaSonde'
ds['instrument'].attrs['serial_number'] = 1
# Create save directory if it doesn't exist.
create_dir(save_dir)
logging.debug('{} - Saving dataset to netCDF4 file: {}'.format(
fname, file_and_path))
ds.to_netcdf(file_and_path,
encoding=encoding,
format='netCDF4',
engine='netcdf4',
unlimited_dims=['time'])
logging.info('{} - netCDF4 file successfully created: {}'.format(
fname, file_and_path))
