def bbox(self, header: PoseHeader):
data = ma.transpose(self.data, axes=POINTS_DIMS)
# Split data by components, `ma` doesn't support ".split"
components = []
idx = 0
for component in header.components:
components.append(data[list(range(idx,
idx + len(component.points)))])
idx += len(component.points)
boxes = [
ma.stack([ma.min(c, axis=0), ma.max(c, axis=0)])
for c in components
]
boxes_cat = ma.concatenate(boxes)
if type(boxes_cat.mask
) == np.bool_: # Sometimes, it doesn't concatenate the mask...
boxes_mask = ma.concatenate([b.mask for b in boxes])
boxes_cat = ma.array(boxes_cat, mask=boxes_mask)
new_data = ma.transpose(boxes_cat, axes=POINTS_DIMS)
confidence_mask = np.split(new_data.mask, [-1], axis=3)[0]
confidence_mask = np.squeeze(confidence_mask, axis=-1)
confidence = np.where(confidence_mask == True, 0, 1)
return NumPyPoseBody(self.fps, new_data, confidence)
def get_points(self, indexes: List[int]):
data = ma.transpose(self.data, axes=POINTS_DIMS)
new_data = ma.transpose(data[indexes], axes=POINTS_DIMS)
confidence_reshape = (2, 1, 0)
confidence = np.transpose(self.confidence, axes=confidence_reshape)
new_confidence = np.transpose(confidence[indexes], axes=confidence_reshape)
return NumPyPoseBody(self.fps, new_data, new_confidence)
def _out2csv(path, fheader, f_name, ignore_fields):
print(f_name)
ignore_fields = ignore_fields if ignore_fields else []
with open(os.path.join(path, f_name + '.csv'), "w") as csvfile:
c = csv.writer(csvfile, delimiter=',')
csv_header = sorted(
list(set(fheader.variables.keys()) - set(ignore_fields)))
insert_indices = {
"time": csv_header.index("time"),
"lat": csv_header.index("lat"),
"lon": csv_header.index("lon")
}
c.writerow(csv_header)
var_matrix = []
set_mask = False
mask_indices = np.array([])
length_dic = []
for var in csv_header:
if var not in ['time', 'lat', 'lon'] and var not in ignore_fields:
if not set_mask:
# we will have lon as the row and lat as the column for kfold split vertically
mask_indices = ma.where(~ma.getmaskarray(
ma.transpose(fheader.variables[var][:], (0, 2, 1))))
set_mask = True
compressed = ma.transpose(fheader.variables[var][:],
(0, 2, 1)).compressed()
length_dic.append(len(compressed))
var_matrix.append(compressed)
assert all(length_dic[0] == length for length in length_dic), \
"Should always be called after all variables are masked to be overlapped"
dim_indices = {
"time": mask_indices[0],
"lon": mask_indices[1],
"lat": mask_indices[2]
} # order of indices changes correspondingly
for dim_name in [
_[0] for _ in sorted(insert_indices.items(),
key=operator.itemgetter(1))
]:
var_matrix.insert(insert_indices[dim_name], [
fheader.variables[dim_name][i] for i in dim_indices[dim_name]
])
num = 0
for row in zip(*var_matrix):
c.writerow(row)
num += 1
def get_rank_minimum_matrix(self, sorted_sample_population):
return sum([
multiply((self.get_steped_difference(sorted_sample) *
transpose(self.get_steped_difference(sorted_sample))),
weight) for weight, sorted_sample in zip(
self.get_adjusted_weights(sorted_sample_population),
sorted_sample_population)
])
def findEout(self, numSamples=1000):
e_out = 0
dataSamples = [self.createDataPoint() for _ in range(numSamples)]
for x, y in dataSamples:
e_out += log(1 +
exp(-1 * multiply(y, np.dot(transpose(self.w), x))))
e_out /= float(numSamples)
return e_out
def __init__(self, data, useAttributeLabels=False):
self.data = data
self.useAttributeLabels = useAttributeLabels
self.attr_labels, self.data_classes = self._data_info(data)
self.attributes = [attr for attr in self.data.domain.attributes if attr.varType in [orange.VarTypes.Continuous, orange.VarTypes.Discrete]]
self.classes = np.array(self.attr_labels if useAttributeLabels else self.data_classes)
self.keys = range(len(data)) if useAttributeLabels else self.attributes
self.array, _, _ = data.toNumpyMA()
if self.useAttributeLabels:
self.array = ma.transpose(self.array)
# self.dim = 1 if useAttributeLabels else 0
self.dim = 0
def __init__(self, data, useAttributeLabels=False):
self.data = data
self.useAttributeLabels = useAttributeLabels
self.attr_labels, self.data_classes = self._data_info(data)
self.attributes = [attr for attr in self.data.domain.attributes \
if isinstance(attr, (ContinuousVariable, DiscreteVariable))]
self.classes = np.array(self.attr_labels if useAttributeLabels else self.data_classes)
self.keys = list(range(len(data))) if useAttributeLabels else self.attributes
self.array, _, _ = data.toNumpyMA()
if self.useAttributeLabels:
self.array = ma.transpose(self.array)
# self.dim = 1 if useAttributeLabels else 0
self.dim = 0
def runLogisticRegression(self):
diff = 1
while diff > 0.01:
permutation = np.random.permutation(self.N)
newWeights = self.w.copy()
for i in permutation:
x, y = self.trainingData[i]
gradient = divide(
multiply(-1.0, multiply(x, y)),
(1.0 + exp(multiply(y, np.dot(transpose(self.w), x)))))
newWeights = subtract(newWeights,
multiply(self.learningRate, gradient))
self.epoch += 1
diff = norm(self.w - newWeights)
self.w = newWeights
def test_testTakeTransposeInnerOuter(self):
# Test of take, transpose, inner, outer products
x = arange(24)
y = np.arange(24)
x[5:6] = masked
x = x.reshape(2, 3, 4)
y = y.reshape(2, 3, 4)
assert_(eq(np.transpose(y, (2, 0, 1)), transpose(x, (2, 0, 1))))
assert_(eq(np.take(y, (2, 0, 1), 1), take(x, (2, 0, 1), 1)))
assert_(eq(np.inner(filled(x, 0), filled(y, 0)), inner(x, y)))
assert_(eq(np.outer(filled(x, 0), filled(y, 0)), outer(x, y)))
y = array(['abc', 1, 'def', 2, 3], object)
y[2] = masked
t = take(y, [0, 3, 4])
assert_(t[0] == 'abc')
assert_(t[1] == 2)
assert_(t[2] == 3)
def __init__(self, data, useAttributeLabels=False):
self.data = data
self.useAttributeLabels = useAttributeLabels
self.attr_labels, self.data_classes = self._data_info(data)
self.attributes = [attr for attr in self.data.domain.attributes \
if isinstance(attr, (ContinuousVariable, DiscreteVariable))]
self.classes = np.array(
self.attr_labels if useAttributeLabels else self.data_classes)
self.keys = list(range(
len(data))) if useAttributeLabels else self.attributes
self.array, _, _ = data.toNumpyMA()
if self.useAttributeLabels:
self.array = ma.transpose(self.array)
# self.dim = 1 if useAttributeLabels else 0
self.dim = 0
def test_testTakeTransposeInnerOuter(self):
# Test of take, transpose, inner, outer products
x = arange(24)
y = np.arange(24)
x[5:6] = masked
x = x.reshape(2, 3, 4)
y = y.reshape(2, 3, 4)
assert_(eq(np.transpose(y, (2, 0, 1)), transpose(x, (2, 0, 1))))
assert_(eq(np.take(y, (2, 0, 1), 1), take(x, (2, 0, 1), 1)))
assert_(eq(np.inner(filled(x, 0), filled(y, 0)), inner(x, y)))
assert_(eq(np.outer(filled(x, 0), filled(y, 0)), outer(x, y)))
y = array(["abc", 1, "def", 2, 3], object)
y[2] = masked
t = take(y, [0, 3, 4])
assert_(t[0] == "abc")
assert_(t[1] == 2)
assert_(t[2] == 3)
#
# This is the ScientificPython method for opening a netCDF file.
#
# ice1 = NetCDFFile(data_dir + "/cdf/fice.nc","r")
#
# Create a masked array to accommodate missing values in the fice variable.
#
fice = ice1.variables["fice"] # fice[120,49,100]
ficea = fice[:, :, :]
fill_value = None
if (hasattr(fice, "missing_value")):
fill_value = fice.missing_value
elif (hasattr(fice, "_FillValue")):
fill_value = fice._FillValue
fice_masked = MA.transpose(MA.masked_values(ficea, fill_value), (1, 2, 0))
hlat = ice1.variables["hlat"] # hlat[49]
hlon = ice1.variables["hlon"] # hlon[100]
dimf = fice.shape # Define an array to hold long-term monthly means.
ntime = fice.shape[0]
nhlat = fice.shape[1]
nhlon = fice.shape[2]
nmo = 0
month = nmo + 1
icemon = MA.zeros((nhlat, nhlon), dtype=float)
for i in range(fice_masked.shape[0]):
for j in range(fice_masked.shape[1]):
import numpy.ma as ma x = ma.arange(4).reshape((2, 2)) x[1, 1] = ma.masked ma.transpose(x)
def get_rank_one_matrix(self):
return multiply(self.anisotropic_evolution_path,
transpose(self.anisotropic_evolution_path))
cmapdiff = CyBuWhRdYl
if var in ['NO3_excess','FG_CO2','STF_O2','FvPER_DIC']:
cmap = CyBuWhRdYl
else:
cmap = pl.cm.jet
for r in range(nreg):
if reg_lname[r] in region_list:
if POPDIAGPY == 'TRUE':
outfile = os.path.join(outdir,'zavg_%s_%s_diff.png'%(var,reg_sname[r]))
else:
outfile = os.path.join(outdir,'zavg_%04d-%04d_%s_%s_diff.png'%(
yrstart,yrend,var,reg_sname[r]))
print('plotting %s'%(os.path.basename(outfile)))
datam = MA.transpose(varsm[var][:,r,:])
datao = MA.transpose(varso[var][:,r,:])
# calculate difference ------------------------------------------
differnce = datam - datao
# plot ----------------------------------------------------------
fig = pl.figure(figsize=(8,11))
# plot experiment ------------------------------------------------
ax = fig.add_subplot(211)
CF = ax.contourf(mon,lat,datam,variables[var]['clev'],cmap=cmap,
norm=colors.BoundaryNorm(variables[var]['clev'],cmap.N),extend='both')
CF.cmap.set_under('black')
CF.cmap.set_over('black')
def fit(self, A):
"""
Fit model parameters U, V.
:param X:
Data matrix of shape (m, n)
Unknown values are assumed to take the value of zero (0).
"""
start_time = time.time()
# permute matrix A, columns minimum increasing
columns_perm = np.argsort(np.min(A, axis = 0))
A = A[:, columns_perm]
m = A.shape[0] # number of rows
n = A.shape[1] # number of columns
# initialization of U matrix
U_initial = self.initialize_U(A, m)
V = min_plus(ma.transpose(np.negative(U_initial)), A)
U = min_plus(A, ma.transpose(np.negative(V)))
D = np.subtract(A, max_plus(U, V))
# initialization of f values needed for convergence test
f_old = 1
f_new = self.b_norm(D)
f = f_new
i_list, j_list, k_list = range(m), range(n), range(self.rank)
U_new, V_new = U, V
comb = get_coordinates(A)
iterations = 0
while abs(f_old - f_new) > self.epsilon:
f = f_new
for x in comb:
i = x[0]
j = x[1]
temp = False
temp_1 = False
for k in k_list:
iterations += 1
V_new = copy.deepcopy(V)
V_new[k, j] = A[i, j] - U[i, k]
U_new = min_plus(A, np.transpose(np.negative(V_new)))
V_new = min_plus(np.transpose(np.negative(U_new)), A)
f_new = self.b_norm(np.subtract(A, max_plus(U_new, V_new)))
if self.criterion == 'iterations' and iterations >= self.max_iter:
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
if f_new < f:
temp_1 = True
break
if temp_1:
break # only executed if the inner loop DID break
for k in k_list:
iterations += 1
U_new = copy.deepcopy(U)
U_new[i, k] = A[i, j] - V[k, j]
V_new = min_plus(np.transpose(np.negative(U_new)), A)
U_new = min_plus(A, np.transpose(np.negative(V_new)))
f_new = self.b_norm(np.subtract(A, max_plus(U_new, V_new)))
if self.criterion == 'iterations' and iterations >= self.max_iter:
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
if f_new < f:
temp = True
break
if temp:
break # only executed if the inner loop DID break
if f_new < f:
U = U_new
V = V_new
f_old = f
f = f_new
if self.criterion == 'iterations' and iterations >= self.max_iter:
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
else:
print("no solution found!")
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
return
self.assign_values(U, V, f, iterations, columns_perm, round(time.time() - start_time, 3))
# # This is the ScientificPython method for opening a netCDF file. # # ice1 = NetCDFFile(data_dir + "/cdf/fice.nc","r") # # Create a masked array to accommodate missing values in the fice variable. # fice = ice1.variables["fice"] # fice[120,49,100] ficea = fice[:,:,:] fill_value = None if (hasattr(fice,"missing_value")): fill_value = fice.missing_value elif (hasattr(fice,"_FillValue")): fill_value = fice._FillVlaue fice_masked = MA.transpose(MA.masked_values(ficea,fill_value),(1,2,0)) hlat = ice1.variables["hlat"] # hlat[49] hlon = ice1.variables["hlon"] # hlon[100] dimf = fice.shape # Define an array to hold long-term monthly means. ntime = fice.shape[0] nhlat = fice.shape[1] nhlon = fice.shape[2] nmo = 0 month = nmo+1 icemon = MA.zeros((nhlat,nhlon),dtype=float) for i in xrange(fice_masked.shape[0]):
# plot figures --------------------------------------------------------
if var in ['NO3_excess','FG_CO2','STF_O2','FvPER_DIC']:
cmap = CyBuWhRdYl
else:
cmap = pl.cm.jet
for r in range(nreg):
if reg_lname[r] in region_list:
if POPDIAGPY == 'TRUE':
outfile = os.path.join(outdir,'zavg_%s_%s.png'%(var,reg_sname[r]))
else:
outfile = os.path.join(outdir,'zavg_%04d-%04d_%s_%s.png'%(
yrstart,yrend,var,reg_sname[r]))
print('plotting %s'%(os.path.basename(outfile)))
data = MA.transpose(vars()[var][:,r,:])
fig = pl.figure()
ax = fig.add_subplot(111)
if var in vintlist: # vertically integrated
CF = ax.contourf(mon,lat,data,variables[var]['cklev'],cmap=cmap,
norm=colors.BoundaryNorm(variables[var]['cklev'],cmap.N),extend='both')
CF.cmap.set_under('black')
CF.cmap.set_over('black')
CS = ax.contour(mon,lat,data,CF.levels,colors='k',
linewidths=0.25)
CB = fig.colorbar(CF,ax=ax,drawedges=True,aspect=15,
ticks=variables[var]['cklev'][1:-1],format='%G')
CB.set_label(variables[var]['kunits'])
for line in CB.ax.get_yticklines(): # remove tick lines
def plot2D(filein,varname,coef=None,units='',lev=None,levunits=None,tmin=None,tmax=None,dtlabel='1h',namefig=None,lbias=False,refdataset=None,error=None,**kwargs):
"""
Do a 2D plot of varname for several MUSC file
"""
title0 = kwargs['title']
data = OrderedDict()
levax = {}
time = {}
timeax = {}
if coef is None:
coef = {}
for k in filein.keys():
coef[k] = 1.
elif isinstance(coef,int) or isinstance(coef,float):
tmp = coef
coef = {}
for k in filein.keys():
coef[k] = tmp
if lev is None:
lev = {}
levunits = {}
for k in filein.keys():
lev[k] = 'zh'
levunits[k] = 'km'
elif isinstance(lev,str):
tmp = lev
lev = {}
for k in filein.keys():
lev[k] = tmp
if levunits is None:
levunits = {}
for k in filein.keys():
if lev[k] == 'zh':
levunits[k] = 'km'
elif lev[k] in['ph','pf']:
levunits[k] = 'hPa'
else:
print 'lev={} not coded yet'.format(lev[k])
sys.exit()
elif isinstance(levunits,str):
tmp = levunits
levunits = {}
for k in filein.keys():
levunits[k] = tmp
datasets = filein.keys()
if lbias:
if refdataset is None:
print 'please provide reference dataset (keyword refdataset) to compute bias)'
sys.exit()
else:
f = cdms2.open(filein[refdataset])
dataref = f(varname[refdataset],squeeze=1)*coef[refdataset]
f.close()
tmp = []
for dd in datasets:
if not(dd == refdataset):
tmp.append(dd)
datasets = tmp
for k in datasets: #filein.keys():
#print k
f = cdms2.open(filein[k])
try:
data[k] = f(varname[k],squeeze=1)*coef[k]
if lbias:
nt,nlev = data[k].shape
tmp = data[k].getTime()
data[k] = data[k]-dataref[0:nt,:]
data[k].setAxis(0,tmp)
except cdms2.error.CDMSError as e:
data[k] = None
f.close()
if verbose:
print 'Variable {2} probably unknown in dataset {0} (file={1})'.format(k,filein[k],varname[k])
print 'Raised error: cdms2.error.CDMSError', e
if error is not None:
if isinstance(error,dict):
if error.has_key(k):
error[k].append(varname[k])
else:
error[k] = [varname[k],]
else:
print 'type of error unexpected:', type(error)
print 'error should be a dictionnary'
sys.exit()
except:
raise
if data[k] is not None:
try:
levax[k] = f(lev[k],squeeze=1)
except:
if lev[k] == 'zh':
levax[k] = f('zf')
if lev[k] == 'ph':
levax[k] = f('pf')
f.close()
if lev[k] == 'zh':
if levunits[k] == 'm':
pass
elif levunits[k] == 'km':
levax[k] = levax[k]/1000.
else:
print 'levunits={} for lev=zh not coded yet'.format(levunits[k])
sys.exit()
elif lev[k] == 'ph':
if levunits[k] == 'Pa':
pass
elif levunits[k] == 'hPa':
levax[k] = levax[k]/100.
else:
print 'levunits={} for lev=ph not coded yet'.format(levunits[k])
sys.exit()
else:
print 'lev={} not coded yet'.format(lev)
sys.exit()
time[k] = data[k].getTime()
if len(levax[k].shape) == 2:
nt,nlev = levax[k].shape
timeax[k] = np.tile(time[k][:],(nlev,1))
X = timeax[k]
Y = np.transpose(levax[k])
else:
nlev, = levax[k].shape
nt, = time[k].shape
timeax[k] = np.tile(time[k][:],(nlev,1))
levax[k] = np.tile(levax[k],(nt,1))
X = np.array(timeax[k][:])
Y = np.transpose(levax[k][:])
if tmin is None:
tmin = cdtime.reltime(time[k][0],time.units)
if tmax is None:
tmax = cdtime.reltime(time[k][-1],time.units)
tt = []
tlabels = []
tminloc = tmin.tocomp()
if dtlabel == '1h':
tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)
t0 = tmin0.add(0,cdtime.Hour)
while t0.cmp(tmax) <= 0:
if t0.cmp(tmin) >= 0:
tt.append(t0.torel(time[k].units).value)
tlabels.append('{0}'.format(t0.hour))
t0 = t0.add(1,cdtime.Hour)
elif dtlabel == '2h':
tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)
t0 = tmin0.add(0,cdtime.Hour)
while t0.cmp(tmax) <= 0:
if t0.cmp(tmin) >= 0:
tt.append(t0.torel(time[k].units).value)
tlabels.append('{0}'.format(t0.hour))
t0 = t0.add(2,cdtime.Hour)
elif dtlabel == '6h':
tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)
t0 = tmin0.add(0,cdtime.Hour)
while t0.cmp(tmax) <= 0:
if t0.cmp(tmin) >= 0:
tt.append(t0.torel(time[k].units).value)
tlabels.append('{0}'.format(t0.hour))
t0 = t0.add(6,cdtime.Hour)
elif dtlabel == '10d':
tmin0 = cdtime.comptime(tminloc.year,tminloc.month,tminloc.day,tminloc.hour)
t0 = tmin0.add(0,cdtime.Hour)
while t0.cmp(tmax) <= 0:
if t0.cmp(tmin) >= 0:
tt.append(t0.torel(time[k].units).value)
tlabels.append('{0}/{1}'.format(t0.month,t0.day))
if t0.day == 1:
t0 = cdtime.comptime(t0.year,t0.month,10,0)
elif t0.day == 10:
t0 = cdtime.comptime(t0.year,t0.month,20,0)
elif t0.day == 20:
if t0.month == 12:
t0 = cdtime.comptime(t0.year+1,1,1,0)
else:
t0 = cdtime.comptime(t0.year,t0.month+1,1,0)
else:
t0 = t0.add(10,cdtime.Day)
#print 't0 unexpected',t0
#sys.exit()
else:
print 'dtlabel={} not coded yet'.format(dtlabel)
sys.exit()
tlabels = tt,tlabels
if isinstance(namefig,str):
tmp = k + '_' + namefig
elif isinstance(namefig,dict):
tmp = namefig[k]
elif namefig is None:
tmp = None
else:
print 'namefig type unexpected:', namefig
sys.exit()
kwargs['title'] = '{0} - {1}'.format(title0,k)
plotutils.plot2D(X,Y,ma.transpose(data[k]),\
xmin = tmin.torel(time[k].units).value,\
xmax = tmax.torel(time[k].units).value,\
xlabels=tlabels,\
namefig=tmp,\
**kwargs)
def load_GPM_IMERG_files_with_spatial_filter(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
user_mask_file=None,
mask_variable_name='mask',
user_mask_values=[10],
longitude_name='lon',
latitude_name='lat'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate a two-dimensional array \
for the masked grid points.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:user_mask_file: user's own gridded mask file(a netCDF file name)
:type name: :mod:`string`
:mask_variable_name: mask variables in user_mask_file
:type name: :mod:`string`
:longitude_name: longitude variable in user_mask_file
:type name: :mod:`string`
:latitude_name: latitude variable in user_mask_file
:type name: :mod:`string`
:param user_mask_values: grid points where mask_variable == user_mask_value will be extracted.
:type user_mask_values: list of strings
:returns: A two-dimensional array with the requested variable's MASKED data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
if ifile == 0 and user_mask_file:
file_object = netCDF4.Dataset(user_mask_file)
mask_variable = file_object.variables[mask_variable_name][:]
mask_longitude = file_object.variables[longitude_name][:]
mask_latitude = file_object.variables[latitude_name][:]
spatial_mask = utils.regrid_spatial_mask(lons,lats,
mask_longitude, mask_latitude,
mask_variable,
user_mask_values)
y_index, x_index = numpy.where(spatial_mask == 0)
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
values0 = ma.transpose(ma.masked_less(
file_object['Grid'][variable_name][:], 0.))
values_masked = values0[y_index, x_index]
values_masked = ma.expand_dims(values_masked, axis=0)
if ifile == 0:
values = values_masked
else:
values = ma.concatenate((values, values_masked))
file_object.close()
return values
def __init__(self, filename):
'''
Handles setting up a reader.
'''
self.fields = {}
self.info = {}
self.nc_dataset = Dataset(filename)
self.filename = filename
self.time = common.ncvar_to_dict(self.nc_dataset.variables['Time'])
self.time['data'] = ma.array(
get_datetime_from_epoch(self.time['data'][:]))
self.time['units'] = "Datetime objects"
del self.time['_FillValue']
self.fields['Nd'] = common.ncvar_to_dict(
self.nc_dataset.variables['VolumetricDrops'])
self.fields['Nd']['data'] = ma.transpose(
np.power(10, self.fields['Nd']['data']))
ma.set_fill_value(self.fields['Nd']['data'], 0.)
self.fields['Nd']['data'] = self.fields['Nd']['data'].filled()
del self.fields['Nd']['_FillValue']
self.fields['Nd']['units'] = "1/m^3 1/mm"
self.fields['RR'] = common.ncvar_to_dict(
self.nc_dataset.variables['ParsivelIntensity'])
self.fields['RR']['data'] = ma.masked_array(self.fields['RR']['data'])
ma.set_fill_value(self.fields['RR']['data'],
self.fields['RR']['_FillValue'])
del self.fields['RR']['_FillValue']
self.fields['Zh'] = common.ncvar_to_dict(
self.nc_dataset.variables['Reflectivity'])
del self.fields['Zh']['_FillValue']
self.fields['num_particles'] = common.ncvar_to_dict(
self.nc_dataset.variables['RawDrops'])
self.fields['num_particles']['data'] = ma.masked_array(
self.fields['num_particles']['data'])
ma.set_fill_value(self.fields['num_particles']['data'],
self.fields['num_particles']['_FillValue'])
del self.fields['num_particles']['_FillValue']
self.fields['terminal_velocity'] = \
common.ncvar_to_dict(self.nc_dataset.variables['VelocityDrops'])
self.fields['terminal_velocity']['data'] = \
ma.transpose(ma.masked_array(self.fields['terminal_velocity']['data']))
ma.set_fill_value(self.fields['terminal_velocity']['data'],
self.fields['terminal_velocity']['_FillValue'])
del self.fields['terminal_velocity']['_FillValue']
self.fields['Precip_Code'] = common.ncvar_to_dict(
self.nc_dataset.variables['PrecipCode'])
diameter = ma.array([
0.0625, 0.1875, 0.3125, 0.4375, 0.5625, 0.6875, 0.8125, 0.9375,
1.0625, 1.1875, 1.375, 1.625, 1.875, 2.125, 2.375, 2.75, 3.25,
3.75, 4.25, 4.75, 5.5, 6.5, 7.5, 8.5, 9.5, 11., 13., 15., 17., 19.,
21.5, 24.5
])
spread = ma.array([
0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
0.125, 0.250, 0.250, 0.250, 0.250, 0.250, 0.500, 0.500, 0.500,
0.500, 0.500, 1.000, 1.000, 1.000, 1.000, 1.000, 2.000, 2.000,
2.000, 2.000, 2.000, 3.000, 3.000
])
# velocity = ma.array([0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3,
# 1.5, 1.7, 1.9, 2.2, 2.6, 3.0, 3.4, 3.8, 4.4, 5.2, 6.0, 6.8, 7.6, 8.8,
# 10.4, 12.0, 13.6, 15.2, 17.6, 20.8])
self.bin_edges = common.var_to_dict(
'bin_edges', np.hstack((0, diameter + np.array(spread) / 2)), 'mm',
'Boundaries of bin sizes')
self.spread = common.var_to_dict('spread', spread, 'mm',
'Bin size spread of bins')
self.diameter = common.var_to_dict('diameter', diameter, 'mm',
'Particle diameter of bins')
for key in self.nc_dataset.ncattrs():
self.info[key] = self.nc_dataset.getncattr(key)
def load_GPM_IMERG_files(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
name='GPM_IMERG'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate an OCW Dataset obejct.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:returns: An OCW Dataset object with the requested variable's data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
variable_unit = "mm/hr"
times = []
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S")
times.append(datetime(*time_struct_parsed[:6]))
values0 = ma.transpose(
ma.masked_less(file_object['Grid'][variable_name][:], 0.))
values0 = ma.expand_dims(values0, axis=0)
if ifile == 0:
values = values0
else:
values = ma.concatenate((values, values0))
file_object.close()
times = numpy.array(times)
return Dataset(lats,
lons,
times,
values,
variable_name,
units=variable_unit,
name=name)
if var in ['NO3_excess', 'FG_CO2', 'STF_O2', 'FvPER_DIC']:
cmap = CyBuWhRdYl
else:
cmap = pl.cm.jet
for r in range(nreg):
if reg_lname[r] in region_list:
if POPDIAGPY == 'TRUE':
outfile = os.path.join(
outdir, 'zavg_%s_%s_diff.png' % (var, reg_sname[r]))
else:
outfile = os.path.join(
outdir, 'zavg_%04d-%04d_%s_%s_diff.png' %
(yrstart, yrend, var, reg_sname[r]))
print('plotting %s' % (os.path.basename(outfile)))
datam = MA.transpose(varsm[var][:, r, :])
datao = MA.transpose(varso[var][:, r, :])
# calculate difference ------------------------------------------
differnce = datam - datao
# plot ----------------------------------------------------------
fig = pl.figure(figsize=(8, 11))
# plot experiment ------------------------------------------------
ax = fig.add_subplot(211)
if var in vintlist: # vertically integrated
CF = ax.contourf(mon,
lat,
def points_perspective(self):
return ma.transpose(self.data, axes=POINTS_DIMS)
def load_GPM_IMERG_files_with_spatial_filter(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
user_mask_file=None,
mask_variable_name='mask',
user_mask_values=[10],
longitude_name='lon',
latitude_name='lat'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate a two-dimensional array \
for the masked grid points.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:user_mask_file: user's own gridded mask file(a netCDF file name)
:type name: :mod:`string`
:mask_variable_name: mask variables in user_mask_file
:type name: :mod:`string`
:longitude_name: longitude variable in user_mask_file
:type name: :mod:`string`
:latitude_name: latitude variable in user_mask_file
:type name: :mod:`string`
:param user_mask_values: grid points where mask_variable == user_mask_value will be extracted.
:type user_mask_values: list of strings
:returns: A two-dimensional array with the requested variable's MASKED data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
if ifile == 0 and user_mask_file:
file_object = netCDF4.Dataset(user_mask_file)
mask_variable = file_object.variables[mask_variable_name][:]
mask_longitude = file_object.variables[longitude_name][:]
mask_latitude = file_object.variables[latitude_name][:]
spatial_mask = utils.regrid_spatial_mask(lons, lats,
mask_longitude,
mask_latitude,
mask_variable,
user_mask_values)
y_index, x_index = numpy.where(spatial_mask == 0)
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
values0 = ma.transpose(
ma.masked_less(file_object['Grid'][variable_name][:], 0.))
values_masked = values0[y_index, x_index]
values_masked = ma.expand_dims(values_masked, axis=0)
if ifile == 0:
values = values_masked
else:
values = ma.concatenate((values, values_masked))
file_object.close()
return values
def load_GPM_IMERG_files(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
name='GPM_IMERG'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate an OCW Dataset obejct.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:returns: An OCW Dataset object with the requested variable's data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
variable_unit = "mm/hr"
times = []
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S")
times.append(datetime(*time_struct_parsed[:6]))
values0 = ma.transpose(ma.masked_less(
file_object['Grid'][variable_name][:], 0.))
values0 = ma.expand_dims(values0, axis=0)
if ifile == 0:
values = values0
else:
values = ma.concatenate((values, values0))
file_object.close()
times = numpy.array(times)
return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
