def kdtree_sample2d(xin, yin, z2d, xout, yout, distance=2.,method='linear'):
""" bin random data points to grids
loc_points: 2 x dpoints, lon, lat
loc_grids: 2 x dgrids, x.ravel, y.ravel
"""
from scipy import spatial, ma
from numpy import meshgrid, exp, array,c_, where
xip,xin = find_overlap(xin, xout)
yip,yin = find_overlap(yin, yout)
z2ds = z2d[where(yip==True)[0],:][:,where(xip==True)[0]]
ismask = ma.is_masked(z2ds)
xin2d,yin2d = meshgrid(xin, yin)
if ismask:
xin1d, yin1d = xin2d[z2ds.mask==False].ravel(), yin2d[z2ds.mask==False].ravel()
z1d = z2ds[z2ds.mask==False]
locs = c_[xin1d, yin1d]
else:
xin1d, yin1d = xin2d.ravel(), yin2d.ravel()
z1d = z2ds.ravel()
locs = c_[xin1d, yin1d]
tree = spatial.cKDTree(locs)
grids = zip(xout, yout)
index = tree.query_ball_point(grids, distance)
Tmis=[]
sample_size=[]
for i in range(xout.size):
ip = index[i]
if len(ip) == 0:
Tmis.append(999999)
sample_size.append(0)
else:
dis = ((xin1d[ip]-xout[i])**2+(yin1d[ip]-yout[i])**2)
if method=='linear':
dis = ma.masked_greater(dis**0.5, distance)
weight = distance - dis**0.5
else:
weight = exp(-(dis/distance**2))
weight = weight/weight.sum()
Tmis.append((weight*z1d[ip]).sum())
sample_size.append(len(ip))
zout = ma.masked_greater(array(Tmis),1e5)
sample_size = ma.masked_equal(array(sample_size),0)
return zout, sample_size
def read(grb_name):
"""Read the MPE data from a grib file into an array.
This function uses wgrib2 to write and then read the MPE
data via an intermediate binary file, which is later
deleted. The non-raining regions are masked and the data
converted into mm/hr.
"""
rows = 3712 # fixed for MPE grid
cols = 3712 # fixed for MPE grid
bin_name = grb_name[0:-4] + '.bin'
exec_str = 'wgrib2 -no_f77 ' + '-bin ' + bin_name + ' ' + grb_name # spaces are N.B.
execute(exec_str)
a = read_bin_data(bin_name)
os.remove(bin_name)
a = a.reshape(rows, cols)
# data in grib file are oriented EW:SN
# need to check that this is correct, the plots look fine...
a = np.fliplr(a)
## a = np.flipud(a)
# mask
a = ma.masked_greater(a, 9E20)
# convert from mm/s to mm/hr (I think...)
a = 3600*a
return a
def _breakList(self, inList, index, parameter):
par = float(parameter)
array = N.empty(shape=[len(inList),],dtype=N.float64)
i = 0
for parameters in inList:
array[i] = parameters[index]
i = i + 1
greater = MA.masked_less(array, par)
less = MA.masked_greater(array, par)
upper = MA.minimum(greater)
lower = MA.maximum(less)
upperArray = MA.masked_inside(array, par, upper)
lowerArray = MA.masked_inside(array, lower, par)
upperList = []
lowerList = []
i = 0
for parameters in inList:
if upperArray.mask[i]:
upperList.append(parameters)
if lowerArray.mask[i]:
lowerList.append(parameters)
i = i + 1
return upperList, lowerList
def action(self, state):
mesh = state.mesh.copy().apply_transform(state.T_obj_world.matrix)
mesh.fix_normals()
vertices, face_ind = sample.sample_surface_even(mesh, self.num_samples)
normals = mesh.face_normals[face_ind]
angles = normals.dot(up)
valid_pushes = np.logical_and(ma.masked_greater(angles, 1.39626), ma.masked_less(angles, 1.74533)) # within 10 degrees of horizontal
if not np.any(valid_pushes):
return LinearPushAction(None, None, metadata={'vertex': np.array([0,0,0]), 'normal': np.array([1,0,0])})
best_valid_ind = np.argmax(normals[valid_pushes][:,2]) # index of best
best_ind = np.arange(self.num_samples)[valid_pushes][best_valid_ind]
start_position = vertices[best_ind] + normals[best_ind] * .015
end_position = vertices[best_ind] - normals[best_ind] * .04
start_pose, end_pose = self.get_hand_pose(start_position, end_position)
return LinearPushAction(
start_pose,
end_pose,
metadata={
'vertex': vertices[best_ind],
'normals': normals[best_ind],
}
)
def import_ijxyz_ascii_tmpl(self, mfile, template):
"""Import OW/DSG IJXYZ ascii format, with a Cube or RegularSurface
instance as template."""
fin = xtgeo._XTGeoCFile(mfile)
if isinstance(template, (xtgeo.cube.Cube, xtgeo.surface.RegularSurface)):
logger.info("OK template")
else:
raise ValueError("Template is of wrong type: {}".format(
type(template)))
nxy = template.ncol * template.nrow
_iok, val = _cxtgeo.surf_import_ijxyz_tmpl(fin.fhandle, template.ilines,
template.xlines, nxy, 0)
val = ma.masked_greater(val, _cxtgeo.UNDEF_LIMIT)
self._xori = template.xori
self._xinc = template.xinc
self._yori = template.yori
self._yinc = template.yinc
self._ncol = template.ncol
self._nrow = template.nrow
self._rotation = template.rotation
self._yflip = template.yflip
self._values = val.reshape((self._ncol, self._nrow))
self._filesrc = mfile
self._ilines = template._ilines.copy()
self._xlines = template._xlines.copy()
fin.close()
def masked_cube_simple(mycube, slicevar, v1, v2, threshold):
"""
Mask function 1 -- simple cube cropping
masking for a specific variable slicevar (string)
arguments: cube, variable, min value, max value, threshold
"""
import numpy.ma as ma
coord_names = [coord.name() for coord in mycube.coords()]
if slicevar in coord_names:
coord = mycube.coord(slicevar)
print('Masking on variable: %s' % coord.standard_name)
cubeslice = mycube.extract(
iris.Constraint(
coord_values={
coord.standard_name: lambda cell: v1 <= cell.point <= v2
}))
if cubeslice is not None:
masked_cubeslice = cubeslice.copy()
masked_cubeslice.data = ma.masked_greater(cubeslice.data,
threshold)
print('Masking cube keeping only what is in between %f and %f' %
(v1, v2))
return masked_cubeslice
else:
print('NOT masking the cube')
return mycube
else:
print('Variable is not a cube dimension, leaving cube untouched')
return mycube
def extract_lai(nc_file):
fh_in = Dataset('../../raw_data/lai/' + nc_file, 'r')
for index, n_days in enumerate(fh_in.variables['time'][:]):
date = (datetime.datetime(2000, 1, 1, 0, 0) + datetime.timedelta(int(n_days))).strftime('%Y%m%d')
print(date)
fh_out = Dataset('../../processed_data/lai/500m/{}.nc'.format(date), 'w')
for name, dim in fh_in.dimensions.items():
if name != 'time':
fh_out.createDimension(name, len(dim) if not dim.isunlimited() else None)
ignore_features = ["time", "crs", "FparExtra_QC", "FparLai_QC"]
mask_value_dic = {'Lai_500m': 10, 'LaiStdDev_500m': 10, 'Fpar_500m': 1, 'FparStdDev_500m': 1}
for v_name, varin in fh_in.variables.items():
if v_name not in ignore_features:
dimensions = varin.dimensions if v_name in ['lat', 'lon'] else ('lat', 'lon')
outVar = fh_out.createVariable(v_name, varin.datatype, dimensions)
if v_name == "lat":
outVar.setncatts({"units": "degree_north"})
outVar[:] = varin[:]
elif v_name == "lon":
outVar.setncatts({"units": "degree_east"})
outVar[:] = varin[:]
else:
outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()})
vin = varin[index, :, :]
vin = ma.masked_greater(vin, mask_value_dic[v_name])
vin = ma.masked_less(vin, 0)
outVar[:] = vin[:]
fh_out.close()
fh_in.close()
def extract_ndvi(nc_file):
fh_in = Dataset('../../raw_data/ndvi/' + nc_file, 'r')
for index, n_days in enumerate(fh_in.variables['time'][:]):
date = (datetime.datetime(2000, 1, 1, 0, 0) + datetime.timedelta(int(n_days))).strftime('%Y%m%d')
print(date)
fh_out = Dataset('../../processed_data/ndvi/1km/{}.nc'.format(date[:-2]), 'w')
for name, dim in fh_in.dimensions.items():
if name != 'time':
fh_out.createDimension(name, len(dim) if not dim.isunlimited() else None)
ignore_features = ["time", "crs", "_1_km_monthly_VI_Quality"]
for v_name, varin in fh_in.variables.items():
if v_name not in ignore_features:
dimensions = varin.dimensions if v_name in ['lat', 'lon'] else ('lat', 'lon')
v_name = v_name if v_name in ['lat', 'lon'] else v_name.split('_')[-1].lower()
outVar = fh_out.createVariable(v_name, varin.datatype, dimensions)
if v_name == "lat":
outVar.setncatts({"units": "degree_north"})
outVar[:] = varin[:]
elif v_name == "lon":
outVar.setncatts({"units": "degree_east"})
outVar[:] = varin[:]
else:
outVar.setncatts({k: varin.getncattr(k) for k in varin.ncattrs()})
vin = varin[index, :, :]
vin = ma.masked_greater(vin, 1.0)
vin = ma.masked_less(vin, -0.2)
outVar[:] = vin[:]
fh_out.close()
fh_in.close()
def get_fence(self, xyfence, sampling="bilinear"):
"""Get surface values along fence."""
cxarr = xyfence[:, 0]
cyarr = xyfence[:, 1]
czarr = xyfence[:, 2].copy()
sampleoptions = {"bilinear": 0, "nearest": 2}
# czarr will be updated "inplace":
istat = _cxtgeo.surf_get_zv_from_xyv(
cxarr,
cyarr,
czarr,
self.ncol,
self.nrow,
self.xori,
self.yori,
self.xinc,
self.yinc,
self.yflip,
self.rotation,
self.get_values1d(),
sampleoptions.get(sampling, 0),
)
if istat != 0:
logger.warning("Seem to be rotten")
xyfence[:, 2] = czarr
xyfence = ma.masked_greater(xyfence, xtgeo.UNDEF_LIMIT)
xyfence = ma.mask_rows(xyfence)
return xyfence
def _import_ijxyz(self, mfile): # pylint: disable=too-many-locals
"""Import OW/DSG IJXYZ ascii format."""
# import of seismic column system on the form:
# 2588 1179 476782.2897888889 6564025.6954 1000.0
# 2588 1180 476776.7181777778 6564014.5058 1000.0
logger.debug("Read data from file... (scan for dimensions)")
cfhandle = mfile.get_cfhandle()
xlist = _cxtgeo.surf_import_ijxyz(cfhandle, 0, 1, 1, 1, 0)
(
ier,
ncol,
nrow,
_,
xori,
yori,
xinc,
yinc,
rot,
iln,
xln,
val,
yflip,
) = xlist
if ier != 0:
mfile.cfclose()
raise RuntimeError("Import from C is wrong...")
# now real read mode
xlist = _cxtgeo.surf_import_ijxyz(cfhandle, 1, ncol, nrow, ncol * nrow, 0)
ier, ncol, nrow, _, xori, yori, xinc, yinc, rot, iln, xln, val, yflip = xlist
if ier != 0:
raise RuntimeError("Import from C is wrong...")
logger.info(xlist)
val = ma.masked_greater(val, xtgeo.UNDEF_LIMIT)
self._xori = xori
self._xinc = xinc
self._yori = yori
self._yinc = yinc
self._ncol = ncol
self._nrow = nrow
self._rotation = rot
self._yflip = yflip
self.values = val.reshape((self._ncol, self._nrow))
self._ilines = iln
self._xlines = xln
mfile.cfclose()
self._metadata.required = self
def import_irap_binary(self, mfile, values=True):
"""Import Irap binary format."""
ifile = xtgeo._XTGeoCFile(mfile)
logger.debug("Enter function...")
# read with mode 0, to get mx my and other metadata
(
ier,
self._ncol,
self._nrow,
_ndef,
self._xori,
self._yori,
self._xinc,
self._yinc,
self._rotation,
val,
) = _cxtgeo.surf_import_irap_bin(ifile.fhandle, 0, 1, 0)
if ier != 0:
ifile.close()
raise RuntimeError("Error in reading Irap binary file")
self._yflip = 1
if self._yinc < 0.0:
self._yinc *= -1
self._yflip = -1
self._filesrc = mfile
self._ilines = np.array(range(1, self._ncol + 1), dtype=np.int32)
self._xlines = np.array(range(1, self._nrow + 1), dtype=np.int32)
# lazy loading, not reading the arrays
if not values:
self._values = None
ifile.close()
return
nval = self._ncol * self._nrow
xlist = _cxtgeo.surf_import_irap_bin(ifile.fhandle, 1, nval, 0)
if xlist[0] != 0:
ifile.close()
raise RuntimeError("Problem in {}, code {}".format(__name__, ier))
val = xlist[-1]
val = np.reshape(val, (self._ncol, self._nrow), order="C")
val = ma.masked_greater(val, _cxtgeo.UNDEF_LIMIT)
if np.isnan(val).any():
logger.info("NaN values are found, will mask...")
val = ma.masked_invalid(val)
self._values = val
ifile.close()
def load(fn):
ff = Dataset(fn, "r")
f = ff.variables
ssh = f["Grid_0001"][:].T
ssh = ma.masked_greater(ssh, 500)
ssh = (ssh[:-1, :] + ssh[1:, :]) / 2.0
ff.close()
return ssh
def load(fn):
ff = Dataset(fn, 'r')
f = ff.variables
ssh = f['Grid_0001'][:].T
ssh = ma.masked_greater(ssh, 500)
ssh = (ssh[:-1, :] + ssh[1:, :]) / 2.
ff.close()
return ssh
def autocorrelation_length_estimate(series, acf=None, M=5, axis=0):
r"""Returns an estimate of the autocorrelation length of the given
series:
.. math::
L = \int_{-\infty}^\infty \rho(t) dt
The estimate is the smallest :math:`L` such that
.. math::
L = \rho(0) + 2 \sum_{j = 1}^{M L} \rho(j)
In words: the ACL is estimated over a window that is at least
:math:`M` ACLs long, with the constraint that :math:`ML < N/2`.
Defined in this way, the ACL gives the reduction factor between
the number of samples and the "effective" number of samples. In
particular, the variance of the estimated mean of the series is
given by
.. math::
\left\langle \left( \frac{1}{N} \sum_{i=0}^{N-1} x_i - \mu
\right)^2 \right\rangle = \frac{\left\langle \left(x_i -
\mu\right)^2 \right\rangle}{N/L}
Returns ``nan`` if there is no such estimate possible (because
the series is too short to fit :math:`2M` ACLs).
For an N-dimensional array, returns an array of ACLs of the same
shape as ``series``, but with the dimension along ``axis``
removed.
"""
if acf is None:
acf = autocorrelation_function(series, axis=axis)
m = [slice(None)] * len(acf.shape)
nmax = acf.shape[axis] / 2
# Generate ACL candidates.
m[axis] = slice(0, nmax)
acl_ests = 2.0 * np.cumsum(acf[m], axis=axis) - 1.0
# Build array of lags (like arange, but N-dimensional).
shape = acf.shape[:axis] + (nmax, ) + acf.shape[axis + 1:]
lags = np.cumsum(np.ones(shape), axis=axis) - 1.0
# Mask out unwanted lags and set corresponding ACLs to nan.
mask = M * acl_ests >= lags
acl_ests[mask] = np.nan
i = ma.masked_greater(mask, lags, copy=False)
# Now get index of smallest unmasked lag -- if all are masked, this will be 0.
j = i.argmin(axis=axis)
k = tuple(np.indices(j.shape))
return acl_ests[k[:axis] + (j, ) + k[axis:]]
def plotOneMonth(dataset, dataVar, month, minval, maxval, cbarTicks = None, cmap = 'viridis'):
"""Plots map of the arctic on North Pole Stereo projection with one month of data overlayed, along with the sea ice edge for each month.
Args:
dataset (xr Dataset): dataset from google bucket
dataVar (str): variable of interest
month (str): month and year of interest, i.e. 'Dec 2019' (does not need to be in any particular format)
minval, maxval (int): minimum and maximum values for the data variable
cbarTicks (list or np array of length 2): ticks to use on colorbar (default to [minval + 1, maxval +1])
cmap (str, optional): color map (default to viridis)
Returns:
Figure displayed in notebook
"""
#define projection and transform
proj = ccrs.NorthPolarStereo(central_longitude = -45)
transform = ccrs.PlateCarree()
#initialize the figure and axes
plt.figure(figsize=(6, 6))
ax = plt.axes(projection = proj)
#define arguments if not inputted
cbarTicks = np.arange(minval, maxval + 1, 1) if cbarTicks is None else cbarTicks
#plot sea ice concentraion
SICarray = dataset['seaice_conc_monthly_cdr'].sel(time = month).where(dataset['region_mask']!=21) #dont plot contour along coastlines
#stackexchange workaround for plotting on a rotated grid
lonGreater = ma.masked_greater(SICarray.longitude.values, -0.01)
lonLesser = ma.masked_less(SICarray.longitude.values, 0)
latGreater = ma.MaskedArray(SICarray.latitude.values, mask = lonGreater.mask)
latLesser = ma.MaskedArray(SICarray.latitude.values, mask = lonLesser.mask)
dataGreater = ma.MaskedArray(SICarray.values[0], mask = lonGreater.mask)
dataLesser = ma.MaskedArray(SICarray.values[0], mask = lonLesser.mask)
#plot contour using each part of the 2 masked data sets
im2a = ax.contour(lonGreater, latGreater, dataGreater, levels = [0.5], transform = transform, colors = 'magenta', linewidths = 0.9, zorder=5, alpha=1)
im2b = ax.contour(lonLesser, latLesser, dataLesser, levels = [0.5], transform = transform, colors = 'magenta', linewidths = 0.9, zorder=5, alpha=1)
#im = ax.contour(SICarray.longitude.values, SICarray.latitude.values, SICarray.values[0], levels = [0.15], transform = transform, colors = 'magenta', linewidths = 0.8, zorder=15, alpha=1)
#plot the data
dataset[dataVar].where(dataset['seaice_conc_monthly_cdr'] > 0.5).sel(time = month).plot(x = 'longitude', y = 'latitude', vmin = minval, vmax = maxval, extend = 'both',
ax = ax, add_colorbar = True, transform = transform, zorder = 2, cmap = cmap,
cbar_kwargs = {'label': "\n".join(wrap(dataset[dataVar].attrs['long_name'] + ' (' + dataset[dataVar].attrs['units'] + ')', 50)), 'orientation': 'horizontal', 'shrink': 0.75, 'pad': 0.025})
#add features to the map
ax.coastlines(linewidth=0.15, color = 'black', zorder = 10) #add coastlines
ax.add_feature(cfeature.LAND, color ='0.95', zorder = 5) #add land
ax.add_feature(cfeature.LAKES, color = 'grey', zorder = 5) #add lakes
ax.gridlines(draw_labels = False, linewidth = 0.25, color = 'gray', alpha = 0.7, linestyle = '--', zorder = 6) #add gridlines
ax.set_extent([-179, 179, 55, 90], crs = transform) #zoom in so map only displays the Arctic
ax.set_title("\n".join(wrap(month + ": " + dataset[dataVar].attrs['long_name'], 38)), fontsize = 'x-large')
#display figure in notebook
plt.show()
def autocorrelation_length_estimate(series, acf=None, M=5, axis=0):
r"""Returns an estimate of the autocorrelation length of the given
series:
.. math::
L = \int_{-\infty}^\infty \rho(t) dt
The estimate is the smallest :math:`L` such that
.. math::
L = \rho(0) + 2 \sum_{j = 1}^{M L} \rho(j)
In words: the ACL is estimated over a window that is at least
:math:`M` ACLs long, with the constraint that :math:`ML < N/2`.
Defined in this way, the ACL gives the reduction factor between
the number of samples and the "effective" number of samples. In
particular, the variance of the estimated mean of the series is
given by
.. math::
\left\langle \left( \frac{1}{N} \sum_{i=0}^{N-1} x_i - \mu
\right)^2 \right\rangle = \frac{\left\langle \left(x_i -
\mu\right)^2 \right\rangle}{N/L}
Returns ``nan`` if there is no such estimate possible (because
the series is too short to fit :math:`2M` ACLs).
For an N-dimensional array, returns an array of ACLs of the same
shape as ``series``, but with the dimension along ``axis``
removed.
"""
if acf is None:
acf = autocorrelation_function(series, axis=axis)
m = [slice(None), ] * len(acf.shape)
nmax = acf.shape[axis]//2
# Generate ACL candidates.
m[axis] = slice(0, nmax)
acl_ests = 2.0*np.cumsum(acf[m], axis=axis) - 1.0
# Build array of lags (like arange, but N-dimensional).
shape = acf.shape[:axis] + (nmax,) + acf.shape[axis+1:]
lags = np.cumsum(np.ones(shape), axis=axis) - 1.0
# Mask out unwanted lags and set corresponding ACLs to nan.
mask = M*acl_ests >= lags
acl_ests[mask] = np.nan
i = ma.masked_greater(mask, lags, copy=False)
# Now get index of smallest unmasked lag -- if all are masked, this will be 0.
j = i.argmin(axis=axis)
k = tuple(np.indices(j.shape))
return acl_ests[k[:axis] + (j,) + k[axis:]]
def _convert_to_xtgeo_prop(self, rox, pname, roxgrid, roxprop, realisation,
faciescodes): # pragma: no cover
"""Collect numpy array and convert to XTGeo fmt"""
indexer = roxgrid.get_grid().grid_indexer
self._ncol, self._nrow, self._nlay = indexer.dimensions
if rox.version_required("1.3"):
logger.info(indexer.ijk_handedness)
else:
logger.info(indexer.handedness)
pvalues = roxprop.get_values(realisation=realisation)
if str(roxprop.type) == "body_facies" and faciescodes:
fmap = roxprop.get_facies_map(realisation=realisation)
pvalues = fmap[pvalues] # numpy magics
self._roxar_dtype = pvalues.dtype
if self._isdiscrete:
mybuffer = np.ndarray(indexer.dimensions, dtype=np.int32)
mybuffer.fill(xtgeo.UNDEF_INT)
else:
mybuffer = np.ndarray(indexer.dimensions, dtype=np.float64)
mybuffer.fill(xtgeo.UNDEF)
cellno = indexer.get_cell_numbers_in_range((0, 0, 0), indexer.dimensions)
ijk = indexer.get_indices(cellno)
iind = ijk[:, 0]
jind = ijk[:, 1]
kind = ijk[:, 2]
mybuffer[iind, jind, kind] = pvalues[cellno]
if self._isdiscrete:
mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_INT_LIMIT)
self.codes = _fix_codes(roxprop.code_names)
logger.info("Fixed codes: %s", self.codes)
else:
mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_LIMIT)
self._values = mybuffer
self._name = pname
def create_anolamy_map(self, snowmap):
'''displays snow cover anomaly data'''
snowmap = ma.masked_greater(snowmap, 2)
snowmap = snowmap.reshape(610, 450)
snowmap = snowmap * 100
fig_anom, ax = plt.subplots(figsize=(8, 10))
cmap_anom = plt.cm.RdBu
cmap_anom.set_bad('black', 0.8)
ax.clear()
ax.text(0.82,
0.98,
'Map: Nico Sun',
fontsize=10,
color='black',
transform=ax.transAxes)
ax.set_title(
'NOAA / NSIDC IMS Snow & Ice Extent Anomaly {}-{}-{}'.format(
self.year, self.month, self.day),
x=0.5)
ax.set_xlabel('Data source: https://nsidc.org/data/g02156', x=0.22)
ax.set_ylabel(
'https://sites.google.com/site/cryospherecomputing/snow-cover',
y=0.25)
ax.text(1.02,
0.22,
'Snow cover anomaly in percent',
transform=ax.transAxes,
rotation='vertical',
color='black',
fontsize=9)
axins1 = inset_axes(ax, width="5%", height="25%", loc=4)
im1 = ax.imshow(snowmap,
interpolation='nearest',
vmin=-100,
vmax=100,
cmap=cmap_anom)
plt.colorbar(im1,
cax=axins1,
orientation='vertical',
ticks=[-100, 0, +100])
axins1.yaxis.set_ticks_position("left")
ax.axes.get_yaxis().set_ticks([])
ax.axes.get_xaxis().set_ticks([])
plt.tight_layout(pad=1)
fig_anom.savefig('X:/Upload/Snow_Cover_Data/NOAA_Snowmap_anomaly.png')
fig_anom.savefig(
'X:/SnowCover/Images/NOAA_Snowmap_anomaly_{}.png'.format(
str(self.day_of_year).zfill(3)))
#
plt.pause(0.01)
def get_mask_for_unphysical(U, cutoffU=2000., fill_value=np.nan):
"""
Returns a mask for masking module. if absolute value of value is greater than cutoff, the value is masked.
Parameters
----------
U: array-like
cutoffU: float
if |value| > cutoff, this method considers those values unphysical.
fill_value:
Returns
-------
mask: multidimensional boolean array
"""
print 'number of invalid values (nan and inf) in the array: ' + str(
np.isnan(U).sum() + np.isinf(U).sum())
print 'number of nan values in U: ' + str(np.isnan(U).sum())
print 'number of inf values in U: ' + str(np.isinf(U).sum()) + '\n'
# a=ma.masked_invalid(U)
# print 'number of masked elements by masked_invalid: '+ str(ma.count_masked(a))
# Replace all nan and inf values with fill_value.
# fix_invalid still enforces a mask on elements with originally invalid values
U_fixed = ma.fix_invalid(U, fill_value=99999)
n_invalid = ma.count_masked(U_fixed)
print 'number of masked elements by masked_invalid: ' + str(n_invalid)
# Update the mask to False (no masking)
U_fixed.mask = False
# Mask unreasonable values of U_fixed
b = ma.masked_greater(U_fixed, cutoffU)
c = ma.masked_less(U_fixed, -cutoffU)
n_greater = ma.count_masked(b) - n_invalid
n_less = ma.count_masked(c)
print 'number of masked elements greater than cutoff: ' + str(n_greater)
print 'number of masked elements less than -cutoff: ' + str(n_less)
# Generate a mask for all nonsense values in the array U
mask = ~(~b.mask * ~c.mask)
d = ma.array(U_fixed, mask=mask)
n_total = ma.count_masked(d)
# U_filled = ma.filled(d, fill_value)
#Total number of elements in U
N = 1
for i in range(len(U.shape)):
N *= U.shape[i]
print 'total number of unphysical values: ' + str(
ma.count_masked(d)) + ' (' + str((float(n_total) / N * 100)) + '%)\n'
return mask
def concentrationshow(self, icemap, Areavalue, extentvalue, outlooktype):
icemap = icemap / 250
icemap = ma.masked_greater(icemap, 1)
icemap = icemap.reshape(448, 304)
icemap = icemap[60:410, 30:260]
# areavalue = int(areavalue*1e6)
# extentvalue = int(extentvalue*1e6)
Areavalue = '{:,}'.format(Areavalue) + ' ' r'$km^2$'
extentvalue = '{:,}'.format(extentvalue) + ' ' r'$km^2$'
cmap = plt.cm.jet
cmap.set_bad('black', 0.6)
self.ax2.clear()
self.ax2.set_title('{}_Forecast , Date: {}-{}-{}'.format(
outlooktype, self.year, self.stringmonth, self.stringday))
#self.ax.set_title('Average Forecast')
self.ax2.set_xlabel('Area: {} / Extent: {}'.format(
Areavalue, extentvalue),
fontsize=14)
self.cax2 = self.ax2.imshow(icemap,
interpolation='nearest',
vmin=0,
vmax=1,
cmap=cmap)
self.ax2.axes.get_yaxis().set_ticks([])
self.ax2.axes.get_xaxis().set_ticks([])
self.ax2.text(2,
8,
r'Data: NSIDC',
fontsize=10,
color='white',
fontweight='bold')
self.ax2.text(2,
18,
r'Map: Nico Sun',
fontsize=10,
color='white',
fontweight='bold')
self.ax2.text(
-0.04,
0.48,
'https://sites.google.com/site/cryospherecomputing/forecast',
transform=self.ax2.transAxes,
rotation='vertical',
color='grey',
fontsize=10)
self.fig2.tight_layout(pad=1)
self.fig2.subplots_adjust(left=0.05)
if self.loopday.month > 9:
self.fig2.savefig(
'X:/Forecast/August_test/SIPN_SIC_{}{}{}.png'.format(
self.year, self.stringmonth, self.stringday))
plt.pause(0.01)
def F(points):
rsq = sd.pdist(points, 'sqeuclidean')
rsq = ma.masked_greater(rsq, 2**(1/3)*sigma**2)
q = 4*epsilon*(-12*sigma**12/rsq**7 + 6*sigma**6/rsq**4)
xs = points[:,0]
ys = points[:,1]
sf = sd.squareform(q.filled(0))
xout = np.subtract.outer(xs, xs)
yout = np.subtract.outer(ys, ys)
return np.array([np.sum(xout*sf, axis=1), np.sum(yout*sf, axis=1)]).T
def get_mask_for_unphysical_using_cutoff(U, cutoff=None, mode='less'):
if mode == 'less' or mode == 'l':
U_masked = ma.masked_less(U, cutoff)
elif mode == 'lesseqal' or mode == 'leq':
U_masked = ma.masked_less_equal(U, cutoff)
elif mode == 'greater' or mode == 'g':
U_masked = ma.masked_greater(U, cutoff)
elif mode == 'greaterequal' or mode == 'geq':
U_masked = ma.masked_greater_equal(U, cutoff)
return U_masked.mask
def projectToLatLong(ifile, outfile, img_bnd_coords, proj4string):
####################################
# Inputs
# For details see http://fcm7/projects/SPS/browser/SPS/trunk/data/products/MSG_Products.nl
ulx_ll, uly_ll, lrx_ll, lry_ll = img_bnd_coords # Bounding coordinates in latlon, although the the image is in mercator projection!!!
# epsg_code = 3395 # Mercator projection code of the imagery
####################################
# For testing
# ifile = '/data/users/hadhy/HyVic/Obs/OLR_noborders/EIAM50_201901210230.png'
# Open the image file
ds = gdal.Open(ifile)
band = ds.GetRasterBand(1)
arr = band.ReadAsArray()
# Mask out pixel values with no data
arr = ma.masked_less(arr, 4)
arr = ma.masked_greater(arr, 254)
# Apply equation to retrieve brightness temperatures. See Chamberlain et al. 2013 https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/met.1403
arrbt = (-0.44 * (arr - 4.)) + 308.
# Fill masked values with -9999 (because the Geotiff format doesn't accept masked arrays)
ma.set_fill_value(arrbt, -9999)
arrbt = arrbt.filled()
# Set some image info ...
## Gets the bounding coordinates in Mercator projection
ulx, uly = latlon2projected(ulx_ll, uly_ll, proj4string)
lrx, lry = latlon2projected(lrx_ll, lry_ll, proj4string)
## Calculate geotransform object
nx, ny = [ds.RasterXSize, ds.RasterYSize]
## NB: The coordinates start in the top left corner (usually), so
## xDist needs to be positive, and yDist negative
xDist = (lrx - ulx) / nx if lrx > ulx else -1 * (lrx - ulx) / nx
yDist = -1 * (uly - lry) / ny if uly > lry else (uly - lry) / ny
rtnX, rtnY = [0, 0]
gt = [ulx, xDist, rtnX, uly, rtnY, yDist]
## Projection information
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4string)
# srs.ImportFromEPSG(epsg_code) # Documentation says it is mercator
# Create dataset in mercator projection
memfile = ''
ds_merc = makeGDALds(arrbt, nx, ny, gt, srs, 'MEM', memfile)
# Regrid to latlon
gdal.Warp(outfile, ds_merc, dstSRS='EPSG:4326', dstNodata=-9999)
# Convert geotiff to a cube and save
timestamp = os.path.basename(outfile).split('.')[0].split('_')[1]
cube = sf.geotiff2cube(outfile, timestamp)
return cube
def make_paperfig():
fig=figure(figsize=(8,8))
gs = gridspec.GridSpec(2, 1, height_ratios=[2.75, 1])
ax=subplot(gs[0],projection=ccrs.NorthPolarStereo(central_longitude=0))
infile=files[9]
with open(infile, 'rb') as fr:
hdr = fr.read(300)
ice = np.fromfile(fr, dtype=np.uint8)
ice = ice.reshape(448, 304)
ice = ice / 250.
ice = ma.masked_greater(ice, 1.0)
# ax = plt.axes(projection=ccrs.NorthPolarStereo(central_longitude=0))
cs = ax.coastlines(resolution='10m', linewidth=0.5)
# ax.gridlines()
ax.set_extent([-90, 10, 60, 90], crs=ccrs.PlateCarree())
kw = dict(central_latitude=90, central_longitude=-45, true_scale_latitude=70)
cs = ax.contourf(x, y, ice,arange(0,1.1,0.1),cmap=plt.cm.Blues,
transform=ccrs.Stereographic(**kw))
ax.contour(x, y, ice,0.2,colors='k',transform=ccrs.Stereographic(**kw))
lon_start=-43#4
lon_end=-42#2
lat_start=60
lat_end=61
x_start,y_start=transform(outProj,inProj,lon_start,lat_start)
x_end,y_end=transform(outProj,inProj,lon_end,lat_end)
# x_start,y_start=transform(outProj,inProj,lon_start,lat_start)
# ax.plot([x_start,x_start,x_end,x_end,x_start],[y_start,y_end,y_end,y_start,y_start],linewidth=2,color='r',transform=ccrs.Stereographic(**kw))
lon_FS1=-18
lon_FS2=0
lat_FS=78.5
x_FS1,y_FS=transform(outProj,inProj,lon_FS1,lat_FS)
x_FS2,y_FS=transform(outProj,inProj,lon_FS2,lat_FS)
ax.plot([x_FS1,x_FS2],[y_FS,y_FS],linewidth=4,color='k',transform=ccrs.Stereographic(**kw))
lon_DS1=-26.5
lon_DS2=-24
lat_DS1=68.5
lat_DS2=67
x_DS1,y_DS1=transform(outProj,inProj,lon_DS1,lat_DS1)
x_DS2,y_DS2=transform(outProj,inProj,lon_DS2,lat_DS2)
ax.plot([x_DS1,x_DS2],[y_DS1,y_DS2],linewidth=4,color='k',transform=ccrs.Stereographic(**kw))
ax.plot([x_start,x_end],[y_start,y_start],linewidth=4,color='k',transform=ccrs.Stereographic(**kw))
x_basin,y_basin=transform(outProj,inProj,en4['lon_basin'],en4['lat_basin'])
ax.plot(x_basin,y_basin,linewidth=2,color='k',transform=ccrs.Stereographic(**kw))
yr=infile[18:22]
mnth=infile[22:24]
# ax.set_title(mnth+'/'+yr,fontsize=14)
# cbaxes = ax.add_axes([0.95, 0.25, 0.04, 0.45])
cb=colorbar(cs)#,cax=cbaxes)
cb.set_label('sea ice concentration\n 05/2015')
ax1=subplot(gs[1])
compseries(daily.date,cc['trans filt'],'Coastal current transport [Sv]','k',
datevec,CFice,'Sea ice concentration','blue',ax1)
savefig('../figures/paperfigs/seaice_4poster.pdf',bbox_inches='tight')
def get_mask_for_unphysical_using_median(U, cutoffratio=0.4, mode='less'):
median = np.median(U)
if mode == 'less' or mode == 'l':
U_masked = ma.masked_less(U, median * cutoffratio)
elif mode == 'lesseqal' or mode == 'leq':
U_masked = ma.masked_less_equal(U, median * cutoffratio)
elif mode == 'greater' or mode == 'g':
U_masked = ma.masked_greater(U, median * cutoffratio)
elif mode == 'greaterequal' or mode == 'geq':
U_masked = ma.masked_greater_equal(U, median * cutoffratio)
return U_masked.mask
def rsxs2nparray(rsns, rsxs):
max_n = np.max(rsns) + 2 # See note above.
rsesxs_cube = np.full((len(rsxs), len(rsns[0]), max_n), -100.0)
for ri, r_esxs in enumerate(rsxs2rsesxs(rsns, rsxs)):
for ei, e_xs in enumerate(r_esxs):
rsesxs_cube[ri, ei] = np.pad(e_xs, (0, max_n - len(e_xs)),
'constant',
constant_values=100.0)
return ma.masked_greater(rsesxs_cube, 99.0)
def test_2Dmasked_array(N=25):
l = N/2
# Ones array
grid = np.linspace(-10, 10, N)
X, Y = np.meshgrid(grid, grid)
data = random((N, N))
thr = np.percentile(data, 70)
data = ma.masked_greater(data, thr)
h = wmean_2D(X, Y, data, l=l)
assert h.mask.any()
def makeAM(vertex, howManyOnes
): #how many vertex there are in the graph, % of connections
adjacencyM = np.random.rand(vertex, vertex) # make random matrix
zeros = ma.masked_greater(
adjacencyM, howManyOnes).mask #find where elements are greater than X
ones = ma.masked_less_equal(
adjacencyM, howManyOnes).mask #find where elements are less than X
adjacencyM[zeros] = 0 # mask = 0
adjacencyM[ones] = 1 # mask = 1
return adjacencyM
def plotHistogramContour(X, Y, Z, xlabel="",ylabel=""):
# Number of bins in each axis in the histogram
bins = 40
# Range of x and y-axis, respectively
r = [[np.min(X), np.max(X)], [np.min(Y), np.max(Y)]]
print('Plotting contour... valid data-points: ', len(Z))
print('[X-range, Y-range] : ', r)
# Weights are temperatures. H is the sum of all temperature at point (X, Y)
H, xedges, yedges = np.histogram2d(X, Y, bins=(bins, bins*5), range=r, weights=Z)
# N is the number of data-points in the bins at position (X, Y)
N, xedges, yedges = np.histogram2d(X, Y, bins=(bins, bins*5), range=r, weights=None)
# Some sanity check and finally taking the average of temperatures
H = ma.masked_less(H, 100)
S = ma.divide(H, N, where=N>0) # S is average temperature as each (X,Y)
S = ma.masked_greater(S, 300)
#print('H: ', H)
#print('N: ', N)
#print('S: ', S)
#print("Shape of S: ", np.shape(S), " max: ", np.max(S), " min: ", np.min(S))
# We had 'buckets' in the histogram for x and y-axis,
# but we need a 'single value' for each bucket. We take their midpoint.
## Using list comprehension :)
xpoints = [(xedges[i]+xedges[i+1])/2.0 for i in range(len(xedges)-1)]
ypoints = [(yedges[i]+yedges[i+1])/2.0 for i in range(len(yedges)-1)]
# Standard for plotting contour, create a meshgrid
(P,Q) = np.meshgrid(xpoints, ypoints)
plt.figure(figsize=(9,9))
# The contour lines at the following temperature will be labeled
V = (140, 160, 180, 200, 220, 240, 260, 280)
# Plot a line contour
contours = plt.contour(P, Q, S.T, V, colors='0.20', corner_mask=True)
# We use transpose of S as S.T because that's what histogram2d() returns
# Plot a filled contour
plt.contourf(P, Q, S.T, 128, cmap=plt.cm.jet)
plt.clabel(contours, inline=True, fontsize=8)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.title("Temperature Contour")
plt.autoscale()
plt.colorbar()
print('Done plotting.')
# Save the contour plot as a high resolution EPS file
global figcount
if(figcount != -1):
savefile = 'contour-%d.eps' %(figcount)
plt.savefig(savefile, format='eps')
figcount = figcount + 1
print("File '%s' saved in the current working directory." %(savefile))
def thicknessshow(self, icemap, Volumevalue, extentvalue, outlooktype):
icemap = ma.masked_greater(icemap, 5)
icemap = icemap.reshape(332, 316)
# areavalue = int(areavalue*1e6)
# extentvalue = int(extentvalue*1e6)
Volumevalue = '{:,}'.format(Volumevalue) + ' ' r'$km^3$'
extentvalue = '{:,}'.format(extentvalue) + ' ' r'$km^2$'
cmap = plt.cm.jet
cmap.set_bad('black', 0.6)
self.ax.clear()
self.ax.set_title('{}_Forecast , Date: {}-{}-{}'.format(
outlooktype, self.year, self.stringmonth, self.stringday))
#self.ax.set_title('Average Forecast')
self.ax.set_xlabel('Volume: {} / Extent: {}'.format(
Volumevalue, extentvalue),
fontsize=14)
self.cax = self.ax.imshow(icemap,
interpolation='nearest',
vmin=0,
vmax=3,
cmap=cmap)
self.ax.axes.get_yaxis().set_ticks([])
self.ax.axes.get_xaxis().set_ticks([])
self.ax.text(2,
8,
r'Data: NSIDC',
fontsize=10,
color='white',
fontweight='bold')
self.ax.text(2,
18,
r'Map: Nico Sun',
fontsize=10,
color='white',
fontweight='bold')
self.ax.text(
-0.04,
0.48,
'https://sites.google.com/site/cryospherecomputing/forecast',
transform=self.ax.transAxes,
rotation='vertical',
color='grey',
fontsize=10)
self.fig.tight_layout(pad=1)
self.fig.subplots_adjust(left=0.05)
# if self.loopday.month > 9:
# self.fig.savefig('X:/Sea_Ice_Forecast/SIPN_south/SIPN_{}{}{}.png'.format(self.year,self.stringmonth,self.stringday))
plt.pause(0.01)
def _convert_to_xtgeo_prop(self, rox, pname, roxgrid, roxprop):
indexer = roxgrid.get_grid().grid_indexer
self._ncol, self._nrow, self._nlay = indexer.dimensions
if rox.version_required("1.3"):
logger.info(indexer.ijk_handedness)
else:
logger.info(indexer.handedness)
pvalues = roxprop.get_values()
self._roxar_dtype = pvalues.dtype
if self._isdiscrete:
mybuffer = np.ndarray(indexer.dimensions, dtype=np.int32)
mybuffer.fill(xtgeo.UNDEF_INT)
else:
mybuffer = np.ndarray(indexer.dimensions, dtype=np.float64)
mybuffer.fill(xtgeo.UNDEF)
cellno = indexer.get_cell_numbers_in_range((0, 0, 0), indexer.dimensions)
ijk = indexer.get_indices(cellno)
iind = ijk[:, 0]
jind = ijk[:, 1]
kind = ijk[:, 2]
mybuffer[iind, jind, kind] = pvalues[cellno]
if self._isdiscrete:
mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_INT_LIMIT)
self.codes = _fix_codes(roxprop.code_names)
logger.info("Fixed codes: %s", self.codes)
else:
mybuffer = ma.masked_greater(mybuffer, xtgeo.UNDEF_LIMIT)
self._values = mybuffer
self._name = pname
def importFromXYZ(self, b, H, RB, alpha):
RT = RB - H * (np.tan(alpha))
x, y, z = b[:, 0], b[:, 1], b[:, 2]
r, tht, z = rec2cyl(x, y, z)
self.setGeometry(RB, H, alpha)
rPerf = self._getRperf(z)
if self.imp_type == 'thick':
imp = b[:, 3]
else:
imp = getGeomImperfection(r, z, rPerf)
tm1 = ma.masked_less(tht, 0.1 * np.pi).mask
tm2 = ma.masked_greater(tht, 1.9 * np.pi).mask
tht = np.hstack((tht, np.pi * 2.0 + tht[tm1], 0.0 + (-1.0) * tht[tm2]))
r = np.hstack((r, r[tm1], r[tm2]))
z = np.hstack((z, z[tm1], z[tm2]))
imp = np.hstack((imp, imp[tm1], imp[tm2]))
ft = np.linspace(0, 2.0 * np.pi, self.samplingRadial)
fz = np.linspace(0, H, self.samplingAxial)
IMPERF = getImperfectionArray(tht, z, imp, ft, fz)
mf = []
for row in IMPERF[0:len(IMPERF)]:
mf.append(np.isnan(np.sum(row)))
row1 = mf.index(False)
mr = []
for i in reversed(mf):
mr.append(i)
row2 = len(mf) - 1 - mr.index(False)
row1 += 1
row2 -= 2
dr1 = row1
rows1 = range(0, row1)
rows1sym = range(row1, row1 + dr1)[::-1]
dr2 = len(mf) - 1 - row2
rows2 = range(len(mf) - 1, row2, -1)
rows2sym = range(row2 - dr2, row2)[::-1]
IMPERF[rows1] = IMPERF[rows1sym].copy()
IMPERF[rows2] = IMPERF[rows2sym].copy()
#EXTRUDE
#IMPERF[0:row1]=IMPERF[row1]
#IMPERF[row2::]=IMPERF[row2]
self.addData(IMPERF, ft, fz)
def importFromXYZ(self,b,H,RB,alpha):
RT=RB-H *( np.tan(alpha ) )
x,y,z=b[:,0],b[:,1],b[:,2]
r,tht,z=rec2cyl(x,y,z)
self.setGeometry(RB,H,alpha)
rPerf=self._getRperf(z)
if self.imp_type == 'thick':
imp=b[:,3]
else:
imp=getGeomImperfection(r,z,rPerf)
tm1=ma.masked_less(tht,0.1*np.pi).mask
tm2=ma.masked_greater(tht,1.9*np.pi).mask
tht=np.hstack((tht, np.pi*2.0+tht[tm1], 0.0+(-1.0)*tht[tm2]))
r=np.hstack((r,r[tm1],r[tm2] ))
z=np.hstack((z,z[tm1],z[tm2]))
imp=np.hstack((imp,imp[tm1],imp[tm2]))
ft=np.linspace(0,2.0*np.pi,self.samplingRadial)
fz=np.linspace(0,H,self.samplingAxial)
IMPERF=getImperfectionArray(tht,z,imp,ft,fz)
mf=[]
for row in IMPERF[0:len(IMPERF)]:
mf.append( np.isnan(np.sum(row)) )
row1=mf.index(False)
mr=[]
for i in reversed(mf):
mr.append(i)
row2= len(mf)-1-mr.index(False)
row1+=1
row2-=2
dr1=row1
rows1=range(0,row1)
rows1sym=range(row1,row1+dr1)[::-1]
dr2=len(mf)-1-row2
rows2=range(len(mf)-1,row2,-1)
rows2sym=range(row2-dr2,row2)[::-1]
IMPERF[rows1]=IMPERF[rows1sym].copy()
IMPERF[rows2]=IMPERF[rows2sym].copy()
#EXTRUDE
#IMPERF[0:row1]=IMPERF[row1]
#IMPERF[row2::]=IMPERF[row2]
self.addData(IMPERF,ft,fz)
def modis_lai_extract(dates_folder, nc_file):
out_path = get_out_path(os.path.join("Data", "MCD15A3H", "500m"))
fh_in = Dataset(
os.path.join("n5eil01u.ecs.nsidc.org", "MCD15A3H", dates_folder,
nc_file + ".nc"), 'r')
for index, n_days in enumerate(fh_in.variables['time'][:]):
date = (datetime.datetime(2000, 1, 1, 0, 0) +
datetime.timedelta(int(n_days))).strftime('%Y%m%d')
print(date)
fh_out = Dataset(os.path.join(out_path, date + '.nc'), 'w')
for name, dim in fh_in.dimensions.items():
if name != 'time':
fh_out.createDimension(
name,
len(dim) if not dim.isunlimited() else None)
ignore_features = ["time", "crs", "FparExtra_QC", "FparLai_QC"]
mask_value_dic = {
'Lai_500m': 10,
'LaiStdDev_500m': 10,
'Fpar_500m': 1,
'FparStdDev_500m': 1
}
for v_name, varin in fh_in.variables.items():
if v_name not in ignore_features:
dimensions = varin.dimensions if v_name in ['lat', 'lon'
] else ('lat',
'lon')
outVar = fh_out.createVariable(v_name, varin.datatype,
dimensions)
if v_name == "lat":
outVar.setncatts({"units": "degree_north"})
outVar[:] = varin[:]
elif v_name == "lon":
outVar.setncatts({"units": "degree_east"})
outVar[:] = varin[:]
else:
outVar.setncatts(
{k: varin.getncattr(k)
for k in varin.ncattrs()})
vin = varin[index, :, :]
vin = ma.masked_greater(vin, mask_value_dic[v_name])
vin = ma.masked_less(vin, 0)
outVar[:] = vin[:]
fh_out.close()
fh_in.close()
def update(val):
# Get parameter
vmin = slider_cutoff.val
borderType = borderTypes[int(np.around(
slider_bordertype.val))] # Border type
d = int(np.around(slider_diameter.val)) # Diameter
sigma = slider_sigma.val # Sigma
sigma_color = slider_sigma_color.val # Sigma color (Bilateral filter)
sigma_space = slider_sigma_space.val # Sigma space (Bilateral filter)
nlm_h = slider_nlm.val
# Apply filters
box = cv2.blur(img, (d, d), borderType=borderType) # Box filter
gaussian = cv2.GaussianBlur(img, (d, d), sigma,
borderType=borderType) # Gaussian
median = cv2.medianBlur(img, d) # Median
bilateral = cv2.bilateralFilter(img,
d,
sigma_color,
sigma_space,
borderType=borderType) # Bilateral
kernel_opening = cv2.getStructuringElement(cv2.MORPH_CROSS, (d, d))
opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel_opening)
masked = ma.masked_greater(opening, 0)
opening2 = np.copy(img)
opening2[~masked.mask] = 0
non_local_means = cv2.fastNlMeansDenoising(img,
None,
h=nlm_h,
templateWindowSize=d,
searchWindowSize=d)
# Show results
original.set_clim(vmin, 255.0)
box_filter.set_data(box)
box_filter.set_clim(vmin, 255.0)
gaussian_filter.set_data(gaussian)
gaussian_filter.set_clim(vmin, 255.0)
median_filter.set_data(median)
median_filter.set_clim(vmin, 255.0)
bilateral_filter.set_data(bilateral)
bilateral_filter.set_clim(vmin, 255.0)
opening_filter.set_data(opening)
opening_filter.set_clim(vmin, 255.0)
opening_filter2.set_data(opening2)
opening_filter2.set_clim(vmin, 255.0)
nlm_filter.set_data(non_local_means)
nlm_filter.set_clim(vmin, 255.0)
def normalshow(self, icemap, icesum):
'''displays sea ice data'''
icemap = ma.masked_greater(icemap, 1)
icemap = icemap.reshape(448, 304)
icemap = icemap[60:410, 30:260]
icesum = round(icesum, 3)
icesum = '{0:.3f}'.format(icesum)
cmap = plt.cm.jet
cmap.set_bad('black', 0.6)
self.ax.clear()
self.ax.set_title('Date: {}-{}-{}'.format(self.year, self.stringmonth,
self.stringday))
#self.ax.set_title('Minimum of Minima')
self.ax.set_xlabel('Area: ' + str(icesum) + ' million km2',
fontsize=14)
self.cax = self.ax.imshow(icemap,
interpolation='nearest',
vmin=0,
vmax=1,
cmap=cmap)
self.ax.axes.get_yaxis().set_ticks([])
self.ax.axes.get_xaxis().set_ticks([])
self.ax.text(2,
8,
r'Data: NSIDC NRT',
fontsize=10,
color='white',
fontweight='bold')
self.ax.text(2,
18,
r'Map: Nico Sun',
fontsize=10,
color='white',
fontweight='bold')
self.ax.text(
-0.04,
0.48,
'https://sites.google.com/site/cryospherecomputing/daily-data',
transform=self.ax.transAxes,
rotation='vertical',
color='grey',
fontsize=10)
self.fig.tight_layout(pad=1)
self.fig.subplots_adjust(left=0.05)
self.fig.savefig('X:/Upload/Arctic_yesterday.png')
plt.pause(0.01)
def processVolume(threshold):
print("Generating Volume Data")
data = dfr.getVolumeData()
print("Applying threshold")
masked = ma.masked_greater(data, threshold)
print("Performing binary opening")
newmask = binary_opening(masked.mask, selem=np.ones((9,9,9)))
seed = np.copy(newmask)
seed[1:-1, 1:-1, 1:-1] = newmask.max()
print("Performing reconstruction")
newmask = reconstruction(seed, newmask, method='erosion').astype(np.int)
#newmask = binary_erosion(newmask, selem=np.ones((29,29,29))).astype(np.int)
masked2 = ma.array(data, mask=np.logical_not(newmask))
return masked, masked2
def contourValuesAbove(lons, lats, rawdata, contourAboveValue):
"""
Contours a grid for lats and lons passed in having values above the given value
Args:
lons : a grid of the longitudes
lats : a grid of the latitudes
rawdata : a grid of values
contourAboveValue : the value of which to contour anything above
Note : the grids must be the same dimension and size
Returns:
A list of polygons
"""
masked_grid = mask.masked_greater(rawdata, contourAboveValue)
return contour(lons, lats, masked_grid)
def test_mask_at_interp():
""" Test the behavior of masked points with interp on|off
As long as the filter is wide enough to capture at least
one data point per point, the interp=True will return
"""
N = 25
t = np.arange(N)
y = random(N)
yn = y[y.argsort()[-5]]
y = ma.masked_greater(y, yn)
# Equivalent to interp=False
h = maud.wmean_1D(y, t=t, l=11)
assert (y.mask == h.mask).all()
h = maud.wmean_1D(y, t=t, l=5, interp=True)
assert (~h.mask).all()
def id_features(data, threshold):
'''
Find the locations where features exist inside a data array
set entries 1=feature, 0=no feature
Separating these out into individual features is application dependent
Returns an array of ints={0,1} of data.shape
'''
#Filter data. notouch masks covers the sections we are not examining. touch is the sections we want
data_notouch = ma.masked_less(data, threshold)
data_touch = ma.masked_greater(data, threshold)
#Extract the mask to get where there are features. We will use this to id features to operate on
regions = ma.getmask(data_touch) #Extract the mask from the touch array as the Trues will line up with the areas more than the threshold
#Create the features map of 1's where we want features (greater than threshold), zeroes otherwise
features = numpy.zeros(data.shape, dtype=numpy.int32)
features[regions] = 1 #Define features
return features
def interp():
""" Test interp option
"""
lon = np.arange(-1, 10.01, 0.1)
lat = np.arange(-5, 1.01, 0.1)
Lon, Lat = np.meshgrid(lon, lat)
h = ma.masked_greater(np.random.random(Lon.shape), 0.7)
h_smooth = wmean_2D_latlon(Lat, Lon, h, l=2e5, interp=False)
h_csmooth = cwindow_mean_2D_latlon(Lat, Lon, h, l=2e5, interp=False)
h_smooth_i = wmean_2D_latlon(Lat, Lon, h, l=2e5, interp=True)
h_csmooth_i = cwindow_mean_2D_latlon(Lat, Lon, h, l=2e5, interp=True)
assert (h_smooth == h_csmooth).all()
assert (h_smooth_i == h_csmooth_i).all()
#assert (abs(h_smooth - h_smooth_i).sum() == 0)
assert ((h_smooth - h_smooth_i) == 0).all()
assert (h_smooth_i.compressed().size >= h_smooth.compressed().size)
def landocmask(icenc,lsmasknc): """ generate an icefree ocean mask, and spits out a land mask for good measure Input: nc file of ice concentration, land-sea mask (as strings) Output: ice free ocean mask (1s and 0s) To use: ocean = ma.masked_where(icexp<1,tempsfc) example: """ lsmask=io.readnc(lsmasknc,'lsm')[0][0,0,:,:] #Create an icefree ocean mask ice=io.readnc(icenc,'iceconc')[0] icesum=ice.sum(axis=0) icelsm=icesum+lsmask ifmask = ma.masked_greater(icelsm,0)+1 icefree = ifmask.data*~ifmask.mask sansice = icefree[0,:,:] #ocean = ma.masked_where(icexp<1,tempsfc) return lsmask, sansice
def _select_working_set(self): return (0,1) I = ma.masked_less( self.alpha, 1e-8 ) grad = self._grad( ma.array(self.X, mask=ma.getmask(I) ) ) i = (-grad).argmax() P = self._P( self.X[ i % self.N ].reshape([1,self.d]), np.vstack( [self.X,]*kappa ) ) # P_IJ a = ma.masked_less( P[0,i] + np.diagonal(P) - 2*P, 0 ) #NOTE the actual equation is -grad < -grad_i - i'm not sure if inversing the signs and equality operator is a problem grad_i = self._grad( self.X[i % self.N] ) b = ma.masked_greater(grad, grad_i) - grad_i j = ( (b**2) / -a ).argmin() return (i,j)
def test_mask_at_interp():
""" Test the behavior of masked points with interp on|off
As long as the filter is wide enough to capture at least
one data point per point, the interp=True will return
"""
N = 25
l = N/2
grid = np.linspace(-10, 10, N)
X, Y = np.meshgrid(grid, grid)
data = np.ones((N, N))
thr = np.percentile(data, 90)
data = ma.masked_greater(data, thr)
# Equivalent to interp=False
h = wmean_2D(X, Y, data, l=l)
assert (data.mask == h.mask).all()
h = wmean_2D(X, Y, data, l=l, interp=True)
assert (~h.mask).all()
def corr_proba(r, ndata, ndataset=2, dof=False):
"""Probability of rejecting correlations
- **r**: Correlation coefficient
- **ndata**: Number of records use for correlations
- **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2]
.. todo::
This must be rewritten using :mod:`scipy.stats`
"""
# Basic tests
ndata = MA.masked_equal(ndata,0,copy=0)
r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0)
# Degree of freedom
if dof:
df = ndata
else:
df = ndata-2-ndataset
# Advanced test: prevent extreme values by locally decreasing the dof
reduc = N.ones(r.shape)
z = None
while z is None or MA.count(MA.masked_greater(z,-600.)):
if z is not None:
imax = MA.argmin(z.ravel())
reduc.flat[imax] += 1
dfr = df/reduc
t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r)))
a = 0.5*dfr
b = 0.5
x = df/(dfr+t**2)
z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x)
# Perfom the test and format the variable
prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100
prob.id = 'corr_proba' ; prob.name = prob.id
prob.long_name = 'Probability of rejection'
prob.units = '%'
return prob
def hardcoded_maskedarray():
"""Test if masked data is not considered in the average
"""
h = np.array([[ 1e9, 1e9, 1e9],
[ 1e9, 3.14, 1e9],
[ 1e9, 1e9, 1e9]])
h = ma.masked_greater(h, 10)
lon = np.array([10.1, 10, 9.9])
lat = np.array([-0.1, -0.09, -0.08])
Lon, Lat = np.meshgrid(lon, lat)
h_smooth = wmean_2D_latlon(Lat, Lon, h, l=1e10)
h_smooth2 = cwindow_mean_2D_latlon(Lat, Lon, h, l=1e10)
# maud and cmaud should return the very same result
assert (h_smooth == h_smooth2).all()
assert (h_smooth.mask == h.mask).all()
#assert (h_smooth.compressed() == h.compressed()).all()
assert (np.absolute(h_smooth - h).sum() == 0.)
def _read(self):
"""Read the field from the given filename."""
basename = os.path.basename(self._filename)
self.version = int(basename[9:11])
self.year = int(basename[12:16])
self.month = basename[16:19]
self.period = basename[19:20]
self.format = basename[21:24]
self.month = month_numbers[self.month]
# Data is 8 bit bigendian.
self.data = np.fromfile(self._filename, dtype='>u1').reshape(X_SIZE,
Y_SIZE)
# Iris' preferred dimensional ordering is (y,x).
self.data = self.data.transpose()
# Flip, for a positive step through the Y dimension.
self.data = self.data[::-1]
# Any percentages greater than 100 represent missing data.
self.data = ma.masked_greater(self.data, 100)
def compare2func(f1, f2):
N = 10
l = N/2
grid = np.linspace(-10, 10, N)
X, Y = np.meshgrid(grid, grid)
data = random(X.shape)
h1 = f1(X, Y, data, l=l)
h2 = f2(X, Y, data, l=l)
assert (h1 == h2).all()
data = ma.array(data)
h = f1(X, Y, data, l=l)
h2 = f2(X, Y, data, l=l)
assert (h1 == h2).all()
thr = np.percentile(data, 70)
data = ma.masked_greater(data, thr)
h = f1(X, Y, data, l=l)
h2 = f2(X, Y, data, l=l)
assert (h1 == h2).all()
def _clean_timestamps(self, track, stime, etime):
'''
Cut track in time and make array with timestamps monotonic.
@param track:
@type track:
@return:
@rtype:
'''
data = ma.masked_inside(track['timestamp'], stime, etime)
track = track.compress(data.mask)
# Eliminate repetitive points.
times, indices = np.unique(track['timestamp'], return_index=True)
track = track[indices]
# Here we still can has points reversed in time. Fix it.
tdifs = np.ediff1d(track['timestamp'], to_begin=1)
# At first let's find ends of track's chunks delimited by timeout,
# if any.
chunk_end_idxs = np.where(tdifs > self.maxtimediff)[0]
# Index of timestamp from which no chunks allowed, just track.
safe_time_idx = np.where(track['timestamp'] <
stime + self.safe_time)[0]
if len(chunk_end_idxs) > 0:
track_start_idx = 0
track_end_idx = chunk_end_idxs[0]
# Situation then there are little tracks exists before window
# open time.
if len(safe_time_idx) > 0:
safe_time_idx = safe_time_idx[-1]
for chunk_end_idx in chunk_end_idxs:
if chunk_end_idx < safe_time_idx:
track_start_idx = chunk_end_idx + 1
else:
track_end_idx = chunk_end_idx
break
track = track[track_start_idx:track_end_idx]
tdifs = tdifs[track_start_idx:track_end_idx]
# Eliminate reverse points.
data = ma.masked_greater(tdifs, 0)
track = track.compress(data.mask)
return track
def dynamic_mask(self, image, sigrange): """ Creates a numpy mask on the image, filtering out any pixel values that are more than sigrange*std from the median value Input: numpy array of the image, sigrange for multiplier on standard dev range Output: Masked numpy array covering any pixels above or below the standard dev range """ # Make a masked array using the static mask and imput image pre_masked = ma.array(image, mask=self.static_mask) # Mask saturated or empty masked1 = ma.masked_greater(pre_masked, 254) masked1 = ma.masked_less(masked1, 0) median = ma.median(masked1) mean = ma.mean(masked1) std = ma.std(masked1) return masked1, median, mean, std
def update(val):
# Get parameter
vmin = slider_cutoff.val
borderType = borderTypes[int(np.around(slider_bordertype.val)) ] # Border type
d = int(np.around(slider_diameter.val)) # Diameter
sigma = slider_sigma.val # Sigma
sigma_color = slider_sigma_color.val # Sigma color (Bilateral filter)
sigma_space = slider_sigma_space.val # Sigma space (Bilateral filter)
nlm_h = slider_nlm.val
# Apply filters
box = cv2.blur(img, (d,d), borderType=borderType) # Box filter
gaussian = cv2.GaussianBlur(img, (d,d), sigma, borderType=borderType) # Gaussian
median = cv2.medianBlur(img, d) # Median
bilateral = cv2.bilateralFilter(img, d, sigma_color, sigma_space, borderType=borderType) # Bilateral
kernel_opening = cv2.getStructuringElement(cv2.MORPH_CROSS,(d,d))
opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel_opening)
masked = ma.masked_greater(opening, 0)
opening2 = np.copy(img)
opening2[~masked.mask] = 0
non_local_means = cv2.fastNlMeansDenoising(img, None, h=nlm_h, templateWindowSize=d, searchWindowSize=d)
# Show results
original.set_clim(vmin, 255.0)
box_filter.set_data(box)
box_filter.set_clim(vmin, 255.0)
gaussian_filter.set_data(gaussian)
gaussian_filter.set_clim(vmin, 255.0)
median_filter.set_data(median)
median_filter.set_clim(vmin, 255.0)
bilateral_filter.set_data(bilateral)
bilateral_filter.set_clim(vmin, 255.0)
opening_filter.set_data(opening)
opening_filter.set_clim(vmin, 255.0)
opening_filter2.set_data(opening2)
opening_filter2.set_clim(vmin, 255.0)
nlm_filter.set_data(non_local_means)
nlm_filter.set_clim(vmin, 255.0)
def random_maskedarray(N=10, res=0.1):
#lon0, lat0 = random(2)
#lon0 = 540*lon0-180 #[180, 360]
#lat0 = 180*lat0-90
grid = np.arange(-N/2, N/2)*res
Lon, Lat = np.meshgrid(grid, grid)
h = random(Lon.shape)
h = ma.masked_greater(h, 0.7)
h_smooth = wmean_2D_latlon(Lat, Lon, h, l=.1)
h_csmooth = cwindow_mean_2D_latlon(Lat, Lon, h, l=.1)
assert (h_smooth == h_csmooth).all()
h_smooth = wmean_2D_latlon(Lat, Lon, h, l=1e10)
h_csmooth = cwindow_mean_2D_latlon(Lat, Lon, h, l=1e10)
# maud and cmaud should return the very same result
assert (h_smooth == h_csmooth).all()
assert (h_smooth.mask == h.mask).all()
#assert (h_smooth.compressed() == h.compressed()).all()
assert (np.absolute(h_smooth - h).sum() == 0.)
def _read(self):
"""Read the field from the given filename."""
basename = os.path.basename(self._filename)
self.version = int(basename[9:11])
self.year = int(basename[12:16])
self.month = basename[16:19]
self.period = basename[19:20]
self.format = basename[21:24]
self.month = month_numbers[self.month]
# Data is 8 bit bigendian.
data = np.fromfile(self._filename, dtype='>u1').reshape(X_SIZE, Y_SIZE)
# Iris' preferred dimensional ordering is (y,x).
data = data.transpose()
# Flip, for a positive step through the Y dimension.
data = data[::-1]
# Any percentages greater than 100 represent missing data.
data = ma.masked_greater(data, 100)
# The default fill value is 999999(!), so we choose something
# more sensible. NB. 999999 % 256 = 63 = bad.
data.fill_value = 255
self.data = data
def dry_spells(rainfall, threshold=0.254, start=True):
"""
Compute the dry spells for a series of precipitations
Parameters
----------
rainfall : TimeSeries
TimeSeries of precipitations.
threshold : float, optional
Minimum amount of precipitation defining a wet day.
start : boolean, optional
Whether the spells are associated with the first or last day of the spell.
Returns
-------
dry_spells : TimeSeries
A :class:`TimeSeries` giving the duration of the spell at either the first
or last date.
"""
rdates = getattr(rainfall, 'dates', None)
rdata = ma.masked_array(rainfall, subok=False)
condition = ma.masked_greater(rdata, threshold)
slices = ma.clump_unmasked(condition)
# Get the durations and starting dates of each spell
durations = [s.stop - s.start for s in slices]
# Give up if we have no dates
if rdates is None:
return np.array(durations)
# Get the dates, then...
if start:
dates = rdates[[s.start for s in slices]]
else:
dates = rdates[[s.stop - 1 for s in slices]]
ensoi = getattr(rainfall, 'ensoindicator', None)
spells = enso.climate_series(durations, dates=dates, ensoindicator=ensoi)
return spells
def test_testOddFeatures(self):
# Test of other odd features
x = arange(20)
x = x.reshape(4, 5)
x.flat[5] = 12
assert_(x[1, 0] == 12)
z = x + 10j * x
assert_(eq(z.real, x))
assert_(eq(z.imag, 10 * x))
assert_(eq((z * conjugate(z)).real, 101 * x * x))
z.imag[...] = 0.0
x = arange(10)
x[3] = masked
assert_(str(x[3]) == str(masked))
c = x >= 8
assert_(count(where(c, masked, masked)) == 0)
assert_(shape(where(c, masked, masked)) == c.shape)
z = where(c, x, masked)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is masked)
assert_(z[7] is masked)
assert_(z[8] is not masked)
assert_(z[9] is not masked)
assert_(eq(x, z))
z = where(c, masked, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
z = masked_where(c, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
assert_(eq(x, z))
x = array([1., 2., 3., 4., 5.])
c = array([1, 1, 1, 0, 0])
x[2] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
c[0] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
assert_(eq(masked_where(greater_equal(x, 2), x),
masked_greater_equal(x, 2)))
assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
assert_(eq(masked_inside(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 1, 3).mask,
[1, 1, 1, 1, 0]))
assert_(eq(masked_outside(array(list(range(5)),
mask=[0, 1, 0, 0, 0]), 1, 3).mask,
[1, 1, 0, 0, 1]))
assert_(eq(masked_equal(array(list(range(5)),
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 0]))
assert_(eq(masked_not_equal(array([2, 2, 1, 2, 1],
mask=[1, 0, 0, 0, 0]), 2).mask,
[1, 0, 1, 0, 1]))
assert_(eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
[99, 99, 3, 4, 5]))
atest = ones((10, 10, 10), dtype=np.float32)
btest = zeros(atest.shape, MaskType)
ctest = masked_where(btest, atest)
assert_(eq(atest, ctest))
z = choose(c, (-x, x))
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
x = arange(6)
x[5] = masked
y = arange(6) * 10
y[2] = masked
c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
cm = c.filled(1)
z = where(c, x, y)
zm = where(cm, x, y)
assert_(eq(z, zm))
assert_(getmask(zm) is nomask)
assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
z = where(c, masked, 1)
assert_(eq(z, [99, 99, 99, 1, 1, 1]))
z = where(c, 1, masked)
assert_(eq(z, [99, 1, 1, 99, 99, 99]))
load_time = t1-t0
print "Loaded file %s in %.3f" % ( cached_file, load_time )
if load_time > max_load_time:
max_load_time = load_time
max_file = cached_file
print "--- "*20
print "MAX Load time: file %s in %.3f" % ( max_file, max_load_time )
if __name__ == "__main__":
# from testing import getVariable
# CacheLevel = 10000.0
# TestVariable = getVariable( 0, CacheLevel )
# data = TestVariable.data
import os, numpy, numpy.ma as ma
cached_file = os.path.expanduser( '~/.cdas/testing/MERRA_mon_atmos_hur' )
data = numpy.fromfile( cached_file, dtype=numpy.float32 ).reshape( (432, 144, 288) )
FillVal = 1.00000002e+20
mdata = ma.masked_greater( data, 0.90000002e+20 )
t0 = time.time()
a = ma.average(data,0)
t1 = time.time()
print "Result computed in %.2f, shape = %s, sample=%s" % ( (t1-t0), str(a.shape), a.flatten()[0:10])
def get_spc(self, *args):
nargs = len(args)
if nargs > 2:
raise TypeError('get_spc expected at most 2 arguments, got %d' % nargs)
item = args[0]
values = []
roles = []
if item.exclude:
item_spc_roles = [k for k in item.iter_species_roles()]
item_spcs = [k[0] for k in item_spc_roles]
for spc, role in self._stoic:
if role == 'u' and spc in item_spcs and (spc, 'u') not in item_spc_roles:
val = self._stoic[spc, role]
if greater(val, 0).all():
role = 'p'
elif less(val, 0).all():
role = 'r'
if not (spc, role) in item_spc_roles:
values.append(1 * self._stoic[spc, role])
roles.append(role)
else:
for spc, props in item.spc_dict.iteritems():
spc_roles = props['role']
for role in spc_roles:
if (spc, role) in self._stoic:
values.append(item.spc_dict[spc]['stoic'] * self._stoic[spc, role])
roles.append(role)
if 'u' not in spc_roles and (spc, 'u') in self._stoic:
val = self._stoic[spc, 'u']
if role == 'p':
values.append(item.spc_dict[spc]['stoic'] * masked_less(val, 0).filled(0))
roles.append(role)
elif role == 'r':
values.append(item.spc_dict[spc]['stoic'] * masked_greater(val, 0).filled(0))
roles.append(role)
if len(values) == 0:
if self._safe:
if nargs == 1:
if item.name in self:
warn("%s not in %s; trying all roles" % (item, self.sum()))
return self.get_spc(Species(item.name, exclude = item.exclude))
raise KeyError, "%s does not contain %s" % (str(self.sum()), str(item))
else:
return args[1]
else:
raise KeyError, "%s does not contain %s" % (str(self.sum()), str(item))
last_role = roles[-1]
same_role = all([last_role == role for role in roles])
if same_role:
role = last_role
else:
role = 'u'
return Stoic(sum(values, axis = 0), role = role)
import numpy as np import numpy.ma as ma a = np.arange(4) a ma.masked_greater(a, 2)
## Constructing masked arrays # # Masked arrays are intended to be able to replace normal np.ndarrays, # and so the same construction functions work: x = ma.array([5, 2, 9, -4]) y = ma.array([5, 2., 9, -4.]) o = ma.ones(5) z = ma.zeros((3, 2)) e = ma.empty(3) a = ma.arange(6) # These functions create masked arrays with all elements not masked # initially. However, masked arrays can be constructed in other ways, # which automatically set more interesting masks. mx = ma.masked_equal([5, 2, 9, -4], 2) my = ma.masked_greater([5, 2., 9, -4.], 3) mis = ma.masked_inside(x, 1, 6) miv = ma.masked_invalid([1, np.nan, 5, np.inf]) ## And more: # ma.greater_equal # ma.masked_less # ma.masked_less_equal # ma.masked_not_equal # ma.masked_object # ma.masked_outside # ma.masked_values # ma.masked_where # To get a masked array's unmasked data, and fill in masked elements, use # 'filled': print "mx, with masked elements filled with 77", mx.filled(77)
#CREATE WAVELENGTH MASK
#edges of masked regions adjusted by hand in TestMasks.ipynb to
#block out the geocoronal lines.
#w1, f1, err1 = np.loadtxt('lcb201010_x1dsum.txt.bin30.unred', unpack=True)
w1, f1, stdev1, err1 = np.loadtxt('../Natalie/lcb201010_x1dsum.txt.bin30.unred.newerrors', unpack=True)
add_disp = 9.1e-18
#adding intrinsic dispersion
toterr1 = np.sqrt(err1**2.0 + add_disp**2.0)
w1_1 = ma.masked_less(w1, 1141)
w1_2 = ma.masked_inside(w1_1, 1178., 1250.)
w1_3 = ma.masked_inside(w1_2, 1292., 1318.)
w1_4 = ma.masked_inside(w1_3, 1348., 1367.)
w1_5 = ma.masked_inside(w1_4, 1332., 1340.)
dataw1 = ma.masked_greater(w1_5, 1725.)
dataw1c = 1.0*dataw1.compressed()
#w2, f2, err2 = np.loadtxt('lcb202010_x1dsum.txt.bin30.unred', unpack=True)
w2, f2, stdev2, err2 = np.loadtxt('../Natalie/lcb202010_x1dsum.txt.bin30.unred.newerrors', unpack=True)
#adding intrinsic dispersion
toterr2 = np.sqrt(err2**2.0 + add_disp**2.0)
w2_1 = ma.masked_less(w2, 1142)
w2_2 = ma.masked_inside(w2_1, 1182., 1250.)
w2_3 = ma.masked_inside(w2_2, 1290., 1318.)
w2_4 = ma.masked_inside(w2_3, 1348., 1368.)
w2_5 = ma.masked_inside(w2_4, 1330., 1340.)
#dataw2 = ma.masked_greater(w2_5, 1533.)
dataw2 = ma.masked_greater(w2_5, 1600.)
dataw2c = 1.0*dataw2.compressed()