def cloud_statistics(file_name):
"""
Return core duration, minimum core base, maximum core height, mean core
mass, formation time, dissipation time, maximum depth, depth evolution and
corresponding times for tracked clouds.
Parameters
----------
file_name : netCDF file name
id_profile file for a tracked core with dimensions double t(t),
double z(z).
Return
------
tuple : cloud_id, lifetime, base, top, mass, l_min, l_max, depths,
max_depth, times
"""
# Read netCDF dataset
data = Dataset(file_name)
# Core ID
cloud_id = int(file_name[-11:-3])
# Core duration (seconds)
times = data.variables['t'][...]
lifetime = len(times)*mc.dt
# Formation time, dissipation time (seconds)
l_min = times.min()*mc.dt
l_max = times.max()*mc.dt
# Minimum core base, maximum core height, maximum depth, depth evolution
# (metres)
area = ma.masked_invalid(data.variables['AREA'][...])
z = data.variables['z'][...]
z = z*np.ones(np.shape(area))
z = ma.masked_array(z, ma.getmask(area))
bases = z.min(axis=1)
tops = z.max(axis=1)
depths = tops - bases + mc.dz
max_depth = depths.max()
base = bases.min()
top = tops.max()
# Mean core mass mass (kilograms)
qn = ma.masked_invalid(data.variables['QN'][...])
rho = ma.masked_invalid(data.variables['RHO'][...])
mass = np.mean(np.sum(area*rho*mc.dz, axis=1))
# Remove missing values
times = ma.masked_array(times, ma.getmask(depths))
depths = depths[~depths.mask]
times = times[~times.mask]
data.close()
return cloud_id, lifetime, base, top, mass, l_min, l_max, depths, \
max_depth, times
def outer (self, a, b):
"Return the function applied to the outer product of a and b."
a = _makeMaskedArg(a)
b = _makeMaskedArg(b)
ma = getmask(a)
mb = getmask(b)
if ma is nomask and mb is nomask:
m = None
else:
ma = getmaskarray(a)
mb = getmaskarray(b)
m = logical_or.outer(ma, mb)
d = numpy.maximum.outer(filled(a), filled(b))
return TransientVariable(d, mask=m)
def test_testCI(self):
# Test of conversions and indexing
x1 = np.array([1, 2, 4, 3])
x2 = array(x1, mask=[1, 0, 0, 0])
x3 = array(x1, mask=[0, 1, 0, 1])
x4 = array(x1)
# test conversion to strings
str(x2) # raises?
repr(x2) # raises?
assert_(eq(np.sort(x1), sort(x2, fill_value=0)))
# tests of indexing
assert_(type(x2[1]) is type(x1[1]))
assert_(x1[1] == x2[1])
assert_(x2[0] is masked)
assert_(eq(x1[2], x2[2]))
assert_(eq(x1[2:5], x2[2:5]))
assert_(eq(x1[:], x2[:]))
assert_(eq(x1[1:], x3[1:]))
x1[2] = 9
x2[2] = 9
assert_(eq(x1, x2))
x1[1:3] = 99
x2[1:3] = 99
assert_(eq(x1, x2))
x2[1] = masked
assert_(eq(x1, x2))
x2[1:3] = masked
assert_(eq(x1, x2))
x2[:] = x1
x2[1] = masked
assert_(allequal(getmask(x2), array([0, 1, 0, 0])))
x3[:] = masked_array([1, 2, 3, 4], [0, 1, 1, 0])
assert_(allequal(getmask(x3), array([0, 1, 1, 0])))
x4[:] = masked_array([1, 2, 3, 4], [0, 1, 1, 0])
assert_(allequal(getmask(x4), array([0, 1, 1, 0])))
assert_(allequal(x4, array([1, 2, 3, 4])))
x1 = np.arange(5) * 1.0
x2 = masked_values(x1, 3.0)
assert_(eq(x1, x2))
assert_(allequal(array([0, 0, 0, 1, 0], MaskType), x2.mask))
assert_(eq(3.0, x2.fill_value))
x1 = array([1, 'hello', 2, 3], object)
x2 = np.array([1, 'hello', 2, 3], object)
s1 = x1[1]
s2 = x2[1]
assert_equal(type(s2), str)
assert_equal(type(s1), str)
assert_equal(s1, s2)
assert_(x1[1:1].shape == (0,))
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result.fill(0)
else:
if clip:
mask = ma.getmask(result)
result = ma.array(np.clip(result.filled(vmax), vmin, vmax),
mask=mask)
resdat = result.data
resdat = self.fn(resdat)
resdat -= self._lower
resdat /= (self._upper - self._lower)
result = np.ma.array(resdat, mask=result.mask, copy=False)
if is_scalar:
result = result[0]
return result
def expand_years(raster, num_years, name="temp", culm=False, save=False):
"""Takes in array of deforestation data and parses it into an set of arrays
stored in dictionary for the yearly deofrestation progression for each
pixel then pickles and returns that dictionary. Array contains 0 for no loss
or 1-n corresponding to the principle year of deforestation
Args:
raster (raster obj): raster from raster class containing the
rasterized data.
name (str): name of file to be saved.
Returns:
dictionary with the status of each year, labeled with a eponymous str
for that year.
"""
shape = raster.shape
print(shape)
raster_mask = ma.getmask(raster)
years = ma.zeros((num_years, *shape))
print(years.shape)
for year in range(1, num_years + 1):
for repl in np.argwhere(raster == year):
years[year - 1, repl[0], repl[1]] = 1
years[year - 1] = ma.masked_where(raster_mask, years[year - 1])
years[year - 1] = years[year - 1].filled(np.nan)
return years
def x_grad_rho(RomsFile, RomsGrd, varname):
"""
Compute x-gradient assuming rectangular grid cells
"""
if type(varname) == str:
#load roms file
RomsNC = nc4(RomsFile, 'r')
#load variable
_var = RomsNC.variables[varname][:]
else:
_var = varname
#get mask
Mask = ma.getmask(_var)
#gradient in x direction on u points
dvar_u = np.diff(_var, n=1, axis=3)
#shift u points to rho points
dvar = GridShift.Upt_to_Rho(dvar_u)
#lon positions [meters]
x_dist = rt.rho_dist_grd(RomsGrd)[0]
#repeat over depth and time space and apply mask
dx = ma.array(rt.AddDepthTime(RomsFile, x_dist), mask=Mask)
#compute gradient
dvar_dx = dvar / dx
return dvar_dx
def y_grad_rho(RomsFile, RomsGrd, varname):
"""
Compute y-gradient assuming rectangular grid cells
"""
if type(varname) == str:
#load roms file
RomsNC = nc4(RomsFile, 'r')
#load variable
_var = RomsNC.variables[varname][:]
else:
_var = varname
#get mask
Mask = ma.getmask(_var)
#compute difference
dvar_v = np.diff(_var, n=1, axis=2)
#shift from v points to rho points
dvar = GridShift.Vpt_to_Rho(dvar_v)
#lon positions [meters]
y_dist = rt.rho_dist_grd(RomsGrd)[1]
#repeat over depth and time space and apply mask
dy = ma.array(rt.AddDepthTime(RomsFile, y_dist), mask=Mask)
#compute gradient
dvar_dy = dvar / dy
return dvar_dy
def subplot(self, axes, subfigure):
"""
Generate the subplot on the figure.
:param axes: :class:`matplotlib.axes` instance where the plot will
be added.
:param tuple subfigure: A tuple that holds all the required elements
of the plot, including the data, x- and y-grids,
title, contour levels, colorbar label and
map keyword arguments.
"""
from mpl_toolkits.basemap import maskoceans
data, xgrid, ygrid, title, lvls, cbarlab, map_kwargs = subfigure
mapobj, mx, my = self.createMap(axes, xgrid, ygrid, map_kwargs)
dmask = data.mask
masked_data = maskoceans(xgrid, ygrid, data, inlands=False)
omask = ma.getmask(masked_data)
nmask = ma.mask_or(dmask, omask)
masked_data.mask = nmask
cmap = selectColormap(lvls)
CS = mapobj.contourf(mx, my, masked_data, levels=lvls,
extend='both', cmap=cmap)
CB = self.colorbar(CS, ticks=lvls[::2], ax=axes, extend='both')
CB.set_label(cbarlab)
axes.set_title(title)
self.labelAxes(axes)
self.addGraticule(axes, mapobj)
self.addCoastline(mapobj)
self.fillContinents(mapobj, fillcolor="#EEEEEE")
self.addMapScale(mapobj)
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
result = ma.masked_less_equal(result, 0, copy=False)
self.autoscale_None(result)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin <= 0:
raise ValueError("values must all be positive")
elif vmin == vmax:
result.fill(0)
else:
if clip:
mask = ma.getmask(result)
result = ma.array(np.clip(result.filled(vmax), vmin, vmax), mask=mask)
# in-place equivalent of above can be much faster
resdat = result.data
mask = result.mask
if mask is np.ma.nomask:
mask = resdat <= 0
else:
mask |= resdat <= 0
cbook._putmask(resdat, mask, 1)
np.log(resdat, resdat)
resdat -= np.log(vmin)
resdat /= np.log(vmax) - np.log(vmin)
result = np.ma.array(resdat, mask=mask, copy=False)
if is_scalar:
result = result[0]
return result
def makeFluxSigMask(flux=None, minThresh=2, maxThresh=5):
"""
Compute the mean total integrated flux value
and its standard deviation.
Find all pixels with a flux in between min/max thresholds and mask them.
Parameters
----------
fluxImage: array_like
The 2D array of the total integrated fluxes.
min/maxThresh: int
Sigma limit thresholds for the min/max.
Return
------
Boolean mask.
"""
sigma = ma.std(flux)
ave = ma.mean(flux)
if sigma > ave:
intervalMin = ave
else:
intervalMin = ave - (minThresh * sigma)
intervalMax = ave + (maxThresh * sigma)
maskedOutside = ma.masked_outside(flux, intervalMin, intervalMax)
maskedZeros = ma.masked_where(maskedOutside == 0,
maskedOutside,
copy=False)
return ma.getmask(maskedZeros)
def __call__(self, value, clip=None):
"""Map value to the interval [0, 1]. The clip argument is unused."""
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vcenter, vmax = self.vmin, self.vcenter, self.vmax
if vmin == vmax == vcenter:
result.fill(0)
elif not vmin <= vcenter <= vmax:
raise ValueError("minvalue must be less than or equal to "
"centervalue which must be less than or "
"equal to maxvalue")
else:
vmin = float(vmin)
vcenter = float(vcenter)
vmax = float(vmax)
# in degenerate cases, prefer the center value to the extremes
degen = (result == vcenter) if vcenter == vmax else None
x, y = [vmin, vcenter, vmax], [0, 0.5, 1]
result = ma.masked_array(np.interp(result, x, y),
mask=ma.getmask(result))
if degen is not None:
result[degen] = 0.5
if is_scalar:
result = np.atleast_1d(result)[0]
return result
def plot_spin_temp_lv(values, ax, key, label, max_clip=None):
scale_vals = values[key]
dataset = values[~ma.getmask(scale_vals)]
if max_clip:
scale_vals = np.clip(dataset[key], 0, max_clip)
base = np.array([dataset['longitude'], dataset['Velocity'], scale_vals],
dtype=[('l', float), ('b', float), ('scale', float)])
sample = base
#vmax = np.max(sample[2,:])
#cm = plt.cm.get_cmap('RdYlBu_r')
cm = plt.cm.get_cmap('viridis')
x = sample[0,:]
y = sample[1,:]
z = sample[2,:]
idx = z.argsort()
x, y, z = x[idx], y[idx], z[idx]
# spin temp plot needs log, min 3
sc = ax.scatter(x, y, c=z, s=25, cmap=cm, norm=matplotlib.colors.PowerNorm(0.5, vmin=3))
formatter = LogFormatter(10, labelOnlyBase=False)
ticks = [0, 5, 10, 20, 40, 70, 100, 150, 200, 300]
cb = plt.colorbar(sc, ticks=ticks, format=formatter)
ax.set_xlim(values['longitude'].max() + 5, values['longitude'].min() - 5)
ax.set_xlabel('Galactic longitude (deg)')
ax.set_ylabel('LSR Velocity (km/s)')
cb.set_label(label)
return None
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin<=0:
raise ValueError("values must all be positive")
elif vmin==vmax:
return 0.0 * val
else:
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = (ma.log(val)-np.log(vmin))/(np.log(vmax)-np.log(vmin))
if vtype == 'scalar':
result = result[0]
return result
def interpolate_time_iso(self, params_dict):
"""
Rounds to nearest hour by adding a timedelta hour if minute >= delta_minutes
"""
try:
_time = None
time_obs = params_dict["timeObs"]
delta_minutes = self.delta / 60
if not ma.getmask(time_obs):
_time = int(ma.compressed(time_obs)[0])
else:
return ""
_time = datetime.utcfromtimestamp(_time)
_time = _time.replace(
second=0, microsecond=0, minute=0,
hour=_time.hour) + timedelta(hours=_time.minute //
delta_minutes)
# convert this iso
return str(_time.isoformat())
except Exception as _e: # pylint:disable=broad-except
logging.error(
"%s handle_data: Exception in named function interpolate_time_iso: error: %s",
self.__class__.__name__,
str(_e),
)
def umask_value_transform(self, params_dict):
"""Retrieves a netcdf value, checking for masking and retrieves the value as a float
Args:
params_dict (dict): named function parameters
Returns:
float: the corresponding value
"""
# Probably need more here....
try:
key = None
rec_num = params_dict["recNum"]
for key in params_dict.keys():
if key != "recNum":
break
nc_value = self.ncdf_data_set[key][rec_num]
if not ma.getmask(nc_value):
value = ma.compressed(nc_value)[0]
return float(value)
else:
return None
except Exception as _e: # pylint:disable=broad-except
logging.error(
"%s umask_value_transform: Exception in named function umask_value_transform for key %s: error: %s",
self.__class__.__name__,
key,
str(_e),
)
return None
def interpolate_time(self, params_dict):
"""
Rounds to nearest hour by adding a timedelta hour if minute >= delta (from the template)
"""
try:
_thistime = None
_time_obs = params_dict["timeObs"]
if not ma.getmask(_time_obs):
_thistime = int(ma.compressed(_time_obs)[0])
else:
return ""
# if I get here process the _thistime
delta_minutes = self.delta / 60
_ret_time = datetime.utcfromtimestamp(_thistime)
_ret_time = _ret_time.replace(
second=0, microsecond=0, minute=0,
hour=_ret_time.hour) + timedelta(hours=_ret_time.minute //
delta_minutes)
return calendar.timegm(_ret_time.timetuple())
except Exception as _e: # pylint:disable=broad-except
logging.error(
"%s handle_data: Exception in named function interpolate_time: error: %s",
self.__class__.__name__,
str(_e),
)
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(numpy.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(numpy.float)
if self.staticrange is None:
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
else:
self.vmin, self.vmax = None, None
self.autoscale_None(val)
vmin, vmax = self.vmax - self.staticrange, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin==vmax:
result = 0.0 * val
else:
vmin = float(vmin)
vmax = float(vmax)
rmin = float(self.rmin)
rmax = float(self.rmax)
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = (val-vmin) * ((rmax-rmin) / (vmax-vmin)) + rmin
if vtype == 'scalar':
result = result[0]
return result
def segmentate_arr(self, seg_dict, msk):
"""
segmentate_arr returns a dict of boolean masks of each segmentated tree.
Parameters
----------
seg_dict (dict): empty dictionary.
msk: array of segmentated trees.
Returns
-------
Dict of boolean masks of each segmentated tree.
"""
seg_trees = np.unique(msk)
i = 0
for tree in seg_trees:
if tree < 65000:
mask = np.zeros((self.size, self.size))
mask = ma.masked_where(msk == tree, mask)
mask = ma.getmask(mask)
seg_dict[self.w][i] = mask
i += 1
if seg_trees.shape[0] == 1:
mask = np.zeros((self.size, self.size)).astype('bool')
seg_dict[self.w][i] = mask
return seg_dict
def openData(self, ccd, ap, save=False, mask=True):
target = self.target.replace(' ', '_')
filters = self.filters[::-1]
ccd = str(ccd)
ap = str(ap)
if ccd not in self.apnames.keys():
raise ValueError('{} not a valid CCD'.format(ccd))
if ap not in self.apnames[ccd]:
raise ValueError('{} not a valid aperture'.format(ap))
data = self.logf.tseries(ccd, ap)
obstime = Time(data.t,
format='mjd',
scale='utc',
location=self.tel_location)
data.t = obstime.tdb.value
exp = self.logf[ccd]['Exptim'] / 86400
weights = np.ones(len(data.t))
m = data.get_mask()
zero_flux_mask = ma.getmask(ma.masked_not_equal(data.y, 0))
data_mask = np.logical_and(~m, zero_flux_mask)
if mask:
out = np.column_stack([
data.t[data_mask], exp[data_mask], data.y[data_mask],
data.ye[data_mask], weights[data_mask], weights[data_mask]
])
else:
out = np.column_stack(
[data.t, exp, data.y, data.ye, weights, weights])
return out, data_mask
def onMouseOver(self, event):
"""
"""
if event.inaxes == self.ax1:
if (event.xdata != None and event.ydata != None):
xidx = (int(event.xdata/
(float(self.omf.parameters['xstepsize'])*1.0e10)))
yidx = (int(event.ydata/
(float(self.omf.parameters['ystepsize'])*1.0e10)))
value = self.angle[xidx,yidx]
if ma.getmask(self.angle)[xidx,yidx]:
self.datavalue.SetStatusText('Angle Value: MASKED')
else:
value = -degrees(value)
if (value < 0.0): value +=360
self.datavalue.SetStatusText('Angle Value: '
+str('%.2f'%value))
else:
self.datavalue.SetLabel('')
return
def __setslice__(self, i, j, value):
"Sets the slice described by [i,j] to `value`."
_localdict = self.__dict__
d = self._data
m = _localdict['_fieldmask']
names = self.dtype.names
if value is masked:
for n in names:
m[i:j][n] = True
elif not self._hardmask:
fval = filled(value)
mval = getmaskarray(value)
for n in names:
d[n][i:j] = fval
m[n][i:j] = mval
else:
mindx = getmaskarray(self)[i:j]
dval = np.asarray(value)
valmask = getmask(value)
if valmask is nomask:
for n in names:
mval = mask_or(m[n][i:j], valmask)
d[n][i:j][~mval] = value
elif valmask.size > 1:
for n in names:
mval = mask_or(m[n][i:j], valmask)
d[n][i:j][~mval] = dval[~mval]
m[n][i:j] = mask_or(m[n][i:j], mval)
self._fieldmask = m
def _draw_pcolormesh(self, ax, z, x=None, y=None, subplot=1, **kwargs):
# NOTE(alexj)stripping out subplot because which subplot we're in is already
# described by ax, and it's not a kwarg to matplotlib's ax.plot. But I
# didn't want to strip it out of kwargs earlier because it should stay
# part of trace['config'].
args = [masked_invalid(arg) for arg in [x, y, z]
if arg is not None]
for arg in args:
if np.all(getmask(arg)):
# if any entire array is masked, don't draw at all
# there's nothing to draw, and anyway it throws a warning
return False
pc = ax.pcolormesh(*args, **kwargs)
if getattr(ax, 'qcodes_colorbar', None):
# update_normal doesn't seem to work...
ax.qcodes_colorbar.update_bruteforce(pc)
else:
# TODO: what if there are several colormeshes on this subplot,
# do they get the same colorscale?
# We should make sure they do, and have it include
# the full range of both.
ax.qcodes_colorbar = self.fig.colorbar(pc, ax=ax)
# ideally this should have been in _update_labels, but
# the colorbar doesn't necessarily exist there.
# I guess we could create the colorbar no matter what,
# and just give it a dummy mappable to start, so we could
# put this where it belongs.
ax.qcodes_colorbar.set_label(self.get_label(z))
return pc
def __init__(self, dim, obs, params=None, E=None):
self.dim = dim
self.all_obs = obs
if type(self.all_obs) != ma.MaskedArray:
self.all_obs = ma.masked_invalid(self.all_obs)
# identify the zeros, nonzeros and nas and store their positions in masks
self.zeros = (self.all_obs == 0)
self.nonzeros = ~self.zeros # this masks includes the nas
self.nas = ma.getmask(self.all_obs)
self.mask = self.nas # TODO mask to be used when calculating variance explained
self.sparsity = self.zeros.sum() / (self.zeros.sum() +
self.nonzeros.sum())
print('using zero inflated noise model with sparsity ', self.sparsity)
# initialise the jaakola node with nas and non zeros masked
obs_jaakola = obs.copy(
) # TODO if obs is already a masked array should we update mask ?
obs_jaakola[self.nonzeros] = np.nan
# instead:?
# self.notnas = np.logical_not(self.nas)
# obs_jaakola[self.nonzeros & self.notnas] = 1 # TOCHECK: for non-zero values put 1 for observed
self.jaakola_node = Bernoulli_PseudoY_Jaakkola(dim, obs_jaakola)
# Initialise a y node where the zeros and nas are masked
obs_normal = obs.copy()
obs_normal[self.zeros] = np.nan # nas are already masked so no need to
self.normal_node = Y_Node(dim, obs_normal)
self.nodes = [self.normal_node, self.jaakola_node]
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result.fill(0)
else:
if clip:
mask = ma.getmask(result)
result = ma.array(np.clip(result.filled(vmax), vmin, vmax),
mask=mask)
# in-place equivalent of above can be much faster
resdat = self._transform(result.data)
resdat -= self._lower
resdat /= (self._upper - self._lower)
if is_scalar:
result = result[0]
return result
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
val = ma.masked_less_equal(val, 0, copy=False)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin<=0:
raise ValueError("values must all be positive")
elif vmin==vmax:
result = 0.0 * val
else:
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = (ma.log(val)-np.log(vmin))/(np.log(vmax)-np.log(vmin))
if vtype == 'scalar':
result = result[0]
return result
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result.fill(0) # Or should it be all masked? Or 0.5?
else:
vmin = float(vmin)
vmax = float(vmax)
if clip:
mask = ma.getmask(result)
result = ma.array(np.clip(result.filled(vmax), vmin, vmax),
mask=mask)
# ma division is very slow; we can take a shortcut
resdat = result.data
resdat -= vmin
resdat /= (vmax - vmin)
result = np.ma.array(resdat, mask=result.mask, copy=False)
if is_scalar:
result = result[0]
return result
def __call__(self, value, clip=None, midpoint=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin==vmax:
return 0.0 * val
else:
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = (val-vmin) * (1.0/(vmax-vmin))
#result = (ma.arcsinh(val)-np.arcsinh(vmin))/(np.arcsinh(vmax)-np.arcsinh(vmin))
result = result**(1./self.nthroot)
if vtype == 'scalar':
result = result[0]
return result
def __call__(self, value, clip=None, midpoint=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
vmid = self.vmid if self.vmid is not None else (vmax + vmin) / 2.0
if midpoint is None:
midpoint = (vmid - vmin) / (vmax - vmin)
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
return 0.0 * val
else:
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = (val - vmin) * (1.0 / (vmax - vmin))
#result = (ma.arcsinh(val)-np.arcsinh(vmin))/(np.arcsinh(vmax)-np.arcsinh(vmin))
result = ma.sinh(result / midpoint) / ma.sinh(1. / midpoint)
if vtype == 'scalar':
result = result[0]
return result
def __call__(self, value, clip=None):
"""Map value to the interval [0, 1]. The clip argument is unused."""
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vcenter, vmax = self.vmin, self.vcenter, self.vmax
if vmin == vmax == vcenter:
result.fill(0)
elif not vmin <= vcenter <= vmax:
raise ValueError("minvalue must be less than or equal to "
"centervalue which must be less than or "
"equal to maxvalue")
else:
vmin = float(vmin)
vcenter = float(vcenter)
vmax = float(vmax)
# in degenerate cases, prefer the center value to the extremes
degen = (result == vcenter) if vcenter == vmax else None
x, y = [vmin, vcenter, vmax], [0, 0.5, 1]
result = ma.masked_array(np.interp(result, x, y),
mask=ma.getmask(result))
if degen is not None:
result[degen] = 0.5
if is_scalar:
result = np.atleast_1d(result)[0]
return(result)
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
cmin, cmax = self.cmin * vmin, self.cmax * vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result = 0.0 * val
else:
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = 0. * val + 0.5
result[val > cmax] = (ma.log10(val[val > cmax]) - ma.log10(cmax)) / (np.log10(vmax) - np.log10(cmax)) / 2. + 0.5
result[val < cmin] = -(ma.log10(-val[val < cmin]) - ma.log10(-cmin)) / (np.log10(-vmin) - np.log10(-cmin)) / 2. + 0.5
if vtype == 'scalar':
result = result[0]
return result
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result.fill(0)
else:
if clip:
mask = ma.getmask(result)
result = ma.array(np.clip(result.filled(vmax), vmin, vmax),
mask=mask)
# in-place equivalent of above can be much faster
resdat = self._transform(result.data)
resdat -= self._lower
resdat /= (self._upper - self._lower)
if is_scalar:
result = result[0]
return result
def __init__(self, centers, time, indices=None):
self.centers = centers
self.time = time
self.indices = indices
# figure out where the sun is
self.solar_coord = ac.get_sun(time)
# subset by the provided indices, if specified
if indices is None:
lats = centers.lat
lons = centers.lon
else:
lats = centers.lat[indices]
lons = centers.lon[indices]
# calculate the solar zenith angle at each pixel center.
# neglect refraction and height above geoid.
pixels = ac.EarthLocation(lons, lats, ellipsoid='GRS80')
solar_vecs = self.solar_coord.transform_to(
ac.AltAz(obstime=time, location=pixels))
# mask out the non earth pixels, unless the user specified which pixels to calculate
if indices is None:
mask = ma.getmask(centers.lon)
else:
mask = None
self.solar_zenith = ma.array(solar_vecs.zen.to_value(u.deg), mask=mask)
self.solar_az = ma.array(solar_vecs.az.to_value(u.deg), mask=mask)
def _process_args(self, *args, **kwargs):
"""
Process args and kwargs.
"""
if not isinstance(args[0], QuadContourSet):
self._corner_mask = kwargs.pop('corner_mask', None)
if self._corner_mask is None:
self._corner_mask = mpl.rcParams['contour.corner_mask']
x, y, z = self._contour_args(args, kwargs)
_mask = ma.getmask(z)
if _mask is ma.nomask or not _mask.any():
_mask = None
if self._corner_mask == 'legacy':
contour_generator = cntr.Cntr(x, y, z.filled(), _mask)
return_args = super(
LegacyContourSet, self)._process_args(*args, **kwargs)
if self._corner_mask == 'legacy':
self.Cntr = contour_generator
self._contour_generator = contour_generator
return return_args
def __init__(self, dim, obs, params=None, E=None):
"""
PARAMETERS
----------
dim (2d tuple): dimensionality of each view
obs (ndarray): observed data
params:
E (ndarray): initial expected value of pseudodata
"""
Unobserved_Variational_Node.__init__(self, dim)
# Initialise observed data
assert obs.shape == dim, "Problems with the dimensionalities"
self.obs = obs
# Initialise parameters
if params is not None:
assert type(self.params) == dict
self.params = params
else:
self.params = {}
# Create a boolean mask of the data to handle missing values
self.mask = ma.getmask(ma.masked_invalid(self.obs))
self.obs[np.isnan(self.obs)] = 0.
# Initialise expectation
if E is not None:
assert E.shape == dim, "Problems with the dimensionalities"
E[self.mask] = 0.
self.E = E
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result.fill(0) # Or should it be all masked? Or 0.5?
else:
vmin = float(vmin)
vmax = float(vmax)
if clip:
mask = ma.getmask(result)
result = ma.array(np.clip(result.filled(vmax), vmin, vmax),
mask=mask)
# ma division is very slow; we can take a shortcut
resdat = result.data
resdat -= vmin
resdat /= (vmax - vmin)
result = np.ma.array(resdat, mask=result.mask, copy=False)
if is_scalar:
result = result[0]
return result
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
result, is_scalar = self.process_value(value)
self.autoscale_None(result)
gamma = self.gamma
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin == vmax:
result.fill(0)
else:
if clip:
mask = ma.getmask(result)
val = ma.array(np.clip(result.filled(vmax), vmin, vmax),
mask=mask)
resdat = result.data
resdat -= vmin
np.power(resdat, gamma, resdat)
resdat /= (vmax - vmin) ** gamma
result = np.ma.array(resdat, mask=result.mask, copy=False)
result[(value < 0)&~result.mask] = 0
if is_scalar:
result = result[0]
return result
def __setslice__(self, i, j, value):
"Sets the slice described by [i,j] to `value`."
_localdict = self.__dict__
d = self._data
m = _localdict['_fieldmask']
names = self.dtype.names
if value is masked:
for n in names:
m[i:j][n] = True
elif not self._hardmask:
fval = filled(value)
mval = getmaskarray(value)
for n in names:
d[n][i:j] = fval
m[n][i:j] = mval
else:
mindx = getmaskarray(self)[i:j]
dval = np.asarray(value)
valmask = getmask(value)
if valmask is nomask:
for n in names:
mval = mask_or(m[n][i:j], valmask)
d[n][i:j][~mval] = value
elif valmask.size > 1:
for n in names:
mval = mask_or(m[n][i:j], valmask)
d[n][i:j][~mval] = dval[~mval]
m[n][i:j] = mask_or(m[n][i:j], mval)
self._fieldmask = m
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin==vmax:
result = 0.0 * val
else:
vmin = float(vmin)
vmax = float(vmax)
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
result = (val-vmin) / (vmax-vmin)
if vtype == 'scalar':
result = result[0]
return result
def __call__(self, X, alpha=1.0, bytes=False):
"""
*X* is either a scalar or an array (of any dimension).
If scalar, a tuple of rgba values is returned, otherwise
an array with the new shape = oldshape+(4,). If the X-values
are integers, then they are used as indices into the array.
If they are floating point, then they must be in the
interval (0.0, 1.0).
Alpha must be a scalar.
If bytes is False, the rgba values will be floats on a
0-1 scale; if True, they will be uint8, 0-255.
"""
if not self._isinit: self._init()
alpha = min(alpha, 1.0) # alpha must be between 0 and 1
alpha = max(alpha, 0.0)
self._lut[:-1, -1] = alpha # Don't assign global alpha to i_bad;
# it would defeat the purpose of the
# default behavior, which is to not
# show anything where data are missing.
mask_bad = None
if not cbook.iterable(X):
vtype = 'scalar'
xa = np.array([X])
else:
vtype = 'array'
# force a copy here -- the ma.array and filled functions
# do force a cop of the data by default - JDH
xma = ma.array(X, copy=True)
xa = xma.filled(0)
mask_bad = ma.getmask(xma)
if xa.dtype.char in np.typecodes['Float']:
np.putmask(xa, xa == 1.0,
0.9999999) #Treat 1.0 as slightly less than 1.
# The following clip is fast, and prevents possible
# conversion of large positive values to negative integers.
if NP_CLIP_OUT:
np.clip(xa * self.N, -1, self.N, out=xa)
else:
xa = np.clip(xa * self.N, -1, self.N)
xa = xa.astype(int)
# Set the over-range indices before the under-range;
# otherwise the under-range values get converted to over-range.
np.putmask(xa, xa > self.N - 1, self._i_over)
np.putmask(xa, xa < 0, self._i_under)
if mask_bad is not None and mask_bad.shape == xa.shape:
np.putmask(xa, mask_bad, self._i_bad)
if bytes:
lut = (self._lut * 255).astype(np.uint8)
else:
lut = self._lut
rgba = np.empty(shape=xa.shape + (4, ), dtype=lut.dtype)
lut.take(xa, axis=0, mode='clip', out=rgba)
# twice as fast as lut[xa];
# using the clip or wrap mode and providing an
# output array speeds it up a little more.
if vtype == 'scalar':
rgba = tuple(rgba[0, :])
return rgba
def __call__(self, X, alpha=1.0, bytes=False):
"""
*X* is either a scalar or an array (of any dimension).
If scalar, a tuple of rgba values is returned, otherwise
an array with the new shape = oldshape+(4,). If the X-values
are integers, then they are used as indices into the array.
If they are floating point, then they must be in the
interval (0.0, 1.0).
Alpha must be a scalar.
If bytes is False, the rgba values will be floats on a
0-1 scale; if True, they will be uint8, 0-255.
"""
if not self._isinit: self._init()
alpha = min(alpha, 1.0) # alpha must be between 0 and 1
alpha = max(alpha, 0.0)
self._lut[:-1,-1] = alpha # Don't assign global alpha to i_bad;
# it would defeat the purpose of the
# default behavior, which is to not
# show anything where data are missing.
mask_bad = None
if not cbook.iterable(X):
vtype = 'scalar'
xa = np.array([X])
else:
vtype = 'array'
# force a copy here -- the ma.array and filled functions
# do force a cop of the data by default - JDH
xma = ma.array(X, copy=True)
xa = xma.filled(0)
mask_bad = ma.getmask(xma)
if xa.dtype.char in np.typecodes['Float']:
np.putmask(xa, xa==1.0, 0.9999999) #Treat 1.0 as slightly less than 1.
# The following clip is fast, and prevents possible
# conversion of large positive values to negative integers.
if NP_CLIP_OUT:
np.clip(xa * self.N, -1, self.N, out=xa)
else:
xa = np.clip(xa * self.N, -1, self.N)
xa = xa.astype(int)
# Set the over-range indices before the under-range;
# otherwise the under-range values get converted to over-range.
np.putmask(xa, xa>self.N-1, self._i_over)
np.putmask(xa, xa<0, self._i_under)
if mask_bad is not None and mask_bad.shape == xa.shape:
np.putmask(xa, mask_bad, self._i_bad)
if bytes:
lut = (self._lut * 255).astype(np.uint8)
else:
lut = self._lut
rgba = np.empty(shape=xa.shape+(4,), dtype=lut.dtype)
lut.take(xa, axis=0, mode='clip', out=rgba)
# twice as fast as lut[xa];
# using the clip or wrap mode and providing an
# output array speeds it up a little more.
if vtype == 'scalar':
rgba = tuple(rgba[0,:])
return rgba
def plot_scatterplot(prefix, feature='asymmetry1', vmin=0, vmax=1, resolution=512, rows=4, cols=4,
dotsize=10, figsize=(12, 10), upload=True, remote_folder = "01_18_Experiment",
bucket='ccurtis.data'):
"""
Plot scatterplot of trajectories in video with colors corresponding to features.
Parameters
----------
prefix: string
Prefix of file name to be plotted e.g. features_P1.csv prefix is P1.
feature: string
Feature to be plotted. See features_analysis.py
vmin: float64
Lower intensity bound for heatmap.
vmax: float64
Upper intensity bound for heatmap.
resolution: int
Resolution of base image. Only needed to calculate bounds of image.
rows: int
Rows of base images used to build tiled image.
cols: int
Columns of base images used to build tiled images.
upload: boolean
True if you want to upload to s3.
"""
# Inputs
# ----------
merged_ft = pd.read_csv('features_{}.csv'.format(prefix))
string = feature
leveler = merged_ft[string]
t_min = vmin
t_max = vmax
ires = resolution
norm = mpl.colors.Normalize(t_min, t_max, clip=True)
mapper = cm.ScalarMappable(norm=norm, cmap=cm.viridis)
zs = ma.masked_invalid(merged_ft[string])
zs = ma.masked_where(zs <= t_min, zs)
zs = ma.masked_where(zs >= t_max, zs)
to_mask = ma.getmask(zs)
zs = ma.compressed(zs)
xs = ma.compressed(ma.masked_where(to_mask, merged_ft['X'].astype(int)))
ys = ma.compressed(ma.masked_where(to_mask, merged_ft['Y'].astype(int)))
fig = plt.figure(figsize=figsize)
plt.scatter(xs, ys, c=zs, s=dotsize)
mapper.set_array(10)
plt.colorbar(mapper)
plt.xlim(0, ires*cols)
plt.ylim(0, ires*rows)
plt.axis('off')
print('Plotted {} scatterplot successfully.'.format(prefix))
outfile = 'scatter_{}_{}.png'.format(feature, prefix)
fig.savefig(outfile, bbox_inches='tight')
if upload == True:
aws.upload_s3(outfile, remote_folder+'/'+outfile, bucket_name=bucket)
def masked_less(self, val):
new = Map.empty()
new.data[0] = ma.masked_less(self.data[0], val)
new.data[1] = ma.masked_where(ma.getmask(new.data[0]), self.data[1])
return new
def sum_by(regions, data): data.mask = np.logical_or(data.mask, regions.mask) regions.mask = ma.getmask(data) regions_idx = regions.compressed().astype(int) summ = np.bincount(regions_idx, data.compressed()) ncells = np.bincount(regions_idx) idx = np.where(ncells > 0) return summ[idx]
def __init__(self, vertices, codes=None):
"""
Create a new path with the given vertices and codes.
vertices is an Nx2 numpy float array, masked array or Python
sequence.
codes is an N-length numpy array or Python sequence of type
Path.code_type.
See the docstring of Path for a description of the various
codes.
These two arrays must have the same length in the first
dimension.
If codes is None, vertices will be treated as a series of line
segments. If vertices contains masked values, the resulting
path will be compressed, with MOVETO codes inserted in the
correct places to jump over the masked regions.
"""
if ma.isMaskedArray(vertices):
is_mask = True
mask = ma.getmask(vertices)
else:
is_mask = False
vertices = np.asarray(vertices, np.float_)
mask = ma.nomask
if codes is not None:
codes = np.asarray(codes, self.code_type)
assert codes.ndim == 1
assert len(codes) == len(vertices)
# The path being passed in may have masked values. However,
# the backends (and any affine transformations in matplotlib
# itself), are not expected to deal with masked arrays, so we
# must remove them from the array (using compressed), and add
# MOVETO commands to the codes array accordingly.
if is_mask:
if mask is not ma.nomask:
mask1d = np.logical_or.reduce(mask, axis=1)
gmask1d = np.invert(mask1d)
if codes is None:
codes = np.empty((len(vertices)), self.code_type)
codes.fill(self.LINETO)
codes[0] = self.MOVETO
vertices = vertices[gmask1d].filled() # ndarray
codes[np.roll(mask1d, 1)] = self.MOVETO
codes = codes[gmask1d] # np.compress is much slower
else:
vertices = np.asarray(vertices, np.float_)
assert vertices.ndim == 2
assert vertices.shape[1] == 2
self.codes = codes
self.vertices = vertices
def _unscented_correct(cross_sigma, mu_pred, sigma2_pred, obs_mu_pred, obs_sigma2_pred, z):
"""Correct predicted state estimates with an observation
Parameters
----------
cross_sigma : [n_dim_state, n_dim_obs] array
cross-covariance between the state at time t given all observations
from timesteps [0, t-1] and the observation at time t
mu_pred : [n_dim_state] array
mean of state at time t given observations from timesteps [0, t-1]
sigma2_pred : [n_dim_state, n_dim_state] array
square root of covariance of state at time t given observations from
timesteps [0, t-1]
obs_mu_pred : [n_dim_obs] array
mean of observation at time t given observations from times [0, t-1]
obs_sigma2_pred : [n_dim_obs] array
square root of covariance of observation at time t given observations
from times [0, t-1]
z : [n_dim_obs] array
observation at time t
Returns
-------
mu_filt : [n_dim_state] array
mean of state at time t given observations from time steps [0, t]
sigma2_filt : [n_dim_state, n_dim_state] array
square root of covariance of state at time t given observations from
time steps [0, t]
"""
n_dim_state = len(mu_pred)
n_dim_obs = len(obs_mu_pred)
if not np.any(ma.getmask(z)):
##############################################
# Same as this, but more stable (supposedly) #
##############################################
# K = cross_sigma.dot(
# linalg.pinv(
# obs_sigma2_pred.T.dot(obs_sigma2_pred)
# )
# )
##############################################
# equivalent to this MATLAB code
# K = (cross_sigma / obs_sigma2_pred.T) / obs_sigma2_pred
K = linalg.lstsq(obs_sigma2_pred, cross_sigma.T)[0]
K = linalg.lstsq(obs_sigma2_pred.T, K)[0]
K = K.T
# correct mu, sigma
mu_filt = mu_pred + K.dot(z - obs_mu_pred)
U = K.dot(obs_sigma2_pred)
sigma2_filt = cholupdate(sigma2_pred, U.T, -1.0)
else:
# no corrections to be made
mu_filt = mu_pred
sigma2_filt = sigma2_pred
return (mu_filt, sigma2_filt)
def approx (a, b, fill_value=True, rtol=1.e-5, atol=1.e-8):
"""Returns true if all components of a and b are equal subject to given tolerances.
If fill_value is True, masked values considered equal. Otherwise, masked values
are considered unequal.
The relative error rtol should be positive and << 1.0
The absolute error atol comes into play for those elements of b that are very
small or zero; it says how small a must be also.
"""
m = mask_or(getmask(a), getmask(b))
d1 = filled(a)
d2 = filled(b)
if d1.dtype.char == "O" or d2.dtype.char == "O":
return np.equal(d1,d2).ravel()
x = filled(masked_array(d1, copy=False, mask=m), fill_value).astype(float_)
y = filled(masked_array(d2, copy=False, mask=m), 1).astype(float_)
d = np.less_equal(umath.absolute(x-y), atol + rtol * umath.absolute(y))
return d.ravel()
def ask_chunks(self, count, price, inplace=False):
assert 0 < count <= self.max_ask
leftover = self.leftover - count * self.min_bet
chunks = self.chunks if inplace else self.chunks.copy()
for _ in range(count):
idx = np.where(ma.getmask(chunks))[0][0]
chunks[idx] = self.chunk_value(price)
chunks.mask[idx] = False
sort_chunks(chunks)
return self._get_ret_val(leftover, chunks, inplace)
def forward_fill(marr, maxgap=None):
"""
Forward fills masked values in a 1-d array when there are less ``maxgap``
consecutive masked values.
Parameters
----------
marr : MaskedArray
Series to fill
maxgap : {int}, optional
Maximum gap between consecutive masked values.
If ``maxgap`` is not specified, all masked values are forward-filled.
Examples
--------
>>> x = ma.arange(20)
>>> x[(x%5)!=0] = ma.masked
>>> print x
[0 -- -- -- -- 5 -- -- -- -- 10 -- -- -- -- 15 -- -- -- --]
>>> print forward_fill(x)
[0 0 0 0 0 5 5 5 5 5 10 10 10 10 10 15 15 15 15 15]
"""
# !!!: We should probably port that to C.
# Initialization ..................
if np.ndim(marr) > 1:
raise ValueError,"The input array should be 1D only!"
a = ma.array(marr, copy=True)
amask = getmask(a)
if amask is nomask or a.size == 0:
return a
#
adata = getdata(a)
# Get the indices of the masked values (except a[0])
idxtofill = amask[1:].nonzero()[0] + 1
currGap = 0
if maxgap is not None:
previdx = -1
for i in idxtofill:
if i != previdx + 1:
currGap = 0
currGap += 1
if currGap <= maxgap and not amask[i-1]:
adata[i] = adata[i-1]
amask[i] = False
previdx = i
else:
amask[i-maxgap:i] = True
else:
for i in idxtofill:
if not amask[i-1]:
adata[i] = adata[i-1]
amask[i] = False
return a
def _numpy_interpolation(self, point_num, eval_points):
"""
Parameters
----------
point_num: int
Index of class position in values list
eval_points: ndarray
Inputs used to evaluate class member function
Returns
-------
ndarray: output from member function
"""
is_masked = ma.is_masked(eval_points)
shape = point_num.shape
ev_shape = eval_points.shape
vals = self.values[point_num.ravel()]
eval_points = np.repeat(eval_points, shape[1], axis=0)
it = np.arange(eval_points.shape[0])
it = np.repeat(it, eval_points.shape[1], axis=0)
eval_points = eval_points.reshape(
eval_points.shape[0] * eval_points.shape[1],
eval_points.shape[-1]
)
scaled_points = eval_points.T
if is_masked:
mask = np.invert(ma.getmask(scaled_points[0]))
else:
mask = np.ones_like(scaled_points[0], dtype=bool)
it = ma.masked_array(it, mask)
scaled_points[0] = (
(scaled_points[0] - (self._bounds[0][0])) /
(self._bounds[0][1] - self._bounds[0][0])
) * (vals.shape[-2] - 1)
scaled_points[1] += (
(scaled_points[1] - (self._bounds[1][0])) /
(self._bounds[1][1] - self._bounds[1][0])
) * (vals.shape[-1] - 1)
scaled_points = np.vstack((it, scaled_points))
output = np.zeros(scaled_points.T.shape[:-1])
output[mask] = map_coordinates(vals, scaled_points.T[mask].T, order=1)
new_shape = (*shape, ev_shape[-2])
output = output.reshape(new_shape)
return ma.masked_array(output, mask=mask)
def __call__(self, value, clip=None):
if clip is None:
clip = self.clip
if cbook.iterable(value):
vtype = 'array'
val = ma.asarray(value).astype(np.float)
else:
vtype = 'scalar'
val = ma.array([value]).astype(np.float)
self.autoscale_None(val)
vmin, vmax = self.vmin, self.vmax
vin, cin = self.vin, self.cin
if vmin > vmax:
raise ValueError("minvalue must be less than or equal to maxvalue")
elif vmin > 0:
raise ValueError("minvalue must be less than 0")
elif vmax < 0:
raise ValueError("maxvalue must be greater than 0")
elif vmin==vmax:
result = 0.0 * val
else:
if clip:
mask = ma.getmask(val)
val = ma.array(np.clip(val.filled(vmax), vmin, vmax),
mask=mask)
ipos = (val > vin)
ineg = (val < -vin)
izero = ~(ipos | ineg)
result = ma.empty_like(val)
result[izero] = 0.5 + cin * val[izero] / vin
result[ipos] = 0.5 + cin + (0.5 - cin) * \
(ma.log(val[ipos]) - np.log(vin)) / (np.log(vmax) - np.log(vin))
result[ineg] = 0.5 - cin - (0.5 - cin) * \
(ma.log(-val[ineg]) - np.log(vin)) / (np.log(-vmin) - np.log(vin))
result.mask = ma.getmask(val)
if vtype == 'scalar':
result = result[0]
return result
def Fill2ThetaMap(data,TA,image):
import numpy.ma as ma
Zmin = data['Zmin']
Zmax = data['Zmax']
tax,tay = TA # 2-theta & yaxis
taz = ma.masked_outside(image.flatten()-Zmin,0,Zmax-Zmin)
tam = ma.getmask(taz)
tax = ma.compressed(ma.array(tax.flatten(),mask=tam))
tay = ma.compressed(ma.array(tay.flatten(),mask=tam))
taz = ma.compressed(ma.array(taz.flatten(),mask=tam))
del(tam)
return tax,tay,taz
def create_value_coverage(input_raster, output_raster, compress='DEFLATE',
verbose=False, logger=None):
""" Create a binary raster based on informative cell values.
All values that have information in input_raster are given value 1 in the
output_raster.
:param input_raster: String path to input raster.
:param output_raster: String path to output raster.
:param compress: String compression level used for the output raster.
:param verbose: Boolean indicating how much information is printed out.
:param logger: logger object to be used.
:return Boolean success.
"""
# 1. Setup --------------------------------------------------------------
all_start = timer()
if not logger:
logging.basicConfig()
llogger = logging.getLogger('create_value_coverage')
llogger.setLevel(logging.DEBUG if verbose else logging.INFO)
else:
llogger = logger
# 2. Read and process raster ---------------------------------------------
# Read raster bands directly to Numpy arrays.
with rasterio.open(input_raster) as raster:
llogger.info("Reading and processing raster {}".format(input_raster))
input_nodata = raster.nodata
# Read in the data
src = raster.read(1, masked=True)
mask = ma.getmask(src)
llogger.debug("Number of informative cells: {}".format(np.sum(~mask)))
# Binarize input where there are values
np.place(src, mask == False, 1)
profile = raster.profile
profile.update(dtype=rasterio.uint8, count=1, compress=compress,
nodata=255)
# Since we're saving a byte, replace old NoData value with 255
np.place(src, src == input_nodata, 255)
with rasterio.open(output_raster, 'w', **profile) as dst:
llogger.info("Writing output raster {}".format(output_raster))
dst.write_mask(mask)
dst.write(src.astype(rasterio.uint8), 1)
all_end = timer()
all_elapsed = round(all_end - all_start, 2)
llogger.info(" [TIME] Binarizing took {} sec".format(all_elapsed))
def azimuthalAverage(image, center=None, maskval=0):
"""
calculate the azimuthally averaged radial profile.
image - 2D image
center - [x,y] pixel coordinates used as the center. the default is
None which then uses the center of the image (including
fractional pixels).
maskval - threshold value for including data in the profile
"""
# calculate the indices from the image
y, x = np.indices(image.shape)
# default to image center if no center given
if not center:
center = np.array([(x.max() - x.min()) / 2.0,
(x.max() - x.min()) / 2.0])
r = np.hypot(x - center[0], y - center[1])
# get sorted radii and sort image accordingly
ind = np.argsort(r.flat)
i_sorted = image.flat[ind]
# for FP data we need to at least mask out data at
# 0 or less so the gaps get ignored.
# also want to mask out area outside of aperture
# so use given maskval to do that.
i_ma = ma.masked_less_equal(i_sorted, maskval)
mask = ma.getmask(i_ma)
# remove masked data points from further analysis
r_sorted = ma.compressed(ma.array(r.flat[ind], mask=mask))
i_mask = ma.compressed(i_ma)
# get the integer part of the radii (bin size = 1)
r_int = r_sorted.astype(int)
# find all pixels that fall within each radial bin.
deltar = r_int[1:] - r_int[:-1] # assumes all radii represented
rind = np.where(deltar)[0] # location of changed radius
nr_tot = rind[1:] - rind[:-1] # total number of points in radius bin
# cumulative sum to figure out sums for each radius bin
csim = ma.cumsum(i_mask, dtype=float)
tbin = csim[rind[1:]] - csim[rind[:-1]]
# calculate and return profile of mean within each bin
radial_prof = tbin / nr_tot
return radial_prof
def interp_data(var, xpts, ypts, xpts2m, ypts2m, int_meth='cubic'):
#interpoalte data onto a regular 2d grid. Used by interp_uv
data = ma.getdata(var)
mask = ma.getmask(var)
index = np.where(mask==False)
data_masked = data[index]
xpoints = xpts[index]
ypoints = ypts[index]
points = (xpoints, ypoints)
var_int = griddata(points,data_masked,(xpts2m,ypts2m),method=int_meth)
var_int = ma.masked_array(var_int,np.isnan(var_int))
return var_int
def reduce (self, target, axis=0):
"""Reduce target along the given axis."""
axes, attributes, id, grid = _extractMetadata(target, omit=axis)
a = _makeMaskedArg(target)
m = getmask(a)
if m is nomask:
t = filled(a)
return masked_array (numpy.maximum.reduce (t, axis))
else:
t = numpy.maximum.reduce(filled(a, numpy.ma.maximum_fill_value(a)), axis)
m = numpy.logical_and.reduce(m, axis)
return TransientVariable(t, mask=m, fill_value=fill_value(a),
axes = axes, grid=grid, id=id)
def assert_array_compare(comparison, x, y, err_msg='', verbose=True, header='',
fill_value=True):
"""Asserts that a comparison relation between two masked arrays is satisfied
elementwise."""
# Fill the data first
# xf = filled(x)
# yf = filled(y)
# Allocate a common mask and refill
m = np.ma.mask_or(getmask(x), getmask(y))
x = masked_array(x, copy=False, mask=m, keep_mask=False, subok=False)
y = masked_array(y, copy=False, mask=m, keep_mask=False, subok=False)
if ((x is masked) and not (y is masked)) or \
((y is masked) and not (x is masked)):
msg = build_err_msg([x, y], err_msg=err_msg, verbose=verbose,
header=header, names=('x', 'y'))
raise ValueError(msg)
# OK, now run the basic tests on filled versions
return utils.assert_array_compare(comparison,
x.filled(fill_value),
y.filled(fill_value),
err_msg=err_msg,
verbose=verbose, header=header)
def compute(self, dataset_pool):
ds = self.get_dataset()
age = ds.get_attribute(self.building_age)
result = ma.filled(age, -1)
mask = ma.getmask(age)
if mask is not ma.nomask and mask.any():
# If there are bad ages then mask won't be ma.nomask and will have some non-False values. In this case compute
# the average age for all buildings, then find the ages of buildings within walking distance, and use those
# to fill in the bad values.
whole_area_average_age = int(age.sum()/float(ds.size() - age.mask.sum()))
age_wwd_values = ma.filled(ds.get_attribute(self.age_wwd), whole_area_average_age)
result[mask] = age_wwd_values[mask]
return result
