def concatenate(self,value,axis=0):
""" Concatentate UncertContainer value to self.
Assumes that if dimensions of self and value do not match, to
add a np.newaxis along axis of value
"""
if isinstance(value,UncertContainer):
if value.vals.ndim == self.vals.ndim:
vals = value.vals
dmin = value.dmin
dmax = value.dmax
wt = value.wt
uncert = value.uncert
mask = value.mask
elif (value.vals.ndim + 1) == self.vals.ndim:
vals = ma.expand_dims(value.vals,axis)
dmin = ma.expand_dims(value.dmin,axis)
dmax = ma.expand_dims(value.dmax,axis)
wt = ma.expand_dims(value.wt,axis)
uncert = ma.expand_dims(value.uncert,axis)
mask = np.expand_dims(value.mask,axis)
else:
raise ValueError('Could not propery match dimensionality')
self.vals = ma.concatenate((self.vals,vals),axis=axis)
self.dmin = ma.concatenate((self.dmin,dmin),axis=axis)
self.dmax = ma.concatenate((self.dmax,dmax),axis=axis)
self.wt = ma.concatenate((self.wt,wt),axis=axis)
self.uncert = ma.concatenate((self.uncert,uncert),axis=axis)
self.mask = np.concatenate((self.mask,mask),axis=axis)
else:
raise ValueError('Can only concatenate with an UncertContainer object')
def period_standardize(self, axis=None, dtype=None, ddof=1, period=None):
"""
Standardizes data by substracting the average over a reference period,
and by dividing by the standard deviation over the same period.
Parameters
----------
%(axis)s
%(dtype)s
%(ddof)s
%(period)s
Warnings
--------
The default ``ddof`` is 1:
by default, the method returns the unbiased estimate of the standard deviation.
"""
if period is None:
period = self.refperiod
elif not isinstance(period, (tuple, list, ndarray)):
msg = "Period should be a tuple (starting date, ending date)!"
raise ValueError, msg
refdata = mask_outside_period(self, period[0], period[1],
include_edges=False)
refavg = refdata.mean(axis=axis, dtype=dtype)
refstd = refdata.std(axis=axis, dtype=dtype, ddof=ddof)
if not axis:
result = (self - refavg) * 1. / refstd
else:
result = (self - ma.expand_dims(refavg)).astype(float)
result /= ma.expand_dims(refstd)
return result
def weighted_average(self, axis=0, expaxis=None):
""" Calculate weighted average of data along axis
after optionally inserting a new dimension into the
shape array at position expaxis
"""
if expaxis is not None:
vals = ma.expand_dims(self.vals, expaxis)
dmin = ma.expand_dims(self.dmin, expaxis)
dmax = ma.expand_dims(self.dmax, expaxis)
wt = ma.expand_dims(self.wt, expaxis)
else:
vals = self.vals
wt = self.wt
dmin = self.dmin
dmax = self.dmax
# Get average value
avg, norm = ma.average(vals, axis=axis, weights=wt, returned=True)
avg_ex = ma.expand_dims(avg, 0)
# Calculate weighted uncertainty
wtmax = ma.max(wt, axis=axis)
neff = norm / wtmax # Effective number of samples based on uncertainties
# Seeking max deviation from the average; if above avg use max, if below use min
term = np.empty_like(vals)
indices = np.where(vals > avg_ex)
i0 = indices[0]
irest = indices[1:]
ii = tuple(x for x in itertools.chain([i0], irest))
jj = tuple(x for x in itertools.chain([np.zeros_like(i0)], irest))
term[ii] = (dmax[ii] - avg_ex[jj])**2
indices = np.where(vals <= avg_ex)
i0 = indices[0]
irest = indices[1:]
ii = tuple(x for x in itertools.chain([i0], irest))
jj = tuple(x for x in itertools.chain([np.zeros_like(i0)], irest))
term[ii] = (avg_ex[jj] - dmin[ii])**2
dsum = ma.sum(term * wt,
axis=0) # Sum for weighted average of deviations
dev = 0.5 * np.sqrt(dsum / (norm * neff))
if isinstance(avg, (float, np.float)):
avg = avg_ex
tmp_min = avg - dev
ii = np.where(tmp_min < 0)
tmp_min[ii] = TOL * avg[ii]
return UncertContainer(avg, tmp_min, avg + dev)
def weighted_average(self,axis=0,expaxis=None):
""" Calculate weighted average of data along axis
after optionally inserting a new dimension into the
shape array at position expaxis
"""
if expaxis is not None:
vals = ma.expand_dims(self.vals,expaxis)
dmin = ma.expand_dims(self.dmin,expaxis)
dmax = ma.expand_dims(self.dmax,expaxis)
wt = ma.expand_dims(self.wt,expaxis)
else:
vals = self.vals
wt = self.wt
dmin = self.dmin
dmax = self.dmax
# Get average value
avg,norm = ma.average(vals,axis=axis,weights=wt,returned=True)
avg_ex = ma.expand_dims(avg,0)
# Calculate weighted uncertainty
wtmax = ma.max(wt,axis=axis)
neff = norm/wtmax # Effective number of samples based on uncertainties
# Seeking max deviation from the average; if above avg use max, if below use min
term = np.empty_like(vals)
indices = np.where(vals > avg_ex)
i0 = indices[0]
irest = indices[1:]
ii = tuple(x for x in itertools.chain([i0],irest))
jj = tuple(x for x in itertools.chain([np.zeros_like(i0)],irest))
term[ii] = (dmax[ii] - avg_ex[jj])**2
indices = np.where(vals <= avg_ex)
i0 = indices[0]
irest = indices[1:]
ii = tuple(x for x in itertools.chain([i0],irest))
jj = tuple(x for x in itertools.chain([np.zeros_like(i0)],irest))
term[ii] = (avg_ex[jj] - dmin[ii])**2
dsum = ma.sum(term*wt,axis=0) # Sum for weighted average of deviations
dev = 0.5*np.sqrt(dsum/(norm*neff))
if isinstance(avg,(float,np.float)):
avg = avg_ex
tmp_min = avg - dev
ii = np.where(tmp_min < 0)
tmp_min[ii] = TOL*avg[ii]
return UncertContainer(avg,tmp_min,avg+dev)
def execute(mp):
"""Read, stretch and return tile."""
# read parameters
resampling = mp.params["resampling"]
scale_method = mp.params.get("scale_method", None)
scales_minmax = mp.params["scales_minmax"]
with mp.open("raster", resampling=resampling) as raster_file:
# exit if input tile is empty
if raster_file.is_empty():
return "empty"
resampled = ()
mask = ()
# actually read data and iterate through bands
raster_data = raster_file.read()
if raster_data.ndim == 2:
raster_data = ma.expand_dims(raster_data, axis=0)
if not scale_method:
scales_minmax = [(i, i) for i in range(len(raster_data))]
for band, scale_minmax in zip(raster_data, scales_minmax):
if scale_method in ["dtype_scale", "minmax_scale"]:
scale_min, scale_max = scale_minmax
resampled += (_stretch_array(band, scale_min, scale_max), )
elif scale_method == "crop":
scale_min, scale_max = scale_minmax
band[band > scale_max] = scale_max
band[band <= scale_min] = scale_min
resampled += (band, )
else:
resampled += (band, )
mask += (band.mask, )
return ma.masked_array(np.stack(resampled), np.stack(mask))
def execute(
mp,
resampling="nearest",
scale_method=None,
scales_minmax=None,
**kwargs
):
"""Read, stretch and return tile."""
with mp.open("raster", resampling=resampling) as raster_file:
# exit if input tile is empty
if raster_file.is_empty():
return "empty"
# actually read data and iterate through bands
scaled = ()
mask = ()
raster_data = raster_file.read()
if raster_data.ndim == 2:
raster_data = ma.expand_dims(raster_data, axis=0)
if not scale_method:
scales_minmax = [(i, i) for i in range(len(raster_data))]
for band, (scale_min, scale_max) in zip(raster_data, scales_minmax):
if scale_method in ["dtype_scale", "minmax_scale"]:
scaled += (_stretch_array(band, scale_min, scale_max), )
elif scale_method == "crop":
scaled += (np.clip(band, scale_min, scale_max), )
else:
scaled += (band, )
mask += (band.mask, )
return ma.masked_array(np.stack(scaled), np.stack(mask))
def ma_mad(x, axis=None):
"""Median absolute deviation"""
median_x = ma.median(x, axis=axis)
if axis is not None:
median_x = ma.expand_dims(median_x, axis=axis)
return ma.median(ma.abs(x - median_x), axis=axis)
def resample_from_array(
in_raster=None, in_affine=None, out_tile=None, resampling="nearest",
nodataval=0
):
"""
Extract and resample from array to target tile.
Parameters
----------
in_raster : array
in_affine : ``Affine``
out_tile : ``BufferedTile``
resampling : string
one of rasterio's resampling methods (default: nearest)
nodataval : integer or float
raster nodata value (default: 0)
Returns
-------
resampled array : array
"""
if isinstance(in_raster, ma.MaskedArray):
pass
if isinstance(in_raster, np.ndarray):
in_raster = ma.MaskedArray(in_raster, mask=in_raster == nodataval)
elif isinstance(in_raster, ReferencedRaster):
in_affine = in_raster.affine
in_raster = in_raster.data
elif isinstance(in_raster, tuple):
in_raster = ma.MaskedArray(
data=np.stack(in_raster),
mask=np.stack([
band.mask
if isinstance(band, ma.masked_array)
else np.where(band == nodataval, True, False)
for band in in_raster
]),
fill_value=nodataval
)
else:
raise TypeError("wrong input data type: %s" % type(in_raster))
if in_raster.ndim == 2:
in_raster = ma.expand_dims(in_raster, axis=0)
elif in_raster.ndim == 3:
pass
else:
raise TypeError("input array must have 2 or 3 dimensions")
if in_raster.fill_value != nodataval:
ma.set_fill_value(in_raster, nodataval)
out_shape = (in_raster.shape[0], ) + out_tile.shape
dst_data = np.empty(out_shape, in_raster.dtype)
in_raster = ma.masked_array(
data=in_raster.filled(), mask=in_raster.mask, fill_value=nodataval)
reproject(
in_raster, dst_data, src_transform=in_affine, src_crs=out_tile.crs,
dst_transform=out_tile.affine, dst_crs=out_tile.crs,
resampling=Resampling[resampling])
return ma.MaskedArray(dst_data, mask=dst_data == nodataval)
def period_standardize(self, axis=None, dtype=None, ddof=1, period=None):
"""
Standardizes data by substracting the average over a reference period,
and by dividing by the standard deviation over the same period.
Parameters
----------
%(axis)s
%(dtype)s
%(ddof)s
%(period)s
Warnings
--------
The default ``ddof`` is 1:
by default, the method returns the unbiased estimate of the standard deviation.
"""
if period is None:
period = self.refperiod
elif not isinstance(period, (tuple, list, ndarray)):
msg = "Period should be a tuple (starting date, ending date)!"
raise ValueError, msg
refdata = mask_outside_period(self,
period[0],
period[1],
include_edges=False)
refavg = refdata.mean(axis=axis, dtype=dtype)
refstd = refdata.std(axis=axis, dtype=dtype, ddof=ddof)
if not axis:
result = (self - refavg) * 1. / refstd
else:
result = (self - ma.expand_dims(refavg)).astype(float)
result /= ma.expand_dims(refstd)
return result
def prepare_array(data, masked=True, nodata=0, dtype="int16"):
"""
Turn input data into a proper array for further usage.
Output array is always 3-dimensional with the given data type. If the output
is masked, the fill_value corresponds to the given nodata value and the
nodata value will be burned into the data array.
Parameters
----------
data : array or iterable
array (masked or normal) or iterable containing arrays
nodata : integer or float
nodata value (default: 0) used if input is not a masked array and
for output array
masked : bool
return a NumPy Array or a NumPy MaskedArray (default: True)
dtype : string
data type of output array (default: "int16")
Returns
-------
array : array
"""
# input is iterable
if isinstance(data, (list, tuple)):
return _prepare_iterable(data, masked, nodata, dtype)
# special case if a 2D single band is provided
elif isinstance(data, np.ndarray) and data.ndim == 2:
data = ma.expand_dims(data, axis=0)
# input is a masked array
if isinstance(data, ma.MaskedArray):
return _prepare_masked(data, masked, nodata, dtype)
# input is a NumPy array
elif isinstance(data, np.ndarray):
if masked:
return ma.masked_values(data.astype(dtype, copy=False),
nodata,
copy=False)
else:
return data.astype(dtype, copy=False)
else:
raise ValueError(
"Data must be array, masked array or iterable containing arrays. "
"Current data: %s (%s)" % (data, type(data)))
def log_objective_function(x, *inner_args, **inner_kwargs):
import numpy.ma as ma
value, jacobian = objective_function(x, *inner_args, **inner_kwargs)
masked_value = ma.masked_equal(value, 0)
log_value = ma.log(masked_value)
# We need expand_dims here because value is lower-dimensional than jacobian, but they must have the same
# dimensionality for numpy broadcasting to work here.
log_jacobian = jacobian / ma.expand_dims(masked_value, -1)
log_value = ma.filled(log_value, -1e100)
log_jacobian = ma.filled(log_jacobian, np.random.randn())
return log_value, log_jacobian
def read(self, indexes=None, **kwargs):
"""
Read reprojected & resampled input data.
Parameters
----------
indexes : integer or list
band number or list of band numbers
Returns
-------
data : array
"""
band_indexes = self._get_band_indexes(indexes)
arr = self.process.get_raw_output(self.tile)
return (arr[band_indexes[0] -
1] if len(band_indexes) == 1 else ma.concatenate(
[ma.expand_dims(arr[i - 1], 0) for i in band_indexes]))
def period_anomalies(self, axis=None, dtype=None, period=None):
"""
Returns data anomalies (deviations from average), where the average
is defined on a reference period, along a given axis.
Parameters
----------
%(axis)s
%(dtype)s
%(period)s
"""
if period is None:
period = self.refperiod
elif not isinstance(period, (tuple, list, ndarray)):
msg = "Period should be a tuple (starting date, ending date)!"
raise ValueError(msg)
period_mean = self.period_mean(axis=axis, dtype=dtype, period=period)
if not axis:
return self - period_mean
else:
return self - ma.expand_dims(period_mean, axis)
def period_anomalies(self, axis=None, dtype=None, period=None):
"""
Returns data anomalies (deviations from average), where the average
is defined on a reference period, along a given axis.
Parameters
----------
%(axis)s
%(dtype)s
%(period)s
"""
if period is None:
period = self.refperiod
elif not isinstance(period, (tuple, list, ndarray)):
msg = "Period should be a tuple (starting date, ending date)!"
raise ValueError(msg)
period_mean = self.period_mean(axis=axis, dtype=dtype, period=period)
if not axis:
return self - period_mean
else:
return self - ma.expand_dims(period_mean, axis)
def _add_axes_back(arr, axis):
'''
Add axes back in again after they've been processed out.
This is not a great implementation. I'm sure it can be improved upon. The
only reason this function exists at all is because the numpy.ma math
functions don't consistently take the keepdims option.
'''
# Here is the scenario. We have some array whose shape was originally
# something like (a,b,c,d). We processed out the second and fourth axes
# by summing over them (axis=(1,3)). At this point, we have an array whose
# shape is now (a,c). Unfortunately, this array shape does not work well
# for broadcasting against our original data. What we need is an array
# with shape (a,1,c,1).
#
# So, this function needs to call ma.expand_dims once for each axis that
# was removed.
# make sure axis is iterable
try:
iter(axis)
except TypeError:
# singleton. Make iterable.
axis = (axis, )
if arr is ma.masked:
# For reasons unknown, ma.expand_dims does nothing for the ma.masked.
# Other singletons work fine. No idea why. I've submitted a bug report:
# https://github.com/numpy/numpy/issues/7424
new_shape = np.ones(len(axis), dtype=int)
new_arr = np.zeros(new_shape)
new_arr = ma.masked_array(data=new_arr, mask=True)
else:
# Otherwise, systematically add back the dimensions removed.
new_arr = ma.copy(arr)
for i in np.sort(axis):
new_arr = ma.expand_dims(new_arr, i)
return new_arr
def combine_signals(X_all,
Y_class,
Y_loc,
defo_m,
APS_topo_m,
APS_turb_m,
heading,
dem,
defo_source,
defo_sources,
loc_list,
outputs,
succesful_generate,
sar_speckle_strength=0.05):
""" Given the synthetic outputs and labels (X and Y) and the parts of the synthetic data, combine into different formats and write to dictionary (X_all)
Inputs:
X_all | dict of masked arrays | keys are formats (e.g. uuu), then rank 4 masked array
Y_class | rank 2 array | class labels, n x 1 (ie not one hot encoding)
Y_loc | rank 2 array | location of deformaiton, nx4 (xy location, xy width)
defo_m | rank 2 array | deformation in metres, not masked
APS_topo_m | rank 2 array | topographically correlated APS, incoherence and water masked out.
APS_turb_m | rank 2 array | tubulent APS, not masked
heading | float | in degrees. e.g. 192 or 012
dem | rank 2 masked array | the DEM. Needed to make radar amplitude.
defo_source_n | int | 0 = no def, 1 = dyke, 2 = sill, 3 = Mogi. No 0's should be passed to this as it makes no deformatio nsignals.
loc_list | list of tuples | xy of centre of location box, and xy width. e.g [(186, 162), (69, 75)]
outputs | list of strings | e.g. ['uuu', 'uud']
succesful_generate | int | which number within a file we've generated so far
sar_speckle_strength | float | strength (variance) of gaussain speckled noise added to SAR real and imaginary
Returns:
X_all | as above, but updated
Y_class | as above, but updated
Y_loc | as above, but updated
succesful | boolean | True if no nans are present. False if nans are
History:
2020/08/20 | MEG | Written from exisiting scripts.
2020/10/19 | MEG | Fix bug that nans_present was being returned, instead of succesful (and they are generally the opposite of each other)
"""
import numpy as np
import numpy.ma as ma
from matplotlib.colors import LightSource # used to make a synthetic Radar return.
def normalise_m1_1(r2_array):
""" Rescale a rank 2 array so that it lies within the range[-1, 1]
"""
import numpy as np
r2_array = r2_array - np.min(r2_array)
r2_array = 2 * (r2_array / np.max(r2_array))
r2_array -= 1
return r2_array
s1_wav = 0.056 # Sentinel-1 wavelength in m
incidence = 30 # ditto - 30 roughly right for S1
# 1 Create ph_all and classes and locations, depending on if we want deformation or not.
if defo_source == 'no_def':
ph_all = ((4 * np.pi) / s1_wav) * (
APS_topo_m + APS_turb_m
) # comnbine the signals in m, then convert to rads
Y_loc[succesful_generate, :] = np.array(
[0, 0, 0, 0]) # 0 0 0 0 for no deformaiton
else:
ph_all = ((4 * np.pi) / s1_wav) * (
APS_topo_m + APS_turb_m + defo_m
) # combine the signals in m, then convert to rads. Here we include deformation.
Y_loc[succesful_generate, :] = np.array(
[loc_list[0][0], loc_list[0][1], loc_list[1][0],
loc_list[1][1]]) # location of deformation
Y_class[succesful_generate,
0] = defo_sources.index(defo_source) # write the label as a number
ph_all_wrap = (ph_all + np.pi) % (2 * np.pi) - np.pi # wrap
#1 Genreate SAR amplitude
look_az = heading - 90
look_in = 90 - incidence #
ls = LightSource(azdeg=look_az, altdeg=look_in)
sar_amplitude = normalise_m1_1(ls.hillshade(dem)) # in range [-1, 1]
# make real and imaginary
ifg_real = sar_amplitude * np.cos(ph_all_wrap)
ifg_imaginary = sar_amplitude * np.sin(ph_all_wrap)
ifg_real = ifg_real + sar_speckle_strength * np.random.randn(
ifg_real.shape[0], ifg_real.shape[1]) # add noise to real
ifg_imaginary = ifg_imaginary + sar_speckle_strength * np.random.randn(
ifg_real.shape[0], ifg_real.shape[1]) # and imaginary
# make things rank 4
unwrapped = ma.expand_dims(ma.expand_dims(ph_all, axis=0), axis=3)
dem = ma.expand_dims(ma.expand_dims(dem, axis=0), axis=3)
ifg_real = ma.expand_dims(ma.expand_dims(ifg_real, axis=0), axis=3)
ifg_imaginary = ma.expand_dims(ma.expand_dims(ifg_imaginary, axis=0),
axis=3)
wrapped = ma.expand_dims(ma.expand_dims(ph_all_wrap, axis=0), axis=3)
# Check for Nans
nans_present = False #initiate
for signal in [unwrapped, dem, ifg_real, ifg_imaginary, wrapped]:
nans_present = np.logical_or(
nans_present,
np.max(np.isnan(signal).astype(int)).astype(bool)
) # check if nans in current signal, and update succesful.
if nans_present:
succesful = False
print(f"| Failed due to Nans ", end='')
else:
succesful = True
for output in outputs:
if output == 'uuu':
X_all[output][succesful_generate, ] = ma.concatenate(
(unwrapped, unwrapped, unwrapped), axis=3) # uuu
elif output == 'uud':
X_all[output][succesful_generate, ] = ma.concatenate(
(unwrapped, unwrapped, dem), axis=3) # uud
elif output == 'rid':
X_all[output][succesful_generate, ] = ma.concatenate(
(ifg_real, ifg_imaginary, dem), axis=3) # rid
elif output == 'www':
X_all[output][succesful_generate, ] = ma.concatenate(
(wrapped, wrapped, wrapped), axis=3) # www
elif output == 'wwd':
X_all[output][succesful_generate, ] = ma.concatenate(
(wrapped, wrapped, dem), axis=3) #wwd
else:
raise Exception(
"Error in output format. Should only be either 'uuu', 'uud', 'rid', 'www', or 'wwd'. Exiting. "
)
return X_all, Y_class, Y_loc, succesful
def getSlicedArray(self, copy=True):
""" Slice the rti using a tuple of slices made from the values of the combo and spin boxes.
:param copy: If True (the default), a copy is made so that inspectors cannot
accidentally modify the underlying of the RTIs. You can set copy=False as a
potential optimization, but only if you are absolutely sure that you don't modify
the the slicedArray in your inspector! Note that this function calls transpose,
which can still make a copy of the array for certain permutations.
:return: Numpy masked array with the same number of dimension as the number of
comboboxes (this can be zero!).
Returns None if no slice can be made (i.e. the RTI is not sliceable).
"""
#logger.debug("getSlicedArray() called")
if not self.rtiIsSliceable:
return None
# The dimensions that are selected in the combo boxes will be set to slice(None),
# the values from the spin boxes will be set as a single integer value
nDims = self.rti.nDims
sliceList = [slice(None)] * nDims
for spinBox in self._spinBoxes:
dimNr = spinBox.property("dim_nr")
sliceList[dimNr] = spinBox.value()
# Make the array slicer. It needs to be a tuple, a list of only integers will be
# interpreted as an index. With a tuple, array[(exp1, exp2, ..., expN)] is equivalent to
# array[exp1, exp2, ..., expN].
# See: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html
logger.debug("Array slice list: {}".format(str(sliceList)))
slicedArray = self.rti[tuple(sliceList)]
# Make a copy to prevent inspectors from modifying the underlying array.
if copy:
slicedArray = ma.copy(slicedArray)
# If there are no comboboxes the sliceList will contain no Slices objects, only ints. Then
# the resulting slicedArray will be a usually a scalar (only structured fields may yield an
# array). We convert this scalar to a zero-dimensional Numpy array so that inspectors
# always get an array (having the same number of dimensions as the dimensionality of the
# inspector, i.e. the number of comboboxes).
if self.maxCombos == 0:
slicedArray = ma.MaskedArray(slicedArray)
# Post-condition type check
check_is_an_array(slicedArray, np.ndarray)
# Enforce the return type to be a masked array.
if not isinstance(slicedArray, ma.MaskedArray):
slicedArray = ma.MaskedArray(slicedArray)
# Add fake dimensions of length 1 so that result.ndim will equal the number of combo boxes
for dimNr in range(slicedArray.ndim, self.maxCombos):
#logger.debug("Adding fake dimension: {}".format(dimNr))
slicedArray = ma.expand_dims(slicedArray, dimNr)
# Post-condition dimension check
assert slicedArray.ndim == self.maxCombos, \
"Bug: getSlicedArray should return a {:d}D array, got: {}D" \
.format(self.maxCombos, slicedArray.ndim)
# Convert to ArrayWithMask class for working around issues with the numpy maskedarray
awm = ArrayWithMask.createFromMaskedArray(slicedArray)
del slicedArray
# Shuffle the dimensions to be in the order as specified by the combo boxes
comboDims = [
self._comboBoxDimensionIndex(cb) for cb in self._comboBoxes
]
permutations = np.argsort(comboDims)
logger.debug("slicedArray.shape: {}".format(awm.data.shape))
logger.debug("Transposing dimensions: {}".format(permutations))
awm = awm.transpose(permutations)
awm.checkIsConsistent()
return awm
def weighted_rms_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict,avg_test_slice,obs_file,nlev):
'''
Computes the weighted rms for a year
@param var The name of the variable that is being averaged.
@reg_name The name of the region to average over.
@reg_num The number of the region in the region_mask.
@mask_var The name of the netCDF variable that contain the region mask.
@wgt_var The name of the netCDF variable that contains the weight information.
@param year The year to average over.
@param hist_dict A dictionary that holds file references for all years/months.
@param ave_info A dictionary of the type of average that is to be done.
Includes: type, months_to_average, fn, and weights
(weights are not used in this function/average)
@param file_dict A dictionary which holds file pointers to the input files that
are needed by this average calculation.
@param avg_test_slice Averaged slice used in this calculation.
@param obs_file Observation file that contains the values to be used in the caluculation.
@param nlev Number of ocean vertical levels
@return nrms The normalized rms results for this variable.
'''
import warnings
# Get the weighted values from the yearly average file
slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False)
# Get the region mask
slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False)
# Since weights and region mask are only one level, we need to expand them to all levels
region_mask = MA.expand_dims(slev_mask, axis=0)
weights = MA.expand_dims(slev_weights, axis=0)
for lev in range(1,nlev):
new_region_mask = MA.expand_dims(slev_mask, axis=0)
region_mask = np.vstack((region_mask,new_region_mask))
new_weights = MA.expand_dims(slev_weights, axis=0)
weights = np.vstack((weights,new_weights))
# Calculate the weighted average
# First, we need to reshape the arrays to average along two dims
if (reg_name == 'Glo'):
temp_mask = MA.masked_where(region_mask<=int(reg_num),avg_test_slice)
else:
temp_mask = MA.masked_where(region_mask!=int(reg_num),avg_test_slice)
ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1)
weights_flattened = weights.reshape(weights.shape[0],-1)
warnings.filterwarnings("ignore")
rms_Ave = MA.sqrt(MA.average((ma_to_average*ma_to_average), axis=1, weights=weights_flattened))
warnings.filterwarnings("default")
#nrms = rms_Ave/(MA.max(rms_Ave) - MA.min(rms_Ave))
nrms = rms_Ave
return nrms
def weighted_hor_avg_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict,nlev):
'''
Computes the weighted hor mean rms diff for a year
@param var The name of the variable that is being averaged.
@reg_name The name of the region to average over.
@reg_num The number of the region in the region_mask.
@mask_var The name of the netCDF variable that contain the region mask.
@wgt_var The name of the netCDF variable that contains the weight information.
@param year The year to average over.
@param hist_dict A dictionary that holds file references for all years/months.
@param nlev Number of ocean vertical levels
@param ave_info A dictionary of the type of average that is to be done.
Includes: type, months_to_average, fn, and weights
(weights are not used in this function/average)
@param file_dict A dictionary which holds file pointers to the input files that
are needed by this average calculation.
@return var_Ave The averaged results for this variable across the designated time frame.
'''
# Get correct data slice from the yearly average file
var_val = rover.fetch_slice(hist_dict,year,0,var,file_dict)
# Get the weighted values from the yearly average file
slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False)
# Get the region mask
slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False)
# Since weights and region mask are only one level, we need to expand them to all levels
region_mask = MA.expand_dims(slev_mask, axis=0)
weights = MA.expand_dims(slev_weights, axis=0)
if var_val.ndim > 2:
for lev in range(1,nlev):
new_region_mask = MA.expand_dims(slev_mask, axis=0)
region_mask = np.vstack((region_mask,new_region_mask))
new_weights = MA.expand_dims(slev_weights, axis=0)
weights = np.vstack((weights,new_weights))
else:
region_mask = np.squeeze(region_mask,axis=0)
# Calculate the weighted average
# First, we need to reshape the arrays to average along two dims
if (reg_name == 'Glo'):
temp_mask = MA.masked_where(region_mask<=int(reg_num),var_val)
else:
temp_mask = MA.masked_where(region_mask!=int(reg_num),var_val)
ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1)
if var_val.ndim > 2:
weights_flattened = weights.reshape(weights.shape[0],-1)
else:
weights_flattened = np.squeeze(weights,axis=0)
var_Ave = MA.average(ma_to_average,axis=1, weights=weights_flattened)
return np.array(var_Ave)
def load_GPM_IMERG_files(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
name='GPM_IMERG'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate an OCW Dataset obejct.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:returns: An OCW Dataset object with the requested variable's data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
variable_unit = "mm/hr"
times = []
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S")
times.append(datetime(*time_struct_parsed[:6]))
values0 = ma.transpose(ma.masked_less(
file_object['Grid'][variable_name][:], 0.))
values0 = ma.expand_dims(values0, axis=0)
if ifile == 0:
values = values0
else:
values = ma.concatenate((values, values0))
file_object.close()
times = numpy.array(times)
return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
def resample_from_array(in_raster=None,
in_affine=None,
out_tile=None,
in_crs=None,
resampling="nearest",
nodataval=None,
nodata=0):
"""
Extract and resample from array to target tile.
Parameters
----------
in_raster : array
in_affine : ``Affine``
out_tile : ``BufferedTile``
resampling : string
one of rasterio's resampling methods (default: nearest)
nodata : integer or float
raster nodata value (default: 0)
Returns
-------
resampled array : array
"""
if nodataval is not None:
warnings.warn("'nodataval' is deprecated, please use 'nodata'")
nodata = nodata or nodataval
# TODO rename function
if isinstance(in_raster, ma.MaskedArray):
pass
elif isinstance(in_raster, np.ndarray):
in_raster = ma.MaskedArray(in_raster, mask=in_raster == nodata)
elif isinstance(in_raster, ReferencedRaster):
in_affine = in_raster.affine
in_crs = in_raster.crs
in_raster = in_raster.data
elif isinstance(in_raster, tuple):
in_raster = ma.MaskedArray(
data=np.stack(in_raster),
mask=np.stack([
band.mask if isinstance(band, ma.masked_array) else np.where(
band == nodata, True, False) for band in in_raster
]),
fill_value=nodata)
else:
raise TypeError("wrong input data type: %s" % type(in_raster))
if in_raster.ndim == 2:
in_raster = ma.expand_dims(in_raster, axis=0)
elif in_raster.ndim == 3:
pass
else:
raise TypeError("input array must have 2 or 3 dimensions")
if in_raster.fill_value != nodata:
ma.set_fill_value(in_raster, nodata)
dst_data = np.empty((in_raster.shape[0], ) + out_tile.shape,
in_raster.dtype)
logger.debug(in_raster)
logger.debug(in_affine)
logger.debug(out_tile.affine)
reproject(in_raster.filled(),
dst_data,
src_transform=in_affine,
src_crs=in_crs or out_tile.crs,
src_nodata=nodata,
dst_transform=out_tile.affine,
dst_crs=out_tile.crs,
dst_nodata=nodata,
resampling=Resampling[resampling])
logger.debug(dst_data)
hanse = ma.MaskedArray(dst_data,
mask=dst_data == nodata,
fill_value=nodata)
logger.debug(hanse)
return hanse
def load_GPM_IMERG_files_with_spatial_filter(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
user_mask_file=None,
mask_variable_name='mask',
user_mask_values=[10],
longitude_name='lon',
latitude_name='lat'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate a two-dimensional array \
for the masked grid points.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:user_mask_file: user's own gridded mask file(a netCDF file name)
:type name: :mod:`string`
:mask_variable_name: mask variables in user_mask_file
:type name: :mod:`string`
:longitude_name: longitude variable in user_mask_file
:type name: :mod:`string`
:latitude_name: latitude variable in user_mask_file
:type name: :mod:`string`
:param user_mask_values: grid points where mask_variable == user_mask_value will be extracted.
:type user_mask_values: list of strings
:returns: A two-dimensional array with the requested variable's MASKED data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
if ifile == 0 and user_mask_file:
file_object = netCDF4.Dataset(user_mask_file)
mask_variable = file_object.variables[mask_variable_name][:]
mask_longitude = file_object.variables[longitude_name][:]
mask_latitude = file_object.variables[latitude_name][:]
spatial_mask = utils.regrid_spatial_mask(lons,lats,
mask_longitude, mask_latitude,
mask_variable,
user_mask_values)
y_index, x_index = numpy.where(spatial_mask == 0)
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
values0 = ma.transpose(ma.masked_less(
file_object['Grid'][variable_name][:], 0.))
values_masked = values0[y_index, x_index]
values_masked = ma.expand_dims(values_masked, axis=0)
if ifile == 0:
values = values_masked
else:
values = ma.concatenate((values, values_masked))
file_object.close()
return values
def weighted_hor_avg_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict):
'''
Computes the weighted hor mean rms diff for a year
@param var The name of the variable that is being averaged.
@reg_name The name of the region to average over.
@reg_num The number of the region in the region_mask.
@mask_var The name of the netCDF variable that contain the region mask.
@wgt_var The name of the netCDF variable that contains the weight information.
@param year The year to average over.
@param hist_dict A dictionary that holds file references for all years/months.
@param ave_info A dictionary of the type of average that is to be done.
Includes: type, months_to_average, fn, and weights
(weights are not used in this function/average)
@param file_dict A dictionary which holds file pointers to the input files that
are needed by this average calculation.
@return var_Ave The averaged results for this variable across the designated time frame.
'''
# Get correct data slice from the yearly average file
var_val = rover.fetch_slice(hist_dict,year,0,var,file_dict)
# Get the weighted values from the yearly average file
slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False)
# Get the region mask
slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False)
# Since weights and region mask are only one level, we need to expand them to all levels
region_mask = MA.expand_dims(slev_mask, axis=0)
weights = MA.expand_dims(slev_weights, axis=0)
if var_val.ndim > 2:
for lev in range(1,60):
new_region_mask = MA.expand_dims(slev_mask, axis=0)
region_mask = np.vstack((region_mask,new_region_mask))
new_weights = MA.expand_dims(slev_weights, axis=0)
weights = np.vstack((weights,new_weights))
else:
region_mask = np.squeeze(region_mask,axis=0)
# Calculate the weighted average
# First, we need to reshape the arrays to average along two dims
if (reg_name == 'Glo'):
temp_mask = MA.masked_where(region_mask<=int(reg_num),var_val)
else:
temp_mask = MA.masked_where(region_mask!=int(reg_num),var_val)
ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1)
if var_val.ndim > 2:
weights_flattened = weights.reshape(weights.shape[0],-1)
else:
weights_flattened = np.squeeze(weights,axis=0)
var_Ave = MA.average(ma_to_average,axis=1, weights=weights_flattened)
return np.array(var_Ave)
def weighted_rms_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict,avg_test_slice,obs_file):
'''
Computes the weighted rms for a year
@param var The name of the variable that is being averaged.
@reg_name The name of the region to average over.
@reg_num The number of the region in the region_mask.
@mask_var The name of the netCDF variable that contain the region mask.
@wgt_var The name of the netCDF variable that contains the weight information.
@param year The year to average over.
@param hist_dict A dictionary that holds file references for all years/months.
@param ave_info A dictionary of the type of average that is to be done.
Includes: type, months_to_average, fn, and weights
(weights are not used in this function/average)
@param file_dict A dictionary which holds file pointers to the input files that
are needed by this average calculation.
@param avg_test_slice Averaged slice used in this calculation.
@param obs_file Observation file that contains the values to be used in the caluculation.
@return nrms The normalized rms results for this variable.
'''
import warnings
# Get the weighted values from the yearly average file
slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False)
# Get the region mask
slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False)
# Since weights and region mask are only one level, we need to expand them to all levels
region_mask = MA.expand_dims(slev_mask, axis=0)
weights = MA.expand_dims(slev_weights, axis=0)
for lev in range(1,60):
new_region_mask = MA.expand_dims(slev_mask, axis=0)
region_mask = np.vstack((region_mask,new_region_mask))
new_weights = MA.expand_dims(slev_weights, axis=0)
weights = np.vstack((weights,new_weights))
# Calculate the weighted average
# First, we need to reshape the arrays to average along two dims
if (reg_name == 'Glo'):
temp_mask = MA.masked_where(region_mask<=int(reg_num),avg_test_slice)
else:
temp_mask = MA.masked_where(region_mask!=int(reg_num),avg_test_slice)
ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1)
weights_flattened = weights.reshape(weights.shape[0],-1)
warnings.filterwarnings("ignore")
rms_Ave = MA.sqrt(MA.average((ma_to_average*ma_to_average), axis=1, weights=weights_flattened))
warnings.filterwarnings("default")
#nrms = rms_Ave/(MA.max(rms_Ave) - MA.min(rms_Ave))
nrms = rms_Ave
return nrms
def interpolate(self, new_fps: int = None, kind='cubic'):
if new_fps is None:
new_fps = self.fps
_frames = self.data.shape[0]
if _frames == 1:
raise ValueError("Can't interpolate single frame")
_new_frames = round(_frames * new_fps / self.fps)
steps = np.linspace(0, 1, _frames)
new_steps = np.linspace(0, 1, _new_frames)
transposed = self.points_perspective() # (points, people, frames, dims)
masked_confidence = ma.array(self.confidence, mask=self.confidence == 0)
confidence = ma.expand_dims(masked_confidence.transpose(), axis=3) # (points, people, frames, 1)
points = ma.concatenate([transposed, confidence], axis=3)
new_people = []
for people in points:
new_frames = []
for frames in people:
mask = frames.transpose()[0].mask
partial_steps = ma.array(steps, mask=mask).compressed()
if partial_steps.shape[0] == 0: # No data for this point
new_frames.append(np.zeros((_new_frames, frames.shape[1])))
else:
partial_frames = frames.compressed().reshape(partial_steps.shape[0], frames.shape[1])
if len(partial_steps) == 1:
f = lambda l: partial_frames
else:
this_kind = kind if len(partial_steps) > 3 \
else "quadratic" if len(partial_steps) > 2 and kind == "cubic" \
else "linear" # Can't do something fancy for 2 points
f = interp1d(partial_steps, partial_frames, axis=0, kind=this_kind)
first_step = partial_steps[0]
last_step = partial_steps[-1]
if first_step == 0 and last_step == 1:
new_frames.append(f(new_steps))
else:
first_step_where = np.argwhere(new_steps >= first_step)
first_step_index = first_step_where[0][0] if len(first_step_where) > 0 else 0
last_step_where = np.argwhere(new_steps > last_step)
last_step_index = last_step_where[0][0] if len(last_step_where) > 0 else len(new_steps)
if first_step_index == last_step_index:
new_frames.append(np.zeros((len(new_steps), frames.shape[1])))
else:
frame_data = f(new_steps[first_step_index:last_step_index])
new_frames.append(np.concatenate([
np.zeros((first_step_index, frames.shape[1])),
np.array(frame_data),
np.zeros((len(new_steps) - last_step_index, frames.shape[1]))
]))
new_people.append(np.stack(new_frames, axis=0))
new_data = np.stack(new_people, axis=0).transpose([2, 1, 0, 3])
dimensions, confidence = np.split(new_data, [-1], axis=3)
confidence = np.squeeze(confidence, axis=3)
mask = confidence == 0
stacked_confidence = np.stack([mask, mask], axis=3)
masked_data = ma.masked_array(dimensions, mask=stacked_confidence)
return NumPyPoseBody(fps=new_fps, data=masked_data, confidence=confidence)
def load_GPM_IMERG_files(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
name='GPM_IMERG'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate an OCW Dataset obejct.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:returns: An OCW Dataset object with the requested variable's data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
variable_unit = "mm/hr"
times = []
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S")
times.append(datetime(*time_struct_parsed[:6]))
values0 = ma.transpose(
ma.masked_less(file_object['Grid'][variable_name][:], 0.))
values0 = ma.expand_dims(values0, axis=0)
if ifile == 0:
values = values0
else:
values = ma.concatenate((values, values0))
file_object.close()
times = numpy.array(times)
return Dataset(lats,
lons,
times,
values,
variable_name,
units=variable_unit,
name=name)
def load_GPM_IMERG_files_with_spatial_filter(file_path=None,
filename_pattern=None,
filelist=None,
variable_name='precipitationCal',
user_mask_file=None,
mask_variable_name='mask',
user_mask_values=[10],
longitude_name='lon',
latitude_name='lat'):
''' Load multiple GPM Level 3 IMEGE files containing calibrated \
precipitation and generate a two-dimensional array \
for the masked grid points.
:param file_path: Directory to the HDF files to load.
:type file_path: :mod:`string`
:param filename_pattern: Path to the HDF files to load.
:type filename_pattern: :mod:`string`
:param filelist: A list of filenames
:type filelist: :mod:`string`
:param variable_name: The variable name to load from the HDF file.
:type variable_name: :mod:`string`
:param name: (Optional) A name for the loaded dataset.
:type name: :mod:`string`
:user_mask_file: user's own gridded mask file(a netCDF file name)
:type name: :mod:`string`
:mask_variable_name: mask variables in user_mask_file
:type name: :mod:`string`
:longitude_name: longitude variable in user_mask_file
:type name: :mod:`string`
:latitude_name: latitude variable in user_mask_file
:type name: :mod:`string`
:param user_mask_values: grid points where mask_variable == user_mask_value will be extracted.
:type user_mask_values: list of strings
:returns: A two-dimensional array with the requested variable's MASKED data from \
the HDF file.
:rtype: :class:`dataset.Dataset`
:raises ValueError:
'''
if not filelist:
GPM_files = []
for pattern in filename_pattern:
GPM_files.extend(glob(file_path + pattern))
else:
GPM_files = [line.rstrip('\n') for line in open(filelist)]
GPM_files.sort()
file_object_first = h5py.File(GPM_files[0])
lats = file_object_first['Grid']['lat'][:]
lons = file_object_first['Grid']['lon'][:]
lons, lats = numpy.meshgrid(lons, lats)
nfile = len(GPM_files)
for ifile, file in enumerate(GPM_files):
if ifile == 0 and user_mask_file:
file_object = netCDF4.Dataset(user_mask_file)
mask_variable = file_object.variables[mask_variable_name][:]
mask_longitude = file_object.variables[longitude_name][:]
mask_latitude = file_object.variables[latitude_name][:]
spatial_mask = utils.regrid_spatial_mask(lons, lats,
mask_longitude,
mask_latitude,
mask_variable,
user_mask_values)
y_index, x_index = numpy.where(spatial_mask == 0)
print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file)
file_object = h5py.File(file)
values0 = ma.transpose(
ma.masked_less(file_object['Grid'][variable_name][:], 0.))
values_masked = values0[y_index, x_index]
values_masked = ma.expand_dims(values_masked, axis=0)
if ifile == 0:
values = values_masked
else:
values = ma.concatenate((values, values_masked))
file_object.close()
return values
def getSlicedArray(self, copy=True):
""" Slice the rti using a tuple of slices made from the values of the combo and spin boxes.
:param copy: If True (the default), a copy is made so that inspectors cannot
accidentally modify the underlying of the RTIs. You can set copy=False as a
potential optimization, but only if you are absolutely sure that you don't modify
the the slicedArray in your inspector! Note that this function calls transpose,
which can still make a copy of the array for certain permutations.
:return: ArrayWithMask array with the same number of dimension as the number of
comboboxes (this can be zero!).
Returns None if no slice can be made (i.e. the RTI is not sliceable).
:rtype ArrayWithMask:
"""
#logger.debug("getSlicedArray() called")
if not self._rti:
self.sigShowMessage.emit("No item selected.")
return None
if not self.rtiIsSliceable:
# This is very common so we don't show a message so the user isn't flooded.
# Also this can have many different causes (compound data, lists, etc_ so it's
# difficult to show a good, descriptive message.
return None
if np.prod(self._rti.arrayShape) == 0:
self.sigShowMessage.emit("Selected item has zero array elements.")
return None
# The dimensions that are selected in the combo boxes will be set to slice(None),
# the values from the spin boxes will be set as a single integer value
nDims = self.rti.nDims
sliceList = [slice(None)] * nDims
for spinBox in self._spinBoxes:
dimNr = spinBox.property("dim_nr")
sliceList[dimNr] = spinBox.value()
# Make the array slicer. It needs to be a tuple, a list of only integers will be
# interpreted as an index. With a tuple, array[(exp1, exp2, ..., expN)] is equivalent to
# array[exp1, exp2, ..., expN].
# See: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html
slicedArray = self.rti[tuple(sliceList)]
# Make a copy to prevent inspectors from modifying the underlying array.
if copy:
if versionStrToTuple(np.__version__) >= (1, 19, 0):
slicedArray = np.copy(slicedArray,
subok=True) # Fixes issue #8
else:
slicedArray = ma.copy(slicedArray)
# If there are no comboboxes the sliceList will contain no Slices objects, only ints. Then
# the resulting slicedArray will be a usually a scalar (only structured fields may yield an
# array). We convert this scalar to a zero-dimensional Numpy array so that inspectors
# always get an array (having the same number of dimensions as the dimensionality of the
# inspector, i.e. the number of comboboxes).
# Also scalar RTIs, which have nDim == 0, can return a scalar which must be converted.
# TODO: perhaps always convert to array.
if self.maxCombos == 0 or self.rti.nDims == 0:
slicedArray = ma.MaskedArray(slicedArray)
# Post-condition type check
check_is_an_array(slicedArray, np.ndarray)
# Enforce the return type to be a masked array.
if not isinstance(slicedArray, ma.MaskedArray):
slicedArray = ma.MaskedArray(slicedArray)
# Add fake dimensions of length 1 so that result.ndim will equal the number of combo boxes
# TODO: Perhaps get rid of this because it fails with masked arrays with fill values.
# The less we do here, the less chance an error occurs. See development/todo.txt
for dimNr in range(slicedArray.ndim, self.maxCombos):
#logger.debug("Adding fake dimension: {}".format(dimNr))
slicedArray = ma.expand_dims(slicedArray, dimNr)
# Post-condition dimension check
assert slicedArray.ndim == self.maxCombos, \
"Bug: getSlicedArray should return a {:d}D array, got: {}D" \
.format(self.maxCombos, slicedArray.ndim)
# Convert to ArrayWithMask class for working around issues with the numpy maskedarray
awm = ArrayWithMask.createFromMaskedArray(slicedArray)
del slicedArray
# Shuffle the dimensions to be in the order as specified by the combo boxes
comboDims = [
self._comboBoxDimensionIndex(cb) for cb in self._comboBoxes
]
permutations = np.argsort(comboDims)
logger.debug("slicedArray.shape: {}".format(awm.data.shape))
logger.debug("Transposing dimensions: {}".format(permutations))
awm = awm.transpose(permutations)
awm.checkIsConsistent()
return awm
def getSlicedArray(self, copy=True):
""" Slice the rti using a tuple of slices made from the values of the combo and spin boxes.
:param copy: If True (the default), a copy is made so that inspectors cannot
accidentally modify the underlying of the RTIs. You can set copy=False as a
potential optimization, but only if you are absolutely sure that you don't modify
the the slicedArray in your inspector! Note that this function calls transpose,
which can still make a copy of the array for certain permutations.
:return: Numpy masked array with the same number of dimension as the number of
comboboxes (this can be zero!).
Returns None if no slice can be made (i.e. the RTI is not sliceable).
"""
#logger.debug("getSlicedArray() called")
if not self.rtiIsSliceable:
return None
# The dimensions that are selected in the combo boxes will be set to slice(None),
# the values from the spin boxes will be set as a single integer value
nDims = self.rti.nDims
sliceList = [slice(None)] * nDims
for spinBox in self._spinBoxes:
dimNr = spinBox.property("dim_nr")
sliceList[dimNr] = spinBox.value()
# Make the array slicer. It needs to be a tuple, a list of only integers will be
# interpreted as an index. With a tuple, array[(exp1, exp2, ..., expN)] is equivalent to
# array[exp1, exp2, ..., expN].
# See: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html
logger.debug("Array slice list: {}".format(str(sliceList)))
slicedArray = self.rti[tuple(sliceList)]
# Make a copy to prevent inspectors from modifying the underlying array.
if copy:
slicedArray = ma.copy(slicedArray)
# If there are no comboboxes the sliceList will contain no Slices objects, only ints. Then
# the resulting slicedArray will be a usually a scalar (only structured fields may yield an
# array). We convert this scalar to a zero-dimensional Numpy array so that inspectors
# always get an array (having the same number of dimensions as the dimensionality of the
# inspector, i.e. the number of comboboxes).
if self.maxCombos == 0:
slicedArray = ma.MaskedArray(slicedArray)
# Post-condition type check
check_is_an_array(slicedArray, np.ndarray)
# Enforce the return type to be a masked array.
if not isinstance(slicedArray, ma.MaskedArray):
slicedArray = ma.MaskedArray(slicedArray)
# Add fake dimensions of length 1 so that result.ndim will equal the number of combo boxes
for dimNr in range(slicedArray.ndim, self.maxCombos):
#logger.debug("Adding fake dimension: {}".format(dimNr))
slicedArray = ma.expand_dims(slicedArray, dimNr)
# Post-condition dimension check
assert slicedArray.ndim == self.maxCombos, \
"Bug: getSlicedArray should return a {:d}D array, got: {}D" \
.format(self.maxCombos, slicedArray.ndim)
# Convert to ArrayWithMask class for working around issues with the numpy maskedarray
awm = ArrayWithMask.createFromMaskedArray(slicedArray)
del slicedArray
# Shuffle the dimensions to be in the order as specified by the combo boxes
comboDims = [self._comboBoxDimensionIndex(cb) for cb in self._comboBoxes]
permutations = np.argsort(comboDims)
logger.debug("slicedArray.shape: {}".format(awm.data.shape))
logger.debug("Transposing dimensions: {}".format(permutations))
awm = awm.transpose(permutations)
awm.checkIsConsistent()
return awm
import numpy as np import numpy.ma as ma x = ma.array([1, 2, 4]) x[1] = ma.masked x np.expand_dims(x, axis=0) ma.expand_dims(x, axis=0) x[np.newaxis, :]
