def concatenate(self,value,axis=0): """ Concatentate UncertContainer value to self. Assumes that if dimensions of self and value do not match, to add a np.newaxis along axis of value """ if isinstance(value,UncertContainer): if value.vals.ndim == self.vals.ndim: vals = value.vals dmin = value.dmin dmax = value.dmax wt = value.wt uncert = value.uncert mask = value.mask elif (value.vals.ndim + 1) == self.vals.ndim: vals = ma.expand_dims(value.vals,axis) dmin = ma.expand_dims(value.dmin,axis) dmax = ma.expand_dims(value.dmax,axis) wt = ma.expand_dims(value.wt,axis) uncert = ma.expand_dims(value.uncert,axis) mask = np.expand_dims(value.mask,axis) else: raise ValueError('Could not propery match dimensionality') self.vals = ma.concatenate((self.vals,vals),axis=axis) self.dmin = ma.concatenate((self.dmin,dmin),axis=axis) self.dmax = ma.concatenate((self.dmax,dmax),axis=axis) self.wt = ma.concatenate((self.wt,wt),axis=axis) self.uncert = ma.concatenate((self.uncert,uncert),axis=axis) self.mask = np.concatenate((self.mask,mask),axis=axis) else: raise ValueError('Can only concatenate with an UncertContainer object')
def period_standardize(self, axis=None, dtype=None, ddof=1, period=None): """ Standardizes data by substracting the average over a reference period, and by dividing by the standard deviation over the same period. Parameters ---------- %(axis)s %(dtype)s %(ddof)s %(period)s Warnings -------- The default ``ddof`` is 1: by default, the method returns the unbiased estimate of the standard deviation. """ if period is None: period = self.refperiod elif not isinstance(period, (tuple, list, ndarray)): msg = "Period should be a tuple (starting date, ending date)!" raise ValueError, msg refdata = mask_outside_period(self, period[0], period[1], include_edges=False) refavg = refdata.mean(axis=axis, dtype=dtype) refstd = refdata.std(axis=axis, dtype=dtype, ddof=ddof) if not axis: result = (self - refavg) * 1. / refstd else: result = (self - ma.expand_dims(refavg)).astype(float) result /= ma.expand_dims(refstd) return result
def weighted_average(self, axis=0, expaxis=None): """ Calculate weighted average of data along axis after optionally inserting a new dimension into the shape array at position expaxis """ if expaxis is not None: vals = ma.expand_dims(self.vals, expaxis) dmin = ma.expand_dims(self.dmin, expaxis) dmax = ma.expand_dims(self.dmax, expaxis) wt = ma.expand_dims(self.wt, expaxis) else: vals = self.vals wt = self.wt dmin = self.dmin dmax = self.dmax # Get average value avg, norm = ma.average(vals, axis=axis, weights=wt, returned=True) avg_ex = ma.expand_dims(avg, 0) # Calculate weighted uncertainty wtmax = ma.max(wt, axis=axis) neff = norm / wtmax # Effective number of samples based on uncertainties # Seeking max deviation from the average; if above avg use max, if below use min term = np.empty_like(vals) indices = np.where(vals > avg_ex) i0 = indices[0] irest = indices[1:] ii = tuple(x for x in itertools.chain([i0], irest)) jj = tuple(x for x in itertools.chain([np.zeros_like(i0)], irest)) term[ii] = (dmax[ii] - avg_ex[jj])**2 indices = np.where(vals <= avg_ex) i0 = indices[0] irest = indices[1:] ii = tuple(x for x in itertools.chain([i0], irest)) jj = tuple(x for x in itertools.chain([np.zeros_like(i0)], irest)) term[ii] = (avg_ex[jj] - dmin[ii])**2 dsum = ma.sum(term * wt, axis=0) # Sum for weighted average of deviations dev = 0.5 * np.sqrt(dsum / (norm * neff)) if isinstance(avg, (float, np.float)): avg = avg_ex tmp_min = avg - dev ii = np.where(tmp_min < 0) tmp_min[ii] = TOL * avg[ii] return UncertContainer(avg, tmp_min, avg + dev)
def weighted_average(self,axis=0,expaxis=None): """ Calculate weighted average of data along axis after optionally inserting a new dimension into the shape array at position expaxis """ if expaxis is not None: vals = ma.expand_dims(self.vals,expaxis) dmin = ma.expand_dims(self.dmin,expaxis) dmax = ma.expand_dims(self.dmax,expaxis) wt = ma.expand_dims(self.wt,expaxis) else: vals = self.vals wt = self.wt dmin = self.dmin dmax = self.dmax # Get average value avg,norm = ma.average(vals,axis=axis,weights=wt,returned=True) avg_ex = ma.expand_dims(avg,0) # Calculate weighted uncertainty wtmax = ma.max(wt,axis=axis) neff = norm/wtmax # Effective number of samples based on uncertainties # Seeking max deviation from the average; if above avg use max, if below use min term = np.empty_like(vals) indices = np.where(vals > avg_ex) i0 = indices[0] irest = indices[1:] ii = tuple(x for x in itertools.chain([i0],irest)) jj = tuple(x for x in itertools.chain([np.zeros_like(i0)],irest)) term[ii] = (dmax[ii] - avg_ex[jj])**2 indices = np.where(vals <= avg_ex) i0 = indices[0] irest = indices[1:] ii = tuple(x for x in itertools.chain([i0],irest)) jj = tuple(x for x in itertools.chain([np.zeros_like(i0)],irest)) term[ii] = (avg_ex[jj] - dmin[ii])**2 dsum = ma.sum(term*wt,axis=0) # Sum for weighted average of deviations dev = 0.5*np.sqrt(dsum/(norm*neff)) if isinstance(avg,(float,np.float)): avg = avg_ex tmp_min = avg - dev ii = np.where(tmp_min < 0) tmp_min[ii] = TOL*avg[ii] return UncertContainer(avg,tmp_min,avg+dev)
def execute(mp): """Read, stretch and return tile.""" # read parameters resampling = mp.params["resampling"] scale_method = mp.params.get("scale_method", None) scales_minmax = mp.params["scales_minmax"] with mp.open("raster", resampling=resampling) as raster_file: # exit if input tile is empty if raster_file.is_empty(): return "empty" resampled = () mask = () # actually read data and iterate through bands raster_data = raster_file.read() if raster_data.ndim == 2: raster_data = ma.expand_dims(raster_data, axis=0) if not scale_method: scales_minmax = [(i, i) for i in range(len(raster_data))] for band, scale_minmax in zip(raster_data, scales_minmax): if scale_method in ["dtype_scale", "minmax_scale"]: scale_min, scale_max = scale_minmax resampled += (_stretch_array(band, scale_min, scale_max), ) elif scale_method == "crop": scale_min, scale_max = scale_minmax band[band > scale_max] = scale_max band[band <= scale_min] = scale_min resampled += (band, ) else: resampled += (band, ) mask += (band.mask, ) return ma.masked_array(np.stack(resampled), np.stack(mask))
def execute( mp, resampling="nearest", scale_method=None, scales_minmax=None, **kwargs ): """Read, stretch and return tile.""" with mp.open("raster", resampling=resampling) as raster_file: # exit if input tile is empty if raster_file.is_empty(): return "empty" # actually read data and iterate through bands scaled = () mask = () raster_data = raster_file.read() if raster_data.ndim == 2: raster_data = ma.expand_dims(raster_data, axis=0) if not scale_method: scales_minmax = [(i, i) for i in range(len(raster_data))] for band, (scale_min, scale_max) in zip(raster_data, scales_minmax): if scale_method in ["dtype_scale", "minmax_scale"]: scaled += (_stretch_array(band, scale_min, scale_max), ) elif scale_method == "crop": scaled += (np.clip(band, scale_min, scale_max), ) else: scaled += (band, ) mask += (band.mask, ) return ma.masked_array(np.stack(scaled), np.stack(mask))
def ma_mad(x, axis=None): """Median absolute deviation""" median_x = ma.median(x, axis=axis) if axis is not None: median_x = ma.expand_dims(median_x, axis=axis) return ma.median(ma.abs(x - median_x), axis=axis)
def resample_from_array( in_raster=None, in_affine=None, out_tile=None, resampling="nearest", nodataval=0 ): """ Extract and resample from array to target tile. Parameters ---------- in_raster : array in_affine : ``Affine`` out_tile : ``BufferedTile`` resampling : string one of rasterio's resampling methods (default: nearest) nodataval : integer or float raster nodata value (default: 0) Returns ------- resampled array : array """ if isinstance(in_raster, ma.MaskedArray): pass if isinstance(in_raster, np.ndarray): in_raster = ma.MaskedArray(in_raster, mask=in_raster == nodataval) elif isinstance(in_raster, ReferencedRaster): in_affine = in_raster.affine in_raster = in_raster.data elif isinstance(in_raster, tuple): in_raster = ma.MaskedArray( data=np.stack(in_raster), mask=np.stack([ band.mask if isinstance(band, ma.masked_array) else np.where(band == nodataval, True, False) for band in in_raster ]), fill_value=nodataval ) else: raise TypeError("wrong input data type: %s" % type(in_raster)) if in_raster.ndim == 2: in_raster = ma.expand_dims(in_raster, axis=0) elif in_raster.ndim == 3: pass else: raise TypeError("input array must have 2 or 3 dimensions") if in_raster.fill_value != nodataval: ma.set_fill_value(in_raster, nodataval) out_shape = (in_raster.shape[0], ) + out_tile.shape dst_data = np.empty(out_shape, in_raster.dtype) in_raster = ma.masked_array( data=in_raster.filled(), mask=in_raster.mask, fill_value=nodataval) reproject( in_raster, dst_data, src_transform=in_affine, src_crs=out_tile.crs, dst_transform=out_tile.affine, dst_crs=out_tile.crs, resampling=Resampling[resampling]) return ma.MaskedArray(dst_data, mask=dst_data == nodataval)
def prepare_array(data, masked=True, nodata=0, dtype="int16"): """ Turn input data into a proper array for further usage. Output array is always 3-dimensional with the given data type. If the output is masked, the fill_value corresponds to the given nodata value and the nodata value will be burned into the data array. Parameters ---------- data : array or iterable array (masked or normal) or iterable containing arrays nodata : integer or float nodata value (default: 0) used if input is not a masked array and for output array masked : bool return a NumPy Array or a NumPy MaskedArray (default: True) dtype : string data type of output array (default: "int16") Returns ------- array : array """ # input is iterable if isinstance(data, (list, tuple)): return _prepare_iterable(data, masked, nodata, dtype) # special case if a 2D single band is provided elif isinstance(data, np.ndarray) and data.ndim == 2: data = ma.expand_dims(data, axis=0) # input is a masked array if isinstance(data, ma.MaskedArray): return _prepare_masked(data, masked, nodata, dtype) # input is a NumPy array elif isinstance(data, np.ndarray): if masked: return ma.masked_values(data.astype(dtype, copy=False), nodata, copy=False) else: return data.astype(dtype, copy=False) else: raise ValueError( "Data must be array, masked array or iterable containing arrays. " "Current data: %s (%s)" % (data, type(data)))
def log_objective_function(x, *inner_args, **inner_kwargs): import numpy.ma as ma value, jacobian = objective_function(x, *inner_args, **inner_kwargs) masked_value = ma.masked_equal(value, 0) log_value = ma.log(masked_value) # We need expand_dims here because value is lower-dimensional than jacobian, but they must have the same # dimensionality for numpy broadcasting to work here. log_jacobian = jacobian / ma.expand_dims(masked_value, -1) log_value = ma.filled(log_value, -1e100) log_jacobian = ma.filled(log_jacobian, np.random.randn()) return log_value, log_jacobian
def read(self, indexes=None, **kwargs): """ Read reprojected & resampled input data. Parameters ---------- indexes : integer or list band number or list of band numbers Returns ------- data : array """ band_indexes = self._get_band_indexes(indexes) arr = self.process.get_raw_output(self.tile) return (arr[band_indexes[0] - 1] if len(band_indexes) == 1 else ma.concatenate( [ma.expand_dims(arr[i - 1], 0) for i in band_indexes]))
def period_anomalies(self, axis=None, dtype=None, period=None): """ Returns data anomalies (deviations from average), where the average is defined on a reference period, along a given axis. Parameters ---------- %(axis)s %(dtype)s %(period)s """ if period is None: period = self.refperiod elif not isinstance(period, (tuple, list, ndarray)): msg = "Period should be a tuple (starting date, ending date)!" raise ValueError(msg) period_mean = self.period_mean(axis=axis, dtype=dtype, period=period) if not axis: return self - period_mean else: return self - ma.expand_dims(period_mean, axis)
def _add_axes_back(arr, axis): ''' Add axes back in again after they've been processed out. This is not a great implementation. I'm sure it can be improved upon. The only reason this function exists at all is because the numpy.ma math functions don't consistently take the keepdims option. ''' # Here is the scenario. We have some array whose shape was originally # something like (a,b,c,d). We processed out the second and fourth axes # by summing over them (axis=(1,3)). At this point, we have an array whose # shape is now (a,c). Unfortunately, this array shape does not work well # for broadcasting against our original data. What we need is an array # with shape (a,1,c,1). # # So, this function needs to call ma.expand_dims once for each axis that # was removed. # make sure axis is iterable try: iter(axis) except TypeError: # singleton. Make iterable. axis = (axis, ) if arr is ma.masked: # For reasons unknown, ma.expand_dims does nothing for the ma.masked. # Other singletons work fine. No idea why. I've submitted a bug report: # https://github.com/numpy/numpy/issues/7424 new_shape = np.ones(len(axis), dtype=int) new_arr = np.zeros(new_shape) new_arr = ma.masked_array(data=new_arr, mask=True) else: # Otherwise, systematically add back the dimensions removed. new_arr = ma.copy(arr) for i in np.sort(axis): new_arr = ma.expand_dims(new_arr, i) return new_arr
def combine_signals(X_all, Y_class, Y_loc, defo_m, APS_topo_m, APS_turb_m, heading, dem, defo_source, defo_sources, loc_list, outputs, succesful_generate, sar_speckle_strength=0.05): """ Given the synthetic outputs and labels (X and Y) and the parts of the synthetic data, combine into different formats and write to dictionary (X_all) Inputs: X_all | dict of masked arrays | keys are formats (e.g. uuu), then rank 4 masked array Y_class | rank 2 array | class labels, n x 1 (ie not one hot encoding) Y_loc | rank 2 array | location of deformaiton, nx4 (xy location, xy width) defo_m | rank 2 array | deformation in metres, not masked APS_topo_m | rank 2 array | topographically correlated APS, incoherence and water masked out. APS_turb_m | rank 2 array | tubulent APS, not masked heading | float | in degrees. e.g. 192 or 012 dem | rank 2 masked array | the DEM. Needed to make radar amplitude. defo_source_n | int | 0 = no def, 1 = dyke, 2 = sill, 3 = Mogi. No 0's should be passed to this as it makes no deformatio nsignals. loc_list | list of tuples | xy of centre of location box, and xy width. e.g [(186, 162), (69, 75)] outputs | list of strings | e.g. ['uuu', 'uud'] succesful_generate | int | which number within a file we've generated so far sar_speckle_strength | float | strength (variance) of gaussain speckled noise added to SAR real and imaginary Returns: X_all | as above, but updated Y_class | as above, but updated Y_loc | as above, but updated succesful | boolean | True if no nans are present. False if nans are History: 2020/08/20 | MEG | Written from exisiting scripts. 2020/10/19 | MEG | Fix bug that nans_present was being returned, instead of succesful (and they are generally the opposite of each other) """ import numpy as np import numpy.ma as ma from matplotlib.colors import LightSource # used to make a synthetic Radar return. def normalise_m1_1(r2_array): """ Rescale a rank 2 array so that it lies within the range[-1, 1] """ import numpy as np r2_array = r2_array - np.min(r2_array) r2_array = 2 * (r2_array / np.max(r2_array)) r2_array -= 1 return r2_array s1_wav = 0.056 # Sentinel-1 wavelength in m incidence = 30 # ditto - 30 roughly right for S1 # 1 Create ph_all and classes and locations, depending on if we want deformation or not. if defo_source == 'no_def': ph_all = ((4 * np.pi) / s1_wav) * ( APS_topo_m + APS_turb_m ) # comnbine the signals in m, then convert to rads Y_loc[succesful_generate, :] = np.array( [0, 0, 0, 0]) # 0 0 0 0 for no deformaiton else: ph_all = ((4 * np.pi) / s1_wav) * ( APS_topo_m + APS_turb_m + defo_m ) # combine the signals in m, then convert to rads. Here we include deformation. Y_loc[succesful_generate, :] = np.array( [loc_list[0][0], loc_list[0][1], loc_list[1][0], loc_list[1][1]]) # location of deformation Y_class[succesful_generate, 0] = defo_sources.index(defo_source) # write the label as a number ph_all_wrap = (ph_all + np.pi) % (2 * np.pi) - np.pi # wrap #1 Genreate SAR amplitude look_az = heading - 90 look_in = 90 - incidence # ls = LightSource(azdeg=look_az, altdeg=look_in) sar_amplitude = normalise_m1_1(ls.hillshade(dem)) # in range [-1, 1] # make real and imaginary ifg_real = sar_amplitude * np.cos(ph_all_wrap) ifg_imaginary = sar_amplitude * np.sin(ph_all_wrap) ifg_real = ifg_real + sar_speckle_strength * np.random.randn( ifg_real.shape[0], ifg_real.shape[1]) # add noise to real ifg_imaginary = ifg_imaginary + sar_speckle_strength * np.random.randn( ifg_real.shape[0], ifg_real.shape[1]) # and imaginary # make things rank 4 unwrapped = ma.expand_dims(ma.expand_dims(ph_all, axis=0), axis=3) dem = ma.expand_dims(ma.expand_dims(dem, axis=0), axis=3) ifg_real = ma.expand_dims(ma.expand_dims(ifg_real, axis=0), axis=3) ifg_imaginary = ma.expand_dims(ma.expand_dims(ifg_imaginary, axis=0), axis=3) wrapped = ma.expand_dims(ma.expand_dims(ph_all_wrap, axis=0), axis=3) # Check for Nans nans_present = False #initiate for signal in [unwrapped, dem, ifg_real, ifg_imaginary, wrapped]: nans_present = np.logical_or( nans_present, np.max(np.isnan(signal).astype(int)).astype(bool) ) # check if nans in current signal, and update succesful. if nans_present: succesful = False print(f"| Failed due to Nans ", end='') else: succesful = True for output in outputs: if output == 'uuu': X_all[output][succesful_generate, ] = ma.concatenate( (unwrapped, unwrapped, unwrapped), axis=3) # uuu elif output == 'uud': X_all[output][succesful_generate, ] = ma.concatenate( (unwrapped, unwrapped, dem), axis=3) # uud elif output == 'rid': X_all[output][succesful_generate, ] = ma.concatenate( (ifg_real, ifg_imaginary, dem), axis=3) # rid elif output == 'www': X_all[output][succesful_generate, ] = ma.concatenate( (wrapped, wrapped, wrapped), axis=3) # www elif output == 'wwd': X_all[output][succesful_generate, ] = ma.concatenate( (wrapped, wrapped, dem), axis=3) #wwd else: raise Exception( "Error in output format. Should only be either 'uuu', 'uud', 'rid', 'www', or 'wwd'. Exiting. " ) return X_all, Y_class, Y_loc, succesful
def getSlicedArray(self, copy=True): """ Slice the rti using a tuple of slices made from the values of the combo and spin boxes. :param copy: If True (the default), a copy is made so that inspectors cannot accidentally modify the underlying of the RTIs. You can set copy=False as a potential optimization, but only if you are absolutely sure that you don't modify the the slicedArray in your inspector! Note that this function calls transpose, which can still make a copy of the array for certain permutations. :return: Numpy masked array with the same number of dimension as the number of comboboxes (this can be zero!). Returns None if no slice can be made (i.e. the RTI is not sliceable). """ #logger.debug("getSlicedArray() called") if not self.rtiIsSliceable: return None # The dimensions that are selected in the combo boxes will be set to slice(None), # the values from the spin boxes will be set as a single integer value nDims = self.rti.nDims sliceList = [slice(None)] * nDims for spinBox in self._spinBoxes: dimNr = spinBox.property("dim_nr") sliceList[dimNr] = spinBox.value() # Make the array slicer. It needs to be a tuple, a list of only integers will be # interpreted as an index. With a tuple, array[(exp1, exp2, ..., expN)] is equivalent to # array[exp1, exp2, ..., expN]. # See: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html logger.debug("Array slice list: {}".format(str(sliceList))) slicedArray = self.rti[tuple(sliceList)] # Make a copy to prevent inspectors from modifying the underlying array. if copy: slicedArray = ma.copy(slicedArray) # If there are no comboboxes the sliceList will contain no Slices objects, only ints. Then # the resulting slicedArray will be a usually a scalar (only structured fields may yield an # array). We convert this scalar to a zero-dimensional Numpy array so that inspectors # always get an array (having the same number of dimensions as the dimensionality of the # inspector, i.e. the number of comboboxes). if self.maxCombos == 0: slicedArray = ma.MaskedArray(slicedArray) # Post-condition type check check_is_an_array(slicedArray, np.ndarray) # Enforce the return type to be a masked array. if not isinstance(slicedArray, ma.MaskedArray): slicedArray = ma.MaskedArray(slicedArray) # Add fake dimensions of length 1 so that result.ndim will equal the number of combo boxes for dimNr in range(slicedArray.ndim, self.maxCombos): #logger.debug("Adding fake dimension: {}".format(dimNr)) slicedArray = ma.expand_dims(slicedArray, dimNr) # Post-condition dimension check assert slicedArray.ndim == self.maxCombos, \ "Bug: getSlicedArray should return a {:d}D array, got: {}D" \ .format(self.maxCombos, slicedArray.ndim) # Convert to ArrayWithMask class for working around issues with the numpy maskedarray awm = ArrayWithMask.createFromMaskedArray(slicedArray) del slicedArray # Shuffle the dimensions to be in the order as specified by the combo boxes comboDims = [ self._comboBoxDimensionIndex(cb) for cb in self._comboBoxes ] permutations = np.argsort(comboDims) logger.debug("slicedArray.shape: {}".format(awm.data.shape)) logger.debug("Transposing dimensions: {}".format(permutations)) awm = awm.transpose(permutations) awm.checkIsConsistent() return awm
def weighted_rms_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict,avg_test_slice,obs_file,nlev): ''' Computes the weighted rms for a year @param var The name of the variable that is being averaged. @reg_name The name of the region to average over. @reg_num The number of the region in the region_mask. @mask_var The name of the netCDF variable that contain the region mask. @wgt_var The name of the netCDF variable that contains the weight information. @param year The year to average over. @param hist_dict A dictionary that holds file references for all years/months. @param ave_info A dictionary of the type of average that is to be done. Includes: type, months_to_average, fn, and weights (weights are not used in this function/average) @param file_dict A dictionary which holds file pointers to the input files that are needed by this average calculation. @param avg_test_slice Averaged slice used in this calculation. @param obs_file Observation file that contains the values to be used in the caluculation. @param nlev Number of ocean vertical levels @return nrms The normalized rms results for this variable. ''' import warnings # Get the weighted values from the yearly average file slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False) # Get the region mask slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False) # Since weights and region mask are only one level, we need to expand them to all levels region_mask = MA.expand_dims(slev_mask, axis=0) weights = MA.expand_dims(slev_weights, axis=0) for lev in range(1,nlev): new_region_mask = MA.expand_dims(slev_mask, axis=0) region_mask = np.vstack((region_mask,new_region_mask)) new_weights = MA.expand_dims(slev_weights, axis=0) weights = np.vstack((weights,new_weights)) # Calculate the weighted average # First, we need to reshape the arrays to average along two dims if (reg_name == 'Glo'): temp_mask = MA.masked_where(region_mask<=int(reg_num),avg_test_slice) else: temp_mask = MA.masked_where(region_mask!=int(reg_num),avg_test_slice) ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1) weights_flattened = weights.reshape(weights.shape[0],-1) warnings.filterwarnings("ignore") rms_Ave = MA.sqrt(MA.average((ma_to_average*ma_to_average), axis=1, weights=weights_flattened)) warnings.filterwarnings("default") #nrms = rms_Ave/(MA.max(rms_Ave) - MA.min(rms_Ave)) nrms = rms_Ave return nrms
def weighted_hor_avg_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict,nlev): ''' Computes the weighted hor mean rms diff for a year @param var The name of the variable that is being averaged. @reg_name The name of the region to average over. @reg_num The number of the region in the region_mask. @mask_var The name of the netCDF variable that contain the region mask. @wgt_var The name of the netCDF variable that contains the weight information. @param year The year to average over. @param hist_dict A dictionary that holds file references for all years/months. @param nlev Number of ocean vertical levels @param ave_info A dictionary of the type of average that is to be done. Includes: type, months_to_average, fn, and weights (weights are not used in this function/average) @param file_dict A dictionary which holds file pointers to the input files that are needed by this average calculation. @return var_Ave The averaged results for this variable across the designated time frame. ''' # Get correct data slice from the yearly average file var_val = rover.fetch_slice(hist_dict,year,0,var,file_dict) # Get the weighted values from the yearly average file slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False) # Get the region mask slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False) # Since weights and region mask are only one level, we need to expand them to all levels region_mask = MA.expand_dims(slev_mask, axis=0) weights = MA.expand_dims(slev_weights, axis=0) if var_val.ndim > 2: for lev in range(1,nlev): new_region_mask = MA.expand_dims(slev_mask, axis=0) region_mask = np.vstack((region_mask,new_region_mask)) new_weights = MA.expand_dims(slev_weights, axis=0) weights = np.vstack((weights,new_weights)) else: region_mask = np.squeeze(region_mask,axis=0) # Calculate the weighted average # First, we need to reshape the arrays to average along two dims if (reg_name == 'Glo'): temp_mask = MA.masked_where(region_mask<=int(reg_num),var_val) else: temp_mask = MA.masked_where(region_mask!=int(reg_num),var_val) ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1) if var_val.ndim > 2: weights_flattened = weights.reshape(weights.shape[0],-1) else: weights_flattened = np.squeeze(weights,axis=0) var_Ave = MA.average(ma_to_average,axis=1, weights=weights_flattened) return np.array(var_Ave)
def load_GPM_IMERG_files(file_path=None, filename_pattern=None, filelist=None, variable_name='precipitationCal', name='GPM_IMERG'): ''' Load multiple GPM Level 3 IMEGE files containing calibrated \ precipitation and generate an OCW Dataset obejct. :param file_path: Directory to the HDF files to load. :type file_path: :mod:`string` :param filename_pattern: Path to the HDF files to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the HDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :returns: An OCW Dataset object with the requested variable's data from \ the HDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: GPM_files = [] for pattern in filename_pattern: GPM_files.extend(glob(file_path + pattern)) else: GPM_files = [line.rstrip('\n') for line in open(filelist)] GPM_files.sort() file_object_first = h5py.File(GPM_files[0]) lats = file_object_first['Grid']['lat'][:] lons = file_object_first['Grid']['lon'][:] lons, lats = numpy.meshgrid(lons, lats) variable_unit = "mm/hr" times = [] nfile = len(GPM_files) for ifile, file in enumerate(GPM_files): print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = h5py.File(file) time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S") times.append(datetime(*time_struct_parsed[:6])) values0 = ma.transpose(ma.masked_less( file_object['Grid'][variable_name][:], 0.)) values0 = ma.expand_dims(values0, axis=0) if ifile == 0: values = values0 else: values = ma.concatenate((values, values0)) file_object.close() times = numpy.array(times) return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
def resample_from_array(in_raster=None, in_affine=None, out_tile=None, in_crs=None, resampling="nearest", nodataval=None, nodata=0): """ Extract and resample from array to target tile. Parameters ---------- in_raster : array in_affine : ``Affine`` out_tile : ``BufferedTile`` resampling : string one of rasterio's resampling methods (default: nearest) nodata : integer or float raster nodata value (default: 0) Returns ------- resampled array : array """ if nodataval is not None: warnings.warn("'nodataval' is deprecated, please use 'nodata'") nodata = nodata or nodataval # TODO rename function if isinstance(in_raster, ma.MaskedArray): pass elif isinstance(in_raster, np.ndarray): in_raster = ma.MaskedArray(in_raster, mask=in_raster == nodata) elif isinstance(in_raster, ReferencedRaster): in_affine = in_raster.affine in_crs = in_raster.crs in_raster = in_raster.data elif isinstance(in_raster, tuple): in_raster = ma.MaskedArray( data=np.stack(in_raster), mask=np.stack([ band.mask if isinstance(band, ma.masked_array) else np.where( band == nodata, True, False) for band in in_raster ]), fill_value=nodata) else: raise TypeError("wrong input data type: %s" % type(in_raster)) if in_raster.ndim == 2: in_raster = ma.expand_dims(in_raster, axis=0) elif in_raster.ndim == 3: pass else: raise TypeError("input array must have 2 or 3 dimensions") if in_raster.fill_value != nodata: ma.set_fill_value(in_raster, nodata) dst_data = np.empty((in_raster.shape[0], ) + out_tile.shape, in_raster.dtype) logger.debug(in_raster) logger.debug(in_affine) logger.debug(out_tile.affine) reproject(in_raster.filled(), dst_data, src_transform=in_affine, src_crs=in_crs or out_tile.crs, src_nodata=nodata, dst_transform=out_tile.affine, dst_crs=out_tile.crs, dst_nodata=nodata, resampling=Resampling[resampling]) logger.debug(dst_data) hanse = ma.MaskedArray(dst_data, mask=dst_data == nodata, fill_value=nodata) logger.debug(hanse) return hanse
def load_GPM_IMERG_files_with_spatial_filter(file_path=None, filename_pattern=None, filelist=None, variable_name='precipitationCal', user_mask_file=None, mask_variable_name='mask', user_mask_values=[10], longitude_name='lon', latitude_name='lat'): ''' Load multiple GPM Level 3 IMEGE files containing calibrated \ precipitation and generate a two-dimensional array \ for the masked grid points. :param file_path: Directory to the HDF files to load. :type file_path: :mod:`string` :param filename_pattern: Path to the HDF files to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the HDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :user_mask_file: user's own gridded mask file(a netCDF file name) :type name: :mod:`string` :mask_variable_name: mask variables in user_mask_file :type name: :mod:`string` :longitude_name: longitude variable in user_mask_file :type name: :mod:`string` :latitude_name: latitude variable in user_mask_file :type name: :mod:`string` :param user_mask_values: grid points where mask_variable == user_mask_value will be extracted. :type user_mask_values: list of strings :returns: A two-dimensional array with the requested variable's MASKED data from \ the HDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: GPM_files = [] for pattern in filename_pattern: GPM_files.extend(glob(file_path + pattern)) else: GPM_files = [line.rstrip('\n') for line in open(filelist)] GPM_files.sort() file_object_first = h5py.File(GPM_files[0]) lats = file_object_first['Grid']['lat'][:] lons = file_object_first['Grid']['lon'][:] lons, lats = numpy.meshgrid(lons, lats) nfile = len(GPM_files) for ifile, file in enumerate(GPM_files): if ifile == 0 and user_mask_file: file_object = netCDF4.Dataset(user_mask_file) mask_variable = file_object.variables[mask_variable_name][:] mask_longitude = file_object.variables[longitude_name][:] mask_latitude = file_object.variables[latitude_name][:] spatial_mask = utils.regrid_spatial_mask(lons,lats, mask_longitude, mask_latitude, mask_variable, user_mask_values) y_index, x_index = numpy.where(spatial_mask == 0) print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = h5py.File(file) values0 = ma.transpose(ma.masked_less( file_object['Grid'][variable_name][:], 0.)) values_masked = values0[y_index, x_index] values_masked = ma.expand_dims(values_masked, axis=0) if ifile == 0: values = values_masked else: values = ma.concatenate((values, values_masked)) file_object.close() return values
def weighted_hor_avg_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict): ''' Computes the weighted hor mean rms diff for a year @param var The name of the variable that is being averaged. @reg_name The name of the region to average over. @reg_num The number of the region in the region_mask. @mask_var The name of the netCDF variable that contain the region mask. @wgt_var The name of the netCDF variable that contains the weight information. @param year The year to average over. @param hist_dict A dictionary that holds file references for all years/months. @param ave_info A dictionary of the type of average that is to be done. Includes: type, months_to_average, fn, and weights (weights are not used in this function/average) @param file_dict A dictionary which holds file pointers to the input files that are needed by this average calculation. @return var_Ave The averaged results for this variable across the designated time frame. ''' # Get correct data slice from the yearly average file var_val = rover.fetch_slice(hist_dict,year,0,var,file_dict) # Get the weighted values from the yearly average file slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False) # Get the region mask slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False) # Since weights and region mask are only one level, we need to expand them to all levels region_mask = MA.expand_dims(slev_mask, axis=0) weights = MA.expand_dims(slev_weights, axis=0) if var_val.ndim > 2: for lev in range(1,60): new_region_mask = MA.expand_dims(slev_mask, axis=0) region_mask = np.vstack((region_mask,new_region_mask)) new_weights = MA.expand_dims(slev_weights, axis=0) weights = np.vstack((weights,new_weights)) else: region_mask = np.squeeze(region_mask,axis=0) # Calculate the weighted average # First, we need to reshape the arrays to average along two dims if (reg_name == 'Glo'): temp_mask = MA.masked_where(region_mask<=int(reg_num),var_val) else: temp_mask = MA.masked_where(region_mask!=int(reg_num),var_val) ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1) if var_val.ndim > 2: weights_flattened = weights.reshape(weights.shape[0],-1) else: weights_flattened = np.squeeze(weights,axis=0) var_Ave = MA.average(ma_to_average,axis=1, weights=weights_flattened) return np.array(var_Ave)
def weighted_rms_var_from_yr(var,reg_name,reg_num,mask_var,wgt_var,year,hist_dict,ave_info,file_dict,avg_test_slice,obs_file): ''' Computes the weighted rms for a year @param var The name of the variable that is being averaged. @reg_name The name of the region to average over. @reg_num The number of the region in the region_mask. @mask_var The name of the netCDF variable that contain the region mask. @wgt_var The name of the netCDF variable that contains the weight information. @param year The year to average over. @param hist_dict A dictionary that holds file references for all years/months. @param ave_info A dictionary of the type of average that is to be done. Includes: type, months_to_average, fn, and weights (weights are not used in this function/average) @param file_dict A dictionary which holds file pointers to the input files that are needed by this average calculation. @param avg_test_slice Averaged slice used in this calculation. @param obs_file Observation file that contains the values to be used in the caluculation. @return nrms The normalized rms results for this variable. ''' import warnings # Get the weighted values from the yearly average file slev_weights = rover.fetch_slice(hist_dict,year,0,wgt_var,file_dict,time=False) # Get the region mask slev_mask = rover.fetch_slice(hist_dict,year,0,mask_var,file_dict,time=False) # Since weights and region mask are only one level, we need to expand them to all levels region_mask = MA.expand_dims(slev_mask, axis=0) weights = MA.expand_dims(slev_weights, axis=0) for lev in range(1,60): new_region_mask = MA.expand_dims(slev_mask, axis=0) region_mask = np.vstack((region_mask,new_region_mask)) new_weights = MA.expand_dims(slev_weights, axis=0) weights = np.vstack((weights,new_weights)) # Calculate the weighted average # First, we need to reshape the arrays to average along two dims if (reg_name == 'Glo'): temp_mask = MA.masked_where(region_mask<=int(reg_num),avg_test_slice) else: temp_mask = MA.masked_where(region_mask!=int(reg_num),avg_test_slice) ma_to_average = temp_mask.reshape(temp_mask.shape[0], -1) weights_flattened = weights.reshape(weights.shape[0],-1) warnings.filterwarnings("ignore") rms_Ave = MA.sqrt(MA.average((ma_to_average*ma_to_average), axis=1, weights=weights_flattened)) warnings.filterwarnings("default") #nrms = rms_Ave/(MA.max(rms_Ave) - MA.min(rms_Ave)) nrms = rms_Ave return nrms
def interpolate(self, new_fps: int = None, kind='cubic'): if new_fps is None: new_fps = self.fps _frames = self.data.shape[0] if _frames == 1: raise ValueError("Can't interpolate single frame") _new_frames = round(_frames * new_fps / self.fps) steps = np.linspace(0, 1, _frames) new_steps = np.linspace(0, 1, _new_frames) transposed = self.points_perspective() # (points, people, frames, dims) masked_confidence = ma.array(self.confidence, mask=self.confidence == 0) confidence = ma.expand_dims(masked_confidence.transpose(), axis=3) # (points, people, frames, 1) points = ma.concatenate([transposed, confidence], axis=3) new_people = [] for people in points: new_frames = [] for frames in people: mask = frames.transpose()[0].mask partial_steps = ma.array(steps, mask=mask).compressed() if partial_steps.shape[0] == 0: # No data for this point new_frames.append(np.zeros((_new_frames, frames.shape[1]))) else: partial_frames = frames.compressed().reshape(partial_steps.shape[0], frames.shape[1]) if len(partial_steps) == 1: f = lambda l: partial_frames else: this_kind = kind if len(partial_steps) > 3 \ else "quadratic" if len(partial_steps) > 2 and kind == "cubic" \ else "linear" # Can't do something fancy for 2 points f = interp1d(partial_steps, partial_frames, axis=0, kind=this_kind) first_step = partial_steps[0] last_step = partial_steps[-1] if first_step == 0 and last_step == 1: new_frames.append(f(new_steps)) else: first_step_where = np.argwhere(new_steps >= first_step) first_step_index = first_step_where[0][0] if len(first_step_where) > 0 else 0 last_step_where = np.argwhere(new_steps > last_step) last_step_index = last_step_where[0][0] if len(last_step_where) > 0 else len(new_steps) if first_step_index == last_step_index: new_frames.append(np.zeros((len(new_steps), frames.shape[1]))) else: frame_data = f(new_steps[first_step_index:last_step_index]) new_frames.append(np.concatenate([ np.zeros((first_step_index, frames.shape[1])), np.array(frame_data), np.zeros((len(new_steps) - last_step_index, frames.shape[1])) ])) new_people.append(np.stack(new_frames, axis=0)) new_data = np.stack(new_people, axis=0).transpose([2, 1, 0, 3]) dimensions, confidence = np.split(new_data, [-1], axis=3) confidence = np.squeeze(confidence, axis=3) mask = confidence == 0 stacked_confidence = np.stack([mask, mask], axis=3) masked_data = ma.masked_array(dimensions, mask=stacked_confidence) return NumPyPoseBody(fps=new_fps, data=masked_data, confidence=confidence)
def load_GPM_IMERG_files(file_path=None, filename_pattern=None, filelist=None, variable_name='precipitationCal', name='GPM_IMERG'): ''' Load multiple GPM Level 3 IMEGE files containing calibrated \ precipitation and generate an OCW Dataset obejct. :param file_path: Directory to the HDF files to load. :type file_path: :mod:`string` :param filename_pattern: Path to the HDF files to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the HDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :returns: An OCW Dataset object with the requested variable's data from \ the HDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: GPM_files = [] for pattern in filename_pattern: GPM_files.extend(glob(file_path + pattern)) else: GPM_files = [line.rstrip('\n') for line in open(filelist)] GPM_files.sort() file_object_first = h5py.File(GPM_files[0]) lats = file_object_first['Grid']['lat'][:] lons = file_object_first['Grid']['lon'][:] lons, lats = numpy.meshgrid(lons, lats) variable_unit = "mm/hr" times = [] nfile = len(GPM_files) for ifile, file in enumerate(GPM_files): print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = h5py.File(file) time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S") times.append(datetime(*time_struct_parsed[:6])) values0 = ma.transpose( ma.masked_less(file_object['Grid'][variable_name][:], 0.)) values0 = ma.expand_dims(values0, axis=0) if ifile == 0: values = values0 else: values = ma.concatenate((values, values0)) file_object.close() times = numpy.array(times) return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
def load_GPM_IMERG_files_with_spatial_filter(file_path=None, filename_pattern=None, filelist=None, variable_name='precipitationCal', user_mask_file=None, mask_variable_name='mask', user_mask_values=[10], longitude_name='lon', latitude_name='lat'): ''' Load multiple GPM Level 3 IMEGE files containing calibrated \ precipitation and generate a two-dimensional array \ for the masked grid points. :param file_path: Directory to the HDF files to load. :type file_path: :mod:`string` :param filename_pattern: Path to the HDF files to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the HDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :user_mask_file: user's own gridded mask file(a netCDF file name) :type name: :mod:`string` :mask_variable_name: mask variables in user_mask_file :type name: :mod:`string` :longitude_name: longitude variable in user_mask_file :type name: :mod:`string` :latitude_name: latitude variable in user_mask_file :type name: :mod:`string` :param user_mask_values: grid points where mask_variable == user_mask_value will be extracted. :type user_mask_values: list of strings :returns: A two-dimensional array with the requested variable's MASKED data from \ the HDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: GPM_files = [] for pattern in filename_pattern: GPM_files.extend(glob(file_path + pattern)) else: GPM_files = [line.rstrip('\n') for line in open(filelist)] GPM_files.sort() file_object_first = h5py.File(GPM_files[0]) lats = file_object_first['Grid']['lat'][:] lons = file_object_first['Grid']['lon'][:] lons, lats = numpy.meshgrid(lons, lats) nfile = len(GPM_files) for ifile, file in enumerate(GPM_files): if ifile == 0 and user_mask_file: file_object = netCDF4.Dataset(user_mask_file) mask_variable = file_object.variables[mask_variable_name][:] mask_longitude = file_object.variables[longitude_name][:] mask_latitude = file_object.variables[latitude_name][:] spatial_mask = utils.regrid_spatial_mask(lons, lats, mask_longitude, mask_latitude, mask_variable, user_mask_values) y_index, x_index = numpy.where(spatial_mask == 0) print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = h5py.File(file) values0 = ma.transpose( ma.masked_less(file_object['Grid'][variable_name][:], 0.)) values_masked = values0[y_index, x_index] values_masked = ma.expand_dims(values_masked, axis=0) if ifile == 0: values = values_masked else: values = ma.concatenate((values, values_masked)) file_object.close() return values
def getSlicedArray(self, copy=True): """ Slice the rti using a tuple of slices made from the values of the combo and spin boxes. :param copy: If True (the default), a copy is made so that inspectors cannot accidentally modify the underlying of the RTIs. You can set copy=False as a potential optimization, but only if you are absolutely sure that you don't modify the the slicedArray in your inspector! Note that this function calls transpose, which can still make a copy of the array for certain permutations. :return: ArrayWithMask array with the same number of dimension as the number of comboboxes (this can be zero!). Returns None if no slice can be made (i.e. the RTI is not sliceable). :rtype ArrayWithMask: """ #logger.debug("getSlicedArray() called") if not self._rti: self.sigShowMessage.emit("No item selected.") return None if not self.rtiIsSliceable: # This is very common so we don't show a message so the user isn't flooded. # Also this can have many different causes (compound data, lists, etc_ so it's # difficult to show a good, descriptive message. return None if np.prod(self._rti.arrayShape) == 0: self.sigShowMessage.emit("Selected item has zero array elements.") return None # The dimensions that are selected in the combo boxes will be set to slice(None), # the values from the spin boxes will be set as a single integer value nDims = self.rti.nDims sliceList = [slice(None)] * nDims for spinBox in self._spinBoxes: dimNr = spinBox.property("dim_nr") sliceList[dimNr] = spinBox.value() # Make the array slicer. It needs to be a tuple, a list of only integers will be # interpreted as an index. With a tuple, array[(exp1, exp2, ..., expN)] is equivalent to # array[exp1, exp2, ..., expN]. # See: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html slicedArray = self.rti[tuple(sliceList)] # Make a copy to prevent inspectors from modifying the underlying array. if copy: if versionStrToTuple(np.__version__) >= (1, 19, 0): slicedArray = np.copy(slicedArray, subok=True) # Fixes issue #8 else: slicedArray = ma.copy(slicedArray) # If there are no comboboxes the sliceList will contain no Slices objects, only ints. Then # the resulting slicedArray will be a usually a scalar (only structured fields may yield an # array). We convert this scalar to a zero-dimensional Numpy array so that inspectors # always get an array (having the same number of dimensions as the dimensionality of the # inspector, i.e. the number of comboboxes). # Also scalar RTIs, which have nDim == 0, can return a scalar which must be converted. # TODO: perhaps always convert to array. if self.maxCombos == 0 or self.rti.nDims == 0: slicedArray = ma.MaskedArray(slicedArray) # Post-condition type check check_is_an_array(slicedArray, np.ndarray) # Enforce the return type to be a masked array. if not isinstance(slicedArray, ma.MaskedArray): slicedArray = ma.MaskedArray(slicedArray) # Add fake dimensions of length 1 so that result.ndim will equal the number of combo boxes # TODO: Perhaps get rid of this because it fails with masked arrays with fill values. # The less we do here, the less chance an error occurs. See development/todo.txt for dimNr in range(slicedArray.ndim, self.maxCombos): #logger.debug("Adding fake dimension: {}".format(dimNr)) slicedArray = ma.expand_dims(slicedArray, dimNr) # Post-condition dimension check assert slicedArray.ndim == self.maxCombos, \ "Bug: getSlicedArray should return a {:d}D array, got: {}D" \ .format(self.maxCombos, slicedArray.ndim) # Convert to ArrayWithMask class for working around issues with the numpy maskedarray awm = ArrayWithMask.createFromMaskedArray(slicedArray) del slicedArray # Shuffle the dimensions to be in the order as specified by the combo boxes comboDims = [ self._comboBoxDimensionIndex(cb) for cb in self._comboBoxes ] permutations = np.argsort(comboDims) logger.debug("slicedArray.shape: {}".format(awm.data.shape)) logger.debug("Transposing dimensions: {}".format(permutations)) awm = awm.transpose(permutations) awm.checkIsConsistent() return awm
def getSlicedArray(self, copy=True): """ Slice the rti using a tuple of slices made from the values of the combo and spin boxes. :param copy: If True (the default), a copy is made so that inspectors cannot accidentally modify the underlying of the RTIs. You can set copy=False as a potential optimization, but only if you are absolutely sure that you don't modify the the slicedArray in your inspector! Note that this function calls transpose, which can still make a copy of the array for certain permutations. :return: Numpy masked array with the same number of dimension as the number of comboboxes (this can be zero!). Returns None if no slice can be made (i.e. the RTI is not sliceable). """ #logger.debug("getSlicedArray() called") if not self.rtiIsSliceable: return None # The dimensions that are selected in the combo boxes will be set to slice(None), # the values from the spin boxes will be set as a single integer value nDims = self.rti.nDims sliceList = [slice(None)] * nDims for spinBox in self._spinBoxes: dimNr = spinBox.property("dim_nr") sliceList[dimNr] = spinBox.value() # Make the array slicer. It needs to be a tuple, a list of only integers will be # interpreted as an index. With a tuple, array[(exp1, exp2, ..., expN)] is equivalent to # array[exp1, exp2, ..., expN]. # See: http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html logger.debug("Array slice list: {}".format(str(sliceList))) slicedArray = self.rti[tuple(sliceList)] # Make a copy to prevent inspectors from modifying the underlying array. if copy: slicedArray = ma.copy(slicedArray) # If there are no comboboxes the sliceList will contain no Slices objects, only ints. Then # the resulting slicedArray will be a usually a scalar (only structured fields may yield an # array). We convert this scalar to a zero-dimensional Numpy array so that inspectors # always get an array (having the same number of dimensions as the dimensionality of the # inspector, i.e. the number of comboboxes). if self.maxCombos == 0: slicedArray = ma.MaskedArray(slicedArray) # Post-condition type check check_is_an_array(slicedArray, np.ndarray) # Enforce the return type to be a masked array. if not isinstance(slicedArray, ma.MaskedArray): slicedArray = ma.MaskedArray(slicedArray) # Add fake dimensions of length 1 so that result.ndim will equal the number of combo boxes for dimNr in range(slicedArray.ndim, self.maxCombos): #logger.debug("Adding fake dimension: {}".format(dimNr)) slicedArray = ma.expand_dims(slicedArray, dimNr) # Post-condition dimension check assert slicedArray.ndim == self.maxCombos, \ "Bug: getSlicedArray should return a {:d}D array, got: {}D" \ .format(self.maxCombos, slicedArray.ndim) # Convert to ArrayWithMask class for working around issues with the numpy maskedarray awm = ArrayWithMask.createFromMaskedArray(slicedArray) del slicedArray # Shuffle the dimensions to be in the order as specified by the combo boxes comboDims = [self._comboBoxDimensionIndex(cb) for cb in self._comboBoxes] permutations = np.argsort(comboDims) logger.debug("slicedArray.shape: {}".format(awm.data.shape)) logger.debug("Transposing dimensions: {}".format(permutations)) awm = awm.transpose(permutations) awm.checkIsConsistent() return awm
import numpy as np import numpy.ma as ma x = ma.array([1, 2, 4]) x[1] = ma.masked x np.expand_dims(x, axis=0) ma.expand_dims(x, axis=0) x[np.newaxis, :]