def plotHeatMap(self): import numpy.ma as ma import numpy dx, dy = self.root.height, 0 fx, fy = self.root.height/len(self.data.domain.attributes), 1.0 data, c, w = self.data.toNumpyMA() data = (data - ma.min(data))/(ma.max(data) - ma.min(data)) x = numpy.arange(data.shape[1] + 1)/float(numpy.max(data.shape)) y = numpy.arange(data.shape[0] + 1)/float(numpy.max(data.shape))*len(self.root) self.heatmap_width = numpy.max(x) X, Y = numpy.meshgrid(x, y - 0.5) self.meshXOffset = numpy.max(X) self.plt.jet() mesh = self.plt.pcolormesh(X, Y, data[self.root.mapping], edgecolor="b", linewidth=2) if self.plot_attr_names: names = [attr.name for attr in self.data.domain.attributes] self.plt.xticks(numpy.arange(data.shape[1] + 1)/float(numpy.max(data.shape)), names) self.plt.gca().xaxis.tick_top() for label in self.plt.gca().xaxis.get_ticklabels(): label.set_rotation(45) for tick in self.plt.gca().xaxis.get_major_ticks(): tick.tick1On = False tick.tick2On = False
def update_ranges(self, x_data, y_data): s_x = ma.mean(x_data) s_y = ma.mean(y_data) bounds = [ ma.min(x_data) - s_x, ma.max(x_data) + s_x, ma.min(y_data) - s_y, ma.max(y_data) + s_y ] plt.axis(bounds)
def bbox(self, header: PoseHeader): data = ma.transpose(self.data, axes=POINTS_DIMS) # Split data by components, `ma` doesn't support ".split" components = [] idx = 0 for component in header.components: components.append(data[list(range(idx, idx + len(component.points)))]) idx += len(component.points) boxes = [ ma.stack([ma.min(c, axis=0), ma.max(c, axis=0)]) for c in components ] boxes_cat = ma.concatenate(boxes) if type(boxes_cat.mask ) == np.bool_: # Sometimes, it doesn't concatenate the mask... boxes_mask = ma.concatenate([b.mask for b in boxes]) boxes_cat = ma.array(boxes_cat, mask=boxes_mask) new_data = ma.transpose(boxes_cat, axes=POINTS_DIMS) confidence_mask = np.split(new_data.mask, [-1], axis=3)[0] confidence_mask = np.squeeze(confidence_mask, axis=-1) confidence = np.where(confidence_mask == True, 0, 1) return NumPyPoseBody(self.fps, new_data, confidence)
def autoscale(self, A): ''' Set *vmin*, *vmax* to min, max of *A*. ''' A = ma.masked_less_equal(A, 0, copy=False) self.vmin = ma.min(A) self.vmax = ma.max(A)
def autoscale_None(self, A): if self.vmin is not None and self.vmax is not None: return if self.vmin is None: self.vmin = ma.min(A) if self.vmax is None: self.vmax = ma.max(A)
def train_step_batch(self, epoch): D1 = ma.dot(self.vectors**2, self.weight_matrix) D2 = ma.dot(self.vectors, self.constant_matrix) Dist = D1 - D2 best_nodes = ma.argmin(Dist, 0) distances = ma.min(Dist, 0) ## print "q error:", ma.mean(ma.sqrt(distances + self.dist_cons)), self.radius(epoch) self.qerror.append(ma.mean(ma.sqrt(distances + self.dist_cons))) if self.neighbourhood == Map.NeighbourhoodGaussian: H = numpy.exp(-self.unit_distances / (2 * self.radius(epoch))) * ( self.unit_distances <= self.radius(epoch)) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: H = 1.0 - (self.unit_distances / self.radius(epoch))**2 H = H * (H >= 0.0) else: H = 1.0 * (self.unit_distances <= self.radius(epoch)) P = numpy.zeros((self.vectors.shape[0], self.data.shape[0])) P[(best_nodes, list(range(len(best_nodes))))] = numpy.ones(len(best_nodes)) S = ma.dot(H, ma.dot(P, self.data)) A = ma.dot(H, ma.dot(P, ~self.data._mask)) ## nonzero = (range(epoch%2, len(self.vectors), 2), ) nonzero = (numpy.array(sorted(set(ma.nonzero(A)[0]))), ) self.vectors[nonzero] = S[nonzero] / A[nonzero]
def _pivot_col(T, tol=1.0E-12, bland=False): """ Given a linear programming simplex tableau, determine the column of the variable to enter the basis. Parameters ---------- T : 2D ndarray The simplex tableau. tol : float Elements in the objective row larger than -tol will not be considered for pivoting. Nominally this value is zero, but numerical issues cause a tolerance about zero to be necessary. bland : bool If True, use Bland's rule for selection of the column (select the first column with a negative coefficient in the objective row, regardless of magnitude). Returns ------- status: bool True if a suitable pivot column was found, otherwise False. A return of False indicates that the linear programming simplex algorithm is complete. col: int The index of the column of the pivot element. If status is False, col will be returned as nan. """ ma = np.ma.masked_where(T[-1, :-1] >= -tol, T[-1, :-1], copy=False) if ma.count() == 0: return False, np.nan if bland: return True, np.where(ma.mask == False)[0][0] return True, np.ma.where(ma == ma.min())[0][0]
def computeAveragesUsingNumpy(): global sizeX, sizeY, sizeZ flattenedArrays = [] for fileName in fileNames: fpath = os.path.join(basepath, fileName) print('processing %s' % fpath) year = fileName.split('_')[-1][:-4] dataset = gdal.Open(fpath) sumArray = ma.zeros((dataset.RasterYSize, dataset.RasterXSize)) total = 0 count = 0 numBands = dataset.RasterCount for bandId in range(numBands): band = ma.masked_outside(dataset.GetRasterBand(bandId + 1).ReadAsArray(), VALUE_RANGE[0], VALUE_RANGE[1]) sumArray += band sumArray /= numBands total = ma.sum(ma.sum(sumArray)) count = sumArray.count() minCell = ma.min(sumArray) maxCell = ma.max(sumArray) sizeX = dataset.RasterXSize sizeY = dataset.RasterYSize flattenedArrays.append(np.ndarray.flatten(sumArray[::-1,:], 0).astype(np.dtype(np.int32))) sizeZ = len(flattenedArrays) return np.ma.concatenate(flattenedArrays)
def train_step_sequential(self, epoch, indices=None): """A single step of sequential training algorithm. """ indices = range(len(self.data)) if indices is None else indices for ind in indices: x = self.data[ind] Dx = self.vectors - self.data[ind] Dist = ma.sum(Dx**2, 1) min_dist = ma.min(Dist) bmu = ma.argmin(Dist) self.distances.append(min_dist) iter = epoch * len(self.data) + ind if self.neighbourhood == Map.NeighbourhoodGaussian: h = numpy.exp(-self.unit_distances[:, bmu]**2 / (2 * self.radius_seq(iter)**2)) * ( self.unit_distances[:, bmu]**2 <= self.radius_seq(iter)**2) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: h = 1.0 - (self.unit_distances[:bmu] / self.radius_seq(iter))**2 h = h * (h >= 0.0) else: h = 1.0 * (self.unit_distances[:, bmu] <= self.radius_seq(iter)) h = h * self.alpha(iter) nonzero = ma.nonzero(h) h = h[nonzero] self.vectors[nonzero] = self.vectors[ nonzero] - Dx[nonzero] * numpy.reshape(h, (len(h), 1))
def autoscale(self, A): """ Set *vmin*, *vmax* to min, max of *A*. """ self.vmin = ma.min(A) self.vmax = ma.max(A) self._transform_vmin_vmax()
def train_step_sequential(self, epoch, indices=None): """A single step of sequential training algorithm. """ indices = range(len(self.data)) if indices == None else indices for ind in indices: x = self.data[ind] Dx = self.vectors - self.data[ind] Dist = ma.sum(Dx**2, 1) min_dist = ma.min(Dist) bmu = ma.argmin(Dist) self.distances.append(min_dist) iter = epoch*len(self.data)+ind if self.neighbourhood == Map.NeighbourhoodGaussian: h = numpy.exp(-self.unit_distances[:, bmu]**2/(2*self.radius_seq(iter)**2)) * (self.unit_distances[:, bmu]**2 <= self.radius_seq(iter)**2) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: h = 1.0 - (self.unit_distances[:bmu]/self.radius_seq(iter))**2 h = h * (h >= 0.0) else: h = 1.0*(self.unit_distances[:, bmu] <= self.radius_seq(iter)) h = h * self.alpha(iter) nonzero = ma.nonzero(h) h = h[nonzero] self.vectors[nonzero] = self.vectors[nonzero] - Dx[nonzero] * numpy.reshape(h, (len(h), 1))
def train_step_batch(self, epoch): """A single step of batch training algorithm. """ D1 = ma.dot(self.vectors**2, self.weight_matrix) D2 = ma.dot(self.vectors, self.constant_matrix) Dist = D1 - D2 best_nodes = ma.argmin(Dist, 0) distances = ma.min(Dist, 0) ## print "q error:", ma.mean(ma.sqrt(distances + self.dist_cons)), self.radius(epoch) self.qerror.append(ma.mean(ma.sqrt(distances + self.dist_cons))) if self.neighbourhood == Map.NeighbourhoodGaussian: H = numpy.exp(-self.unit_distances**2/(2*self.radius(epoch)**2)) * (self.unit_distances**2 <= self.radius(epoch)**2) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: H = 1.0 - (self.unit_distances/self.radius(epoch))**2 H = H * (H >= 0.0) else: H = 1.0*(self.unit_distances <= self.radius(epoch)) P = numpy.zeros((self.vectors.shape[0], self.data.shape[0])) P[(best_nodes, range(len(best_nodes)))] = numpy.ones(len(best_nodes)) S = ma.dot(H, ma.dot(P, self.data)) A = ma.dot(H, ma.dot(P, ~self.data._mask)) ## nonzero = (range(epoch%2, len(self.vectors), 2), ) nonzero = (numpy.array(sorted(set(ma.nonzero(A)[0]))), ) self.vectors[nonzero] = S[nonzero] / A[nonzero]
def train_step_sequential(self, epoch, indices=None): indices = list(range(len(self.data))) if indices == None else indices for ind in indices: x = self.data[ind] Dx = self.vectors - self.data[ind] Dist = ma.sum(Dx**2, 1) min_dist = ma.min(Dist) bmu = ma.argmin(Dist) self.distances.append(min_dist) if self.neighbourhood == Map.NeighbourhoodGaussian: h = numpy.exp( -self.unit_distances[:, bmu] / (2 * self.radius(epoch))) * (self.unit_distances[:, bmu] <= self.radius(epoch)) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: h = 1.0 - (self.unit_distances[:bmu] / self.radius(epoch))**2 h = h * (h >= 0.0) else: h = 1.0 * (self.unit_distances[:, bmu] <= self.radius(epoch)) h = h * self.alpha(epoch) nonzero = ma.nonzero(h) h = h[nonzero] self.vectors[nonzero] = self.vectors[ nonzero] - Dx[nonzero] * numpy.reshape(h, (len(h), 1))
def min(self): ''' return minimum unmasked value in the data ''' res = ma.min(self.data) return res
def test_variable_values(self): assert 900 < self.nc.variables["wavelength"][:] < 1065 assert 0 <= self.nc.variables["zenith_angle"][:] < 90 assert np.all( (self.nc.variables["height"][:] - self.site_meta["altitude"] - self.nc.variables["range"][:]) <= 1e-3) assert ma.min(self.nc.variables["beta"][:]) > 0
def min_(self): """ Calculates the minimum of the image over the segmentation :return: """ return ma.min(self.masked_img, 0)
def statistics(numpy_array): return {'mean' : ma.mean(numpy_array), 'median' : ma.median(numpy_array.real)+1j*ma.median(numpy_array.imag), 'max' : ma.max(abs(array)), 'min' : ma.min(abs(array)), 'std' : ma.std(array), 'stdmean': ma.std(numpy_array)/sqrt(sum(logical_not(numpy_array.mask))-1)}
def _attvalues(attribute, stacked): """Attribute values computed in numpy.ma stack.""" if attribute == "max": attvalues = ma.max(stacked, axis=2) elif attribute == "min": attvalues = ma.min(stacked, axis=2) elif attribute == "rms": attvalues = np.sqrt(ma.mean(np.square(stacked), axis=2)) elif attribute == "var": attvalues = ma.var(stacked, axis=2) elif attribute == "mean": attvalues = ma.mean(stacked, axis=2) elif attribute == "maxpos": stacked = ma.masked_less(stacked, 0.0, copy=True) attvalues = ma.max(stacked, axis=2) elif attribute == "maxneg": # ~ minimum of negative values? stacked = ma.masked_greater_equal(stacked, 0.0, copy=True) attvalues = ma.min(stacked, axis=2) elif attribute == "maxabs": attvalues = ma.max(abs(stacked), axis=2) elif attribute == "sumpos": stacked = ma.masked_less(stacked, 0.0, copy=True) attvalues = ma.sum(stacked, axis=2) elif attribute == "sumneg": stacked = ma.masked_greater_equal(stacked, 0.0, copy=True) attvalues = ma.sum(stacked, axis=2) elif attribute == "sumabs": attvalues = ma.sum(abs(stacked), axis=2) elif attribute == "meanabs": attvalues = ma.mean(abs(stacked), axis=2) elif attribute == "meanpos": stacked = ma.masked_less(stacked, 0.0, copy=True) attvalues = ma.mean(stacked, axis=2) elif attribute == "meanneg": stacked = ma.masked_greater_equal(stacked, 0.0, copy=True) attvalues = ma.mean(stacked, axis=2) else: etxt = "Invalid attribute applied: {}".format(attribute) raise ValueError(etxt) if not attvalues.flags["C_CONTIGUOUS"]: mask = ma.getmaskarray(attvalues) mask = np.asanyarray(mask, order="C") attvalues = np.asanyarray(attvalues, order="C") attvalues = ma.array(attvalues, mask=mask, order="C") return attvalues
def getchanindex(self, chan): """ """ if ma.max(self._chans) >= chan >= ma.min(self._chans): return ma.where(self._chans == int(chan))[0][0] else: return -1
def autoscale_None(self, A): ' autoscale only None-valued vmin or vmax' if self.vmin is None: self.vmin = ma.min(A) if self.vmax is None: self.vmax = ma.max(A) if self.vmid is None: self.vmid = (self.vmax + self.vmin) / 2.0
def autoscale_None(self, A): ' autoscale only None-valued vmin or vmax' if self.vmin is None and np.size(A) > 0: self.vmin = ma.min(A) if self.vmax is None and np.size(A) > 0: self.vmax = ma.max(A) if self.vcenter is None: self.vcenter = (self.vmax + self.vmin) * 0.5
def autoscale_None(self, A): ' autoscale only None-valued vmin or vmax' if self.vmin is None: self.vmin = ma.min(A) if self.vmax is None: self.vmax = ma.max(A) if self.vmid is None: self.vmid = (self.vmax+self.vmin)/2.0
def autoscale(self, A): ''' Set *vmin*, *vmax* to min, max of *A*. ''' A = ma.masked_less_equal(np.abs(A), 1e-16, copy=False) self.vmin = -ma.max(A) self.vmax = ma.max(A) self.vin = ma.min(A)
def calc_chrom_fast(self, index, coords_vals): self.population[index]['fitness'] = \ np.abs(self.array_mean - ma.mean(coords_vals[0])) + \ np.abs(self.array_stdev - ma.std(coords_vals[0])) + \ np.abs(self.array_range - (ma.max(coords_vals[0])-ma.min(coords_vals[0])))/10 + \ np.abs((self.chromosome_size-1) - coords_vals[2]) #locations #~ print "Chromosome size: ",self.chromosome_size print "Number of locations is: ", coords_vals[2]
def normalizado(c): min = np.min(c) max = np.max(c) new = (c - min) / (max - min) OldRange = (max - min) NewRange = (1 - 0.1) new = (((c - min) * NewRange) / OldRange) + 0.1 return new
def autoscale_None(self, A): ' autoscale only None-valued vmin or vmax' if self.vmin is None and np.size(A) > 0: self.vmin = ma.min(A) if self.vmax is None and np.size(A) > 0: self.vmax = ma.max(A) self._transform_vmin_vmax()
def create_graph(self, iv=None, dv=None, my_data=None, result=None): try: self.fig.delaxes(self.ax) except AttributeError: pass if len(iv) == 1: # plot iff we have a single independent variable self.ax = self.fig.add_subplot(111) self.ax.set_xlabel(iv[0]['name']) self.ax.set_ylabel(dv[0]['name']) axis_type = self.axis_type.GetString(self.axis_type.GetSelection()) # Type of axis if axis_type == 'Fit All': self.ax.set_xlim(iv[0]['min'], iv[0]['max']) self.ax.set_ylim(dv[0]['min'], dv[0]['max']) elif axis_type == 'Zoom': self.plot_zoom_graph(iv[0]['min'], iv[0]['max'], dv[0]['min'], dv[0]['max']) self.zoom.set_bounds_absolute(iv[0]['min'], iv[0]['max'], dv[0]['min'], dv[0]['max']) self.ax.grid() dv_plot_data = my_data[0] iv_plot_data = my_data[1] plot_type = self.plot_type.GetString(self.plot_type.GetSelection()) if plot_type == 'Scatterplot': # adjust marker size and alpha based on how many points we're plotting marker_size = mpl.rcParams['lines.markersize']**2 marker_size *= min(1, max(.12, 200 / len(iv_plot_data))) alpha = min(1, max(.002, 500 / len(iv_plot_data))) self.ax.scatter(iv_plot_data, dv_plot_data, s=marker_size, alpha=alpha) else: # heatmap bins = 200 heatmap, iv_edges, dv_edges = np.histogram2d(iv_plot_data, dv_plot_data, bins=bins) x_min, x_max = iv_edges[0], iv_edges[-1] y_min, y_max = dv_edges[0], dv_edges[-1] self.ax.imshow(np.log(heatmap.transpose() + 1), extent=[x_min, x_max, y_min, y_max], cmap='Blues', origin='lower', aspect='auto') # plot regression line extent = [ma.min(iv_plot_data), ma.max(iv_plot_data)] intercept, slope = result.params[0:2] self.ax.plot(extent, [intercept + slope * x for x in extent], 'r--') self.fig.tight_layout() self.canvas.draw()
def _compute_variable_stats(variable, axis, weights, calc_avg, calc_min, calc_max, calc_stddev, calc_count): ''' Calculate statistics for a single variable. ''' # Get the data out. Note: scale_factor and add_offset are automatically # applied. data = variable[:] # Make sure data is a masked array data = ma.masked_array(data) # Broadcast the weights before we try to combine the masks for data and # weights weights = ma.masked_array(data=np.broadcast_to(weights.data, data.shape), mask=np.broadcast_to(ma.getmaskarray(weights), data.shape)) # We want all our calculations to happen over areas that are unmasked in # both the weights and data combined_mask = np.logical_or(ma.getmaskarray(data), ma.getmaskarray(weights)) data = ma.masked_array(data.data, mask=combined_mask) weights = ma.masked_array(weights.data, mask=combined_mask) out = {} if calc_count: # Irritatingly, the ma.count function can only take one value at a time # for the axis. So, instead, construct an array of ones ones = np.ones(data.shape) # Set the masked areas to 0 ones[combined_mask] = 0 out["count"] = ma.sum(ones, axis=axis) if calc_min: out["min"] = ma.min(data, axis=axis) if calc_max: out["max"] = ma.max(data, axis=axis) # Note: standard deviation needs the weighted average and the weights sum if calc_avg or calc_stddev: sum_weights = _add_axes_back(ma.sum(weights, axis=axis), axis) weighted_avg_numerator = _add_axes_back( ma.sum(weights * data, axis=axis), axis) weighted_avg = weighted_avg_numerator / sum_weights if calc_avg: out["avg"] = ma.squeeze(weighted_avg, axis=axis) if calc_stddev: # calculate the anomaly anomaly = data - weighted_avg # calculate the standard deviation variance = ma.sum(weights * (anomaly**2) / sum_weights, axis=axis) out["stddev"] = np.sqrt(variance) return out
def autoscale_None(self, A): ' autoscale only None-valued vmin or vmax' if self.vmin is not None and self.vmax is not None: return A = ma.masked_less_equal(A, 0, copy=False) if self.vmin is None: self.vmin = ma.min(A) if self.vmax is None: self.vmax = ma.max(A)
def autoscale_None(self, A): """ autoscale only None-valued vmin or vmax """ if self.vmin is not None and self.vmax is not None: pass if self.vmin is None: self.vmin = ma.min(A) if self.vmax is None: self.vmax = ma.max(A) self._transform_vmin_vmax()
def custom_range_for_CNN(r4_array, min_max, mean_centre=False): """ Rescale a rank 4 array so that each channel's image lies in custom range e.g. input with range of (-5, 15) is rescaled to (-125 125) or (-1 1) for use with VGG16. Designed for use with masked arrays. Inputs: r4_array | r4 masked array | works with masked arrays? min_max | dict | 'min' and 'max' of range desired as a dictionary. mean_centre | boolean | if True, each image's channels are mean centered. Returns: r4_array | rank 4 numpy array | masked items are set to zero, rescaled so that each channel for each image lies between min_max limits. History: 2019/03/20 | now includes mean centering so doesn't stretch data to custom range. Instead only stretches until either min or max touches, whilst mean is kept at 0 2020/11/02 | MEG | Update so range can have a min and max, and not just a range 2021/01/06 | MEG | Upate to work with masked arrays. Not test with normal arrays. """ import numpy as np import numpy.ma as ma if mean_centre: im_channel_means = ma.mean( r4_array, axis=(1, 2)) # get the average for each image (in all thre channels) im_channel_means = expand_to_r4(im_channel_means, r4_array[ 0, :, :, 0].shape) # expand to r4 so we can do elementwise manipulation r4_array -= im_channel_means # do mean centering im_channel_min = ma.min( r4_array, axis=(1, 2)) # get the minimum of each image and each of its channels im_channel_min = expand_to_r4( im_channel_min, r4_array[0, :, :, 0].shape) # exapnd to rank 4 for elementwise applications r4_array -= im_channel_min # set so lowest channel for each image is 0 im_channel_max = ma.max( r4_array, axis=(1, 2)) # get the maximum of each image and each of its channels im_channel_max = expand_to_r4( im_channel_max, r4_array[0, :, :, 0].shape) # make suitable for elementwise applications r4_array /= im_channel_max # should now be in range [0, 1] r4_array *= (min_max['max'] - min_max['min'] ) # should now be in range [0, new max-min] r4_array += min_max['min'] # and now in range [new min, new max] r4_nparray = r4_array.filled( fill_value=0 ) # convert to numpy array, maksed incoherent areas are set to zero. return r4_nparray
def initialize_map_random(self, data=None, dimension=5): """ Initialize the map nodes vectors randomly, by supplying either training data or dimension of the data """ if data is not None: min, max = ma.min(data, 0), ma.max(data, 0) dimension = data.shape[1] else: min, max = numpy.zeros(dimension), numpy.ones(dimension) for node in self: node.vector = min + numpy.random.rand(dimension) * (max - min)
def get_mask(self): self.array_mean = ma.mean(self.array) self.array_stdev = ma.std(self.array) self.array_range = ma.max(self.array) - ma.min(self.array) print "The mean is %f, the stdev is %f, the range is %f." %(self.array_mean, self.array_stdev, self.array_range) from scipy.io.netcdf import netcdf_file as NetCDFFile ### get landmask nc = NetCDFFile(os.getcwd()+ '/../data/netcdf_files/ORCA2_landmask.nc','r') self.mask = ma.masked_values(nc.variables['MASK'][:, :self.time_len, :self.lat_len, :180], -9.99999979e+33) nc.close() self.xxx, self.yyy, self.zzz = np.lib.index_tricks.mgrid[0:self.time_len, 0:self.lat_len, 0:180]
def plot(self, mask=None): """ Plot the isotope, with desired mask. :param mask: Mask to be used. :type mask: numpy bool array """ # Get plot data if type(mask) is np.ndarray: plot_data = ma.masked_array(self._data, mask=mask) vmin = ma.min(plot_data) vmax = ma.max(plot_data) else: plot_data = self._data vmin = np.min(plot_data) vmax = np.max(plot_data) x = range(np.shape(plot_data)[1]) y = range(np.shape(plot_data)[2]) z_max = np.shape(plot_data)[0] X, Y = np.meshgrid(x, y) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') for z in range(z_max): x_y_data = plot_data[z, :, :] contour = ax.contourf(X, Y, x_y_data, zdir='z', offset=-z, alpha=1, cmap="CMRmap", vmin=vmin, vmax=vmax) cbar = fig.colorbar(contour, ax=ax) cbar.ax.set_xlabel("Counts") ax.set_zlim((-z_max, 0)) ax.set_zlim((-z_max, 0)) ax.set_xlabel("X (px)") ax.set_ylabel("Y (px)") ax.set_zlabel("Planes") ax.set_aspect('equal') ax.invert_xaxis() ax.view_init(elev=45, azim=45) values = "Min value: " + str(vmin) + " Max value: " + str(vmax) fig.text(x=0.2, y=0.02, s=values) plt.show()
def autoscale_None(self, A): ' autoscale only None-valued vmin or vmax' if self.vmin is None and np.size(A) > 0: self.vmin = ma.min(A) if self.vmin < 0: self.vmin = 0 warnings.warn("Power-law scaling on negative values is " "ill-defined, clamping to 0.") if self.vmax is None and np.size(A) > 0: self.vmax = ma.max(A)
def autoscale(self, A): """ Set *vmin*, *vmax* to min, max of *A*. """ self.vmin = ma.min(A) if self.vmin < 0: self.vmin = 0 warnings.warn("Power-law scaling on negative values is " "ill-defined, clamping to 0.") self.vmax = ma.max(A)
def __call__(self, value): if self.vmin is None: self.vmin = ma.min(value) self.vmax = ma.max(value) result = ma.array(value).astype(np.float) # ma division is slow, take a shortcut resdat = result.data resdat -= self.vmin resdat /= (self.vmax - self.vmin) # remask result = ma.array(resdat, mask=result.mask, copy=False) return result
def initialize_map_random(self, data=None, dimension=5): """Initialize the map nodes vectors randomly, by supplying either training data or dimension of the data. """ if data is not None: min, max = ma.min(data, 0), ma.max(data, 0) dimension = data.shape[1] else: min, max = numpy.zeros(dimension), numpy.ones(dimension) for node in self: # node.vector = min + numpy.random.rand(dimension) * (max - min) node.vector = min + random.randint(0, dimension) * (max - min)
def ol_normalize(x): """ Normalize layer based on occurrence levels in the array. The value of each element is divided by the sum off all elements. Input must be a numpy ndarray, no coercion is tried. :param x: numpy ndarray to be rescaled. :return: numpy ndarray with transformed values. """ if type(x) is not np.ndarray and type(x) is not ma.core.MaskedArray: raise TypeError("x must be a numpy.ndarray or numpy.ma.MaskedArray") min_val = ma.min(x) return (x - min_val) / (ma.sum(x - min_val))
def crossmatch(cat0, cat1, threshold=1., racol0='ra', deccol0='dec', racol1='ra', deccol1='dec', right_join=False): dra = cat0[racol0] - cat1[racol1][:, newaxis] ddec = cat0[deccol0] - cat1[deccol1][:, newaxis] sep = np.sqrt(dra**2 + ddec**2) * 3600. matches = np.min(sep, axis=1) < threshold inds = np.argmin(sep, axis=1) out = Table(cat0[inds], masked=True) if right_join: for col in out.colnames: out[col].mask = ~matches cat1 = cat1.copy() else: out = out[matches] cat1 = cat1[matches] return out, cat1
def normalize(x): """ Rescale all numeric values in range [0, 1]. Input must be a numpy ndarray, no coercion is tried. :param x: numpy ndarray to be rescaled. :return: numpy ndarray with rescaled values. """ if type(x) is not np.ndarray and type(x) is not ma.core.MaskedArray: raise TypeError("x must be a numpy.ndarray or numpy.ma.MaskedArray") # NOTE: the approach commented out below would be more memory efficient, # but doesn't work as such with masked arrays # np.true_divide(x, np.max(np.abs(x)), out=x, casting='unsafe') # Data may have negative values, thus first add the abs(min(x)) to # everything. x_min = ma.min(x) x_max = ma.max(x) return (x - x_min) / (x_max - x_min)
def regridToCoarse(fine,fac,mode,missValue): nr,nc = np.shape(fine) coarse = np.zeros(nr/fac * nc / fac).reshape(nr/fac,nc/fac) + MV nr,nc = np.shape(coarse) for r in range(0,nr): for c in range(0,nc): ar = fine[r * fac : fac * (r+1),c * fac: fac * (c+1)] m = np.ma.masked_values(ar,missValue) if ma.count(m) == 0: coarse[r,c] = MV else: if mode == 'average': coarse [r,c] = ma.average(m) elif mode == 'median': coarse [r,c] = ma.median(m) elif mode == 'sum': coarse [r,c] = ma.sum(m) elif mode =='min': coarse [r,c] = ma.min(m) elif mode == 'max': coarse [r,c] = ma.max(m) return coarse
nc.close() data_cflux_5day = ma.masked_values(data_cflux_5day, masked_value) data_cflux_5day = ma.array(data_cflux_5day, dtype=np.float32) data_name = data_cflux_5day_name data = data_cflux_5day*unit_changer year_stack = np.split(data, 10) year_stack = ma.array(year_stack) print "Year stack has shape: ", np.shape(year_stack) decadal_mean = ma.mean(data, 0) dec_mean = ma.mean(decadal_mean) dec_stdev = ma.std(decadal_mean) dec_range = np.abs(ma.max(decadal_mean) - ma.min(decadal_mean)) samples = [] for item in coords_and_values: samples.append(decadal_mean[ item[1], item[2]]) samples = ma.array(samples) samples_mean = ma.mean(samples) samples_stdev = ma.std(samples) samples_range = np.abs(ma.max(samples) - ma.min(samples)) original_fitness = np.abs(dec_mean - samples_mean) + np.abs(dec_stdev - samples_stdev) \ + np.abs(dec_range - samples_range) #~ x = 0 year_sample_dict_data = {} year_sample_list_data = {}
def generate_plots(fname="/tmp/bitstead.hdf5", figbase='/tmp/figures'): with h5py.File("/tmp/bitstead.hdf5") as f: for fieldname,grp in f.iteritems(): years = grp.keys() years.sort() print fieldname ylds = [] profits = [] for year in years: print '\t%s' % year for commodity in grp[year]: print '\t\t%s' % commodity data = grp[year][commodity]['yield'] xmin = data.attrs['xmin'] ymin = data.attrs['ymin'] xmax = data.attrs['xmax'] ymax = data.attrs['ymax'] stride = data.attrs['stride'] map_extents = [xmin,xmax,ymin,ymax] # Map data mask = data['mask'].value == 0 yld = ma.masked_less(ma.masked_invalid(ma.masked_array(data['yield'], mask)), 0) moisture = ma.masked_array(data['moisture'], mask) elevation = ma.masked_array(data['elevation'], mask) # Areas grid_cell_area = (stride ** 2) / 4046.86 areas = ma.masked_array(np.ones(mask.shape), mask) * grid_cell_area # Convert square meters to acres # Calculating profit/loss costs = get_inputs(fieldname, year, commodity, areas) * areas income = yld * areas * get_market_price(fieldname, year, commodity) profit = income - costs ppa = profit/areas # Normalized yield ynorm = (yld - ma.mean(yld)) / ma.std(yld) # For summary info ylds.append(ynorm) profits.append(ppa) try: # Mask (for outline of field) # fig = plt.figure() # plt.ticklabel_format(useOffset=False, axis='y') # plt.ticklabel_format(useOffset=False, axis='x') # plt.imshow(mask, extent=map_extents) # fig.savefig('%s/%s-%s-%s-mask.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Yield Map fig = plt.figure() plt.title('%s %s Yield (Bu/Acre)' % (year, commodity) ) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(yld),ma.max(yld),10), cmap.N) plt.gca().axis('off') plot_grid_map(yld, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-yield-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Yield Histogram fig = plt.figure() plt.title('%s %s Yield Distribution' % (year, commodity) ) a = np.ravel(ma.compressed(yld)) Y,X = np.histogram(a, 10, normed=0, weights=(np.ones(a.shape) * grid_cell_area)) x_span = X.max()-X.min() C = [cmap(((x-X.min())/x_span)) for x in X] plt.bar(X[:-1],Y,color=C,width=X[1]-X[0]) plt.xlabel('Yield (Bushel/Acre)') plt.ylabel('Acres') fig.savefig('%s/%s-%s-%s-yield-histogram.pdf' %(figbase, fieldname, year, commodity), format='pdf') # Normalized Yield Map fig = plt.figure() plt.title('%s %s Yield (Bu/Acre) Normalized' % (year, commodity) ) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(-5,5,11), cmap.N) plot_grid_map(ynorm, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-yield-map-normalized.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Input Costs Map # fig = plt.figure() # plt.title('%s %s Costs ($/Acre)' % (year, commodity) ) # cmap = mpl.cm.get_cmap('RdYlGn') # norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(costs/areas),ma.max(costs/areas),10), cmap.N) # plot_grid_map(costs/areas, cmap=cmap, norm=norm, interpolation='none') # plt.colorbar() # fig.savefig('%s/%s-%s-%s-costs-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Profit/Loss Map fig = plt.figure() plt.title('%s %s Profit/Loss ($/Acre)' % (year, commodity) ) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(ppa),ma.max(ppa),10), cmap.N) plot_grid_map(ppa, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-profit-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Profit/Loss Histogram fig = plt.figure() plt.title('%s %s Profit/Loss Distribution' % (year, commodity) ) a = np.ravel(ma.compressed(profit / areas)) Y,X = np.histogram(a, 10, normed=0, weights=(np.ones(a.shape) * grid_cell_area)) def pmap(x): if x < 0: return cmap(X.min()) else: return cmap(X.max()) x_span = X.max()-X.min() C = [pmap(x) for x in X] plt.bar(X[:-1],Y,color=C,width=X[1]-X[0]) plt.xlabel('Profit/Loss ($/Acre)') plt.ylabel('Acres') fig.savefig('%s/%s-%s-%s-profit-histogram.pdf' %(figbase, fieldname, year, commodity), format='pdf') # A map distinguishing what is (or is not) profitable. fig = plt.figure() plot_grid_map(profit > 0, cmap=cmap) plt.title('%s %s Profit or Loss' % (year,commodity)) fig.savefig('%s/%s-%s-%s-profit-or-loss.pdf' %(figbase, fieldname, year, commodity), format='pdf') except Exception as e: print e print yld.shape print ma.min(yld) print ma.max(yld) print '%s-%s' % (np.min(yld), np.max(yld)) print 'Could not generate maps for %s/%s/%s' % (fieldname, year, commodity) # raise e # Field summary stats # First, Profits. vals = ma.dstack(profits) mvals = ma.mean(vals, axis=2) vvars = ma.std(vals, axis=2) fig = plt.figure() plt.title('Profit, mean across years ($/Acre)') norm = mpl.colors.BoundaryNorm(np.linspace(mvals.min(),mvals.max(),10), cmap.N) plot_grid_map(mvals, cmap=cmap, norm=norm) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-mean.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Profit, std deviation across years') norm = mpl.colors.BoundaryNorm(np.linspace(vvars.min(),vvars.max(),10), cmap.N) plot_grid_map(vvars, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-std.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Profit, max across years ($/Acre)') ymax = ma.max(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymax.min(),ymax.max(),10), cmap.N) plot_grid_map(ymax, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-max.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Profit, min across years ($/Acre)') ymin = ma.min(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymin.min(),ymin.max(),10), cmap.N) plot_grid_map(ymin, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-min.pdf' % (figbase, fieldname)) # Now yields vals = ma.dstack(ylds) mvals = ma.mean(vals, axis=2) vvars = ma.std(vals, axis=2) fig = plt.figure() plt.title('Normalized Yield, mean across years') norm = mpl.colors.BoundaryNorm(np.linspace(mvals.min(),mvals.max(),10), cmap.N) plot_grid_map(mvals, cmap=cmap, norm=norm) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-mean.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, std deviation across years') norm = mpl.colors.BoundaryNorm(np.linspace(vvars.min(),vvars.max(),10), cmap.N) plot_grid_map(vvars, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-std.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, max across years') ymax = ma.max(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymax.min(),ymax.max(),10), cmap.N) plot_grid_map(ymax, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-max.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, min across years') ymin = ma.min(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymin.min(),ymin.max(),10), cmap.N) plot_grid_map(ymin, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-min.pdf' % (figbase, fieldname)) # Log the number of years of data that we have. fig = plt.figure() plt.title('Number of years of data') cnt = ma.count(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(0,cnt.max(),2*(cnt.max()+1)), cmap.N) plot_grid_map(cnt, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-number-years-map.pdf' % (figbase, fieldname))
def measure(mode, x, y, x0, x1, thresh=0, slopewin=1.0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1)# .compressed() ym = ma.array(y, mask = ma.getmask(xm))# .compressed() if mode == 'mean': r1 = np.mean(ym) r2 = np.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'minormax': r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == '1090': #measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1*r1 y90 = 0.9*r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': slope = [] win = ma.flatnotmasked_contiguous(ym) dt = x[1]-x[0] st = int(slopewin/dt) # use slopewin duration window for fit. print('st: ', st) for k, w in enumerate(win): # move through the slope measurementwindow tb = range(k-st, k+st) # get tb array ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope.append(ppars[0]) # keep track of max slope r1 = np.max(slope) r2 = np.argmax(slope) return(r1, r2)
def autoscale(self, A): ''' Set *vmin*, *vmax* to min, max of *A*. ''' self.vmin = ma.min(A) self.vmax = ma.max(A)
def autoscale_None(self, A): " autoscale only None-valued vmin or vmax" if self.vmin is None: self.vmin = ma.min(A) if self.vmax is None: self.vmax = ma.max(A)
def measure(mode, x, y, x0, x1, thresh=0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1) # .compressed() ym = ma.array(y, mask=ma.getmask(xm)) # .compressed() if mode == "mean": r1 = np.mean(ym) r2 = np.std(ym) if mode == "max" or mode == "maximum": r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == "min" or mode == "minimum": r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == "minormax": r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == "median": r1 = ma.median(ym) r2 = 0 if mode == "p2p": # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == "std": # standard deviation r1 = ma.std(ym) r2 = 0 if mode == "var": # variance r1 = ma.var(ym) r2 = 0 if mode == "cumsum": # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == "anom": # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == "sum": r1 = ma.sum(ym) r2 = 0 if mode == "area" or mode == "charge": r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm)) r2 = 0 if mode == "latency": # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == "1090": # measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1 * r1 y90 = 0.9 * r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == "count": r1 = ma.count(ym) r2 = 0 if mode == "maxslope": return (0, 0) slope = np.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win) / 20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k - st, k + st) # get tb array newa = np.array(self.dat[i][j, thisaxis, tb]) ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = np.append(slope, ppars[0]) # keep track of max slope r1 = np.amax(slope) r2 = np.argmax(slope) return (r1, r2)
def measure(mode, x, y, x0, x1, thresh = 0): """ return the a measure of y in the window x0 to x1 """ xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster! v = y.view(numpy.ndarray) xm = ma.masked_outside(xt, x0, x1).T ym = ma.array(v, mask = ma.getmask(xm)) if mode == 'mean': r1 = ma.mean(ym) r2 = ma.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': return(0,0) slope = numpy.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win)/20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k-st, k+st) # get tb array newa = numpy.array(self.dat[i][j, thisaxis, tb]) ppars = numpy.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = numpy.append(slope, ppars[0]) # keep track of max slope r1 = numpy.amax(slope) r2 = numpy.argmax(slope) return(r1, r2)