def concatenate(self,value,axis=0): """ Concatentate UncertContainer value to self. Assumes that if dimensions of self and value do not match, to add a np.newaxis along axis of value """ if isinstance(value,UncertContainer): if value.vals.ndim == self.vals.ndim: vals = value.vals dmin = value.dmin dmax = value.dmax wt = value.wt uncert = value.uncert mask = value.mask elif (value.vals.ndim + 1) == self.vals.ndim: vals = ma.expand_dims(value.vals,axis) dmin = ma.expand_dims(value.dmin,axis) dmax = ma.expand_dims(value.dmax,axis) wt = ma.expand_dims(value.wt,axis) uncert = ma.expand_dims(value.uncert,axis) mask = np.expand_dims(value.mask,axis) else: raise ValueError('Could not propery match dimensionality') self.vals = ma.concatenate((self.vals,vals),axis=axis) self.dmin = ma.concatenate((self.dmin,dmin),axis=axis) self.dmax = ma.concatenate((self.dmax,dmax),axis=axis) self.wt = ma.concatenate((self.wt,wt),axis=axis) self.uncert = ma.concatenate((self.uncert,uncert),axis=axis) self.mask = np.concatenate((self.mask,mask),axis=axis) else: raise ValueError('Can only concatenate with an UncertContainer object')
def test_testUfuncs1(self): # Test various functions such as sin, cos. (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d assert_(eq(np.cos(x), cos(xm))) assert_(eq(np.cosh(x), cosh(xm))) assert_(eq(np.sin(x), sin(xm))) assert_(eq(np.sinh(x), sinh(xm))) assert_(eq(np.tan(x), tan(xm))) assert_(eq(np.tanh(x), tanh(xm))) with np.errstate(divide='ignore', invalid='ignore'): assert_(eq(np.sqrt(abs(x)), sqrt(xm))) assert_(eq(np.log(abs(x)), log(xm))) assert_(eq(np.log10(abs(x)), log10(xm))) assert_(eq(np.exp(x), exp(xm))) assert_(eq(np.arcsin(z), arcsin(zm))) assert_(eq(np.arccos(z), arccos(zm))) assert_(eq(np.arctan(z), arctan(zm))) assert_(eq(np.arctan2(x, y), arctan2(xm, ym))) assert_(eq(np.absolute(x), absolute(xm))) assert_(eq(np.equal(x, y), equal(xm, ym))) assert_(eq(np.not_equal(x, y), not_equal(xm, ym))) assert_(eq(np.less(x, y), less(xm, ym))) assert_(eq(np.greater(x, y), greater(xm, ym))) assert_(eq(np.less_equal(x, y), less_equal(xm, ym))) assert_(eq(np.greater_equal(x, y), greater_equal(xm, ym))) assert_(eq(np.conjugate(x), conjugate(xm))) assert_(eq(np.concatenate((x, y)), concatenate((xm, ym)))) assert_(eq(np.concatenate((x, y)), concatenate((x, y)))) assert_(eq(np.concatenate((x, y)), concatenate((xm, y)))) assert_(eq(np.concatenate((x, y, x)), concatenate((x, ym, x))))
def computeStatistics(self): dates = [] values = [] for ts in self.itervalues(): dates.append(ts.dates) values.append(ts.values) dates = concatenate(dates).astype(int) values = ma.concatenate(values) minDate = dates.min() nDates = dates.max() - minDate + 1 hist = histogram(dates*(-values.mask), bins=arange(nDates)+minDate) values = values.compressed() if values.size == 0: raise Exception("empty dataset") lowend = values.min()*0.99 highend = values.max()*1.01 #cumfreqs, lowlim, binsize, extrapoints = cumfreq(values, 40, (lowend, highend)) #normcumfreqs = cumfreqs/values.size #ind = ((normcumfreqs > 0.02) & (normcumfreqs < 0.98)).nonzero()[0] #if ind.size == 0: # raise Exception("empty dataset") #min = ind[0]*binsize + lowlim #max = ind[-1]*binsize + lowlim #return min, max, hist return values.min(),values.max(),hist
def add_solver(self, fname): # Reg exp: Any line starting (ignoring white-space) # starting with a comment character. Also col sep. comment = re.compile(r'[\s]*[%#]') column = re.compile(self.sep) # Grab the column from the file metrics = [] file = open(fname, 'r') for line in file.readlines(): if not comment.match(line): line = line.strip() cols = column.split( line ) data = atof(cols[opts.datacol - 1]) metrics.append(data) file.close() if self.metric is not None: self.metric = ma.concatenate((self.metric, [metrics])) else: self.metric = ma.array([metrics]) # Current num of probs grabbed nprobs = len(metrics) if not self.nprobs: self.nprobs = nprobs elif self.nprobs <> nprobs: commandline_error("All files must have same num of problems.")
def rect2sphere(vector, degree=True): """\ Convert vector (x,y,z) from rect to sphere coordinates. If degree is ``True`` the unit will be in degree. Examples -------- >>> convert.rect2sphere([1,1,1], degree=False) array([ 0.78539816, 0.61547971, 1.73205081]) >>> convert.rect2sphere(numpy.array([[1,2],[1,0],[1,3]]), degree=True) array([[ 45. , 0. ], [ 35.26438968, 56.30993247], [ 1.73205081, 3.60555128]]) """ x, y, z = vector r = np.sqrt(x**2 + y**2 + z**2) lon = np.arctan2(y,x) lat = np.arcsin(z/r) if degree: lon = np.rad2deg(lon) lat = np.rad2deg(lat) return ma.concatenate([ lon[np.newaxis], lat[np.newaxis], r[np.newaxis] ])
def _map_common(draw_method_name, arg_func, mode, cube, data, *args, **kwargs): """ Draw the given cube on a map using its points or bounds. "Mode" parameter will switch functionality between POINT or BOUND plotting. """ # get the 2d x and 2d y from the CS if mode == iris.coords.POINT_MODE: x, y = cartography.get_xy_grids(cube) else: try: x, y = cartography.get_xy_contiguous_bounded_grids(cube) # Exception translation. except iris.exceptions.CoordinateMultiDimError: raise ValueError("Could not get XY grid from bounds. " "X or Y coordinate not 1D.") except ValueError: raise ValueError("Could not get XY grid from bounds. " "X or Y coordinate doesn't have 2 bounds " "per point.") # take a copy of the data so that we can make modifications to it data = data.copy() # If we are global, then append the first column of data the array to the # last (and add 360 degrees) NOTE: if it is found that this block of code # is useful in anywhere other than this plotting routine, it may be better # placed in the CS. x_coord = cube.coord(axis="X") if getattr(x_coord, 'circular', False): _, direction = iris.util.monotonic(x_coord.points, return_direction=True) y = np.append(y, y[:, 0:1], axis=1) x = np.append(x, x[:, 0:1] + 360 * direction, axis=1) data = ma.concatenate([data, data[:, 0:1]], axis=1) # Replace non-cartopy subplot/axes with a cartopy alternative. cs = cube.coord_system('CoordSystem') if cs: cartopy_proj = cs.as_cartopy_projection() else: cartopy_proj = cartopy.crs.PlateCarree() ax = _get_cartopy_axes(cartopy_proj) draw_method = getattr(ax, draw_method_name) # Set the "from transform" keyword. # NB. While cartopy doesn't support spherical contours, just use the # projection as the source CRS. assert 'transform' not in kwargs, 'Transform keyword is not allowed.' kwargs['transform'] = cartopy_proj if arg_func is not None: new_args, kwargs = arg_func(x, y, data, *args, **kwargs) else: new_args = (x, y, data) + args # Draw the contour lines/filled contours. return draw_method(*new_args, **kwargs)
def _map_common(draw_method_name, arg_func, mode, cube, plot_defn, *args, **kwargs): """ Draw the given cube on a map using its points or bounds. "Mode" parameter will switch functionality between POINT or BOUND plotting. """ # Generate 2d x and 2d y grids. y_coord, x_coord = plot_defn.coords if mode == iris.coords.POINT_MODE: if x_coord.ndim == y_coord.ndim == 1: x, y = np.meshgrid(x_coord.points, y_coord.points) elif x_coord.ndim == y_coord.ndim == 2: x = x_coord.points y = y_coord.points else: raise ValueError("Expected 1D or 2D XY coords") else: try: x, y = np.meshgrid(x_coord.contiguous_bounds(), y_coord.contiguous_bounds()) # Exception translation. except iris.exceptions.CoordinateMultiDimError: raise ValueError("Could not get XY grid from bounds. " "X or Y coordinate not 1D.") except ValueError: raise ValueError("Could not get XY grid from bounds. " "X or Y coordinate doesn't have 2 bounds " "per point.") # Obtain the data array. data = cube.data if plot_defn.transpose: data = data.T # If we are global, then append the first column of data the array to the # last (and add 360 degrees) NOTE: if it is found that this block of code # is useful in anywhere other than this plotting routine, it may be better # placed in the CS. if getattr(x_coord, 'circular', False): _, direction = iris.util.monotonic(x_coord.points, return_direction=True) y = np.append(y, y[:, 0:1], axis=1) x = np.append(x, x[:, 0:1] + 360 * direction, axis=1) data = ma.concatenate([data, data[:, 0:1]], axis=1) # Replace non-cartopy subplot/axes with a cartopy alternative and set the # transform keyword. kwargs = _ensure_cartopy_axes_and_determine_kwargs(x_coord, y_coord, kwargs) if arg_func is not None: new_args, kwargs = arg_func(x, y, data, *args, **kwargs) else: new_args = (x, y, data) + args # Draw the contour lines/filled contours. return getattr(plt, draw_method_name)(*new_args, **kwargs)
def fix_pop_grid(tlon,tlat,data): """ Pad coordinates and data on CCSM/POP gx3v5 grid so it can be plotted with matplotlib/basemap tlon,tlat,data must be 2D arrays Inputs: tlon, tlat, data Outputs: lon, lat, data """ # make lon monotonic and pad coordinate and data arrays along lon axis tlon = N.where(N.greater_equal(tlon,tlon[:,0].min()),tlon-360,tlon) lon = N.concatenate((tlon,tlon+360,tlon+2*360),1) lat = N.concatenate((tlat,tlat,tlat),1) if hasattr(data,'mask'): data = MA.concatenate((data,data,data),-1) else: data = N.concatenate((data,data,data),-1) lon = lon - 360 # clip grid - this allows to clip map anywhere between -360 and 360 ind1 = N.searchsorted(lon[0,:],-360) ind2 = N.searchsorted(lon[0,:],360) lon = lon[:,ind1-1:ind2+1] lat = lat[:,ind1-1:ind2+1] data = data[...,ind1-1:ind2+1] return lon, lat, data
def extend_interp(datafield): # add masked values at southernmost end southernlimitmask = ma.masked_all(len(self.olon)) olat_ext = np.append(-82.1,self.olat) dfield_ext = ma.concatenate([ma.column_stack(southernlimitmask), datafield], 0) # f = interp2d(self.olon, olat_ext, dfield_ext) # return f(self.pismlon, self.pismlat) return interp(dfield_ext, self.olon, olat_ext, self.pismlon, self.pismlat)
def recache(self, always=False): if always or self._invalidx: xconv = self.convert_xunits(self._xorig) if ma.isMaskedArray(self._xorig): x = ma.asarray(xconv, np.float_) else: x = np.asarray(xconv, np.float_) x = x.ravel() else: x = self._x if always or self._invalidy: yconv = self.convert_yunits(self._yorig) if ma.isMaskedArray(self._yorig): y = ma.asarray(yconv, np.float_) else: y = np.asarray(yconv, np.float_) y = y.ravel() else: y = self._y if len(x) == 1 and len(y) > 1: x = x * np.ones(y.shape, np.float_) if len(y) == 1 and len(x) > 1: y = y * np.ones(x.shape, np.float_) if len(x) != len(y): raise RuntimeError("xdata and ydata must be the same length") x = x.reshape((len(x), 1)) y = y.reshape((len(y), 1)) if ma.isMaskedArray(x) or ma.isMaskedArray(y): self._xy = ma.concatenate((x, y), 1) else: self._xy = np.concatenate((x, y), 1) self._x = self._xy[:, 0] # just a view self._y = self._xy[:, 1] # just a view self._subslice = False if ( self.axes and len(x) > 100 and self._is_sorted(x) and self.axes.name == "rectilinear" and self.axes.get_xscale() == "linear" and self._markevery is None ): self._subslice = True if hasattr(self, "_path"): interpolation_steps = self._path._interpolation_steps else: interpolation_steps = 1 self._path = Path(self._xy, None, interpolation_steps) self._transformed_path = None self._invalidx = False self._invalidy = False
def test_flags2bin(n=100): flag = ma.concatenate([np.random.randint(0,5,n), ma.masked_all(2, dtype='int8')]) binflags = flags2bin(flag) assert type(binflags) == ma.MaskedArray assert binflags.dtype == 'bool' assert binflags.shape == (n+2,) assert binflags.mask[flag.mask].all(), \ "All masked flags records should be also masked at binflags"
def test_testCopySize(self): # Tests of some subtle points of copying and sizing. n = [0, 0, 1, 0, 0] m = make_mask(n) m2 = make_mask(m) assert_(m is m2) m3 = make_mask(m, copy=1) assert_(m is not m3) x1 = np.arange(5) y1 = array(x1, mask=m) assert_(y1._data is not x1) assert_(allequal(x1, y1._data)) assert_(y1._mask is m) y1a = array(y1, copy=0) # For copy=False, one might expect that the array would just # passed on, i.e., that it would be "is" instead of "==". # See gh-4043 for discussion. assert_(y1a._mask.__array_interface__ == y1._mask.__array_interface__) y2 = array(x1, mask=m3, copy=0) assert_(y2._mask is m3) assert_(y2[2] is masked) y2[2] = 9 assert_(y2[2] is not masked) assert_(y2._mask is m3) assert_(allequal(y2.mask, 0)) y2a = array(x1, mask=m, copy=1) assert_(y2a._mask is not m) assert_(y2a[2] is masked) y2a[2] = 9 assert_(y2a[2] is not masked) assert_(y2a._mask is not m) assert_(allequal(y2a.mask, 0)) y3 = array(x1 * 1.0, mask=m) assert_(filled(y3).dtype is (x1 * 1.0).dtype) x4 = arange(4) x4[2] = masked y4 = resize(x4, (8,)) assert_(eq(concatenate([x4, x4]), y4)) assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0])) y5 = repeat(x4, (2, 2, 2, 2), axis=0) assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3])) y6 = repeat(x4, 2, axis=0) assert_(eq(y5, y6))
def densitystep(S, T, P): """ """ assert S.shape == T.shape assert S.shape == P.shape try: import gsw rho0 = gsw.pot_rho_t_exact(S, T, P, 0) assert S.ndim == 1, "Not able to densitystep an array ndim > 1" ds = ma.concatenate([ma.masked_all(1), np.sign(np.diff(P))*np.diff(rho0)]) return ma.fix_invalid(ds) except ImportError: print("Package gsw is required and is not available.")
def addcyclic(data): """ Adds cyclic points to an array in rightmost dimension. data = input 2D array. """ if data.ndim != 2: print('ERROR: Input array is not two-dimensional') return if MA.isMA(data): newdata = MA.concatenate((data,data[:,0,N.newaxis]),axis=-1) else: newdata = N.concatenate((data,data[:,0,N.newaxis]),axis=-1) return newdata
def append(self, na): """ Append tableDict na to self. """ # for key in na.cols: # if key == 'keys': continue # if key in self.cols: # newDataVec = self.__getitem__(key) # for ele in na.data[key]: ##newDataVec.append(ele) # np.append(newDataVec, ele) # self.__setitem__(key, newDataVec) for col in self.cols: self.data[col] = ma.concatenate((self.data[col][:], na.data[col][:])) self.nRows = self.nRows + na.nRows
def fix_pop_grid_test(tlon,tlat,data): """ Pad coordinates and data on CCSM/POP gx3v5 grid so it can be plotted with matplotlib/basemap tlon,tlat,data must be 2D arrays """ # make lon monotonic and pad coordinate and data arrays along lon axis tlon = N.where(N.greater_equal(tlon,min(tlon[:,0])),tlon-360,tlon) lon = N.concatenate((tlon,tlon+360),1) lat = N.concatenate((tlat,tlat),1) if hasattr(data,'mask'): data = MA.concatenate((data,data),1) else: data = N.concatenate((data,data),1) return lon, lat, data
def test_i2b_flags(n=100): flag = ma.concatenate([np.random.randint(0,5,n), ma.masked_all(2, dtype='int8')]) binflags = i2b_flags(flag) assert type(binflags) == ma.MaskedArray assert binflags.dtype == 'bool' assert binflags.shape == (n+2,) assert binflags.mask[flag.mask].all(), \ "All masked flags records should be also masked at binflags" # FIXME: Improve this. Include cases with dict as input. # Check the output in case of dict input and differente combinations # of True/False, masked or not. assert (i2b_flags([1, 2, 3, 4]) == ma.array([True, True, False, False])).all()
def _make_verts(self, U, V): uv = ma.asarray(U+V*1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a/(self.scale*self.width) X, Y = self._h_arrows(length) # There seems to be a ma bug such that indexing # a masked array with one element converts it to # an ndarray. theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) xy = (X+Y*1j) * np.exp(1j*theta)*self.width xy = xy[:,:,np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def test_testCopySize(self): # Tests of some subtle points of copying and sizing. n = [0, 0, 1, 0, 0] m = make_mask(n) m2 = make_mask(m) assert_(m is m2) m3 = make_mask(m, copy=1) assert_(m is not m3) x1 = np.arange(5) y1 = array(x1, mask=m) assert_(y1._data is not x1) assert_(allequal(x1, y1._data)) assert_(y1.mask is m) y1a = array(y1, copy=0) assert_(y1a.mask is y1.mask) y2 = array(x1, mask=m3, copy=0) assert_(y2.mask is m3) assert_(y2[2] is masked) y2[2] = 9 assert_(y2[2] is not masked) assert_(y2.mask is m3) assert_(allequal(y2.mask, 0)) y2a = array(x1, mask=m, copy=1) assert_(y2a.mask is not m) assert_(y2a[2] is masked) y2a[2] = 9 assert_(y2a[2] is not masked) assert_(y2a.mask is not m) assert_(allequal(y2a.mask, 0)) y3 = array(x1 * 1.0, mask=m) assert_(filled(y3).dtype is (x1 * 1.0).dtype) x4 = arange(4) x4[2] = masked y4 = resize(x4, (8,)) assert_(eq(concatenate([x4, x4]), y4)) assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0])) y5 = repeat(x4, (2, 2, 2, 2), axis=0) assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3])) y6 = repeat(x4, 2, axis=0) assert_(eq(y5, y6))
def _build_data(self): """ Generate the data payload for the new concatenated cube. Returns: The concatenated :class:`iris.cube.Cube` data payload. """ skeletons = self._skeletons data = [skeleton.data for skeleton in skeletons] if self._data_is_masked: data = ma.concatenate(tuple(data), axis=self.axis) else: data = np.concatenate(tuple(data), axis=self.axis) return data
def _build_data(self): """ Generate the data payload for the new concatenated cube. Returns: The concatenated :class:`iris.cube.Cube` data payload. """ skeletons = self._skeletons data = [skeleton.data for skeleton in skeletons] if self._cube_signature.mdi is not None: # Preserve masked entries. data = ma.concatenate(tuple(data), axis=self.axis) else: data = np.concatenate(tuple(data), axis=self.axis) return data
def test_testAddSumProd(self): # Test add, sum, product. (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d assert_(eq(np.add.reduce(x), add.reduce(x))) assert_(eq(np.add.accumulate(x), add.accumulate(x))) assert_(eq(4, sum(array(4), axis=0))) assert_(eq(4, sum(array(4), axis=0))) assert_(eq(np.sum(x, axis=0), sum(x, axis=0))) assert_(eq(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0))) assert_(eq(np.sum(x, 0), sum(x, 0))) assert_(eq(np.product(x, axis=0), product(x, axis=0))) assert_(eq(np.product(x, 0), product(x, 0))) assert_(eq(np.product(filled(xm, 1), axis=0), product(xm, axis=0))) if len(s) > 1: assert_(eq(np.concatenate((x, y), 1), concatenate((xm, ym), 1))) assert_(eq(np.add.reduce(x, 1), add.reduce(x, 1))) assert_(eq(np.sum(x, 1), sum(x, 1))) assert_(eq(np.product(x, 1), product(x, 1)))
def read(self, indexes=None, **kwargs): """ Read reprojected & resampled input data. Parameters ---------- indexes : integer or list band number or list of band numbers Returns ------- data : array """ band_indexes = self._get_band_indexes(indexes) arr = self.process.get_raw_output(self.tile) return (arr[band_indexes[0] - 1] if len(band_indexes) == 1 else ma.concatenate( [ma.expand_dims(arr[i - 1], 0) for i in band_indexes]))
def test_testAddSumProd(self): # Test add, sum, product. (x, y, a10, m1, m2, xm, ym, z, zm, xf, s) = self.d self.assertTrue(eq(np.add.reduce(x), add.reduce(x))) self.assertTrue(eq(np.add.accumulate(x), add.accumulate(x))) self.assertTrue(eq(4, sum(array(4), axis=0))) self.assertTrue(eq(4, sum(array(4), axis=0))) self.assertTrue(eq(np.sum(x, axis=0), sum(x, axis=0))) self.assertTrue(eq(np.sum(filled(xm, 0), axis=0), sum(xm, axis=0))) self.assertTrue(eq(np.sum(x, 0), sum(x, 0))) self.assertTrue(eq(np.product(x, axis=0), product(x, axis=0))) self.assertTrue(eq(np.product(x, 0), product(x, 0))) self.assertTrue(eq(np.product(filled(xm, 1), axis=0), product(xm, axis=0))) if len(s) > 1: self.assertTrue(eq(np.concatenate((x, y), 1), concatenate((xm, ym), 1))) self.assertTrue(eq(np.add.reduce(x, 1), add.reduce(x, 1))) self.assertTrue(eq(np.sum(x, 1), sum(x, 1))) self.assertTrue(eq(np.product(x, 1), product(x, 1)))
def read_v0_0(cls, header: PoseHeader, reader: BufferReader): fps, _frames = reader.unpack(ConstStructs.double_ushort) _dims = max([len(c.format) for c in header.components]) - 1 _points = sum([len(c.points) for c in header.components]) frames_d = [] frames_c = [] for _ in range(_frames): _people = reader.unpack(ConstStructs.ushort) people_d = [] people_c = [] for pid in range(_people): reader.advance(ConstStructs.short) # Skip Person ID person_d = [] person_c = [] for component in header.components: points = np.array( reader.unpack_numpy( ConstStructs.float, (len(component.points), len(component.format)))) dimensions, confidence = np.split(points, [-1], axis=1) boolean_confidence = np.where(confidence > 0, 0, 1) # To create the mask mask = np.column_stack( tuple([boolean_confidence] * (len(component.format) - 1))) person_d.append(ma.masked_array(dimensions, mask=mask)) person_c.append(np.squeeze(confidence, axis=-1)) if pid == 0: people_d.append(ma.concatenate(person_d)) people_c.append(np.concatenate(person_c)) # In case no person, should all be zeros if len(people_d) == 0: people_d.append(np.zeros((_points, _dims))) people_c.append(np.zeros(_points)) frames_d.append(ma.stack(people_d)) frames_c.append(np.stack(people_c)) return cls(fps, ma.stack(frames_d), ma.stack(frames_c))
def test_testCopySize(self): # Tests of some subtle points of copying and sizing. with suppress_warnings() as sup: sup.filter( np.ma.core.MaskedArrayFutureWarning, "setting an item on a masked array which has a " "shared mask will not copy") n = [0, 0, 1, 0, 0] m = make_mask(n) m2 = make_mask(m) self.assertTrue(m is m2) m3 = make_mask(m, copy=1) self.assertTrue(m is not m3) x1 = np.arange(5) y1 = array(x1, mask=m) self.assertTrue(y1._data is not x1) self.assertTrue(allequal(x1, y1._data)) self.assertTrue(y1.mask is m) y1a = array(y1, copy=0) self.assertTrue(y1a.mask is y1.mask) y2 = array(x1, mask=m, copy=0) self.assertTrue(y2.mask is m) self.assertTrue(y2[2] is masked) y2[2] = 9 self.assertTrue(y2[2] is not masked) self.assertTrue(y2.mask is not m) self.assertTrue(allequal(y2.mask, 0)) y3 = array(x1 * 1.0, mask=m) self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype) x4 = arange(4) x4[2] = masked y4 = resize(x4, (8,)) self.assertTrue(eq(concatenate([x4, x4]), y4)) self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0])) y5 = repeat(x4, (2, 2, 2, 2), axis=0) self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3])) y6 = repeat(x4, 2, axis=0) self.assertTrue(eq(y5, y6))
def _geo_globe(x, y, z, xmin=-180, modulo=False): """ Ensure global coverage by fixing gaps over poles and across longitude seams. Increases the size of the arrays. """ # Cover gaps over poles by appending polar data with np.errstate(all='ignore'): p1 = np.mean(z[0, :]) # do not ignore NaN if present p2 = np.mean(z[-1, :]) ps = (-90, 90) if (y[0] < y[-1]) else (90, -90) z1 = np.repeat(p1, z.shape[1]) z2 = np.repeat(p2, z.shape[1]) y = ma.concatenate((ps[:1], y, ps[1:])) z = ma.concatenate((z1[None, :], z, z2[None, :]), axis=0) # Cover gaps over cartopy longitude seam # Ensure coordinates span 360 after modulus if modulo: if x[0] % 360 != (x[-1] + 360) % 360: x = ma.concatenate((x, (x[0] + 360,))) z = ma.concatenate((z, z[:, :1]), axis=1) # Cover gaps over basemap longitude seam # Ensure coordinates span exactly 360 else: # Interpolate coordinate centers to seam. Size possibly augmented by 2 if x.size == z.shape[1]: if x[0] + 360 != x[-1]: xi = np.array([x[-1], x[0] + 360]) # input coordinates xq = xmin + 360 # query coordinate zq = ma.concatenate((z[:, -1:], z[:, :1]), axis=1) zq = (zq[:, :1] * (xi[1] - xq) + zq[:, 1:] * (xq - xi[0])) / (xi[1] - xi[0]) # noqa: E501 x = ma.concatenate(((xmin,), x, (xmin + 360,))) z = ma.concatenate((zq, z, zq), axis=1) # Extend coordinate edges to seam. Size possibly augmented by 1. elif x.size - 1 == z.shape[1]: if x[0] != xmin: x = ma.append(xmin, x) x[-1] = xmin + 360 z = ma.concatenate((z[:, -1:], z), axis=1) else: raise ValueError('Unexpected shapes of coordinates or data arrays.') return x, y, z
def _make_verts(self, U, V): uv = ma.asarray(U + V * 1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a / (self.scale * self.width) X, Y = self._h_arrows(length) if self.angles == 'xy': theta = self._angles(U, V).filled(0)[:, np.newaxis] elif self.angles == 'uv': theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) else: theta = ma.asarray(self.angles * np.pi / 180.0).filled(0) xy = (X + Y * 1j) * np.exp(1j * theta) * self.width xy = xy[:, :, np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def y_grad_v(RomsFile, RomsGrd, varname): """ Compute y-gradient on v points """ if type(varname) == str: #load roms file RomsNC = nc4(RomsFile, 'r') #load variable _var = RomsNC.variables[varname][:] else: _var = varname #[v points] #get mask Mask = ma.getmask(_var) #compute difference [rho points] dvar_rho = ma.diff(_var, n=1, axis=2) #pad mask_pad = ma.notmasked_edges(dvar_rho, axis=2) dvar_ = dvar_rho dvar_[:, :, mask_pad[0][2][0] - 1, :] = dvar_rho[:, :, mask_pad[0][2][0], :] dvar_[:, :, mask_pad[1][2][0] + 1, :] = dvar_rho[:, :, mask_pad[1][2][0], :] dvar_pad = ma.concatenate((dvar_[:,:,0:1, :],\ dvar_, \ dvar_[:,:,-2:-1, :]), axis = 2) #shift to V points dvar_v = GridShift.Rho_to_Vpt(dvar_pad) #dy y_dist = rt.rho_dist_grd(RomsGrd)[1] dy = rt.AddDepthTime(RomsFile, y_dist) dy_v = ma.array(GridShift.Rho_to_Vpt(dy), mask=Mask) #compute gradient dvar_dy = dvar_v / dy_v return dvar_dy
def sphere2rect(vector, degree=True): """\ Convert vector (lon, lat, r) from sphere to rect coordinates. If degree is True, the unit of vector has to be in degree. Examples -------- >>> convert.sphere2rect([120, 30, 1], degree=True) array([-0.4330127, 0.75 , 0.5 ]) """ lon, lat, r = vector if degree: lon = np.deg2rad(lon) lat = np.deg2rad(lat) return ma.concatenate([(r * np.cos(lat) * np.cos(lon))[np.newaxis], (r * np.cos(lat) * np.sin(lon))[np.newaxis], (r * np.sin(lat))[np.newaxis]])
def add_cyclic_point(data, axis=-1): """ Add a cyclic point to an array. Args: * data: An n-dimensional array of data to add a cyclic point to. Kwargs: * axis: Specifies the axis of the data array to add the cyclic point to. Defaults to the right-most axis. Returns: * cyclic_data: The data array with a cyclic point added. Examples: Adding a cyclic point to a data array, where the cyclic dimension is the right-most dimension >>> import numpy as np >>> data = np.ones([5, 6]) * np.arange(6) >>> cyclic_data = add_cyclic_point(data) >>> print(cyclic_data) [[ 0. 1. 2. 3. 4. 5. 0.] [ 0. 1. 2. 3. 4. 5. 0.] [ 0. 1. 2. 3. 4. 5. 0.] [ 0. 1. 2. 3. 4. 5. 0.] [ 0. 1. 2. 3. 4. 5. 0.]] """ slicer = [slice(None)] * data.ndim try: slicer[axis] = slice(0, 1) except IndexError: raise ValueError('specified axis does not exist in the input') new_data = ma.concatenate((data, data[slicer]), axis=axis) return new_data
def data(self): """ This is a getter for the data property. It caches the raw data if it has not already been read. Throws a MemoryError when reading for the first time if the data is too large. """ import numpy.ma as ma if self._data is None: try: # If we were given a list of data managers then we need to concatenate them now... self._data = self.retrieve_raw_data(self._data_manager[0]) if len(self._data_manager) > 1: for manager in self._data_manager[1:]: self._data = ma.concatenate((self._data, self.retrieve_raw_data(manager)), axis=0) self._post_process() except MemoryError: raise MemoryError( "Failed to read the ungridded data as there was not enough memory available.\n" "Consider freeing up variables or indexing the cube before getting its data.") return self._data
def _make_verts(self, U, V): uv = ma.asarray(U+V*1j) a = ma.absolute(uv) if self.scale is None: sn = max(10, math.sqrt(self.N)) scale = 1.8 * a.mean() * sn / self.span # crude auto-scaling self.scale = scale length = a/(self.scale*self.width) X, Y = self._h_arrows(length) if self.angles == 'xy': theta = self._angles(U, V).filled(0)[:,np.newaxis] elif self.angles == 'uv': theta = np.angle(ma.asarray(uv[..., np.newaxis]).filled(0)) else: theta = ma.asarray(self.angles*np.pi/180.0).filled(0) xy = (X+Y*1j) * np.exp(1j*theta)*self.width xy = xy[:,:,np.newaxis] XY = ma.concatenate((xy.real, xy.imag), axis=2) return XY
def Vpt_to_Rho(Vpt_variable): """ Convert variable on v point to rho point """ if np.ma.is_masked(Vpt_variable) == True: import numpy.ma as ma else: import numpy as ma _dy_pad = ma.concatenate((Vpt_variable[0:1,:], Vpt_variable,\ Vpt_variable[-2:-1,:]), axis = 0) d_y = 0.5*(_dy_pad[0:_dy_pad.shape[0]-1, :] + \ _dy_pad[1:_dy_pad.shape[0], :]) return d_y
def Upt_to_Rho(Upt_variable): """ Converts variables on u points to rho points """ if np.ma.is_masked(Upt_variable) == True: import numpy.ma as ma else: import numpy as ma _dx_pad = ma.concatenate((Upt_variable[:,:,:,0:1], Upt_variable,\ Upt_variable[:, :,:,-2:-1]), axis = 3) d_x = 0.5*(_dx_pad[:,:,:, 0:_dx_pad.shape[3]-1] + \ _dx_pad[:,:,:, 1:_dx_pad.shape[3]]) return d_x
def regrid(data_array): nc_grid = netcdf_file('/home/nicholas/thesis/data/netcdf_files/ORCA2.0_grid.nc','r') lon = nc_grid.variables['lon'][0:50,:] lat = nc_grid.variables['lat'][0:50,:] area = nc_grid.variables['area'][0:50,:] mask = nc_grid.variables['mask'][0,0:50,:] nc_grid.close() lon_min = lon.copy() i,j = np.where(lon_min >= 180.) lon_min[i,j] = lon_min[i,j] - 360. iw = np.where(lon_min[0,:] >= lon_min[0][0])[0] ie = np.where(lon_min[0,:] < lon_min[0][0])[0] lon = np.concatenate((np.take(lon_min,ie,axis=1)[:,:-1],np.take(lon_min,iw,axis=1)),axis=1)[:,:-1] lat = np.concatenate((np.take(lat,ie,axis=1)[:,:-1],np.take(lat,iw,axis=1)),axis=1)[:,:-1] lon_list = np.round(lon[0]) lat_list = np.round(lat[:, 0]) timesteps = np.shape(data_array)[0] bm_array = [ma.concatenate((ma.take(data_array[i, :, :],ie,axis=1),ma.take(data_array[i, :, :],iw,axis=1)),axis=1)[:,:-1] for i in range(timesteps)] bm_array = ma.array(bm_array) return bm_array, lon, lat
def test_split_data_groups(n=100): flag = ma.concatenate([np.random.randint(0,5,n), ma.masked_all(2, dtype='int8')]) indices = split_data_groups(flag) assert type(indices) is dict assert sorted(indices.keys()) == ['err', 'fit', 'test'] for k in indices: indices[k].size == n assert indices[k].dtype == 'bool' # Necessarily each group have trues and falses assert indices[k].any(), "%s are all True" % k assert (~indices[k]).any(), "%s are all False" % k # Indices return only valid data. Ignore masked. assert ~flag[indices[k]].mask.any() # Fit group is all True, but err & test must have both assert sorted(np.unique(flag[indices['fit']])) == [1,2] for k in ['err', 'test']: assert sorted(np.unique(flag[indices[k]])) == [1,2,3,4]
def insertRow(self, rowIndex, msk = True): ''' At the moment, insert empty row before index rowIndex rowindex starts counting at zeros (as usual) New row is masked by default ''' for col in self.cols: dt = self.data[col].dtype newEle = ma.empty((1,), dtype = dt) #print(self.data[col]) #print(self.data[col][:rowIndex].shape) #print(newEle.shape) #print(self.data[col][rowIndex:].shape) self.data[col] = ma.concatenate((self.data[col][:rowIndex], newEle, self.data[col][rowIndex:])) #if rowIndex <= self.nRows: #self.data[col] = ma.concatenate((self.data[col][:rowIndex], newEle, self.data[col][rowIndex:])) #else: #self.data[col] = ma.concatenate((self.data[col], newEle)) if msk: self.data[col][rowIndex] = ma.masked self.nRows += 1
def sphere2rect(vector, degree=True): """\ Convert vector (lon, lat, r) from sphere to rect coordinates. If degree is True, the unit of vector has to be in degree. Examples -------- >>> convert.sphere2rect([120, 30, 1], degree=True) array([-0.4330127, 0.75 , 0.5 ]) """ lon, lat, r = vector if degree: lon = np.deg2rad(lon) lat = np.deg2rad(lat) return ma.concatenate([ (r * np.cos(lat) * np.cos(lon))[np.newaxis], (r * np.cos(lat) * np.sin(lon))[np.newaxis], (r * np.sin(lat))[np.newaxis] ])
def test_split_data_groups(n=100): flag = ma.concatenate([np.random.random(n) > .5, ma.masked_all(2, dtype='bool')]) indices = split_data_groups(flag) assert type(indices) is dict assert sorted(indices.keys()) == ['err', 'fit', 'test'] for k in indices: indices[k].size == n assert indices[k].dtype == 'bool' # Necessarily each group have trues and falses assert indices[k].any(), "%s are all True" % k assert (~indices[k]).any(), "%s are all False" % k # Indices return only valid data. Ignore masked. assert ~flag[indices[k]].mask.any() # Fit group is all True, but err & test must have both assert sorted(np.unique(flag[indices['fit']])) == [True] for k in ['err', 'test']: assert sorted(np.unique(flag[indices[k]])) == [False, True]
def get_gates_from_tar(nexrad_archive): time_list = [] alt_list = [] lat_list = [] lon_list = [] ref_list = [] with tarfile.open(nexrad_archive) as tar: # Loop iver each element and inspect to see if they are actual radar archive files (there is also metadata in the tar) for item in [ name for name in tar.getnames() if name[-9:] == '_V06.ar2v' ]: try: radar = pyart.io.read_nexrad_archive( tar.extractfile(tar.getmember(item)), include_fields=['reflectivity'], delay_field_loading=True) except IOError: pass else: alt_list.append(radar.gate_altitude['data']) lat_list.append(radar.gate_latitude['data']) lon_list.append(radar.gate_longitude['data']) ref_list.append(radar.fields['reflectivity']['data']) start_time = parse_date(item[4:19], fuzzy=True) time_list.append([ start_time + timedelta(seconds=t) for t in radar.time['data'] ]) del radar times = np.concatenate(time_list, 0) alts = np.concatenate(alt_list, 0) lats = np.concatenate(lat_list, 0) lons = np.concatenate(lon_list, 0) refs = ma.concatenate(ref_list, 0) return times, alts, lats, lons, refs
def densitystep(S, T, P): """Estimates the potential density step of successive mesurements Expects the data to be recorded along the time, i.e. first measurement was recorded first. This makes difference since the first measurement has no reference to define the delta change. This is relevant for the type of instrument. For instance: XBTs are always measured surface to bottom, CTDs are expected the same, but Spray underwater gliders measure bottom to surface. """ assert T.shape == P.shape assert T.shape == S.shape assert T.ndim == 1, "Not ready to densitystep an array ndim > 1" try: import gsw except ImportError: print("Package gsw is required and is not available.") rho0 = gsw.pot_rho_t_exact(S, T, P, 0) ds = ma.concatenate([ma.masked_all(1), np.sign(np.diff(P)) * np.diff(rho0)]) return ma.fix_invalid(ds)
def insertRow(self, rowIndex, msk=True): """ At the moment, insert empty row before index rowIndex rowindex starts counting at zeros (as usual) New row is masked by default """ for col in self.cols: dt = self.data[col].dtype newEle = ma.empty((1,), dtype=dt) # print(self.data[col]) # print(self.data[col][:rowIndex].shape) # print(newEle.shape) # print(self.data[col][rowIndex:].shape) self.data[col] = ma.concatenate((self.data[col][:rowIndex], newEle, self.data[col][rowIndex:])) # if rowIndex <= self.nRows: # self.data[col] = ma.concatenate((self.data[col][:rowIndex], newEle, self.data[col][rowIndex:])) # else: # self.data[col] = ma.concatenate((self.data[col], newEle)) if msk: self.data[col][rowIndex] = ma.masked self.nRows += 1
def regrid_array(data=data_cflux): ''' #Could be put with plotting tools??? # Regrid array to be used with Basemap # Only works if the same latitudes and longitudes are selected from netdcf file and grid # Uses the ORCA netcdf file ### transform the longitude of ORCA onto something that basemap can read ### The ORCA grid starts at 80 and goes to 440 ### What we want: starts at 80 and goes to 180 and then switches to -180 and goes to 80 ### this method ''' from Scientific.IO.NetCDF import NetCDFFile #nc_grid_file = choose_netcdf_file() #~ indir = raw_input('Where is the ORCA netcdf file located? \n') nc_grid = NetCDFFile(NC_PATH+ 'ORCA2.0_grid.nc','r') lon = nc_grid.variables['lon'][0:40,:] lat = nc_grid.variables['lat'][0:40,:] area = nc_grid.variables['area'][0:40,:] mask = nc_grid.variables['mask'][0,0:40,:] nc_grid.close() lon_min = lon.copy() i,j = np.where(lon_min >= 180.) # elements of lon_min that are over 180 lon_min[i,j] = lon_min[i,j] - 360. # takes those elements and subtracts 360 from them ### ============================================================================================================== ### get rid of the funny extra lon and do the same for the lat array ! iw = np.where(lon_min[0,:] >= lon_min[0][0])[0] # are the elements that are greater or equal to the first element ie. 78.000038 ie = np.where(lon_min[0,:] < lon_min[0][0])[0] # are the elements less than 78.000038 ### puts the lon in order from -180 to 180 and removes the extra 80 at the end lon = np.concatenate((np.take(lon_min,ie,axis=1),np.take(lon_min,iw,axis=1)),axis=1)[:,:-1] lat = np.concatenate((np.take(lat,ie,axis=1),np.take(lat,iw,axis=1)),axis=1)[:,:-1] # The data that is to be plotted needs to be regridded bm_array = [ma.concatenate((ma.take(data[i, :, :],ie,axis=1),ma.take(data[i, :, :],iw,axis=1)),axis=1)[:,:-1] for i in range(3650)] bm_array = ma.array(bm_array) return bm_array
def test_testCopySize(self): # Tests of some subtle points of copying and sizing. n = [0, 0, 1, 0, 0] m = make_mask(n) m2 = make_mask(m) self.assertTrue(m is m2) m3 = make_mask(m, copy=1) self.assertTrue(m is not m3) x1 = np.arange(5) y1 = array(x1, mask=m) self.assertTrue(y1._data is not x1) self.assertTrue(allequal(x1, y1._data)) self.assertTrue(y1.mask is m) y1a = array(y1, copy=0) self.assertTrue(y1a.mask is y1.mask) y2 = array(x1, mask=m, copy=0) self.assertTrue(y2.mask is m) self.assertTrue(y2[2] is masked) y2[2] = 9 self.assertTrue(y2[2] is not masked) self.assertTrue(y2.mask is not m) self.assertTrue(allequal(y2.mask, 0)) y3 = array(x1 * 1.0, mask=m) self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype) x4 = arange(4) x4[2] = masked y4 = resize(x4, (8, )) self.assertTrue(eq(concatenate([x4, x4]), y4)) self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0])) y5 = repeat(x4, (2, 2, 2, 2), axis=0) self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3])) y6 = repeat(x4, 2, axis=0) self.assertTrue(eq(y5, y6))
def recache(self): #if self.axes is None: print 'recache no axes' #else: print 'recache units', self.axes.xaxis.units, self.axes.yaxis.units if ma.isMaskedArray(self._xorig) or ma.isMaskedArray(self._yorig): x = ma.asarray(self.convert_xunits(self._xorig), float) y = ma.asarray(self.convert_yunits(self._yorig), float) x = ma.ravel(x) y = ma.ravel(y) else: x = np.asarray(self.convert_xunits(self._xorig), float) y = np.asarray(self.convert_yunits(self._yorig), float) x = np.ravel(x) y = np.ravel(y) if len(x) == 1 and len(y) > 1: x = x * np.ones(y.shape, float) if len(y) == 1 and len(x) > 1: y = y * np.ones(x.shape, float) if len(x) != len(y): raise RuntimeError('xdata and ydata must be the same length') x = x.reshape((len(x), 1)) y = y.reshape((len(y), 1)) if ma.isMaskedArray(x) or ma.isMaskedArray(y): self._xy = ma.concatenate((x, y), 1) else: self._xy = np.concatenate((x, y), 1) self._x = self._xy[:, 0] # just a view self._y = self._xy[:, 1] # just a view # Masked arrays are now handled by the Path class itself self._path = Path(self._xy) self._transformed_path = TransformedPath(self._path, self.get_transform()) self._invalid = False
def concatenate(arrays, axis=0): """ Concatenate a list of numpy arrays into one larger array along the axis specified (the default axis is zero). If any of the arrays are masked arrays then the returned array will be a masked array with the correct mask, otherwise a numpy array is returned. :param arrays: A list of numpy arrays (masked or not) :param axis: The axis along which to concatenate (the default is 0) :return: The concatenated array """ from numpy.ma import MaskedArray if any(isinstance(array, MaskedArray) for array in arrays): from numpy.ma import concatenate else: from numpy import concatenate res = arrays[0] if len(arrays) > 1: for array in arrays[1:]: res = concatenate((res, array), axis) return res
def __init__(self, contour, contour_height, data, interval): # Height of the contour self.h_0 = contour_height # Step minimal to consider amplitude self.interval_min = interval * 2 # Indices of all pixels in contour self.contour = contour # Link on original grid (local view) or copy if it's on bound (x_start, x_stop), (y_start, y_stop) = contour.bbox_slice on_bounds = x_start > x_stop if on_bounds: self.grid_extract = ma.concatenate( (data[x_start:, y_start:y_stop], data[:x_stop, y_start:y_stop])) if self.grid_extract.mask.size == 1: self.grid_extract = ma.array( self.grid_extract, mask=ones(self.grid_extract.shape, dtype="bool") * self.grid_extract.mask, ) else: self.grid_extract = data[x_start:x_stop, y_start:y_stop] # => maybe replace pixel out of contour by nan? self.pixel_mask = zeros(self.grid_extract.shape, dtype="bool") i_x = contour.pixels_index[0] - x_start if on_bounds: i_x %= data.shape[0] self.pixel_mask[i_x, contour.pixels_index[1] - y_start] = True self.nb_pixel = i_x.shape[0] # Only pixel in contour self.sla = data[contour.pixels_index] # Amplitude which will be provide self.amplitude = 0 # Maximum local extrema accepted self.mle = 1
def _map_common(draw_method_name, arg_func, mode, cube, plot_defn, *args, **kwargs): """ Draw the given cube on a map using its points or bounds. "Mode" parameter will switch functionality between POINT or BOUND plotting. """ # Generate 2d x and 2d y grids. y_coord, x_coord = plot_defn.coords if mode == iris.coords.POINT_MODE: if x_coord.ndim == y_coord.ndim == 1: x, y = np.meshgrid(x_coord.points, y_coord.points) elif x_coord.ndim == y_coord.ndim == 2: x = x_coord.points y = y_coord.points else: raise ValueError("Expected 1D or 2D XY coords") else: try: x, y = np.meshgrid(x_coord.contiguous_bounds(), y_coord.contiguous_bounds()) # Exception translation. except iris.exceptions.CoordinateMultiDimError: raise ValueError("Could not get XY grid from bounds. " "X or Y coordinate not 1D.") except ValueError: raise ValueError("Could not get XY grid from bounds. " "X or Y coordinate doesn't have 2 bounds " "per point.") # Obtain the data array. data = cube.data if plot_defn.transpose: data = data.T # If we are global, then append the first column of data the array to the # last (and add 360 degrees) NOTE: if it is found that this block of code # is useful in anywhere other than this plotting routine, it may be better # placed in the CS. if getattr(x_coord, 'circular', False): _, direction = iris.util.monotonic(x_coord.points, return_direction=True) y = np.append(y, y[:, 0:1], axis=1) x = np.append(x, x[:, 0:1] + 360 * direction, axis=1) data = ma.concatenate([data, data[:, 0:1]], axis=1) # Replace non-cartopy subplot/axes with a cartopy alternative and set the # transform keyword. kwargs = _ensure_cartopy_axes_and_determine_kwargs(x_coord, y_coord, kwargs) if arg_func is not None: new_args, kwargs = arg_func(x, y, data, *args, **kwargs) else: new_args = (x, y, data) + args # Draw the contour lines/filled contours. axes = kwargs.pop('axes', None) plotfn = getattr(axes if axes else plt, draw_method_name) return plotfn(*new_args, **kwargs)
def add_cyclic_point(data, coord=None, axis=-1): """ Add a cyclic point to an array and optionally a corresponding coordinate. Parameters ---------- data An n-dimensional array of data to add a cyclic point to. coord : optional A 1-dimensional array which specifies the coordinate values for the dimension the cyclic point is to be added to. The coordinate values must be regularly spaced. Defaults to None. axis : optional Specifies the axis of the data array to add the cyclic point to. Defaults to the right-most axis. Returns ------- cyclic_data The data array with a cyclic point added. cyclic_coord The coordinate with a cyclic point, only returned if the coord keyword was supplied. Examples -------- Adding a cyclic point to a data array, where the cyclic dimension is the right-most dimension. >>> import numpy as np >>> data = np.ones([5, 6]) * np.arange(6) >>> cyclic_data = add_cyclic_point(data) >>> print(cyclic_data) # doctest: +NORMALIZE_WHITESPACE [[0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.]] Adding a cyclic point to a data array and an associated coordinate >>> lons = np.arange(0, 360, 60) >>> cyclic_data, cyclic_lons = add_cyclic_point(data, coord=lons) >>> print(cyclic_data) # doctest: +NORMALIZE_WHITESPACE [[0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.] [0. 1. 2. 3. 4. 5. 0.]] >>> print(cyclic_lons) [ 0 60 120 180 240 300 360] """ if coord is not None: if coord.ndim != 1: raise ValueError('The coordinate must be 1-dimensional.') if len(coord) != data.shape[axis]: raise ValueError(f'The length of the coordinate does not match ' f'the size of the corresponding dimension of ' f'the data array: len(coord) = {len(coord)}, ' f'data.shape[{axis}] = {data.shape[axis]}.') delta_coord = np.diff(coord) if not np.allclose(delta_coord, delta_coord[0]): raise ValueError('The coordinate must be equally spaced.') new_coord = ma.concatenate((coord, coord[-1:] + delta_coord[0])) slicer = [slice(None)] * data.ndim try: slicer[axis] = slice(0, 1) except IndexError: raise ValueError('The specified axis does not correspond to an ' 'array dimension.') new_data = ma.concatenate((data, data[tuple(slicer)]), axis=axis) if coord is None: return_value = new_data else: return_value = new_data, new_coord return return_value
def join_by(key, r1, r2, jointype='inner', r1postfix='1', r2postfix='2', defaults=None, usemask=True, asrecarray=False): """ Join arrays `r1` and `r2` on key `key`. The key should be either a string or a sequence of string corresponding to the fields used to join the array. An exception is raised if the `key` field cannot be found in the two input arrays. Neither `r1` nor `r2` should have any duplicates along `key`: the presence of duplicates will make the output quite unreliable. Note that duplicates are not looked for by the algorithm. Parameters ---------- key : {string, sequence} A string or a sequence of strings corresponding to the fields used for comparison. r1, r2 : arrays Structured arrays. jointype : {'inner', 'outer', 'leftouter'}, optional If 'inner', returns the elements common to both r1 and r2. If 'outer', returns the common elements as well as the elements of r1 not in r2 and the elements of not in r2. If 'leftouter', returns the common elements and the elements of r1 not in r2. r1postfix : string, optional String appended to the names of the fields of r1 that are present in r2 but absent of the key. r2postfix : string, optional String appended to the names of the fields of r2 that are present in r1 but absent of the key. defaults : {dictionary}, optional Dictionary mapping field names to the corresponding default values. usemask : {True, False}, optional Whether to return a MaskedArray (or MaskedRecords is `asrecarray==True`) or a ndarray. asrecarray : {False, True}, optional Whether to return a recarray (or MaskedRecords if `usemask==True`) or just a flexible-type ndarray. Notes ----- * The output is sorted along the key. * A temporary array is formed by dropping the fields not in the key for the two arrays and concatenating the result. This array is then sorted, and the common entries selected. The output is constructed by filling the fields with the selected entries. Matching is not preserved if there are some duplicates... """ # Check jointype if jointype not in ('inner', 'outer', 'leftouter'): raise ValueError("The 'jointype' argument should be in 'inner', "\ "'outer' or 'leftouter' (got '%s' instead)" % jointype) # If we have a single key, put it in a tuple if isinstance(key, basestring): key = (key, ) # Check the keys for name in key: if name not in r1.dtype.names: raise ValueError('r1 does not have key field %s' % name) if name not in r2.dtype.names: raise ValueError('r2 does not have key field %s' % name) # Make sure we work with ravelled arrays r1 = r1.ravel() r2 = r2.ravel() (nb1, nb2) = (len(r1), len(r2)) (r1names, r2names) = (r1.dtype.names, r2.dtype.names) # Check the names for collision if (set.intersection(set(r1names), set(r2names)).difference(key) and not (r1postfix or r2postfix)): msg = "r1 and r2 contain common names, r1postfix and r2postfix " msg += "can't be empty" raise ValueError(msg) # Make temporary arrays of just the keys r1k = drop_fields(r1, [n for n in r1names if n not in key]) r2k = drop_fields(r2, [n for n in r2names if n not in key]) # Concatenate the two arrays for comparison aux = ma.concatenate((r1k, r2k)) idx_sort = aux.argsort(order=key) aux = aux[idx_sort] # # Get the common keys flag_in = ma.concatenate(([False], aux[1:] == aux[:-1])) flag_in[:-1] = flag_in[1:] + flag_in[:-1] idx_in = idx_sort[flag_in] idx_1 = idx_in[(idx_in < nb1)] idx_2 = idx_in[(idx_in >= nb1)] - nb1 (r1cmn, r2cmn) = (len(idx_1), len(idx_2)) if jointype == 'inner': (r1spc, r2spc) = (0, 0) elif jointype == 'outer': idx_out = idx_sort[~flag_in] idx_1 = np.concatenate((idx_1, idx_out[(idx_out < nb1)])) idx_2 = np.concatenate((idx_2, idx_out[(idx_out >= nb1)] - nb1)) (r1spc, r2spc) = (len(idx_1) - r1cmn, len(idx_2) - r2cmn) elif jointype == 'leftouter': idx_out = idx_sort[~flag_in] idx_1 = np.concatenate((idx_1, idx_out[(idx_out < nb1)])) (r1spc, r2spc) = (len(idx_1) - r1cmn, 0) # Select the entries from each input (s1, s2) = (r1[idx_1], r2[idx_2]) # # Build the new description of the output array ....... # Start with the key fields ndtype = [list(_) for _ in r1k.dtype.descr] # Add the other fields ndtype.extend(list(_) for _ in r1.dtype.descr if _[0] not in key) # Find the new list of names (it may be different from r1names) names = list(_[0] for _ in ndtype) for desc in r2.dtype.descr: desc = list(desc) name = desc[0] # Have we seen the current name already ? if name in names: nameidx = ndtype.index(desc) current = ndtype[nameidx] # The current field is part of the key: take the largest dtype if name in key: current[-1] = max(desc[1], current[-1]) # The current field is not part of the key: add the suffixes else: current[0] += r1postfix desc[0] += r2postfix ndtype.insert(nameidx + 1, desc) #... we haven't: just add the description to the current list else: names.extend(desc[0]) ndtype.append(desc) # Revert the elements to tuples ndtype = [tuple(_) for _ in ndtype] # Find the largest nb of common fields : r1cmn and r2cmn should be equal, but... cmn = max(r1cmn, r2cmn) # Construct an empty array output = ma.masked_all((cmn + r1spc + r2spc, ), dtype=ndtype) names = output.dtype.names for f in r1names: selected = s1[f] if f not in names or (f in r2names and not r2postfix and not f in key): f += r1postfix current = output[f] current[:r1cmn] = selected[:r1cmn] if jointype in ('outer', 'leftouter'): current[cmn:cmn + r1spc] = selected[r1cmn:] for f in r2names: selected = s2[f] if f not in names or (f in r1names and not r1postfix and f not in key): f += r2postfix current = output[f] current[:r2cmn] = selected[:r2cmn] if (jointype == 'outer') and r2spc: current[-r2spc:] = selected[r2cmn:] # Sort and finalize the output output.sort(order=key) kwargs = dict(usemask=usemask, asrecarray=asrecarray) return _fix_output(_fix_defaults(output, defaults), **kwargs)
def stack_arrays(arrays, defaults=None, usemask=True, asrecarray=False, autoconvert=False): """ Superposes arrays fields by fields Parameters ---------- seqarrays : array or sequence Sequence of input arrays. defaults : dictionary, optional Dictionary mapping field names to the corresponding default values. usemask : {True, False}, optional Whether to return a MaskedArray (or MaskedRecords is `asrecarray==True`) or a ndarray. asrecarray : {False, True}, optional Whether to return a recarray (or MaskedRecords if `usemask==True`) or just a flexible-type ndarray. autoconvert : {False, True}, optional Whether automatically cast the type of the field to the maximum. Examples -------- >>> from numpy.lib import recfunctions as rfn >>> x = np.array([1, 2,]) >>> rfn.stack_arrays(x) is x True >>> z = np.array([('A', 1), ('B', 2)], dtype=[('A', '|S3'), ('B', float)]) >>> zz = np.array([('a', 10., 100.), ('b', 20., 200.), ('c', 30., 300.)], ... dtype=[('A', '|S3'), ('B', float), ('C', float)]) >>> test = rfn.stack_arrays((z,zz)) >>> test masked_array(data = [('A', 1.0, --) ('B', 2.0, --) ('a', 10.0, 100.0) ('b', 20.0, 200.0) ('c', 30.0, 300.0)], mask = [(False, False, True) (False, False, True) (False, False, False) (False, False, False) (False, False, False)], fill_value = ('N/A', 1e+20, 1e+20), dtype = [('A', '|S3'), ('B', '<f8'), ('C', '<f8')]) """ if isinstance(arrays, ndarray): return arrays elif len(arrays) == 1: return arrays[0] seqarrays = [np.asanyarray(a).ravel() for a in arrays] nrecords = [len(a) for a in seqarrays] ndtype = [a.dtype for a in seqarrays] fldnames = [d.names for d in ndtype] # dtype_l = ndtype[0] newdescr = dtype_l.descr names = [_[0] for _ in newdescr] for dtype_n in ndtype[1:]: for descr in dtype_n.descr: name = descr[0] or '' if name not in names: newdescr.append(descr) names.append(name) else: nameidx = names.index(name) current_descr = newdescr[nameidx] if autoconvert: if np.dtype(descr[1]) > np.dtype(current_descr[-1]): current_descr = list(current_descr) current_descr[-1] = descr[1] newdescr[nameidx] = tuple(current_descr) elif descr[1] != current_descr[-1]: raise TypeError("Incompatible type '%s' <> '%s'" % \ (dict(newdescr)[name], descr[1])) # Only one field: use concatenate if len(newdescr) == 1: output = ma.concatenate(seqarrays) else: # output = ma.masked_all((np.sum(nrecords), ), newdescr) offset = np.cumsum(np.r_[0, nrecords]) seen = [] for (a, n, i, j) in zip(seqarrays, fldnames, offset[:-1], offset[1:]): names = a.dtype.names if names is None: output['f%i' % len(seen)][i:j] = a else: for name in n: output[name][i:j] = a[name] if name not in seen: seen.append(name) # return _fix_output(_fix_defaults(output, defaults), usemask=usemask, asrecarray=asrecarray)
def load_GPM_IMERG_files_with_spatial_filter(file_path=None, filename_pattern=None, filelist=None, variable_name='precipitationCal', user_mask_file=None, mask_variable_name='mask', user_mask_values=[10], longitude_name='lon', latitude_name='lat'): ''' Load multiple GPM Level 3 IMEGE files containing calibrated \ precipitation and generate a two-dimensional array \ for the masked grid points. :param file_path: Directory to the HDF files to load. :type file_path: :mod:`string` :param filename_pattern: Path to the HDF files to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the HDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :user_mask_file: user's own gridded mask file(a netCDF file name) :type name: :mod:`string` :mask_variable_name: mask variables in user_mask_file :type name: :mod:`string` :longitude_name: longitude variable in user_mask_file :type name: :mod:`string` :latitude_name: latitude variable in user_mask_file :type name: :mod:`string` :param user_mask_values: grid points where mask_variable == user_mask_value will be extracted. :type user_mask_values: list of strings :returns: A two-dimensional array with the requested variable's MASKED data from \ the HDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: GPM_files = [] for pattern in filename_pattern: GPM_files.extend(glob(file_path + pattern)) else: GPM_files = [line.rstrip('\n') for line in open(filelist)] GPM_files.sort() file_object_first = h5py.File(GPM_files[0]) lats = file_object_first['Grid']['lat'][:] lons = file_object_first['Grid']['lon'][:] lons, lats = numpy.meshgrid(lons, lats) nfile = len(GPM_files) for ifile, file in enumerate(GPM_files): if ifile == 0 and user_mask_file: file_object = netCDF4.Dataset(user_mask_file) mask_variable = file_object.variables[mask_variable_name][:] mask_longitude = file_object.variables[longitude_name][:] mask_latitude = file_object.variables[latitude_name][:] spatial_mask = utils.regrid_spatial_mask(lons, lats, mask_longitude, mask_latitude, mask_variable, user_mask_values) y_index, x_index = numpy.where(spatial_mask == 0) print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = h5py.File(file) values0 = ma.transpose( ma.masked_less(file_object['Grid'][variable_name][:], 0.)) values_masked = values0[y_index, x_index] values_masked = ma.expand_dims(values_masked, axis=0) if ifile == 0: values = values_masked else: values = ma.concatenate((values, values_masked)) file_object.close() return values
def load_GPM_IMERG_files(file_path=None, filename_pattern=None, filelist=None, variable_name='precipitationCal', name='GPM_IMERG'): ''' Load multiple GPM Level 3 IMEGE files containing calibrated \ precipitation and generate an OCW Dataset obejct. :param file_path: Directory to the HDF files to load. :type file_path: :mod:`string` :param filename_pattern: Path to the HDF files to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the HDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :returns: An OCW Dataset object with the requested variable's data from \ the HDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: GPM_files = [] for pattern in filename_pattern: GPM_files.extend(glob(file_path + pattern)) else: GPM_files = [line.rstrip('\n') for line in open(filelist)] GPM_files.sort() file_object_first = h5py.File(GPM_files[0]) lats = file_object_first['Grid']['lat'][:] lons = file_object_first['Grid']['lon'][:] lons, lats = numpy.meshgrid(lons, lats) variable_unit = "mm/hr" times = [] nfile = len(GPM_files) for ifile, file in enumerate(GPM_files): print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = h5py.File(file) time_struct_parsed = strptime(file[-39:-23], "%Y%m%d-S%H%M%S") times.append(datetime(*time_struct_parsed[:6])) values0 = ma.transpose( ma.masked_less(file_object['Grid'][variable_name][:], 0.)) values0 = ma.expand_dims(values0, axis=0) if ifile == 0: values = values0 else: values = ma.concatenate((values, values0)) file_object.close() times = numpy.array(times) return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
def load_WRF_2d_files_RAIN(file_path=None, filename_pattern=None, filelist=None, name=''): ''' Load multiple WRF (or nuWRF) original output files containing 2D \ fields such as precipitation and surface variables into a Dataset. \ The dataset can be spatially subset. :param file_path: Directory to the NetCDF file to load. :type file_path: :mod:`string` :param filename_pattern: Path to the NetCDF file to load. :type filename_pattern: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :returns: An OCW Dataset object with the requested variable's data from \ the NetCDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: WRF_files = [] for pattern in filename_pattern: WRF_files.extend(glob(file_path + pattern)) WRF_files.sort() else: WRF_files = [line.rstrip('\n') for line in open(filelist)] file_object_first = netCDF4.Dataset(WRF_files[0]) lats = file_object_first.variables['XLAT'][0, :] lons = file_object_first.variables['XLONG'][0, :] times = [] nfile = len(WRF_files) for ifile, file in enumerate(WRF_files): print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = netCDF4.Dataset(file) time_struct_parsed = strptime(file[-19:], "%Y-%m-%d_%H:%M:%S") for ihour in range(24): times.append( datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour)) temp_value = file_object.variables['RAINC'][:] + file_object.variables[ 'RAINNC'][:] if isinstance(temp_value, numpy.ndarray): temp_value = ma.array(temp_value, mask=numpy.zeros(temp_value.shape)) if ifile == 0: values0 = temp_value else: values0 = ma.concatenate((values0, temp_value)) file_object.close() times = numpy.array(times) years = numpy.array([d.year for d in times]) ncycle = numpy.unique(years).size print('ncycle=', ncycle) nt, ny, nx = values0.shape values = numpy.zeros([nt - ncycle * 24, ny, nx]) values = ma.array(values, mask=numpy.zeros(values.shape)) times2 = [] nt2 = nt / ncycle # remove the first day in each year nt3 = nt2 - 24 t_index = 0 for icycle in numpy.arange(ncycle): for it in numpy.arange(nt3) + 24: values[t_index, :] = values0[icycle * nt2 + it, :] - \ values0[icycle * nt2 + it - 1, :] times2.append(times[icycle * nt2 + it]) t_index = t_index + 1 variable_name = 'PREC' variable_unit = 'mm/hr' times2 = numpy.array(times2) return Dataset(lats, lons, times2, values, variable_name, units=variable_unit, name=name)
def load_WRF_2d_files(file_path=None, filename_pattern=None, filelist=None, variable_name='T2', name=''): ''' Load multiple WRF (or nuWRF) original output files containing 2D \ fields such as precipitation and surface variables into a Dataset. \ The dataset can be spatially subset. :param file_path: Directory to the NetCDF file to load. :type file_path: :mod:`string` :param filename_pattern: Path to the NetCDF file to load. :type filename_pattern: :mod:`string` :param filelist: A list of filenames :type filelist: :mod:`string` :param variable_name: The variable name to load from the NetCDF file. :type variable_name: :mod:`string` :param name: (Optional) A name for the loaded dataset. :type name: :mod:`string` :returns: An OCW Dataset object with the requested variable's data from \ the NetCDF file. :rtype: :class:`dataset.Dataset` :raises ValueError: ''' if not filelist: WRF_files = [] for pattern in filename_pattern: WRF_files.extend(glob(file_path + pattern)) else: WRF_files = [line.rstrip('\n') for line in open(filelist)] WRF_files.sort() file_object_first = netCDF4.Dataset(WRF_files[0]) lats = file_object_first.variables['XLAT'][0, :] lons = file_object_first.variables['XLONG'][0, :] times = [] nfile = len(WRF_files) for ifile, file in enumerate(WRF_files): print('Reading file ' + str(ifile + 1) + '/' + str(nfile), file) file_object = netCDF4.Dataset(file) time_struct_parsed = strptime(file[-19:], "%Y-%m-%d_%H:%M:%S") for ihour in numpy.arange(24): times.append( datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour)) values0 = file_object.variables[variable_name][:] if isinstance(values0, numpy.ndarray): values0 = ma.array(values0, mask=numpy.zeros(values0.shape)) if ifile == 0: values = values0 variable_unit = file_object.variables[variable_name].units else: values = ma.concatenate((values, values0)) file_object.close() times = numpy.array(times) return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
import numpy.ma as ma a = ma.arange(3) a[1] = ma.masked b = ma.arange(2, 5) a b ma.concatenate([a, b])
'*.pkl'))) # get a list of the paths to all the VolcNet files if len(VolcNet_files) == 0: raise Exception( 'No VolcNet files have been found. Perhaps the path is wrong? Or perhaps you only want to use synthetic data? In which case, this section can be removed. Exiting...' ) X_1s = [] Y_class_1s = [] Y_loc_1s = [] for VolcNet_file in VolcNet_files: X_1, Y_class_1, Y_loc_1 = open_VolcNet_file( VolcNet_file, synthetic_ifgs_settings['defo_sources']) X_1s.append(X_1) Y_class_1s.append(Y_class_1) Y_loc_1s.append(Y_loc_1) X = ma.concatenate(X_1s, axis=0) Y_class = np.concatenate(Y_class_1s, axis=0) Y_loc = np.concatenate(Y_loc_1s, axis=0) del X_1s, Y_class_1s, Y_loc_1s, X_1, Y_class_1, Y_loc_1 plot_data_class_loc_caller( X[:30, ], Y_class[:30, ], Y_loc[:30, ], source_names=['dyke', 'sill', 'no def'], window_title='02 Sample of Real data' ) # plot the data in it (note that this can be across multiople windows) print('Done. ') n_augmented_files = int( (X.shape[0] * real_ifg_settings['augmentation_factor']) / ifg_settings['n_per_file']
def test_masked_data(self): new_data = ma.masked_less(self.data2d, 3) c_data = add_cyclic_point(new_data) r_data = ma.concatenate((self.data2d, self.data2d[:, :1]), axis=1) assert_array_equal(c_data, r_data)