def createPolygonsAndColors(latDeg, lonDeg, rgb, colorMode=None): """ Returns polygons (in lat/lon coords) and a color for each polygon. :param latDeg: latitude for each pixel corner (h+1,w+1) :param lonDeg: longitude for each pixel corner (h+1,w+1) :param rgb: RGB array of (h,w,3) shape :param colorMode: 'matplotlib' normalizes colors to [0,1] :rtype: verts of shape (h*w,4,2), colors of shape (h*w,3) """ latLonDeg = ma.dstack((latDeg, lonDeg)) # adapted from matplotlib.collections.QuadMesh.convert_mesh_to_paths verts = ma.concatenate( ( latLonDeg[0:-1, 0:-1], latLonDeg[0:-1, 1:], latLonDeg[1:, 1:], latLonDeg[1:, 0:-1], #latLonDeg[0:-1, 0:-1] # matplotlib automatically closes the polygon ), axis=2) verts = verts.reshape(rgb.shape[0] * rgb.shape[1], 4, 2) if colorMode == ColorMode.matplotlib: rgb = image2mpl(rgb) colors = rgb.reshape(-1, 3) return verts, colors
def __call__(self, energy, impact, xmax, xb, yb): """ Evaluate interpolated templates for a set of shower parameters and pixel positions Parameters ---------- energy: array-like Energy of interpolated template impact: array-like Impact distance of interpolated template xmax: array-like Depth of maximum of interpolated templates xb: array-like Pixel X position at which to evaluate template yb: array-like Pixel X position at which to evaluate template Returns ------- ndarray: Pixel amplitude expectation values """ array = np.stack((energy, impact, xmax), axis=-1) points = ma.dstack((xb, yb)) interpolated_value = self.interpolator(array, points) interpolated_value[interpolated_value<0] = 0 interpolated_value = interpolated_value return interpolated_value
def __call__(self, energy, impact, xmax, xb, yb): """ Evaluate interpolated templates for a set of shower parameters and pixel positions Parameters ---------- energy: array-like Energy of interpolated template impact: array-like Impact distance of interpolated template xmax: array-like Depth of maximum of interpolated templates xb: array-like Pixel X position at which to evaluate template yb: array-like Pixel X position at which to evaluate template Returns ------- ndarray: Pixel amplitude expectation values """ array = np.stack((energy, impact, xmax), axis=-1) points = ma.dstack((xb, yb)) interpolated_value = self.interpolator(array, points) interpolated_value[interpolated_value < 0] = 0 interpolated_value = interpolated_value return interpolated_value
def ReadAndAgg_AgMERRA(var, year, lon_min, lon_max, lat_min, lat_max): f = netCDF4.Dataset("Data/AgMERRA/" + var + "/AgMERRA_" + \ str(year) + "_" + var + ".nc4") # time = f.variables['time'][:] # days since start of year (366) lats = f.variables['latitude'][:] # degrees north (31) lons = f.variables['longitude'][:] # degrees east (59) data = f.variables[var][:] f.close() # reducing to region of West Africa data_lonpos = data[:, :, lons < 180] data_lonneg = data[:, :, lons > 180] data_changed = np_ma.dstack([data_lonneg, data_lonpos]) data_changed = np.flip(data_changed, axis=1) lons = np.arange(-179.875, 180, 0.25) lats = np.flip(lats) data_rel = data_changed[:,((lats>=lat_min) & (lats<=lat_max)),:] \ [:,:,((lons>=lon_min) & (lons<=lon_max))] # aggregating to 0.5 degree resolution [n_t, n_lat, n_lon] = data_rel.shape data_rel_agg = np.zeros([n_t, int(n_lat / 2), int(n_lon / 2)]) for t in range(0, n_t): for lat in range(0, int(n_lat / 2)): for lon in range(0, int(n_lon / 2)): x = np.nanmin(data_rel[t, (2*lat):(2*lat+2), \ (2*lon):(2*lon+2)]) if np.isnan(float(x)): data_rel_agg[t, lat, lon] = np.nan else: data_rel_agg[t, lat, lon] = np.nanmean(data_rel[t, \ (2*lat):(2*lat+2), (2*lon):(2*lon+2)]) return (data_rel_agg)
def interpusblposition(usbl): # ok lest make sure the GPS is on the second usbl = usbl.drop_duplicates() timesteps = np.diff(usbl.index) / np.timedelta64(1, 's') Transition_Matrix = np.array([[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]]) temp = np.zeros((len(timesteps), 4, 4)) for i in range(len(timesteps)): Transition_Matrix[0, 2] = timesteps[i] Transition_Matrix[1, 3] = timesteps[i] temp[i] = Transition_Matrix Transition_Matrix = temp mask = usbl['UsblNorthing'][:-1].isna() lat = ma.array(usbl['UsblNorthing'][:-1], mask=mask) lon = ma.array(usbl['UsblEasting'][:-1], mask=mask) #print(timesteps.min()) lonSpeed = ma.array(np.diff(usbl.ShipEasting) / timesteps) lonSpeed[np.isnan(lonSpeed)] = 0 latSpeed = ma.array(np.diff(usbl.ShipNorthing) / timesteps) latSpeed[np.isnan(latSpeed)] = 0 coord = ma.dstack((lon, lat, lonSpeed, latSpeed))[0] Observation_Matrix = np.eye(4) xinit = lon[0] yinit = lat[0] vxinit = lonSpeed[0] vyinit = latSpeed[0] initstate = [xinit, yinit, vxinit, vyinit] #print(initstate) initcovariance = 1.0e-3 * np.eye(4) transistionCov = 1.0e-3 * np.eye(4) observationCov = np.eye(4) observationCov[0, 0] = 15 observationCov[1, 1] = 15 observationCov[2, 2] = 0.01 observationCov[3, 3] = 0.01 #print('kman') kf = KalmanFilter(transition_matrices=Transition_Matrix, observation_matrices=Observation_Matrix, initial_state_mean=initstate, initial_state_covariance=initcovariance, transition_covariance=transistionCov, observation_covariance=observationCov) output = kf.smooth(coord)[0] result = usbl result['KfUsblNorthing'] = np.append(output[:, 1], output[-1, 1]) result['KfUsblEasting'] = np.append(output[:, 0], output[-1, 0]) myProj = Proj( "+proj=utm +zone=55F, +south +ellps=WGS84 +datum=WGS84 +units=m +no_defs" ) result['KfUsblLongitude'], result['KfUsblLatitude'] = myProj( result['KfUsblEasting'].values, result['KfUsblNorthing'].values, inverse='True') return result
def get_vectormap(self, pafs, number=None): from numpy import ma if number is None: for i in range(26): u = pafs[2 * i, :, :] * -1 # u = cv2.resize(u, (1280, 720)) v = pafs[2 * i + 1, :, :] # v = cv2.resize(v, (1280, 720)) # print(i) # print(u.max(), v.max()) x, y = np.meshgrid(np.arange(u.shape[1]), np.arange(u.shape[0])) M = np.zeros(u.shape, dtype='bool') M[u**2 + v**2 < 0.2 * 0.2] = True u = ma.masked_array(u, mask=M) v = ma.masked_array(v, mask=M) if i == 0: U = u V = v X = x Y = y else: U = ma.dstack((U, u)) V = ma.dstack((V, v)) X = np.dstack((X, x)) Y = np.dstack((Y, y)) # print(U.shape) U = U.transpose(2, 0, 1) V = V.transpose(2, 0, 1) X = X.transpose(2, 0, 1) Y = Y.transpose(2, 0, 1) else: U = pafs[2 * number, :, :] * -1 # U = cv2.resize(U, (1280, 720)) V = pafs[2 * number + 1, :, :] # V = cv2.resize(V, (1280, 720)) # print(U.max(), V.max()) X, Y = np.meshgrid(np.arange(U.shape[1]), np.arange(U.shape[0])) M = np.zeros(U.shape, dtype='bool') M[U**2 + V**2 < 0.2 * 0.2] = True U = ma.masked_array(U, mask=M) V = ma.masked_array(V, mask=M) # self.frame = cv2.resize(self.frame, (82, 64)) return X, Y, U, V
def __init__(self, cdfPath, i=0): with pycdf.CDF(cdfPath) as root: var = root altitude = var[self.var_altitude][...] / 1000 cameraPosGCRS = var[self.var_cameraPos][i] photoTime = var[self.var_photoTime][i] # for three channels (RGB), each channel is stored as a # separate variable: img_red, img_green, img_blue # for grayscale, the single variable is called 'img' try: fillval = var[self.var_img].attrs['FILLVAL'] img = np.atleast_3d(var[self.var_img][i]) img = _convertImgDtype(img, fillval) except: fillval = var[self.var_img_red].attrs['FILLVAL'] img_red = _convertImgDtype(var[self.var_img_red][i], fillval) img_green = _convertImgDtype(var[self.var_img_green][i], fillval) img_blue = _convertImgDtype(var[self.var_img_blue][i], fillval) img = ma.dstack((img_red, img_green, img_blue)) latsCenter = var[self.var_latsCenter][i] lonsCenter = var[self.var_lonsCenter][i] lats = var[var[self.var_latsCenter].attrs['bounds']][i] lons = var[var[self.var_lonsCenter].attrs['bounds']][i] # TODO read in MLat/MLT as well if available self._latsCenter = ma.masked_invalid(latsCenter) self._lonsCenter = ma.masked_invalid(lonsCenter) self._lats = ma.masked_invalid(lats) self._lons = ma.masked_invalid(lons) self._elevation = ma.masked_invalid(90 - var[self.var_zenithAngle][i]) metadata = root.attrs assert var[self.var_altitude].attrs['UNITS'] == 'meters' assert var[self.var_cameraPos].attrs['UNITS'] == 'kilometers' identifier = os.path.splitext(os.path.basename(cdfPath))[0] BaseMapping.__init__(self, altitude, cameraPosGCRS, photoTime, identifier, metadata=metadata) ArrayImageMixin.__init__(self, img)
def retrieve_lst_space_time(data_files, reliability_files, date_regex, sanity_path): space_list = [] for raster, rel in zip(data_files, reliability_files): if re.compile(date_regex).search(raster).group() != re.compile( date_regex).search(rel).group(): sys.exit("Data and reliability files do not match.") logging.info("Processing data: " + raster) logging.info("Applying reliability mask: " + rel) masked_array = create_lst_masked_array(raster, rel, sanity_path) logging.warn("Data totally masked: " + str(masked_array.mask.all())) space_list.append(masked_array) # Lon, Lat, Time. space_time = ma.dstack(space_list) logging.debug("Space-time shape:" + str(space_time.shape)) logging.warn("Space-time totally masked: " + str(space_time.mask.all())) return space_time
def __init__(self, cdfPath, i=0): with pycdf.CDF(cdfPath) as root: var = root altitude = var[self.var_altitude][...]/1000 cameraPosGCRS = var[self.var_cameraPos][i] photoTime = var[self.var_photoTime][i] # for three channels (RGB), each channel is stored as a # separate variable: img_red, img_green, img_blue # for grayscale, the single variable is called 'img' try: fillval = var[self.var_img].attrs['FILLVAL'] img = np.atleast_3d(var[self.var_img][i]) img = _convertImgDtype(img, fillval) except: fillval = var[self.var_img_red].attrs['FILLVAL'] img_red = _convertImgDtype(var[self.var_img_red][i], fillval) img_green = _convertImgDtype(var[self.var_img_green][i], fillval) img_blue = _convertImgDtype(var[self.var_img_blue][i], fillval) img = ma.dstack((img_red, img_green, img_blue)) latsCenter = var[self.var_latsCenter][i] lonsCenter = var[self.var_lonsCenter][i] lats = var[var[self.var_latsCenter].attrs['bounds']][i] lons = var[var[self.var_lonsCenter].attrs['bounds']][i] # TODO read in MLat/MLT as well if available self._latsCenter = ma.masked_invalid(latsCenter) self._lonsCenter = ma.masked_invalid(lonsCenter) self._lats = ma.masked_invalid(lats) self._lons = ma.masked_invalid(lons) self._elevation = ma.masked_invalid(90 - var[self.var_zenithAngle][i]) metadata = root.attrs assert var[self.var_altitude].attrs['UNITS'] == 'meters' assert var[self.var_cameraPos].attrs['UNITS'] == 'kilometers' identifier = os.path.splitext(os.path.basename(cdfPath))[0] BaseMapping.__init__(self, altitude, cameraPosGCRS, photoTime, identifier, metadata=metadata) ArrayImageMixin.__init__(self, img)
def get_masked_arr_stack(window_size, correction_switch, degree): pol=get_padded_feature_stack(window_size, correction_switch, degree) s_mask=get_slick_wise_mask(window_size) ma_cond=s_mask[...,0] shp=ma_cond.shape num_features=pol.shape[-1] #for i in range(7): #a=ma.masked_where(np.repeat(s_mask[...,0]==0,num_features).reshape(shp[0],shp[1],num_features), pol) #print (pol.shape) #print(s_mask.shape) P_ma=ma.masked_where(np.repeat(s_mask[...,0]<1,num_features).reshape(shp[0],shp[1],num_features), pol) E40_ma=ma.masked_where(np.repeat(s_mask[...,1]<1,num_features).reshape(shp[0],shp[1],num_features), pol) E60_ma=ma.masked_where(np.repeat(s_mask[...,2]<1,num_features).reshape(shp[0],shp[1],num_features), pol) E80_ma=ma.masked_where(np.repeat(s_mask[...,3]<1,num_features).reshape(shp[0],shp[1],num_features), pol) W_near_ma=ma.masked_where(np.repeat(s_mask[...,-3]<1,num_features).reshape(shp[0],shp[1],num_features), pol) W_mid_ma=ma.masked_where(np.repeat(s_mask[...,-2]<1,num_features).reshape(shp[0],shp[1],num_features), pol) W_far_ma=ma.masked_where(np.repeat(s_mask[...,-1]<1,num_features).reshape(shp[0],shp[1],num_features), pol) #return P_ma return ma.dstack((P_ma,E40_ma,E60_ma,E80_ma, W_near_ma, W_mid_ma, W_far_ma))
print(inv_gt_thp) offset_thp = gdal.ApplyGeoTransform(inv_gt_thp, 1399618.9749825108, 705060.6257949192) xoff2, yoff2 = map(int, offset_thp) array_thp = thp.ReadAsArray(xoff2, yoff2, 599, 1240) print("Array THP ", array_thp) comp_dem = ma.masked_where(array_dem >= 3000, array_dem).copy() comp_slope = ma.masked_where(array_slope < 0, array_slope).copy() comp_thp = ma.masked_where(array_thp >= 10000, array_thp).copy() print("DEM", comp_dem) print("SLOPE", comp_slope) print(comp_thp) print("Mean of DEM: ", np.mean(comp_demy), "Min of DEM: ", np.min(comp_dem), "Max of DEM: ", np.max(comp_dem)) print("Mean of slope: ", np.mean(comp_slope), "Min of slope: ", np.min(comp_slope), "Max of slope: ", np.max(comp_slope)) arr_dem_slope = ma.dstack((comp_dem_copy, comp_slope_copy)) print("STACK:",arr_dem_slope) print(arr_dem_slope.shape) a = arr_dem_slope[:,:,0].copy() b = arr_dem_slope[:,:,1].copy() print("DEM:", a) print("SLOPE:",b) req_dem_slope = (a < 1000) & (b < 30) print("Test:", req_dem_slope) print("Shape of test", req_dem_slope.shape) req_dem_slope = req_dem_slope * 1 #print(ma.max(req_dem_slope))
def _zone_averaging( xprop, yprop, zoneprop, zone_minmax, coarsen, zone_avg, dzprop, mprop, summing=False ): # General preprocessing, and... # Change the 3D numpy array so they get layers by # averaging across zones. This may speed up a lot, # but will reduce the resolution. # The x y coordinates shall be averaged (ideally # with thickness weighting...) while e.g. hcpfzprop # must be summed. # Somewhat different processing whether this is a hc thickness # or an average. xpr = xprop ypr = yprop zpr = zoneprop dpr = dzprop mpr = mprop if coarsen > 1: xpr = xprop[::coarsen, ::coarsen, ::].copy(order="C") ypr = yprop[::coarsen, ::coarsen, ::].copy(order="C") zpr = zoneprop[::coarsen, ::coarsen, ::].copy(order="C") dpr = dzprop[::coarsen, ::coarsen, ::].copy(order="C") mpr = mprop[::coarsen, ::coarsen, ::].copy(order="C") zpr.astype(np.int32) if zone_avg: zmin = int(zone_minmax[0]) zmax = int(zone_minmax[1]) if zpr.min() > zmin: zmin = zpr.min() if zpr.max() < zmax: zmax = zpr.max() newx = [] newy = [] newz = [] newm = [] newd = [] for izv in range(zmin, zmax + 1): logger.info("Averaging for zone %s ...", izv) xpr2 = ma.masked_where(zpr != izv, xpr) ypr2 = ma.masked_where(zpr != izv, ypr) zpr2 = ma.masked_where(zpr != izv, zpr) dpr2 = ma.masked_where(zpr != izv, dpr) mpr2 = ma.masked_where(zpr != izv, mpr) # get the thickness and normalize along axis 2 (vertical) # to get normalized thickness weights lay_sums = dpr2.sum(axis=2) normed_dz = dpr2 / lay_sums[:, :, np.newaxis] # assume that coordinates have equal weights within a zone xpr2 = ma.average(xpr2, axis=2) ypr2 = ma.average(ypr2, axis=2) zpr2 = ma.average(zpr2, axis=2) # avg zone dpr2 = ma.sum(dpr2, axis=2) if summing: mpr2 = ma.sum(mpr2, axis=2) else: mpr2 = ma.average(mpr2, weights=normed_dz, axis=2) # avg zone newx.append(xpr2) newy.append(ypr2) newz.append(zpr2) newd.append(dpr2) newm.append(mpr2) xpr = ma.dstack(newx) ypr = ma.dstack(newy) zpr = ma.dstack(newz) dpr = ma.dstack(newd) mpr = ma.dstack(newm) zpr.astype(np.int32) xpr = ma.filled(xpr, fill_value=xtgeo.UNDEF) ypr = ma.filled(ypr, fill_value=xtgeo.UNDEF) zpr = ma.filled(zpr, fill_value=0) dpr = ma.filled(dpr, fill_value=0.0) mpr = ma.filled(mpr, fill_value=0.0) return xpr, ypr, zpr, mpr, dpr
def _slice_between_surfaces( this, cube, sampling, other, other_position, zrange, ndiv, mask, attrlist, mthreshold, snapxy, showprogress=False, deadtraces=True, deletecube=False, ): """Slice and find values between two surfaces.""" npcollect = [] zincr = zrange / float(ndiv) zcenter = this.copy() zcenter.slice_cube(cube, sampling=sampling, mask=mask, snapxy=snapxy, deadtraces=deadtraces) npcollect.append(zcenter.values) # collect below or above the original surface if other_position == "above": mul = -1 else: mul = 1 # collect above the original surface progress = XTGShowProgress(ndiv, show=showprogress, leadtext="progress: ") for idv in range(ndiv): progress.flush(idv) ztmp = this.copy() ztmp.values += zincr * (idv + 1) * mul zvalues = ztmp.values.copy() ztmp.slice_cube(cube, sampling=sampling, mask=mask, snapxy=snapxy, deadtraces=deadtraces) diff = mul * (other.values - zvalues) values = ztmp.values values = ma.masked_where(diff < 0.0, values) npcollect.append(values) stacked = ma.dstack(npcollect) del npcollect if deletecube: del cube # for cases with erosion, the two surfaces are equal isovalues = mul * (other.values - this.values) attvalues = dict() for attr in attrlist: attvaluestmp = _attvalues(attr, stacked) attvalues[attr] = ma.masked_where(isovalues < mthreshold, attvaluestmp) progress.finished() return attvalues # this is dict with numpies, one per attribute
def _slice_constant_window( this, cube, sampling, zrange, ndiv, mask, attrlist, snapxy, showprogress=False, deadtraces=True, deletecube=False, ): """Slice a window, (constant in vertical extent).""" npcollect = [] zcenter = this.copy() logger.info("Mean W of depth no MIDDLE slice is %s", zcenter.values.mean()) zcenter.slice_cube(cube, sampling=sampling, mask=mask, snapxy=snapxy, deadtraces=deadtraces) logger.info("Mean of cube slice is %s", zcenter.values.mean()) npcollect.append(zcenter.values) zincr = zrange / float(ndiv) logger.info("ZINCR is %s", zincr) # collect above the original surface progress = XTGShowProgress(ndiv * 2, show=showprogress, leadtext="progress: ", skip=1) for idv in range(ndiv): progress.flush(idv) ztmp = this.copy() ztmp.values -= zincr * (idv + 1) ztmp.slice_cube(cube, sampling=sampling, mask=mask, snapxy=snapxy, deadtraces=deadtraces) npcollect.append(ztmp.values) # collect below the original surface for idv in range(ndiv): progress.flush(ndiv + idv) ztmp = this.copy() ztmp.values += zincr * (idv + 1) ztmp.slice_cube(cube, sampling=sampling, mask=mask, snapxy=snapxy, deadtraces=deadtraces) npcollect.append(ztmp.values) logger.info("Make a stack of the maps...") stacked = ma.dstack(npcollect) del npcollect if deletecube: del cube attvalues = dict() for attr in attrlist: logger.info("Running attribute %s", attr) attvalues[attr] = _attvalues(attr, stacked) progress.finished() return attvalues # this is dict with numpies, one per attribute
def __init__(self, cdfPath): with Dataset(cdfPath, 'r') as root: var = root.variables altitude = var[self.var_altitude][:] / 1000 cameraPosGCRS = var[self.var_cameraPos][:] photoTime = _readDate(var[self.var_photoTime]) # for three channels (RGB), each channel is stored as a # separate variable: img_red, img_green, img_blue # for grayscale, the single variable is called 'img' try: img = np.atleast_3d(var[self.var_img][:]) img = _convertImgDtype(img) except: img_red = _convertImgDtype(var[self.var_img_red][:]) img_green = _convertImgDtype(var[self.var_img_green][:]) img_blue = _convertImgDtype(var[self.var_img_blue][:]) img = ma.dstack((img_red, img_green, img_blue)) latsCenter = var[self.var_latsCenter][:] lonsCenter = var[self.var_lonsCenter][:] latBounds = var[var[self.var_latsCenter].bounds][:] lonBounds = var[var[self.var_lonsCenter].bounds][:] # TODO read in MLat/MLT as well if available if latsCenter.ndim == 1: latsCenter, lonsCenter = np.dstack( np.meshgrid(latsCenter, lonsCenter)).T assert np.all(latBounds[:-1, 1] == latBounds[1:, 0]) assert np.all(lonBounds[:-1, 1] == lonBounds[1:, 0]) latBounds = np.concatenate((latBounds[:, 0], [latBounds[-1, 1]])) lonBounds = np.concatenate((lonBounds[:, 0], [lonBounds[-1, 1]])) lats, lons = np.dstack(np.meshgrid(latBounds, lonBounds)).T else: lats = np.empty( (latsCenter.shape[0] + 1, latsCenter.shape[1] + 1), latBounds.dtype) lons = np.empty_like(lats) for grid, bounds in [(lats, latBounds), (lons, lonBounds)]: # have to use numpy's assert to handle NaN's correctly assert_array_equal(bounds[:-1, :-1, 2], bounds[:-1, 1:, 3]) assert_array_equal(bounds[:-1, :-1, 2], bounds[1:, 1:, 0]) assert_array_equal(bounds[:-1, :-1, 2], bounds[1:, :-1, 1]) grid[:-1, :-1] = bounds[:, :, 0] grid[-1, :-1] = bounds[-1, :, 3] grid[:-1, -1] = bounds[:, -1, 1] grid[-1, -1] = bounds[-1, -1, 2] self._latsCenter = ma.masked_invalid(latsCenter) self._lonsCenter = ma.masked_invalid(lonsCenter) self._lats = ma.masked_invalid(lats) self._lons = ma.masked_invalid(lons) self._elevation = ma.masked_invalid(90 - var[self.var_zenithAngle][:]) metadata = root.__dict__ # ordered dict of all global attributes assert var[self.var_altitude].units == 'meters' assert var[self.var_cameraPos].units == 'kilometers' identifier = os.path.splitext(os.path.basename(cdfPath))[0] BaseMapping.__init__(self, altitude, cameraPosGCRS, photoTime, identifier, metadata=metadata) ArrayImageMixin.__init__(self, img)
def generate_plots(fname="/tmp/bitstead.hdf5", figbase='/tmp/figures'): with h5py.File("/tmp/bitstead.hdf5") as f: for fieldname,grp in f.iteritems(): years = grp.keys() years.sort() print fieldname ylds = [] profits = [] for year in years: print '\t%s' % year for commodity in grp[year]: print '\t\t%s' % commodity data = grp[year][commodity]['yield'] xmin = data.attrs['xmin'] ymin = data.attrs['ymin'] xmax = data.attrs['xmax'] ymax = data.attrs['ymax'] stride = data.attrs['stride'] map_extents = [xmin,xmax,ymin,ymax] # Map data mask = data['mask'].value == 0 yld = ma.masked_less(ma.masked_invalid(ma.masked_array(data['yield'], mask)), 0) moisture = ma.masked_array(data['moisture'], mask) elevation = ma.masked_array(data['elevation'], mask) # Areas grid_cell_area = (stride ** 2) / 4046.86 areas = ma.masked_array(np.ones(mask.shape), mask) * grid_cell_area # Convert square meters to acres # Calculating profit/loss costs = get_inputs(fieldname, year, commodity, areas) * areas income = yld * areas * get_market_price(fieldname, year, commodity) profit = income - costs ppa = profit/areas # Normalized yield ynorm = (yld - ma.mean(yld)) / ma.std(yld) # For summary info ylds.append(ynorm) profits.append(ppa) try: # Mask (for outline of field) # fig = plt.figure() # plt.ticklabel_format(useOffset=False, axis='y') # plt.ticklabel_format(useOffset=False, axis='x') # plt.imshow(mask, extent=map_extents) # fig.savefig('%s/%s-%s-%s-mask.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Yield Map fig = plt.figure() plt.title('%s %s Yield (Bu/Acre)' % (year, commodity) ) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(yld),ma.max(yld),10), cmap.N) plt.gca().axis('off') plot_grid_map(yld, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-yield-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Yield Histogram fig = plt.figure() plt.title('%s %s Yield Distribution' % (year, commodity) ) a = np.ravel(ma.compressed(yld)) Y,X = np.histogram(a, 10, normed=0, weights=(np.ones(a.shape) * grid_cell_area)) x_span = X.max()-X.min() C = [cmap(((x-X.min())/x_span)) for x in X] plt.bar(X[:-1],Y,color=C,width=X[1]-X[0]) plt.xlabel('Yield (Bushel/Acre)') plt.ylabel('Acres') fig.savefig('%s/%s-%s-%s-yield-histogram.pdf' %(figbase, fieldname, year, commodity), format='pdf') # Normalized Yield Map fig = plt.figure() plt.title('%s %s Yield (Bu/Acre) Normalized' % (year, commodity) ) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(-5,5,11), cmap.N) plot_grid_map(ynorm, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-yield-map-normalized.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Input Costs Map # fig = plt.figure() # plt.title('%s %s Costs ($/Acre)' % (year, commodity) ) # cmap = mpl.cm.get_cmap('RdYlGn') # norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(costs/areas),ma.max(costs/areas),10), cmap.N) # plot_grid_map(costs/areas, cmap=cmap, norm=norm, interpolation='none') # plt.colorbar() # fig.savefig('%s/%s-%s-%s-costs-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Profit/Loss Map fig = plt.figure() plt.title('%s %s Profit/Loss ($/Acre)' % (year, commodity) ) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(ppa),ma.max(ppa),10), cmap.N) plot_grid_map(ppa, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-profit-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Profit/Loss Histogram fig = plt.figure() plt.title('%s %s Profit/Loss Distribution' % (year, commodity) ) a = np.ravel(ma.compressed(profit / areas)) Y,X = np.histogram(a, 10, normed=0, weights=(np.ones(a.shape) * grid_cell_area)) def pmap(x): if x < 0: return cmap(X.min()) else: return cmap(X.max()) x_span = X.max()-X.min() C = [pmap(x) for x in X] plt.bar(X[:-1],Y,color=C,width=X[1]-X[0]) plt.xlabel('Profit/Loss ($/Acre)') plt.ylabel('Acres') fig.savefig('%s/%s-%s-%s-profit-histogram.pdf' %(figbase, fieldname, year, commodity), format='pdf') # A map distinguishing what is (or is not) profitable. fig = plt.figure() plot_grid_map(profit > 0, cmap=cmap) plt.title('%s %s Profit or Loss' % (year,commodity)) fig.savefig('%s/%s-%s-%s-profit-or-loss.pdf' %(figbase, fieldname, year, commodity), format='pdf') except Exception as e: print e print yld.shape print ma.min(yld) print ma.max(yld) print '%s-%s' % (np.min(yld), np.max(yld)) print 'Could not generate maps for %s/%s/%s' % (fieldname, year, commodity) # raise e # Field summary stats # First, Profits. vals = ma.dstack(profits) mvals = ma.mean(vals, axis=2) vvars = ma.std(vals, axis=2) fig = plt.figure() plt.title('Profit, mean across years ($/Acre)') norm = mpl.colors.BoundaryNorm(np.linspace(mvals.min(),mvals.max(),10), cmap.N) plot_grid_map(mvals, cmap=cmap, norm=norm) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-mean.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Profit, std deviation across years') norm = mpl.colors.BoundaryNorm(np.linspace(vvars.min(),vvars.max(),10), cmap.N) plot_grid_map(vvars, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-std.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Profit, max across years ($/Acre)') ymax = ma.max(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymax.min(),ymax.max(),10), cmap.N) plot_grid_map(ymax, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-max.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Profit, min across years ($/Acre)') ymin = ma.min(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymin.min(),ymin.max(),10), cmap.N) plot_grid_map(ymin, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-min.pdf' % (figbase, fieldname)) # Now yields vals = ma.dstack(ylds) mvals = ma.mean(vals, axis=2) vvars = ma.std(vals, axis=2) fig = plt.figure() plt.title('Normalized Yield, mean across years') norm = mpl.colors.BoundaryNorm(np.linspace(mvals.min(),mvals.max(),10), cmap.N) plot_grid_map(mvals, cmap=cmap, norm=norm) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-mean.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, std deviation across years') norm = mpl.colors.BoundaryNorm(np.linspace(vvars.min(),vvars.max(),10), cmap.N) plot_grid_map(vvars, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-std.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, max across years') ymax = ma.max(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymax.min(),ymax.max(),10), cmap.N) plot_grid_map(ymax, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-max.pdf' %(figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, min across years') ymin = ma.min(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(ymin.min(),ymin.max(),10), cmap.N) plot_grid_map(ymin, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-min.pdf' % (figbase, fieldname)) # Log the number of years of data that we have. fig = plt.figure() plt.title('Number of years of data') cnt = ma.count(vals,axis=2) norm = mpl.colors.BoundaryNorm(np.linspace(0,cnt.max(),2*(cnt.max()+1)), cmap.N) plot_grid_map(cnt, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-number-years-map.pdf' % (figbase, fieldname))
def __init__(self, cdfPath): with Dataset(cdfPath, 'r') as root: var = root.variables altitude = var[self.var_altitude][:]/1000 cameraPosGCRS = var[self.var_cameraPos][:] photoTime = _readDate(var[self.var_photoTime]) # for three channels (RGB), each channel is stored as a # separate variable: img_red, img_green, img_blue # for grayscale, the single variable is called 'img' try: img = np.atleast_3d(var[self.var_img][:]) img = _convertImgDtype(img) except: img_red = _convertImgDtype(var[self.var_img_red][:]) img_green = _convertImgDtype(var[self.var_img_green][:]) img_blue = _convertImgDtype(var[self.var_img_blue][:]) img = ma.dstack((img_red, img_green, img_blue)) latsCenter = var[self.var_latsCenter][:] lonsCenter = var[self.var_lonsCenter][:] latBounds = var[var[self.var_latsCenter].bounds][:] lonBounds = var[var[self.var_lonsCenter].bounds][:] # TODO read in MLat/MLT as well if available if latsCenter.ndim == 1: latsCenter, lonsCenter = np.dstack(np.meshgrid(latsCenter, lonsCenter)).T assert np.all(latBounds[:-1,1] == latBounds[1:,0]) assert np.all(lonBounds[:-1,1] == lonBounds[1:,0]) latBounds = np.concatenate((latBounds[:,0], [latBounds[-1,1]])) lonBounds = np.concatenate((lonBounds[:,0], [lonBounds[-1,1]])) lats, lons = np.dstack(np.meshgrid(latBounds, lonBounds)).T else: lats = np.empty((latsCenter.shape[0]+1, latsCenter.shape[1]+1), latBounds.dtype) lons = np.empty_like(lats) for grid, bounds in [(lats, latBounds), (lons, lonBounds)]: # have to use numpy's assert to handle NaN's correctly assert_array_equal(bounds[:-1,:-1,2], bounds[:-1,1:,3]) assert_array_equal(bounds[:-1,:-1,2], bounds[1:,1:,0]) assert_array_equal(bounds[:-1,:-1,2], bounds[1:,:-1,1]) grid[:-1,:-1] = bounds[:,:,0] grid[-1,:-1] = bounds[-1,:,3] grid[:-1,-1] = bounds[:,-1,1] grid[-1,-1] = bounds[-1,-1,2] self._latsCenter = ma.masked_invalid(latsCenter) self._lonsCenter = ma.masked_invalid(lonsCenter) self._lats = ma.masked_invalid(lats) self._lons = ma.masked_invalid(lons) self._elevation = ma.masked_invalid(90 - var[self.var_zenithAngle][:]) metadata = root.__dict__ # ordered dict of all global attributes assert var[self.var_altitude].units == 'meters' assert var[self.var_cameraPos].units == 'kilometers' identifier = os.path.splitext(os.path.basename(cdfPath))[0] BaseMapping.__init__(self, altitude, cameraPosGCRS, photoTime, identifier, metadata=metadata) ArrayImageMixin.__init__(self, img)
def generate_plots(fname="/tmp/bitstead.hdf5", figbase='/tmp/figures'): with h5py.File("/tmp/bitstead.hdf5") as f: for fieldname, grp in f.iteritems(): years = grp.keys() years.sort() print fieldname ylds = [] profits = [] for year in years: print '\t%s' % year for commodity in grp[year]: print '\t\t%s' % commodity data = grp[year][commodity]['yield'] xmin = data.attrs['xmin'] ymin = data.attrs['ymin'] xmax = data.attrs['xmax'] ymax = data.attrs['ymax'] stride = data.attrs['stride'] map_extents = [xmin, xmax, ymin, ymax] # Map data mask = data['mask'].value == 0 yld = ma.masked_less( ma.masked_invalid(ma.masked_array(data['yield'], mask)), 0) moisture = ma.masked_array(data['moisture'], mask) elevation = ma.masked_array(data['elevation'], mask) # Areas grid_cell_area = (stride**2) / 4046.86 areas = ma.masked_array( np.ones(mask.shape), mask ) * grid_cell_area # Convert square meters to acres # Calculating profit/loss costs = get_inputs(fieldname, year, commodity, areas) * areas income = yld * areas * get_market_price( fieldname, year, commodity) profit = income - costs ppa = profit / areas # Normalized yield ynorm = (yld - ma.mean(yld)) / ma.std(yld) # For summary info ylds.append(ynorm) profits.append(ppa) try: # Mask (for outline of field) # fig = plt.figure() # plt.ticklabel_format(useOffset=False, axis='y') # plt.ticklabel_format(useOffset=False, axis='x') # plt.imshow(mask, extent=map_extents) # fig.savefig('%s/%s-%s-%s-mask.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Yield Map fig = plt.figure() plt.title('%s %s Yield (Bu/Acre)' % (year, commodity)) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm( np.linspace(ma.min(yld), ma.max(yld), 10), cmap.N) plt.gca().axis('off') plot_grid_map(yld, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-yield-map.pdf' % (figbase, fieldname, year, commodity), format='pdf') # Yield Histogram fig = plt.figure() plt.title('%s %s Yield Distribution' % (year, commodity)) a = np.ravel(ma.compressed(yld)) Y, X = np.histogram(a, 10, normed=0, weights=(np.ones(a.shape) * grid_cell_area)) x_span = X.max() - X.min() C = [cmap(((x - X.min()) / x_span)) for x in X] plt.bar(X[:-1], Y, color=C, width=X[1] - X[0]) plt.xlabel('Yield (Bushel/Acre)') plt.ylabel('Acres') fig.savefig('%s/%s-%s-%s-yield-histogram.pdf' % (figbase, fieldname, year, commodity), format='pdf') # Normalized Yield Map fig = plt.figure() plt.title('%s %s Yield (Bu/Acre) Normalized' % (year, commodity)) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm(np.linspace(-5, 5, 11), cmap.N) plot_grid_map(ynorm, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-yield-map-normalized.pdf' % (figbase, fieldname, year, commodity), format='pdf') # Input Costs Map # fig = plt.figure() # plt.title('%s %s Costs ($/Acre)' % (year, commodity) ) # cmap = mpl.cm.get_cmap('RdYlGn') # norm = mpl.colors.BoundaryNorm(np.linspace(ma.min(costs/areas),ma.max(costs/areas),10), cmap.N) # plot_grid_map(costs/areas, cmap=cmap, norm=norm, interpolation='none') # plt.colorbar() # fig.savefig('%s/%s-%s-%s-costs-map.pdf' % (figbase, fieldname, year, commodity) , format='pdf') # Profit/Loss Map fig = plt.figure() plt.title('%s %s Profit/Loss ($/Acre)' % (year, commodity)) cmap = mpl.cm.get_cmap('RdYlGn') norm = mpl.colors.BoundaryNorm( np.linspace(ma.min(ppa), ma.max(ppa), 10), cmap.N) plot_grid_map(ppa, cmap=cmap, norm=norm, interpolation='none') plt.colorbar() fig.savefig('%s/%s-%s-%s-profit-map.pdf' % (figbase, fieldname, year, commodity), format='pdf') # Profit/Loss Histogram fig = plt.figure() plt.title('%s %s Profit/Loss Distribution' % (year, commodity)) a = np.ravel(ma.compressed(profit / areas)) Y, X = np.histogram(a, 10, normed=0, weights=(np.ones(a.shape) * grid_cell_area)) def pmap(x): if x < 0: return cmap(X.min()) else: return cmap(X.max()) x_span = X.max() - X.min() C = [pmap(x) for x in X] plt.bar(X[:-1], Y, color=C, width=X[1] - X[0]) plt.xlabel('Profit/Loss ($/Acre)') plt.ylabel('Acres') fig.savefig('%s/%s-%s-%s-profit-histogram.pdf' % (figbase, fieldname, year, commodity), format='pdf') # A map distinguishing what is (or is not) profitable. fig = plt.figure() plot_grid_map(profit > 0, cmap=cmap) plt.title('%s %s Profit or Loss' % (year, commodity)) fig.savefig('%s/%s-%s-%s-profit-or-loss.pdf' % (figbase, fieldname, year, commodity), format='pdf') except Exception as e: print e print yld.shape print ma.min(yld) print ma.max(yld) print '%s-%s' % (np.min(yld), np.max(yld)) print 'Could not generate maps for %s/%s/%s' % ( fieldname, year, commodity) # raise e # Field summary stats # First, Profits. vals = ma.dstack(profits) mvals = ma.mean(vals, axis=2) vvars = ma.std(vals, axis=2) fig = plt.figure() plt.title('Profit, mean across years ($/Acre)') norm = mpl.colors.BoundaryNorm( np.linspace(mvals.min(), mvals.max(), 10), cmap.N) plot_grid_map(mvals, cmap=cmap, norm=norm) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-mean.pdf' % (figbase, fieldname)) fig = plt.figure() plt.title('Profit, std deviation across years') norm = mpl.colors.BoundaryNorm( np.linspace(vvars.min(), vvars.max(), 10), cmap.N) plot_grid_map(vvars, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-std.pdf' % (figbase, fieldname)) fig = plt.figure() plt.title('Profit, max across years ($/Acre)') ymax = ma.max(vals, axis=2) norm = mpl.colors.BoundaryNorm( np.linspace(ymax.min(), ymax.max(), 10), cmap.N) plot_grid_map(ymax, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-max.pdf' % (figbase, fieldname)) fig = plt.figure() plt.title('Profit, min across years ($/Acre)') ymin = ma.min(vals, axis=2) norm = mpl.colors.BoundaryNorm( np.linspace(ymin.min(), ymin.max(), 10), cmap.N) plot_grid_map(ymin, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-profit-min.pdf' % (figbase, fieldname)) # Now yields vals = ma.dstack(ylds) mvals = ma.mean(vals, axis=2) vvars = ma.std(vals, axis=2) fig = plt.figure() plt.title('Normalized Yield, mean across years') norm = mpl.colors.BoundaryNorm( np.linspace(mvals.min(), mvals.max(), 10), cmap.N) plot_grid_map(mvals, cmap=cmap, norm=norm) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-mean.pdf' % (figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, std deviation across years') norm = mpl.colors.BoundaryNorm( np.linspace(vvars.min(), vvars.max(), 10), cmap.N) plot_grid_map(vvars, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-std.pdf' % (figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, max across years') ymax = ma.max(vals, axis=2) norm = mpl.colors.BoundaryNorm( np.linspace(ymax.min(), ymax.max(), 10), cmap.N) plot_grid_map(ymax, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-max.pdf' % (figbase, fieldname)) fig = plt.figure() plt.title('Normalized Yield, min across years') ymin = ma.min(vals, axis=2) norm = mpl.colors.BoundaryNorm( np.linspace(ymin.min(), ymin.max(), 10), cmap.N) plot_grid_map(ymin, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-yield-min.pdf' % (figbase, fieldname)) # Log the number of years of data that we have. fig = plt.figure() plt.title('Number of years of data') cnt = ma.count(vals, axis=2) norm = mpl.colors.BoundaryNorm( np.linspace(0, cnt.max(), 2 * (cnt.max() + 1)), cmap.N) plot_grid_map(cnt, norm=norm, cmap=cmap) plt.colorbar(shrink=0.5) fig.savefig('%s/%s-number-years-map.pdf' % (figbase, fieldname))