def to_dataset(lyr: str) -> xr.DataArray: with rio.MemoryFile() as memfile: memfile.write(r_dict[lyr]) with memfile.open() as src: geom = [_geometry.intersection(box(*src.bounds))] if geom[0].is_empty: msk, transform, _ = rio_mask.raster_geometry_mask( src, [_geometry], invert=True) else: msk, transform, _ = rio_mask.raster_geometry_mask( src, geom, invert=True) meta = src.meta meta.update({ "driver": "GTiff", "height": msk.shape[0], "width": msk.shape[1], "transform": transform, "nodata": nodata, }) with rio.vrt.WarpedVRT(src, **meta) as vrt: ds = xr.open_rasterio(vrt) try: ds = ds.squeeze("band", drop=True) except ValueError: pass coords = { ds_dims[0]: ds.coords[ds_dims[0]], ds_dims[1]: ds.coords[ds_dims[1]] } msk_da = xr.DataArray(msk, coords, dims=ds_dims) ds = ds.where(msk_da, drop=True) ds.attrs["crs"] = r_crs.to_string() ds.name = var_name[lyr] return ds
def test_raster_geometrymask_crop_no_overlap(path_rgb_byte_tif, basic_geometry): """If there is no overlap with crop=True, an Exception should be raised""" with rasterio.open(path_rgb_byte_tif) as src: with pytest.raises(ValueError) as excinfo: raster_geometry_mask(src, [basic_geometry], crop=True) assert 'shapes do not overlap raster' in repr(excinfo)
def test_raster_geometrymask_no_overlap(path_rgb_byte_tif, basic_geometry): """If there is no overlap, a warning should be raised""" with rasterio.open(path_rgb_byte_tif) as src: with pytest.warns(UserWarning) as warning: raster_geometry_mask(src, [basic_geometry]) assert 'outside bounds of raster' in warning[0].message.args[0]
def test_raster_geometrymask_crop_no_overlap(path_rgb_byte_tif, basic_geometry): """If there is no overlap with crop=True, an Exception should be raised""" with rasterio.open(path_rgb_byte_tif) as src: with pytest.raises(ValueError) as excinfo: raster_geometry_mask(src, [basic_geometry], crop=True) assert 'shapes do not overlap raster' in repr(excinfo)
def test_raster_geometrymask_crop_invert(basic_image_file, basic_geometry): """crop and invert cannot be combined""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: with pytest.raises(ValueError): raster_geometry_mask(src, geometries, crop=True, invert=True)
def test_raster_geometrymask_no_overlap(path_rgb_byte_tif, basic_geometry): """If there is no overlap, a warning should be raised""" with rasterio.open(path_rgb_byte_tif) as src: with pytest.warns(UserWarning) as warning: raster_geometry_mask(src, [basic_geometry]) assert 'outside bounds of raster' in warning[0].message.args[0]
def test_raster_geometrymask_crop_invert(basic_image_file, basic_geometry): """crop and invert cannot be combined""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: with pytest.raises(ValueError): raster_geometry_mask(src, geometries, crop=True, invert=True)
def extract_image(rst, polygon): with MemoryFile() as memfile: meta = rst.meta.copy() meta["count"] = 4 rgb = mask(rst, [polygon])[0] a = raster_geometry_mask(rst, [polygon], invert=True)[0].astype(rio.uint8) a = np.where(a == 1, 255, 0).astype(rio.uint8) img_data = np.stack((rgb[0], rgb[1], rgb[2], a)) with memfile.open(**meta) as masked: masked.write(img_data) r = masked.read(1, window=from_bounds(*polygon.bounds, rst.transform)) g = masked.read(2, window=from_bounds(*polygon.bounds, rst.transform)) b = masked.read(3, window=from_bounds(*polygon.bounds, rst.transform)) a = masked.read(4, window=from_bounds(*polygon.bounds, rst.transform)) img = Image.fromarray(np.dstack((r, g, b, a))) return img
def construct_input_data(self, tile_location, selected_big_tile): print("=========== CONSTRUCTING TRAINING SET ===========") input_data = np.zeros((self.tile_dimension, self.tile_dimension, self.nb_bands, self.nb_images+1)) #last dimension (images) also holds Region of interest true/false mask. listing = os.listdir(self.data_dir) listing.sort() j=0 for file in listing[1:]: if(not(selected_big_tile in file)): continue data_source = rasterio.open(self.data_dir+'/'+file) for i in range(-1, self.nb_bands): if(i==-1): #Read and add ROI shapefile to input_data tensor for masking out ROI in batch_generator function in order to avoid label mismatch. Region_of_interest = shapefile.Reader(self.region_of_interest_shapefile) ROI_geometry = shape(Region_of_interest.shapes()) masked = raster_geometry_mask(data_source, ROI_geometry) ROI_mask = ~ np.array(masked[0][tile_location[0]*self.tile_dimension:(tile_location[0]+1)*self.tile_dimension,tile_location[1]*self.tile_dimension:(tile_location[1]+1)*self.tile_dimension]) input_data[:self.tile_dimension,:self.tile_dimension,0,self.nb_images] = ROI_mask else: #Preprocess band, then tile it into 8x8 windows and add to input_data. band = data_source.read(1+i) tile = band[tile_location[0]*self.tile_dimension:(tile_location[0]+1)*self.tile_dimension,tile_location[1]*self.tile_dimension:(tile_location[1]+1)*self.tile_dimension] tile -= np.median(tile) tile /= np.percentile(tile,99) np.putmask(tile, tile > 1, 1) np.putmask(tile, tile < -1, -1) input_data[:self.tile_dimension,:self.tile_dimension,i,j] = tile j+=1 print('X_tr size: '+str(sys.getsizeof(input_data))) return(input_data)
def test_raster_geometrymask_invert(basic_image_2x2, basic_image_file, basic_geometry): """Pixels inside the geometry are True in the mask""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries, invert=True) assert np.array_equal(geometrymask, basic_image_2x2) assert transform == Affine.identity()
def test_raster_geometrymask(basic_image_2x2, basic_image_file, basic_geometry): """Pixels inside the geometry are False in the mask""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries) assert np.array_equal(geometrymask, (basic_image_2x2 == 0)) assert transform == Affine.identity() assert window is None
def test_raster_geometrymask_invert(basic_image_2x2, basic_image_file, basic_geometry): """Pixels inside the geometry are True in the mask""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries, invert=True) assert np.array_equal(geometrymask, basic_image_2x2) assert transform == Affine.identity()
def test_raster_geometrymask(basic_image_2x2, basic_image_file, basic_geometry): """Pixels inside the geometry are False in the mask""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries) assert np.array_equal(geometrymask, (basic_image_2x2 == 0)) assert transform == Affine.identity() assert window is None
def extract_pixels(self, raster_fn, ranch=None, pasture=None): if ranch is not None: ranch = ranch.replace(' ', '_') if pasture is not None: pasture = pasture.replace(' ', '_') if pasture is None: raise NotImplementedError("Cannot process request") ds = rasterio.open(raster_fn) ds_proj4 = ds.crs.to_proj4() loc_path = self.loc_path _d = self._d sf_fn = _join(loc_path, _d['sf_fn']) sf_feature_properties_key = _d['sf_feature_properties_key'] sf_fn = os.path.abspath(sf_fn) sf = fiona.open(sf_fn, 'r') features = [] for feature in sf: properties = feature['properties'] key = properties[sf_feature_properties_key].replace(' ', '_') _pasture, _ranch = key.split(self.key_delimiter) if self.reverse_key: _ranch, _pasture = _pasture, _ranch if ranch is not None: if _ranch.lower() != ranch.lower(): continue _features = transform_geom(sf.crs_wkt, ds_proj4, feature['geometry']) if pasture is None: features.append(_features) elif _pasture.lower() == pasture.lower(): features.append(_features) data = ds.read(1, masked=True) if 'biomass' in raster_fn: data = np.ma.masked_values(data, 0) pasture_mask, _, _ = raster_geometry_mask(ds, features) x = np.ma.MaskedArray(data, mask=pasture_mask) return x.compressed().tolist(), int(x.count())
def maskRasterIm(img, GT, roi_analysis, hogs=False): with imtools.convertraster(img, GT) as raster: out, _ = mask.mask(raster, roi_analysis.geometry, invert=False) m, _, _ = mask.raster_geometry_mask(raster, roi_analysis.geometry, invert=False) if hogs: temp = np.zeros((m.shape[0] // 16, m.shape[1] // 16), dtype=bool) for i, row in enumerate(range(0, m.shape[0], 16)): for j, col in enumerate(range(0, m.shape[1], 16)): temp[i, j] = m[row, col] m = ~temp.copy() out = out.transpose([1, 2, 0]).astype('uint8') return out, m
def main(): parser = argparse.ArgumentParser(description=__doc__) add_local_args(parser) args = parser.parse_args() logging.basicConfig(level=args.debug) # currently handling a single geometry at a time geoms = [read_geometry(args.geometry)] if args.method == 'standard': zonal = dict(mean=[], min=[], max=[], std=[]) else: zonal = {args.method: []} with rasterio.open(args.raster) as src: masked, transform, window = raster_geometry_mask(src, geoms, crop=True, all_touched=True) zonal['window'] = window zonal['count'] = src.count zonal['band'] = range(src.count) for i in range(src.count): data = src.read(i + 1, window=window) values = np.ma.array(data=data, mask=np.logical_or(np.equal(data, src.nodata), masked)) if args.method == 'mean' or args.method == 'standard': zonal['mean'].append(np.mean(values)) if args.method == 'min' or args.method == 'standard': zonal['min'].append(np.min(values)) if args.method == 'max' or args.method == 'standard': zonal['max'].append(np.max(values)) if args.method == 'std' or args.method == 'standard': zonal['std'].append(np.std(values)) if args.json: write_json(zonal) elif args.valonly: write_values(zonal, args.method) else: write_gdallocationinfo(zonal, args.method)
def test_raster_geometrymask_crop_pad(basic_image_2x2, basic_image_file, basic_geometry): """Mask returned will be cropped to extent of geometry plus 1/2 pixel on all sides, and transform is transposed 1 down and 1 over""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries, crop=True, pad=True) image = basic_image_2x2[1:5, 1:5] == 0 # invert because invert=False assert geometrymask.shape == (4, 4) assert np.array_equal(geometrymask, image) assert transform == Affine(1, 0, 1, 0, 1, 1) assert window is not None and window.flatten() == (1, 1, 4, 4)
def test_raster_geometrymask_crop(basic_image_2x2, basic_image_file, basic_geometry): """Mask returned will be cropped to extent of geometry, and transform is transposed 2 down and 2 over""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries, crop=True) image = basic_image_2x2[2:5, 2:5] == 0 # invert because invert=False assert geometrymask.shape == (3, 3) assert np.array_equal(geometrymask, image) assert transform == Affine(1, 0, 2, 0, 1, 2) assert window is not None and window.flatten() == (2, 2, 3, 3)
def test_raster_geometrymask_crop_all_touched(basic_image, basic_image_file, basic_geometry): """Mask returned will be cropped to extent of geometry, and transform is transposed 2 down and 2 over""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask( src, geometries, crop=True, all_touched=True) image = basic_image[2:5, 2:5] == 0 # invert because invert=False assert geometrymask.shape == (3, 3) assert np.array_equal(geometrymask, image) assert transform == Affine(1, 0, 2, 0, 1, 2) assert window is not None and window.flatten() == (2, 2, 3, 3)
def test_raster_geometrymask_crop_pad(basic_image_2x2, basic_image_file, basic_geometry): """Mask returned will be cropped to extent of geometry plus 1/2 pixel on all sides, and transform is transposed 1 down and 1 over""" geometries = [basic_geometry] with rasterio.open(basic_image_file) as src: geometrymask, transform, window = raster_geometry_mask(src, geometries, crop=True, pad=True) image = basic_image_2x2[1:5, 1:5] == 0 # invert because invert=False assert geometrymask.shape == (4, 4) assert np.array_equal(geometrymask, image) assert transform == Affine(1, 0, 1, 0, 1, 1) assert window is not None and window.flatten() == (1, 1, 4, 4)
def get_raster_inds(shp_file, fname): # Grab vector file of region of interest df_shp = gpd.read_file(shp_file) extract_shp = df_shp[df_shp[shp_dic['colname']] == shp_dic['geoname']] # Convert extracted feature geometry to geojson for mask extract_shp = extract_shp['geometry'].values[0] # Use Rasterio to open raster file again with rasterio.open(os.path.join(base_dir, fname)) as src: crop_mask, crop_transform, window = raster_geometry_mask( src, [extract_shp], invert=True, all_touched=False) inds = np.where(crop_mask) inds_tpl = [(i, j) for i, j in zip(inds[0], inds[1])] return inds, inds_tpl
def get_pasture_indx(self, raster_fn, pasture, ranch): if ranch is not None: ranch = ranch.replace(' ', '_') if pasture is not None: pasture = pasture.replace(' ', '_') ds = rasterio.open(raster_fn) ds_proj4 = ds.crs.to_proj4() target_ranch = ranch.replace(' ', '_').lower().strip() target_pasture = pasture.replace(' ', '_').lower().strip() loc_path = self.loc_path _d = self._d sf_fn = _join(loc_path, _d['sf_fn']) sf_feature_properties_key = _d['sf_feature_properties_key'] sf_fn = os.path.abspath(sf_fn) sf = fiona.open(sf_fn, 'r') features = [] for feature in sf: properties = feature['properties'] _key = properties[sf_feature_properties_key].replace(' ', '_') _pasture, _ranch = _key.split(self.key_delimiter) if self.reverse_key: _ranch, _pasture = _pasture, _ranch if _ranch.lower() == target_ranch and _pasture.lower( ) == target_pasture: _features = transform_geom(sf.crs_wkt, ds_proj4, feature['geometry']) features.append(_features) if len(features) == 0: raise KeyError((ranch, pasture)) pasture_mask, _, _ = raster_geometry_mask(ds, features) indx = np.where(pasture_mask == False) return indx
def extract_pixels_by_pasture(self, raster_fn, ranch=None): if ranch is not None: ranch = ranch.replace(' ', '_') ds = rasterio.open(raster_fn) ds_proj4 = ds.crs.to_proj4() loc_path = self.loc_path _d = self._d sf_fn = _join(loc_path, _d['sf_fn']) sf_feature_properties_key = _d['sf_feature_properties_key'] sf_fn = os.path.abspath(sf_fn) sf = fiona.open(sf_fn, 'r') data = ds.read(1, masked=True) # if 'biomass' in raster_fn: # data = np.ma.masked_values(data, 0) _data = {} for feature in sf: properties = feature['properties'] key = properties[sf_feature_properties_key].replace(' ', '_') _pasture, _ranch = key.split(self.key_delimiter) if self.reverse_key: _ranch, _pasture = _pasture, _ranch if ranch is not None: if _ranch.lower() != ranch.lower(): continue _features = transform_geom(sf.crs_wkt, ds_proj4, feature['geometry']) pasture_mask, _, _ = raster_geometry_mask(ds, [_features]) x = np.ma.MaskedArray(data, mask=pasture_mask) _data[(_ranch, _pasture)] = x.compressed().tolist(), int(x.count()) return _data
def worker(img, polygon, func, records): dist = Distance('hav') with open(img, 'r') as src: ma, *_ = raster_geometry_mask(src, [polygon], crop=True) data, transform = mask(src, [polygon], crop=True, indexes=1) count = np.ma.masked_equal(ma, True).count() kwargs = { 'img': data, 'transform': transform, 'count': count, 'geometry': polygon, 'distance': dist, } rec = func(**kwargs) if rec: records.append(rec)
def overlap_mask(shapes, filename, profile, tiff_map, target_shape): transform, crs = tiff_map.profile['transform'], tiff_map.profile['crs'] mask_arr = riomask.raster_geometry_mask(tiff_map, shapes, invert=True) mask_arr = mask_arr[0] * 255 i = 0 j = 0 while i < mask_arr.shape[0]: while j < mask_arr.shape[1]: img_arr = mask_arr[i:i + target_shape[0], j:j + target_shape[1]] if np.argmax(img_arr.ravel()) == 0 or img_arr.shape != ( target_shape[0], target_shape[1]): j += target_shape[0] continue img_arr = np.array(img_arr, dtype='uint8') img_arr = Image.fromarray(img_arr) img_arr.save(os.path.join('mask', f'{filename}_{i}_{j}.png')) j += target_shape[0] i += target_shape[1] j = 0
def get_mask_window(shape, raster, max_dims=None): # Get the boolean mask and window of shape try: bool_mask, transform, bb_window = raster_geometry_mask(raster, [shape], crop=True) except ValueError: return None if max_dims is None: return bb_window else: # Turn mask into int array with 1 at farm pixels int_mask = np.bitwise_not(bool_mask).astype(int) # Get the shape of the bounding box window bb_shape = (bb_window.height, bb_window.width) # Get number of pixels to add to x and y dims pad_x = int(np.ceil((max_dims[1] - bb_shape[1]) / 2)) pad_y = int(np.ceil((max_dims[0] - bb_shape[0]) / 2)) # Get a window with padding around it and the desired shape in the center # Depending on the shape, this window can be a different size than max_dims window_ = rasterio.mask.geometry_window( raster, [shape], pad_x=pad_x, pad_y=pad_y, pixel_precision=2, # I found this fixes some rounding errors ) # To fix sizing issues, create a new window that starts at the same top left anchor, but of a fixed width and height window = rasterio.windows.Window( col_off=window_.col_off, row_off=window_.row_off, width=max_dims[1], height=max_dims[0], ) return window
def create_one_mask(file, geom): '''Create a mask for an image given the path and a list of geometries. Args: file: path to filename geom: geometry list from a shapefile Returns: Tuples of image mask and metadata associated to image ''' with rasterio.open(file) as src: out_image, out_transform, window = raster_geometry_mask(src, geom, crop=False, invert=True) out_meta = src.meta.copy() out_meta.update({ "driver": "GTiff", "height": out_image.shape[0], "width": out_image.shape[1], "transform": out_transform, "count": 1 }) return out_image, out_meta
def detect_indicators(geometries, indicators): """Check area of interest against coarse resolution indicator mask for each indicator to see if indicator is present in this area. Parameters ---------- geometries : list-like of geometry objects that provide __geo_interface__ indicators : list-like of indicator IDs Returns ------- list of indicator IDs present in area """ if not indicators: return [] with rasterio.open( src_dir / indicators[0]["filename"].replace(".tif", "_mask.tif") ) as src: geometry_mask, transform, window = raster_geometry_mask( src, geometries, crop=True, all_touched=False ) indicators_with_data = [] for indicator in indicators: with rasterio.open( src_dir / indicator["filename"].replace(".tif", "_mask.tif") ) as src: data = src.read(1, window=window) nodata = src.nodatavals[0] mask = (data == nodata) | geometry_mask # if there are unmasked areas, keep this indicator if not mask.min(): indicators_with_data.append(indicator) return indicators_with_data
def mask_image(id, training=True): if training: dataset = rio.open( './AOI_4_Shanghai_Train/RGB-PanSharpen/RGB-PanSharpen_AOI_4_Shanghai_img{}.tif' .format(id)) data_read = dataset.read() raw_image = parse_rgb(data_read) save_image(raw_image, './train_data/rgb/Shanghai_img{}.png'.format(id), cv2.INTER_LINEAR) else: dataset = rio.open( './AOI_4_Shanghai_Test/RGB-PanSharpen/RGB-PanSharpen_AOI_4_Shanghai_img{}.tif' .format(id)) data_read = dataset.read() raw_image = parse_rgb(data_read) save_image(raw_image, './test_data/rgb/Shanghai_img{}.png'.format(id), cv2.INTER_LINEAR) return geom = json.loads( open( './AOI_4_Shanghai_Train/geojson/buildings/buildings_AOI_4_Shanghai_img{}.geojson' .format(id)).read()) geom = [p['geometry'] for p in geom['features']] if len(geom) == 0: empty_mask = np.zeros(raw_image.shape) masked_image = empty_mask out_mask = empty_mask else: out_image, _ = mask(dataset, geom, crop=False) masked_image = parse_rgb(out_image) out_mask, _, _ = raster_geometry_mask(dataset, geom) save_image(masked_image, './train_data/clipped/Shanghai_img{}.png'.format(id), cv2.INTER_LINEAR) save_image(255 - np.float32(out_mask) * 255, './train_data/mask/Shanghai_img{}.png'.format(id), cv2.INTER_NEAREST)
def analyze_pastures(self, sf, sf_feature_properties_key, sf_feature_properties_delimiter='+'): """ Iterate over each pasture and determine the biomass, etc. for each model :param sf: :return: """ ls = self.ls sat = self.ls.satellite cellsize = ls.cellsize qa_snow = self.qa_snow qa_water = self.qa_water aerosol_mask = self.aerosol_mask not_qa_mask = self.not_qa_mask biomass = self.biomass models = self.models summer_vi = self.summer_vi fall_vi = self.fall_vi summer_mask = self.summer_mask res = [] # becomes a list of dictionary objects for each pasture valid_pastures_cnt = 0 for feature in sf: key = feature['properties'][sf_feature_properties_key] features = [ transform_geom(sf.crs_wkt, ls.proj4, feature['geometry']) ] # true where valid pasture_mask, _, _ = raster_geometry_mask(ls.template_ds, features) not_pasture_mask = np.logical_not(pasture_mask) if np.sum(not_pasture_mask) == 0: warnings.warn('{} in {} has zero pixels in mask'.format( key, ls.product_id)) total_px = np.sum(not_pasture_mask) snow_px = np.sum(np.ma.array(qa_snow, mask=pasture_mask)) water_px = np.sum(np.ma.array(qa_water, mask=pasture_mask)) aerosol_px = np.sum(np.ma.array(aerosol_mask, mask=pasture_mask)) valid_px = np.sum(np.ma.array(not_qa_mask, mask=pasture_mask)) # catch the case where all the pasture grid cells are masked if isinstance(valid_px, np.ma.core.MaskedConstant): valid_px = 0 # not really sure why this happens, it is very infrequent if not total_px > 0: coverage = 0.0 else: coverage = float(valid_px) / float(total_px) area_ha = total_px * cellsize * cellsize * 0.0001 pasture, ranch = key.split(sf_feature_properties_delimiter) if pasture in 'ABCDEFGHIJKLMNO': area_ha = 2.0 coverage = 1.0 model_stats = {} # dictionary of dictionaries for each model for m in models: m_sat = m[sat] d = ModelStat(model=m.name) if coverage > m_sat.required_coverage and area_ha > m_sat.minimum_area_ha: # get masked array of the biomass pasture_biomass = np.ma.array(biomass[m.name], mask=pasture_mask) # calculate the average biomass of each pixel in grams/meter^2 d.biomass_mean_gpm = np.mean(pasture_biomass) # calculate the total estimated biomass based on the area of the pasture d.biomass_total_kg = d.biomass_mean_gpm * area_ha * 10 # determine the 10th, 75th and 90th percentiles of the distribution percentiles = _quantile(pasture_biomass, [0.1, 0.5, 0.75, 0.9]) d.biomass_10pct_gpm = percentiles[0] d.biomass_10pct_gpm = percentiles[1] d.biomass_75pct_gpm = percentiles[2] d.biomass_90pct_gpm = percentiles[3] # calculate the standard deviation of biomass in the pasture d.biomass_sd_gpm = np.std(pasture_biomass) # calculate a 90% confidence interval for biomass_mean_gpm d.biomass_ci90_gpm = 1.645 * (d.biomass_sd_gpm / sqrt(valid_px)) # calculate summer and winter mean gpms d.summer_vi_mean_gpm = np.mean( np.ma.array(summer_vi[m.name], mask=pasture_mask)) d.fall_vi_mean_gpm = np.mean( np.ma.array(fall_vi[m.name], mask=pasture_mask)) # calculate the fraction of the pasture that is above the ndvi_threshold if summer_mask[m.name] is not None: d.fraction_summer = np.sum( np.ma.array(summer_mask[m.name], mask=pasture_mask)) d.fraction_summer /= float(total_px) else: d.fraction_summer = None # keep track of the number of valid pastures models as a quality measure for the scene # this can be more than the number of pastures if there is more than 1 model valid_pastures_cnt += 1 # store the model results model_stats[m.name] = d ls_stats = {} _ndvi = np.ma.array(self.ndvi, mask=pasture_mask) ls_stats['ndvi_mean'] = np.mean(_ndvi) ls_stats['ndvi_sd'] = np.std(_ndvi) _ndvi_percentiles = _quantile(_ndvi, [0.1, 0.5, 0.75, 0.9]) ls_stats['ndvi_10pct'] = _ndvi_percentiles[0] ls_stats['ndvi_50pct'] = _ndvi_percentiles[1] ls_stats['ndvi_75pct'] = _ndvi_percentiles[2] ls_stats['ndvi_90pct'] = _ndvi_percentiles[3] ls_stats['ndvi_ci90'] = 1.645 * (ls_stats['ndvi_sd'] / sqrt(valid_px)) _nbr = np.ma.array(self.nbr, mask=pasture_mask) ls_stats['nbr_mean'] = np.mean(_nbr) ls_stats['nbr_sd'] = np.std(_nbr) _nbr_percentiles = _quantile(_nbr, [0.1, 0.5, 0.75, 0.9]) ls_stats['nbr_10pct'] = _nbr_percentiles[0] ls_stats['nbr_50pct'] = _nbr_percentiles[1] ls_stats['nbr_75pct'] = _nbr_percentiles[2] ls_stats['nbr_90pct'] = _nbr_percentiles[3] ls_stats['nbr_ci90'] = 1.645 * (ls_stats['nbr_sd'] / sqrt(valid_px)) _nbr2 = np.ma.array(self.nbr2, mask=pasture_mask) ls_stats['nbr2_mean'] = np.mean(_nbr2) ls_stats['nbr2_sd'] = np.std(_nbr2) _nbr2_percentiles = _quantile(_nbr2, [0.1, 0.5, 0.75, 0.9]) ls_stats['nbr2_10pct'] = _nbr2_percentiles[0] ls_stats['nbr2_50pct'] = _nbr2_percentiles[1] ls_stats['nbr2_75pct'] = _nbr2_percentiles[2] ls_stats['nbr2_90pct'] = _nbr2_percentiles[3] ls_stats['nbr2_ci90'] = 1.645 * (ls_stats['nbr2_sd'] / sqrt(valid_px)) # store the pasture results res.append( dict(product_id=ls.product_id, key=key, total_px=total_px, area_ha=area_ha, snow_px=snow_px, water_px=water_px, aerosol_px=aerosol_px, valid_px=valid_px, coverage=coverage, valid_pastures_cnt=valid_pastures_cnt, model_stats=model_stats, ls_stats=ls_stats)) return res
def rk45(i_ver, f_ver, init_flg): # --------------------- # Files and directories # --------------------- for ver in range(i_ver, f_ver): base_dir = '/data/vp/pop_migration/comb_rasters_red' shp_file = '/data/shapefiles/yemen_ic.shp' shp_dic = {'colname': 'name', 'geoname': 'initial_condition'} fname = f'yemen_resistance_smoothed_{ver}.tif' # fname = f'yemen_resistance_{ver}.tif' save_dir = f'/data/vp/pop_migration/figs_rk45_{ver}_red' # --------- # Constants # --------- # Scale factor to apply to resistance surface scl_fct = 0.01 # Acceptable tolerance for error e_tol = 4e-3 # Max allowable time step # del_t_max = 1.0 del_t_max = 60.0 # Min allowable time step del_t_min = 0.0001 # Initial time step to try del_t = 0.01 # Time interval for output t_out_int = 25000 # Iteration interval for output i_int = 10 # Upper bound for del_t scaling factor s_upper = 4.0 # Lower bound for del_t scaling factor s_lower = 0.125 # Max number of iterations max_iters = 100000 # Set the boundary conditions t_cool = 0 t_hot = 200 # Test if save directory exists, if not create it if not os.path.isdir(save_dir): os.mkdir(save_dir) # Get projection from reference raster ref_proj = get_raster_proj(os.path.join(base_dir, fname)) # ---------------------------- # Find region to set condition # ---------------------------- # Grab vector file of region of interest df_shp = gpd.read_file(shp_file) extract_shp = df_shp[df_shp[shp_dic['colname']] == shp_dic['geoname']] # Convert extracted feature geometry to geojson for mask extract_shp = extract_shp['geometry'].values[0] # Use Rasterio to open raster file again with rasterio.open(os.path.join(base_dir, fname)) as src: crop_mask, crop_transform, window = raster_geometry_mask( src, [extract_shp], invert=True, all_touched=False) source_inds = np.where(crop_mask) # ------------------------ # Read in friction surface # ------------------------ # Get Friction surface from raster k = raster_2array(os.path.join(base_dir, fname), band=1, replace_nodata_val=None) k = k.astype(float) # Inverse friction to represent thermal conductivity k = 1 / k * scl_fct # Find partials of thermal conductivity k_x, k_y = partial_k(k) k_x[np.isnan(k_x)] = 0 k_y[np.isnan(k_y)] = 0 # Number of rows and columns n_rows = np.shape(k)[0] n_cols = np.shape(k)[1] # Initialize temperature matrix if not init_flg: # Load latest temperature field and set as ic flist = [ os.path.basename(f) for f in glob.glob(os.path.join(save_dir, "*.tif")) ] regex = re.compile(r'\d+') flist.sort(key=lambda x: int(regex.findall(x)[1])) last_f = os.path.join(save_dir, flist[-1]) t_total = int(regex.findall(flist[-1])[1]) t_out_ref = t_total u_0 = raster_2array(last_f, band=1) else: u_0 = t_cool * np.ones((n_rows, n_cols)) # Set initial conditions u_0[source_inds] = t_hot # ----------- # Calculation # ----------- t_total = 0 t_out_ref = 0 for i in range(max_iters): # Calculate first slope k_1 = del_t * f_star(u_0, k, k_x, k_y) # Calculate intermediate estimate of function (1/4) u_1 = u_0 + 0.25 * k_1 # Calculate second slope k_2 = del_t * f_star(u_1, k, k_x, k_y) # Calculate intermediate estimate of function (3/8) u_2 = u_0 + (3 / 32) * k_1 + (9 / 32) * k_2 # Calculate third slope k_3 = del_t * f_star(u_2, k, k_x, k_y) # Calculate intermediate estimate of function (12/13) u_3 = u_0 + (1932 / 2197) * k_1 - (7200 / 2197) * k_2 + ( 7296 / 2197) * k_3 # Calculate fourth slope k_4 = del_t * f_star(u_3, k, k_x, k_y) # Calculate intermediate estimate of function (12/13) u_4 = u_0 + (439 / 216) * k_1 - 8 * k_2 + (3680 / 513) * k_3 - ( 845 / 4104) * k_4 # Calculate fifth slope k_5 = del_t * f_star(u_4, k, k_x, k_y) # Calculate intermediate estimate of function (12/13) u_5 = (u_0 - (8 / 27) * k_1 + 2 * k_2 - (3544 / 2565) * k_3 + (1859 / 4104) * k_4 - (11 / 40) * k_5) # Calculate sixth slope k_6 = del_t * f_star(u_5, k, k_x, k_y) # 4th order approximation u4_update = u_0 + (25 / 216) * k_1 + (1408 / 2565) * k_3 + ( 2197 / 4101) * k_4 - 0.2 * k_5 # 5th order approximation u5_update = (u_0 + (16 / 135) * k_1 + (6656 / 12825) * k_3 + (28561 / 56430) * k_4 - (9 / 50) * k_5 + (2 / 55) * k_6) # ---------------------- # Find optimal step size # ---------------------- # Find error between 4th and 5th order RK u_diff = np.abs(u4_update - u5_update) err_cal = u_diff.max() # Calculate scaling factor s = np.power(((e_tol * del_t) / (2 * err_cal)), 0.25) # Check value of s s = np.min((s, s_upper)) s = np.max((s, s_lower)) # Apply scaling factor to time-step del_t_new = s * del_t # Check if calculated time-step is in bounds del_t_new = np.min((del_t_new, del_t_max)) del_t_new = np.max((del_t_new, del_t_min)) # ------------------------------------------- # Check calculated error compared to min # error tolerance to decide if to accept step # ------------------------------------------- # import ipdb; ipdb.set_trace() if ((err_cal / del_t) < e_tol) or (del_t == del_t_min): t_total += del_t # u_0 = u4_update.copy() u_0 = u5_update.copy() del_t_old = del_t del_t = del_t_new.copy() if i % i_int == 0: print(f'Completed Iteration = {i}') print(f'Total time = {t_total}') print(f'Calculated error per second = {err_cal / del_t_old}') print(f'Current time step = {del_t}') print(f'Minimum value of array = {u_0.min()}') print(f'Maximum value of array = {u_0.max()}') print(f'---------------------------------------------------') # -------------------------- # Output array, svg, and tif # -------------------------- if (t_total >= (t_out_ref + t_out_int)) or (i == 0): t_out_ref = t_total t_out = np.round(t_total, decimals=0) print(f'Saving output for time {t_total}') print(f'---------------------------------------------------') # Save array as raster u_plot = np.reshape(u4_update.copy(), (n_rows, n_cols)) array_2raster( ref_proj, u_plot, fname_out=os.path.join( save_dir, f'yemen_heat_transient_ver_{ver}_time_{t_out:.0f}.tif') ) create_output(t_cool, t_hot, save_dir, ver, t_out, u_plot)
def get_masks_n_imgs(base_dir, band, CROP_SIZE, geoms=None, no_cut=False): """ The data structure is expected to resemble with what it has been tested: BASE_DIR | |_WV | | | |_20190427T083601 | | | | | |_20190427T083601_agriculture.jp2 | | |_20190427T083601_false_color.jp2 | | ... | |_20190427T083601 | | | | | |_20190427T083601_geology.jp2 | | ... |_XA | | | |_20190427T083601 | | | .... """ tiles = [] for d in os.walk(base_dir): if len(d[2]) > 0: l = [i for i in d[2] if '_' + str(band) in i] if len(l) > 0: tiles.append(os.path.join(d[0], l[0])) for tile in tiles: i = j = 0 file = rio.open(tile, driver='GTiff') file_array = file.read() mask_arr, mask_transform, window = riomask.raster_geometry_mask( file, geoms, invert=True) transform, crs = file.profile['transform'], file.profile['crs'] file.close() if no_cut: mask_arr.save(os.path.join('mask', filename)) tag_arr = np.ma.transpose(file_array, [1, 2, 0]) tag_img.save(os.path.join(band, filename)) return while i < file_array.shape[1]: while j < file_array.shape[2]: img_arr = mask_arr[i:i + CROP_SIZE, j:j + CROP_SIZE] * 255 if np.argmax( img_arr.ravel()) == 0 or img_arr.shape != (CROP_SIZE, CROP_SIZE): print('No overlaping masks found in tile {}'.format( (i, i + CROP_SIZE, j, j + CROP_SIZE))) j += CROP_SIZE continue base_name = os.path.join( tile.split('/')[0], tile.split('/')[1], tile.split('/')[2], tile.split('/')[3]) print(base_name) if not os.path.isdir(os.path.join(base_name, 'masks')): os.system('mkdir -p {}'.format( os.path.join(base_name, 'masks'))) if not os.path.isdir(os.path.join(base_name, 'images')): os.system('mkdir -p {}'.format( os.path.join(base_name, 'images'))) img_filename = '{}/images/{}_{}_{}.png'.format( base_name, tile.split('/')[3].split('.')[0], i, j) mask_filename = '{}/masks/{}_{}_{}.png'.format( base_name, tile.split('/')[3].split('.')[0], i, j) print(mask_filename, 'will be added to masks and images') img_arr = Image.fromarray(np.uint8(img_arr)) img_arr.save(mask_filename) tag_arr = file_array[:, i:i + CROP_SIZE, j:j + CROP_SIZE] tag_arr = np.ma.transpose(tag_arr, [1, 2, 0]) tag_img = Image.fromarray(tag_arr.astype(np.uint8)) tag_img.save(img_filename) j += CROP_SIZE i += CROP_SIZE j = 0
def make_pastures_mask(self, raster_fn, ranch, dst_fn, nodata=-9999): """ :param raster_fn: utm raster :param ranches: :param dst_fn: :param nodata: :return: """ assert _exists(_split(dst_fn)[0]) assert dst_fn.endswith('.tif') ds = rasterio.open(raster_fn) ds_proj4 = ds.crs.to_proj4() loc_path = self.loc_path _d = self._d sf_fn = _join(loc_path, _d['sf_fn']) sf_feature_properties_key = _d['sf_feature_properties_key'] sf_fn = os.path.abspath(sf_fn) sf = fiona.open(sf_fn, 'r') reverse_key = self.reverse_key pastures = {} pastures_mask = np.zeros(ds.shape, dtype=np.uint16) for feature in sf: properties = feature['properties'] key = properties[sf_feature_properties_key].replace(' ', '_') if not reverse_key: _pasture, _ranch = key.split(self.key_delimiter) else: _ranch, _pasture = key.split(self.key_delimiter) if _ranch.lower() != ranch.lower(): continue if _pasture not in pastures: pastures[_pasture] = len(pastures) + 1 # true where valid _features = transform_geom(sf.crs_wkt, ds_proj4, feature['geometry']) _mask, _, _ = raster_geometry_mask(ds, [_features]) k = pastures[_pasture] # update pastures_mask pastures_mask[np.where(_mask == False)] = k utm_dst_fn = '' try: head, tail = _split(dst_fn) utm_dst_fn = _join(head, tail.replace('.tif', '.utm.tif')) dst_vrt_fn = _join(head, tail.replace('.tif', '.wgs.vrt')) dst_wgs_fn = _join(head, tail.replace('.tif', '.wgs.tif')) with rasterio.Env(): profile = ds.profile dtype = rasterio.uint16 profile.update(count=1, dtype=rasterio.uint16, nodata=nodata, compress='lzw') with rasterio.open(utm_dst_fn, 'w', **profile) as dst: dst.write(pastures_mask.astype(dtype), 1) assert _exists(utm_dst_fn) except: raise try: if _exists(dst_vrt_fn): os.remove(dst_vrt_fn) cmd = [ 'gdalwarp', '-t_srs', 'EPSG:4326', '-of', 'vrt', utm_dst_fn, dst_vrt_fn ] p = Popen(cmd) p.wait() assert _exists(dst_vrt_fn) except: if _exists(dst_vrt_fn): os.remove(dst_vrt_fn) raise try: if _exists(dst_fn): os.remove(dst_fn) cmd = [ 'gdal_translate', '-co', 'COMPRESS=LZW', '-of', 'GTiff', dst_vrt_fn, dst_wgs_fn ] p = Popen(cmd) p.wait() assert _exists(dst_wgs_fn) except: if _exists(dst_wgs_fn): os.remove(dst_wgs_fn) raise if _exists(dst_vrt_fn): os.remove(dst_vrt_fn) for OUTPUT_RASTER in (dst_wgs_fn, utm_dst_fn): # https://gdal.org/python/osgeo.gdal.RasterAttributeTable-class.html # https://gdal.org/python/osgeo.gdalconst-module.html ds = gdal.Open(OUTPUT_RASTER) rb = ds.GetRasterBand(1) # Create and populate the RAT rat = gdal.RasterAttributeTable() rat.CreateColumn('VALUE', gdal.GFT_Integer, gdal.GFU_Generic) rat.CreateColumn('PASTURE', gdal.GFT_String, gdal.GFU_Generic) for i, (pasture, key) in enumerate(pastures.items()): rat.SetValueAsInt(i, 0, key) rat.SetValueAsString(i, 1, pasture) # Associate with the band rb.SetDefaultRAT(rat) # Close the dataset and persist the RAT ds = None
def mask_ranches(self, raster_fn, ranches, dst_fn, nodata=-9999): """ :param raster_fn: utm raster :param ranches: :param dst_fn: :param nodata: :return: """ assert _exists(_split(dst_fn)[0]) assert dst_fn.endswith('.tif') ds = rasterio.open(raster_fn) ds_proj4 = ds.crs.to_proj4() loc_path = self.loc_path _d = self._d sf_fn = _join(loc_path, _d['sf_fn']) sf_feature_properties_key = _d['sf_feature_properties_key'] sf_fn = os.path.abspath(sf_fn) sf = fiona.open(sf_fn, 'r') reverse_key = self.reverse_key features = [] for feature in sf: properties = feature['properties'] key = properties[sf_feature_properties_key].replace(' ', '_') if not reverse_key: _pasture, _ranch = key.split(self.key_delimiter) else: _ranch, _pasture = key.split(self.key_delimiter) if any(_ranch.lower() == r.lower() for r in ranches): _features = transform_geom(sf.crs_wkt, ds_proj4, feature['geometry']) features.append(_features) utm_dst_fn = '' try: # true where valid pasture_mask, _, _ = raster_geometry_mask(ds, features) data = np.ma.array(ds.read(1, masked=True), mask=pasture_mask) head, tail = _split(dst_fn) utm_dst_fn = _join(head, '_utm_' + tail) if isinstance(data, np.ma.core.MaskedArray): data.fill_value = nodata _data = data.filled() else: _data = data with rasterio.Env(): profile = ds.profile dtype = profile.get('dtype') profile.update(count=1, nodata=nodata, compress='lzw') with rasterio.open(utm_dst_fn, 'w', **profile) as dst: dst.write(_data.astype(dtype), 1) assert _exists(utm_dst_fn) except: if _exists(utm_dst_fn): os.remove(utm_dst_fn) raise dst_vrt_fn = '' try: dst_vrt_fn = dst_fn.replace('.tif', '.vrt') if _exists(dst_vrt_fn): os.remove(dst_vrt_fn) cmd = [ 'gdalwarp', '-t_srs', 'EPSG:4326', '-of', 'vrt', utm_dst_fn, dst_vrt_fn ] p = Popen(cmd) p.wait() assert _exists(dst_vrt_fn) except: if _exists(dst_vrt_fn): os.remove(dst_vrt_fn) raise try: if _exists(dst_fn): os.remove(dst_fn) cmd = [ 'gdal_translate', '-co', 'COMPRESS=LZW', '-of', 'GTiff', dst_vrt_fn, dst_fn ] p = Popen(cmd) p.wait() assert _exists(dst_fn) except: if _exists(dst_fn): os.remove(dst_fn) raise if _exists(utm_dst_fn): os.remove(utm_dst_fn) if _exists(dst_vrt_fn): os.remove(dst_vrt_fn)
def extract_image_features(self, feat_keys=None): """ Extract features from plot polygons specified by plot_data_gdf in image Returns ------- geopandas.GeoDataFrame of features in subindex 'feats' and data in 'data' """ plot_data_gdf = self._plot_data_gdf plot_data_gdf = plot_data_gdf.to_crs( self._image_reader.crs ) # convert plot co-ordinates to image projection im_plot_data_dict = dict() # dict to store plot features etc im_plot_count = 0 max_thumbnail_vals = np.zeros( self._image_reader.count ) # max vals for each image band to scale thumbnails im_feat_dict = {} im_data_dict = {} for plot_id, plot in plot_data_gdf.iterrows( ): # loop through plot polygons # convert polygon to raster mask plot_mask, plot_transform, plot_window = raster_geometry_mask( self._image_reader, [plot['geometry']], crop=True, all_touched=False) plot_cnrs_pixel = np.array(plot_window.toranges()) plot_mask = ~plot_mask # TODO: can we lose this? # check plot window lies inside image if not (np.all(plot_cnrs_pixel[1, :] < self._image_reader.width) and np.all(plot_cnrs_pixel[0, :] < self._image_reader.height) and np.all(plot_cnrs_pixel >= 0) ): # and plot.has_key('Yc') and plot['Yc'] > 0.: logger.warning( f'Excluding plot {plot["ID"]} - outside image extent') continue im_buf = self._image_reader.read( window=plot_window) # read plot ROI from image if np.all(im_buf == 0): logger.warning( f'Excluding plot {plot["ID"]} - all pixels are zero') continue plot_mask = plot_mask & np.all(im_buf > 0, axis=0) & np.all( ~np.isnan(im_buf), axis=0) # exclude any nan or -ve pixels im_feat_dict = self._patch_feature_extractor.extract_features( im_buf, mask=plot_mask, fn_keys=feat_keys) # extract image features for this plot # create plot thumbnail for visualisation in numpy format if self._store_thumbnail: thumbnail = np.float32(np.moveaxis(im_buf, 0, 2)) thumbnail[~plot_mask] = 0. im_data_dict = { **im_feat_dict, **plot, 'thumbnail': thumbnail } # combine features and other plot data # calc max thumbnail vals for scaling later max_val = np.percentile(thumbnail, 98., axis=(0, 1)) max_thumbnail_vals[max_val > max_thumbnail_vals] = max_val[ max_val > max_thumbnail_vals] else: im_data_dict = { **im_feat_dict, **plot } # combine features and other plot data im_plot_data_dict[ plot_id] = im_data_dict # add to dict of all plots im_plot_count += 1 log_dict = { 'ABC': 'Abc' in plot, 'Num zero pixels': (im_buf == 0).sum(), 'Num -ve pixels': (im_buf < 0).sum(), 'Num nan pixels': np.isnan(im_buf).sum() } logger.info( ', '.join([f'Plot {plot_id}'] + ['{}: {}'.format(k, v) for k, v in log_dict.items()])) logger.info('Processed {0} plots'.format(im_plot_count)) # scale thumbnails for display if self._store_thumbnail: for im_data_dict in im_plot_data_dict.values(): thumbnail = im_data_dict['thumbnail'] for b in range(0, self._image_reader.count): thumbnail[:, :, b] /= max_thumbnail_vals[b] thumbnail[:, :, b][thumbnail[:, :, b] > 1.] = 1. im_data_dict['thumbnail'] = thumbnail # create MultiIndex column labels that separate features from other data data_labels = ['feats'] * len(im_feat_dict) + ['data'] * ( len(im_data_dict) - len(im_feat_dict)) columns = pd.MultiIndex.from_arrays( [data_labels, list(im_data_dict.keys())], names=['high', 'low']) # create geodataframe of results self.im_plot_data_gdf = gpd.GeoDataFrame.from_dict(im_plot_data_dict, orient='index') self.im_plot_data_gdf = self.im_plot_data_gdf.set_crs( self._image_reader.crs) self.im_plot_data_gdf.columns = columns self.im_plot_data_gdf[('data', 'ID')] = self.im_plot_data_gdf.index return self.im_plot_data_gdf