def burn_vector_mask_into_raster(raster_path, vector_path, out_path, vector_field=None): """Create a new raster based on the input raster with vector features burned into the raster. To be used as a mask for pixels in the vector. Parameters ---------- raster_path : str vector_path : str out_path : str Path for output raster. Format and Datatype are the same as ``ras``. vector_field : str or None Name of a field in the vector to burn values from. If None, all vector features are burned with a constant value of 1. Returns ------- gdal.Dataset Single band raster with vector geometries burned. """ ras = open_raster(raster_path) vec = open_vector(vector_path) # Check EPSG are same, if not reproject vector. if not same_epsg(ras, vec): raise ValueError( 'Raster and vector are not the same EPSG.\n' '{} != {}'.format(get_epsg(ras), get_epsg(vec)) ) # Create an empty for GDALRasterize to burn vector values to. out_ds = _create_empty_raster(ras, out_path, n_bands=1, no_data_value=0) if vector_field is None: # Use a constant value for all features. burn_values = [1] attribute = None else: # Use the values given in the vector field. burn_values = None attribute = vector_field # Options for Rasterize. # note: burnValues and attribute are exclusive. rasterize_opts = gdal.RasterizeOptions( bands=[1], burnValues=burn_values, attribute=attribute, allTouched=True) _ = gdal.Rasterize(out_ds, vector_path, options=rasterize_opts) # Explicitly close raster to ensure it is saved. out_ds.FlushCache() out_ds = None return open_raster(out_path)
def shape_rasterize(shp_path, out_path, tif_path): """ 统计矢量转换为栅格 :param shp_path: 转换矢量 :param out_path: 输出路径 若设置为空字符串则默认以MEM模式输出 :param tif_path: 为转栅格提供xy向分辨率 :return: """ tif_obj = GeoTiffFile(tif_path) tif_obj.readTif() x_res = tif_obj.getXRes() y_res = tif_obj.getYRes() shp_obj = ShapeFile(shp_path) shp_obj.readShp() shp_extend = shp_obj.getExtent() out_bounds = (shp_extend["xMin"], shp_extend["yMin"], shp_extend["xMax"], shp_extend["yMax"]) if out_path == '': options = gdal.RasterizeOptions(outputBounds=out_bounds, outputType=gdal.GDT_UInt16, noData=65535, attribute="obj_id", useZ=False, xRes=x_res, yRes=y_res, format="MEM") ds = gdal.Rasterize(out_path, shp_path, options=options) else: options = gdal.RasterizeOptions(outputBounds=out_bounds, outputType=gdal.GDT_UInt16, noData=65535, attribute="obj_id", useZ=False, xRes=x_res, yRes=y_res, format="GTiff") ds = gdal.Rasterize(out_path, shp_path, options=options) return ds
def PolygonToRaster(in_path, out_path, x_res, y_res, field_name, out_bounds=None, no_data=65535): """ 面矢量转为栅格 :param in_path: str 输入矢量文件路径 :param out_path: str 输出栅格文件路径 :param x_res: float 输出栅格x向分辨率 :param y_res: float 输出栅格y向分辨率 :param field_name: str 栅格赋值的矢量字段名 :param out_bounds: tuple 输出栅格范围 [xmin, ymin, xmax, ymax] :param no_data: int 无效值大小 :return: """ try: if not out_bounds: gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8", "NO") # 为了支持中文路径,请添加 gdal.SetConfigOption("SHAPE_ENCODING", "GBK") # 为了使属性表字段支持中文,请添加 ogr.RegisterAll() # 注册所有的驱动 layer = ogr.Open(in_path, gdal.GA_ReadOnly).GetLayer(0) shp_extent = layer.GetExtent() out_bounds = (shp_extent[0], shp_extent[2], shp_extent[1], shp_extent[3]) options = gdal.RasterizeOptions(outputBounds=out_bounds, outputType=gdal.GDT_UInt16, noData=no_data, attribute=field_name, useZ=False, xRes=x_res, yRes=y_res, format="GTiff") ds = gdal.Rasterize(out_path, in_path, options=options) if not ds: return False del ds return True except Exception as e: print(e) return False
def shp_rasterize(in_path, out_path, xres, yres, field_name, out_bounds=None, nodataValue=65535): """ 矢量文件栅格化 :param in_path: str: 输入矢量文件路径 :param out_path: str: 输出栅格文件路径 :param xres: float: x向分辨率 :param yres: float: y向分辨率 :param field_name: str: 栅格化字段 :return: """ try: shp_obj = ShapeFile(in_path) shp_obj.readShp() if not out_bounds: shp_extend = shp_obj.getExtent() out_bounds = (shp_extend["xMin"], shp_extend["yMin"], shp_extend["xMax"], shp_extend["yMax"]) # outputBounds [xmin, ymin, xmax, ymax] options = gdal.RasterizeOptions(outputBounds=out_bounds, outputType=gdal.GDT_UInt16, noData=nodataValue, attribute=field_name, useZ=False, xRes=xres, yRes=yres, format="GTiff") ds = gdal.Rasterize(out_path, in_path, options=options) if not ds: return False del ds return True except Exception as e: print(e) return False
def gdal_rasterize(src, dst, options): """ a simple wrapper for :osgeo:func:`gdal.Rasterize` Parameters ---------- src: str or :osgeo:class:`ogr.DataSource` the input data set dst: str the output data set options: dict additional parameters passed to :osgeo:func:`gdal.Rasterize`; see :osgeo:func:`gdal.RasterizeOptions` Returns ------- """ out = gdal.Rasterize(dst, src, options=gdal.RasterizeOptions(**options)) out = None
def rasterize_feat_shp_ds(shp_ds, layer_name, feat_name, feat_id, raster_ds, all_touched=False, exclude=False): if not exclude: str_eq = "='" else: str_eq = "!='" sql_stat = 'SELECT * FROM ' + layer_name + ' WHERE ' + feat_name + str_eq + feat_id + "'" opts = gdal.RasterizeOptions(burnValues=[1], bands=[1], SQLStatement=sql_stat, allTouched=all_touched) gdal.Rasterize(raster_ds, shp_ds, options=opts)
def gdal_rasterize(src, dst, options): """ a simple wrapper for gdal.Rasterize Parameters ---------- src: str or ogr.DataSource the input data set dst: str the output data set options: dict additional parameters passed to gdal.Rasterize; see http://gdal.org/python/osgeo.gdal-module.html#RasterizeOptions Returns ------- """ out = gdal.Rasterize(dst, src, options=gdal.RasterizeOptions(**options)) out = None
def rasterize_shp(fn_shp, fn_raster_ref, all_touched=False): #create memory raster based on reference raster raster_ds = create_mem_raster_on_ref(fn_raster_ref) #open .shp with GDAL shp_ds = gdal.OpenEx(fn_shp, gdal.OF_VECTOR) #rasterize opts = gdal.RasterizeOptions(burnValues=[1], bands=[1], allTouched=all_touched) gdal.Rasterize(raster_ds, shp_ds, options=opts) #get rasterized array rast_array = raster_ds.GetRasterBand(1).ReadAsArray() rast_array = rast_array.astype(float) rast_array[rast_array != 1] = np.nan #close shp shp_ds = None raster_ds = None return ~np.isnan(rast_array)
for region in REGIONS: output_folder = f"data/tif/proximity/{region.get('name')}" if not os.path.exists(output_folder): os.makedirs(output_folder) match_fn = f"data/nc/MODIS/MCD64A1/{region.get('name')}/MCD64A1_500m.nc" match_ds = xarray.open_dataset(match_fn) stack = [] for feature in ACCESSIBILITY_FEATURES: # Create temporal raster rasterize_options = gdal.RasterizeOptions( xRes=xres, yRes=yres, allTouched=True, burnValues=[1] ) temp1 = gdal.Rasterize( "/vsimem/temp", feature.get("path"), options=rasterize_options ) temp1_band = temp1.GetRasterBand(1) # Create temporal proximity raster driver = gdal.GetDriverByName("MEM") temp2 = driver.Create("temp2", temp1.RasterXSize, temp1.RasterYSize) temp2.SetGeoTransform(temp1.GetGeoTransform()) temp2.SetProjection(temp1.GetProjection()) temp2_band = temp2.GetRasterBand(1) gdal.ComputeProximity(temp1_band, temp2_band, ["VALUES=1"]) temp2_band.FlushCache()
def gdal_rasterize(src, dst, options): out = gdal.Rasterize(dst, src, options=gdal.RasterizeOptions(**options)) out = None
def prep_mask(product_dict, maskfilename, bbox_file, prods_TOTbbox, proj, amp_thresh=None, arrshape=None, workdir='./', outputFormat='ENVI'): ''' Function to load and export mask file. If "Download" flag is specified, GSHHS water mask will be donwloaded on the fly. ''' # Import functions from ARIAtools.vrtmanager import renderVRT import glob _world_watermask = [ ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L1.shp', ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L2.shp', ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L3.shp', ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L4.shp', ' /vsizip/vsicurl/https://osmdata.openstreetmap.de/download/land-polygons-complete-4326.zip/land-polygons-complete-4326/land_polygons.shp' ] # Get bounds of user bbox_file bounds = open_shapefile(bbox_file, 0, 0).bounds # File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format. if outputFormat == 'VRT': outputFormat = 'ENVI' # Download mask if maskfilename.lower() == 'download': maskfilename = os.path.join(workdir, 'watermask' + '.msk') ###Make coastlines/islands union VRT os.system('ogrmerge.py -o ' + os.path.join(workdir, 'watermsk_shorelines.vrt') + ''.join(_world_watermask[::2]) + ' -field_strategy Union -f VRT -single') ###Make lakes/ponds union VRT os.system('ogrmerge.py -o ' + os.path.join(workdir, 'watermsk_lakes.vrt') + ''.join(_world_watermask[1::2]) + ' -field_strategy Union -f VRT -single') ###Initiate water-mask with coastlines/islands union VRT gdal.Rasterize(maskfilename, os.path.join(workdir, 'watermsk_shorelines.vrt'), options=gdal.RasterizeOptions(format=outputFormat, outputBounds=bounds, outputType=gdal.GDT_Byte, width=arrshape[1], height=arrshape[0], burnValues=[1], layers='merged')) gdal.Open(maskfilename, gdal.GA_Update).SetProjection(proj) gdal.Translate(maskfilename + '.vrt', maskfilename, options=gdal.TranslateOptions(format="VRT")) ###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land) lake_masks = gdal.Rasterize( '', os.path.join(workdir, 'watermsk_lakes.vrt'), options=gdal.RasterizeOptions(format='MEM', outputBounds=bounds, outputType=gdal.GDT_Byte, width=arrshape[1], height=arrshape[0], burnValues=[1], layers='merged', inverse=True)) lake_masks.SetProjection(proj) lake_masks = lake_masks.ReadAsArray() if amp_thresh: ###Make average amplitude mask # Iterate through all IFGs for i, j in enumerate(product_dict[0]): amp_file = gdal.Warp('', j, options=gdal.WarpOptions( format="MEM", cutlineDSName=prods_TOTbbox, outputBounds=bounds)) amp_file_arr = np.ma.masked_where( amp_file.ReadAsArray() == amp_file.GetRasterBand( 1).GetNoDataValue(), amp_file.ReadAsArray()) # Iteratively update average amplitude file # If looping through first amplitude file, nothing to sum so just save to file if os.path.exists(os.path.join(workdir, 'avgamplitude.msk')): amp_file = gdal.Open( os.path.join(workdir, 'avgamplitude.msk'), gdal.GA_Update) amp_file = amp_file.GetRasterBand(1).WriteArray( amp_file_arr + amp_file.ReadAsArray()) else: renderVRT( os.path.join(workdir, 'avgamplitude.msk'), amp_file_arr, geotrans=amp_file.GetGeoTransform(), drivername=outputFormat, gdal_fmt=amp_file_arr.dtype.name, proj=amp_file.GetProjection(), nodata=amp_file.GetRasterBand(1).GetNoDataValue()) amp_file = coh_val = amp_file_arr = None # Take average of amplitude sum amp_file = gdal.Open(os.path.join(workdir, 'avgamplitude.msk'), gdal.GA_Update) arr_mean = amp_file.ReadAsArray() / len(product_dict[0]) arr_mean = np.where(arr_mean < float(amp_thresh), 0, 1) amp_file = amp_file.GetRasterBand(1).WriteArray(arr_mean) amp_file = None arr_mean = None amp_file = gdal.Open(os.path.join( workdir, 'avgamplitude.msk')).ReadAsArray() else: amp_file = np.ones((lake_masks.shape[0], lake_masks.shape[1])) ###Update water-mask with lakes/ponds union and average amplitude update_file = gdal.Open(maskfilename, gdal.GA_Update) update_file = update_file.GetRasterBand(1).WriteArray( update_file.ReadAsArray() * lake_masks * amp_file) print('Downloaded water-mask here: ' + maskfilename) update_file = None lake_masks = None amp_file = None #Delete temp files os.remove(os.path.join(workdir, 'watermsk_shorelines.vrt')) os.remove(os.path.join(workdir, 'watermsk_lakes.vrt')) for i in glob.glob(os.path.join(workdir, 'avgamplitude.msk*')): os.remove(i) # Load mask try: mask = gdal.Warp('', maskfilename, options=gdal.WarpOptions(format="MEM", cutlineDSName=prods_TOTbbox, outputBounds=bounds, dstNodata=0)) mask.SetProjection(proj) # If no data value if mask.GetRasterBand(1).GetNoDataValue(): mask = np.ma.masked_where( mask.ReadAsArray() == mask.GetRasterBand(1).GetNoDataValue(), mask.ReadAsArray()) else: mask = mask.ReadAsArray() except: raise Exception('Failed to open user mask') return mask
def rasterize_shp2raster_extent(ogr_ds, gdal_ds, attribute=None, burnValues=None, where=None, write_rasterized=False, out_path=None, nodata_val=None): """ Rasterize a ogr datasource to the extent, projection, resolution of a given gdal datasource object. Optionally write out the rasterized product. ogr_ds : osgeo.ogr.DataSource OR os.path.abspath gdal_ds : osgeo.gdal.Dataset write_rasterised : True to write rasterized product, must provide out_path out_path : Path to write rasterized product Writes Rasterized dataset to file. Returns osgeo.gdal.Dataset or None """ logger.debug('Rasterizing OGR DataSource: {}'.format(ogr_ds)) # If datasources are not open, open them if isinstance(ogr_ds, ogr.DataSource): pass else: ogr_ds = gdal.OpenEx(ogr_ds) if isinstance(gdal_ds, gdal.Dataset): pass else: gdal_ds = gdal.Open(gdal_ds) # Get DEM attributes if nodata_val is None: nodata_val = gdal_ds.GetRasterBand(1).GetNoDataValue() dem_sr = gdal_ds.GetProjection() dem_gt = gdal_ds.GetGeoTransform() x_min = dem_gt[0] y_max = dem_gt[3] x_res = dem_gt[1] y_res = dem_gt[5] x_sz = gdal_ds.RasterXSize y_sz = gdal_ds.RasterYSize x_max = x_min + x_res * x_sz y_min = y_max + y_res * y_sz gdal_ds = None ## Open shapefile # ogr_lyr = ogr_ds.GetLayer() # Create new raster in memory if write_rasterized is False: out_path = r'/vsimem/rasterized.tif' if os.path.exists(out_path): os.remove(out_path) ro = gdal.RasterizeOptions(format='GTiff', xRes=x_res, yRes=y_res, width=x_sz, height=y_sz, attribute=attribute, burnValues=burnValues, where=where) # driver = gdal.GetDriverByName('GTiff') # out_ds = driver.Create(out_path, x_sz, y_sz, 1, gdal.GDT_Float32) # out_ds.SetGeoTransform((x_min, x_res, 0, y_min, 0, -y_res)) # out_ds.SetProjection(dem_sr) # band = out_ds.GetRasterBand(1) # band.SetNoDataValue( # dem_no_data_val) # fix to get no_data_val before(?) clipping rasters # band.FlushCache() out_ds = gdal.Rasterize(out_path, ogr_ds, options=ro) out_ds.GetRasterBand(1).SetNoDataValue(nodata_val) # if write_rasterized is False: # return out_ds # else: # out_ds = None # return out_path out_ds = None return out_path
def image_analysis(f): # for f in tqdm(glob.glob("../Data/LongTermStudy/Orthomosaics/Georeferenced/"+"*.tif")): ## Translating RGB image into 16-bit template rgb_img = gdal.Open(f) #Getting filenames fullname = os.path.split(f)[1] filename = os.path.splitext(fullname)[0] #Output to new format template = gdal.Translate( "../Data/LongTermStudy/Templates/NewGeoref/" + filename + "_ITC.tif", rgb_img, format="GTiff", outputType=gdal.GDT_UInt16, creationOptions=['COMPRESS=PACKBITS']) #Properly close the datasets to flush to disk rgb_img = None template = None ##Rasterizing tree crown polygons onto template #Open RGB image, raster template and polygons to burn bgr_img = cv2.imread(f, cv2.IMREAD_COLOR) #Getting dimension of rgb image [height, width, dim] = bgr_img.shape #Burn tree crown polygons onto template if argv[2] == 'Manual': #For manual delineation rasterizeOptions = gdal.RasterizeOptions( format="GTiff", width=width, height=height, attribute="Tree_ID", outputType=gdal.GDT_UInt16) rasterpoly = gdal.Rasterize( "../Data/LongTermStudy/Templates/NewGeoref/" + filename + "_ITC.tif", "../Data/LongTermStudy/ITCSegmentation/Manual/ManualDelineation.shp", options=rasterizeOptions) elif argv[2] == 'VTree': #For VTrees rasterizeOptions = gdal.RasterizeOptions( format="GTiff", width=width, height=height, attribute="id", outputType=gdal.GDT_UInt16) rasterpoly = gdal.Rasterize( "../Data/LongTermStudy/Templates/NewGeoref/" + filename + "_ITC.tif", "../Data/LongTermStudy/ITCSegmentation/VTreeDelineation.shp", options=rasterizeOptions) elif argv[2] == '2DSFM': #For 3D Point Cloud delineation rasterizeOptions = gdal.RasterizeOptions( format="GTiff", width=width, height=height, attribute="ID", outputType=gdal.GDT_UInt16) rasterpoly = gdal.Rasterize( "../Data/LongTermStudy/Templates/NewGeoref/" + filename + "_ITC.tif", "../Data/LongTermStudy/ITCSegmentation/itc2DSFM/2DSFMDelineation.shp", options=rasterizeOptions) elif argv[2] == 'LiDAR': #For LiDAR delineation rasterizeOptions = gdal.RasterizeOptions( format="GTiff", width=width, height=height, attribute="ID", outputType=gdal.GDT_UInt16) rasterpoly = gdal.Rasterize( "../Data/LongTermStudy/Templates/NewGeoref/" + filename + "_ITC.tif", "../Data/LongTermStudy/ITCSegmentation/itcLiDAR/LiDARDelineation.shp", options=rasterizeOptions) #Properly close the datasets to flush to disk rasterpoly = None #~ rgb_img = None bgr_img = None print("Tree crown delineation successful.") #----------------------------------------------------------------------# #----------------------------------------------------------------------# ##################################### ## EXTRACTING FILENAME INFORMATION ## ##################################### ##Create empty arrays for the data Date = [] Start_Time = [] FlightNo = [] Location = [] Software = [] Light = [] Cloud = [] Wind = [] Height = [] ##Splitting path from filename fullname = os.path.split(f)[1] filename = os.path.splitext(fullname)[0] ##When the image was taken #Date find_date = re.compile(r"_([0-9]+-[0-9]+-[0-9]+)_") date = find_date.search(filename).group(1) Date.append(date) #Start Time find_time = re.compile(r"_([0-9]+.[0-9]+)_") start_time = find_time.search(filename).group(1) start_time.replace(".", ":") Start_Time.append(start_time) ##Where the image was taken find_where = re.compile(r"^([a-zA-Z]+\d+){1}_") where = find_where.search(filename).group(1) where = re.split('(\d+)', where) FlightNo.append(where[1]) Location.append(where[0]) ##Software used to stitch find_software = re.compile(r"_([a-z]+)_") software = find_software.search(filename).group(1) if software[0] == "o": Software.append("OpenDroneMap") if software[0] == "p": Software.append("Pix4D") if software[0] == "d": Software.append("DroneDeploy") ##Conditions it was taken in find_cond = re.compile(r"_([A-Z]+)") cond = find_cond.search(filename).group(1) #Explaning conditions if cond[0] == "D": Light.append("Dull") if cond[0] == "B": Light.append("Bright") if cond[0] == "S": Light.append("Sunny") if cond[1] == "N": Cloud.append("None") if cond[1] == "S": Cloud.append("Some") if cond[1] == "C": Cloud.append("Cloudy") if cond[1] == "O": Cloud.append("Overcast") if cond[2] == "N": Wind.append("None") if cond[2] == "L": Wind.append("Light") if cond[2] == "M": Wind.append("Medium") if cond[2] == "H": Wind.append("High") ##() Height of flight in ft or m find_height = re.compile(r"_(\d+[a-zA-Z]+)_") height = find_height.search(filename).group(1) Height.append(height) print("Variables have been added.") #----------------------------------------------------------------------# #----------------------------------------------------------------------# #################################### ## PIXEL ANALYSIS OF ITC SEGMENTS ## #################################### ##Reading 16 bit ITC tiff file and bgr image treecrowns = tiff.imread("../Data/LongTermStudy/Templates/NewGeoref/" + filename + "_ITC.tif") img = cv2.imread(f) #Converting bgr to rgb b, g, r = cv2.split(img) img = cv2.merge([r, g, b]) ##Empty lists to populate Trees = [] R_mean = [] G_mean = [] B_mean = [] R_SD = [] G_SD = [] B_SD = [] RCC = [] GCC = [] BCC = [] ExG = [] ##Pixel Analysis print("Starting pixel analysis.") def pixel_analysis(i): # for i in tqdm(range(1, np.amax(treecrowns+1))): #Individual Tree Number Trees.append(i) #Finding tree crowns in array locations = np.where(treecrowns == i) #Extract based on tree crown cells values = img[locations] values = np.ma.masked_equal(values, 0) values_table = pd.DataFrame(values, columns=["R", "G", "B"]) #Edit to speed up #Calculating mean for each colour channel Rmean = values_table["R"].mean() Gmean = values_table["G"].mean() Bmean = values_table["B"].mean() #Calculating standard deviation for each colour channel Rsd = values_table["R"].std() Gsd = values_table["G"].std() Bsd = values_table["B"].std() #Appending results R_mean.append(Rmean) G_mean.append(Gmean) B_mean.append(Bmean) R_SD.append(Rsd) G_SD.append(Gsd) B_SD.append(Bsd) #Calculating overall brightness rgb = (Rmean + Gmean + Bmean) #Calculating chromatic coordinates for each channel rcc = Rmean / rgb gcc = Gmean / rgb bcc = Bmean / rgb exg = (2 * Gmean) / (Rmean + Bmean) #Appending chromatic coordinates to lists GCC.append(gcc) RCC.append(rcc) BCC.append(bcc) ExG.append(exg) # *map(pixel_analysis, tqdm(range(1, np.amax(treecrowns+1)))), [pixel_analysis(i) for i in tqdm(range(1, np.amax(treecrowns + 1)))] print("Pixel Analysis Completed.") #~ #----------------------------------------------------------------------# #~ #----------------------------------------------------------------------# ######################### ## CREATING DATAFRAMES ## ######################### ##Converting results table to dataframe pixels_df = pd.DataFrame({ "Tree_Crown_ID": Trees, "R_Mean": R_mean, "G_Mean": G_mean, "B_Mean": B_mean, "R_StDev": R_SD, "G_StDev": G_SD, "B_StDev": B_SD, "RCC": RCC, "GCC": GCC, "BCC": BCC, "ExG": ExG }) variables_df = pd.DataFrame({ "Date": Date, "Start_Time": Start_Time, "Flight_Number": FlightNo, "Location": Location, "Software": Software, "Light": Light, "Cloud": Cloud, "Wind": Wind, "Height": Height }) ##Matching dataframe lenghts repeat_variables_df = pd.concat([variables_df] * len(pixels_df), ignore_index=True) ##Combining dataframe combined_df = pd.concat([repeat_variables_df, pixels_df], axis=1) ##Rearranging dataframe #~ results_table = results_table[["Number", "Date", "Location", "Software", "Start_Time", "Height", "Light", "Cloud", "Wind", "Mean_Green", "GCC", "Mean_Red", "RCC", "Mean_Blue", "BCC", "ExG"]] ## Saving dataframe to new csv or existing csv #Command line arguments to name the csv file if not os.path.isfile(argv[1]): combined_df.to_csv(argv[1], index=False) else: with open(argv[1], "a") as f: combined_df.to_csv(f, header=False, index=False)
def prep_mask(product_dict, maskfilename, bbox_file, prods_TOTbbox, proj, amp_thresh=None, arrshape=None, workdir='./', outputFormat='ENVI', num_threads='2'): ''' Function to load and export mask file. If "Download" flag is specified, GSHHS water mask will be donwloaded on the fly. If the full resolution NLCD landcover data is given (NLCD...img) it will be cropped to match product ''' # Import functions from ARIAtools.vrtmanager import renderOGRVRT _world_watermask = [ ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L1.shp', ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L2.shp', ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L3.shp', ' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L4.shp', ' /vsizip/vsicurl/https://osmdata.openstreetmap.de/download/land-polygons-complete-4326.zip/land-polygons-complete-4326/land_polygons.shp' ] # If specified DEM subdirectory exists, delete contents workdir = os.path.join(workdir, 'mask') if os.path.exists( workdir) and os.path.abspath(maskfilename) != os.path.abspath( os.path.join( workdir, os.path.basename(maskfilename).split('.') [0].split('uncropped')[0] + '.msk')) and os.path.abspath( maskfilename) != os.path.abspath( os.path.join( workdir, os.path.basename(maskfilename).split('.')[0] + '.msk')) or maskfilename.lower() == 'download': for i in glob.glob(os.path.join(workdir, '*.*')): os.remove(i) if not os.path.exists(workdir): os.mkdir(workdir) # Get bounds of user bbox_file bounds = open_shapefile(bbox_file, 0, 0).bounds # File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format. if outputFormat == 'VRT': outputFormat = 'ENVI' # Download mask if maskfilename.lower() == 'download': log.info("***Downloading water mask... ***") maskfilename = os.path.join(workdir, 'watermask' + '.msk') os.environ['CPL_ZIP_ENCODING'] = 'UTF-8' ###Make coastlines/islands union VRT renderOGRVRT(os.path.join(workdir, 'watermsk_shorelines.vrt'), _world_watermask[::2]) ###Make lakes/ponds union VRT renderOGRVRT(os.path.join(workdir, 'watermsk_lakes.vrt'), _world_watermask[1::2]) ###Initiate water-mask with coastlines/islands union VRT # save uncropped mask gdal.Rasterize(os.path.join(workdir, 'watermask_uncropped.msk'), os.path.join(workdir, 'watermsk_shorelines.vrt'), options=gdal.RasterizeOptions(format=outputFormat, outputBounds=bounds, outputType=gdal.GDT_Byte, width=arrshape[1], height=arrshape[0], burnValues=[1], layers='merged')) gdal.Translate(os.path.join(workdir, 'watermask_uncropped.msk.vrt'), os.path.join(workdir, 'watermask_uncropped.msk'), options=gdal.TranslateOptions(format="VRT")) # save cropped mask gdal.Warp(maskfilename, os.path.join(workdir, 'watermask_uncropped.msk.vrt'), options=gdal.WarpOptions( format=outputFormat, outputBounds=bounds, outputType=gdal.GDT_Byte, width=arrshape[1], height=arrshape[0], multithread=True, options=['NUM_THREADS=%s' % (num_threads)])) update_file = gdal.Open(maskfilename, gdal.GA_Update) update_file.SetProjection(proj) update_file.GetRasterBand(1).SetNoDataValue(0.) del update_file gdal.Translate(maskfilename + '.vrt', maskfilename, options=gdal.TranslateOptions(format="VRT")) ###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land) lake_masks = gdal.Rasterize( '', os.path.join(workdir, 'watermsk_lakes.vrt'), options=gdal.RasterizeOptions(format='MEM', outputBounds=bounds, outputType=gdal.GDT_Byte, width=arrshape[1], height=arrshape[0], burnValues=[1], layers='merged', inverse=True)) lake_masks.SetProjection(proj) lake_masks = lake_masks.ReadAsArray() ###Update water-mask with lakes/ponds union and average amplitude mask_file = gdal.Open(maskfilename, gdal.GA_Update) mask_file.GetRasterBand(1).WriteArray( lake_masks * gdal.Open(maskfilename).ReadAsArray()) #Delete temp files del lake_masks, mask_file if os.path.basename(maskfilename).lower().startswith('nlcd'): log.info("***Accessing and cropping the NLCD mask...***") maskfilename = NLCDMasker(os.path.dirname(workdir))(proj, bounds, arrshape, outputFormat) # Make sure to apply amplitude mask to downloaded products if os.path.exists(os.path.join(workdir, 'watermsk_shorelines.vrt') ) or os.path.basename(maskfilename) == 'NLCD_crop.msk': ###Make average amplitude mask if amp_thresh: amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'), product_dict[0], bounds, prods_TOTbbox, outputFormat=outputFormat, thresh=amp_thresh) ###Update mask with average amplitude mask_file = gdal.Open(maskfilename, gdal.GA_Update) mask_file.GetRasterBand(1).WriteArray( gdal.Open(maskfilename).ReadAsArray() * amp_file) #Delete temp files del mask_file, amp_file if os.path.exists(os.path.join(workdir, 'watermsk_shorelines.vrt')): os.remove(os.path.join(workdir, 'watermsk_shorelines.vrt')) if os.path.exists(os.path.join(workdir, 'watermsk_lakes.vrt')): os.remove(os.path.join(workdir, 'watermsk_lakes.vrt')) # Load mask try: # Check if uncropped/cropped maskfiles exist in 'mask' subdirectory if not os.path.exists( os.path.join( workdir, os.path.basename(maskfilename).split('.')[0] + '.msk')): # save uncropped masfile gdal.BuildVRT(os.path.join( workdir, os.path.basename(maskfilename).split('.')[0] + '_uncropped.msk.vrt'), maskfilename, options=gdal.BuildVRTOptions(outputBounds=bounds)) # update maskfilename maskfilename = os.path.join( workdir, os.path.basename(maskfilename).split('.')[0].split('uncropped') [0] + '.msk') # save cropped maskfile gdal.Warp(maskfilename, os.path.join( workdir, os.path.basename(maskfilename).split('.')[0] + '_uncropped.msk.vrt'), options=gdal.WarpOptions( format=outputFormat, cutlineDSName=prods_TOTbbox, outputBounds=bounds, width=arrshape[1], height=arrshape[0], multithread=True, options=['NUM_THREADS=%s' % (num_threads)])) mask_file = gdal.Open(maskfilename, gdal.GA_Update) mask_file.SetProjection(proj) del mask_file gdal.Translate(maskfilename + '.vrt', maskfilename, options=gdal.TranslateOptions(format="VRT")) ###Make average amplitude mask if amp_thresh: amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'), product_dict[0], bounds, prods_TOTbbox, outputFormat=outputFormat, thresh=amp_thresh) ###Update mask with average amplitude mask_file = gdal.Open(maskfilename, gdal.GA_Update) mask_file.GetRasterBand(1).WriteArray( gdal.Open(maskfilename).ReadAsArray() * amp_file) #Delete temp files del mask_file, amp_file #pass cropped DEM mask = gdal.Warp('', maskfilename, options=gdal.WarpOptions( format="MEM", cutlineDSName=prods_TOTbbox, outputBounds=bounds, width=arrshape[1], height=arrshape[0], multithread=True, options=['NUM_THREADS=%s' % (num_threads)])) mask.SetProjection(proj) mask.SetDescription(maskfilename) except: raise Exception('Failed to open user mask') return mask
def proximity_shp(fn_shp, raster_ref, type_prox='both'): # create memory raster based on reference raster raster_ds = create_mem_raster_on_ref(raster_ref) # open .shp with GDAL shp_ds = gdal.OpenEx(fn_shp, gdal.OF_VECTOR) # rasterize opts = gdal.RasterizeOptions(burnValues=[1], bands=[1]) gdal.Rasterize(raster_ds, shp_ds, options=opts) # close shp shp_ds = None drv = gdal.GetDriverByName('MEM') proxy_ds = drv.Create('', raster_ds.RasterXSize, raster_ds.RasterYSize, 1, gdal.GetDataTypeByName('Float32')) if type_prox == 'exterior': gdal.ComputeProximity(raster_ds.GetRasterBand(1), proxy_ds.GetRasterBand(1), ["VALUES=1", "DISTUNITS=GEO"]) proxy_array = proxy_ds.GetRasterBand(1).ReadAsArray() elif type_prox == 'interior': raster_arr = raster_ds.GetRasterBand(1).ReadAsArray() mask = (raster_arr == 1) raster_arr[mask] = raster_ds.GetRasterBand(1).GetNoDataValue() raster_arr[~mask] = 1 raster_ds.GetRasterBand(1).WriteArray(raster_arr) gdal.ComputeProximity(raster_ds.GetRasterBand(1), proxy_ds.GetRasterBand(1), ["VALUES=1", "DISTUNITS=GEO"]) proxy_array = proxy_ds.GetRasterBand(1).ReadAsArray() elif type_prox == 'both': gdal.ComputeProximity(raster_ds.GetRasterBand(1), proxy_ds.GetRasterBand(1), ["VALUES=1", "DISTUNITS=GEO"]) proxy_ext = proxy_ds.GetRasterBand(1).ReadAsArray() raster_arr = raster_ds.GetRasterBand(1).ReadAsArray() mask = (raster_arr == 1) raster_arr[mask] = raster_ds.GetRasterBand(1).GetNoDataValue() raster_arr[~mask] = 1 raster_ds.GetRasterBand(1).WriteArray(raster_arr) gdal.ComputeProximity(raster_ds.GetRasterBand(1), proxy_ds.GetRasterBand(1), ["VALUES=1", "DISTUNITS=GEO"]) proxy_int = proxy_ds.GetRasterBand(1).ReadAsArray() proxy_array = proxy_ext + proxy_int else: sys.exit('Type of proximity:' + type_prox + ' not recognized. Must be "interior", "exterior" or "both".') return proxy_array
def get_training_data(image_path, shape_path, attribute="CODE", shape_projection_id=4326): """ Given an image and a shapefile with categories, returns training data and features suitable for fitting a scikit-learn classifier. For full details of how to create an appropriate shapefile, see [here](../index.html#training_data). Parameters ---------- image_path : str The path to the raster image to extract signatures from shape_path : str The path to the shapefile containing labelled class polygons attribute : str, optional The shapefile field containing the class labels. Defaults to "CODE". shape_projection_id : int, optional The EPSG number of the projection of the shapefile. Defaults to EPSG 4326. Returns ------- training_data : array_like A numpy array of shape (n_pixels, bands), where n_pixels is the number of pixels covered by the training polygons features : array_like A 1-d numpy array of length (n_pixels) containing the class labels for the corresponding pixel in training_data Notes ----- For performance, this uses scikit's sparse.nonzero() function to get the location of each training data pixel. This means that this will ignore any classes with a label of '0'. """ # TODO: WRITE A TEST FOR THIS TOO; if this goes wrong, it'll go wrong # quietly and in a way that'll cause the most issues further on down the line FILL_VALUE = -9999 with TemporaryDirectory() as td: # Step 1; rasterise shapefile into .tif of class values shape_projection = osr.SpatialReference() shape_projection.ImportFromEPSG(shape_projection_id) image = gdal.Open(image_path) image_gt = image.GetGeoTransform() x_res, y_res = image_gt[1], image_gt[5] ras_path = os.path.join(td, "poly_ras") ras_params = gdal.RasterizeOptions(noData=0, attribute=attribute, xRes=x_res, yRes=y_res, outputType=gdal.GDT_Int16, outputSRS=shape_projection) # This produces a rasterised geotiff that's right, but not perfectly aligned to pixels. # This can probably be fixed. gdal.Rasterize(ras_path, shape_path, options=ras_params) rasterised_shapefile = gdal.Open(ras_path) shape_array = rasterised_shapefile.GetVirtualMemArray() local_x, local_y = get_local_top_left(image, rasterised_shapefile) shape_sparse = sp.coo_matrix(np.asarray(shape_array).squeeze()) y, x, features = sp.find(shape_sparse) training_data = np.empty((len(features), image.RasterCount)) image_array = image.GetVirtualMemArray() image_view = image_array[:, local_y:local_y + rasterised_shapefile.RasterYSize, local_x:local_x + rasterised_shapefile.RasterXSize] for index in range(len(features)): training_data[index, :] = image_view[:, y[index], x[index]] image_view = None image_array = None shape_array = None rasterised_shapefile = None return training_data, features
def cellneigh_ctry(raster=None, region=None, vector=None, csize=10, rank=1): """ Compute number of spatial cells and neighbours. :param raster: Path to raster file to compute region. :param vector: Path to vector file with country borders. :param region: List/tuple of region coordinates (east, west, south, north). :param csize: Spatial cell size (in km). :param rank: Rank of the neighborhood (1 for chess king's move). :return: List of length 2 with number of neighbours for each cell \ and adjacent cells. """ # Region if raster is not None: r = gdal.Open(raster) ncol_r = r.RasterXSize nrow_r = r.RasterYSize gt = r.GetGeoTransform() Xmin = gt[0] Xmax = gt[0] + gt[1] * ncol_r Ymin = gt[3] + gt[5] * nrow_r Ymax = gt[3] elif region is not None: Xmin = region[0] Xmax = region[1] Ymin = region[2] Ymax = region[3] else: print("raster or region must be specified") sys.exit(1) # Cell number from region print("Compute number of {} x {} km spatial cells".format(csize, csize)) csize = csize * 1000 # Transform km in m ncol = np.int(np.ceil((Xmax - Xmin) / csize)) nrow = np.int(np.ceil((Ymax - Ymin) / csize)) Xmax_new = Xmin + ncol * csize Ymin_new = Ymax + nrow * (-csize) ncell = ncol * nrow print("... {} cells ({} x {})".format(ncell, nrow, ncol)) # Cells within country borders (rasterizing method) cb_ds = gdal.OpenEx(vector, gdal.OF_VECTOR) rOptions = gdal.RasterizeOptions( xRes=csize, yRes=-csize, allTouched=True, outputBounds=[Xmin, Ymin_new, Xmax_new, Ymax], burnValues=1, noData=0) outfile = "/vsimem/tmpfile" ds = gdal.Rasterize(outfile, cb_ds, options=rOptions) mask = ds.ReadAsArray() ds = None gdal.Unlink(outfile) y_in, x_in = np.where(mask == 1) cell_in = y_in * ncol + x_in # Adjacent cells and number of neighbors print("Identify adjacent cells and compute number of neighbors") nneigh = [] adj = [] adj_sort = [] around = np.arange(-rank, rank + 1) for i in range(nrow): for j in range(ncol): if mask[i, j] == 1: I = i + around Iprim = I[(I >= 0) & (I < nrow)] J = j + around Jprim = J[(J >= 0) & (J < ncol)] # Loop on potential neighbors nneighbors = 0 for cy in Iprim: for cx in Jprim: if (not (cy == i and cx == j)) and (mask[cy, cx] == 1): adj.append(cy * ncol + cx) nneighbors += 1 nneigh.append(nneighbors) nneigh = np.array(nneigh) adj = np.array(adj) for i in adj: adj_sort.append(np.flatnonzero(cell_in == i)[0]) adj_sort = np.array(adj_sort) return (nneigh, adj_sort, cell_in, ncell)
def gdalos_rasterize(in_filename: str, shp_filename_or_ds: str, out_filename: str = None, shp_layer_name: str = None, shp_z_attribute: str = 'Height', add: bool = True, extent: GeoRectangle = ..., **kwargs): """rasterize a vector layer into a given raster layer, overriding or adding the values Parameters ---------- in_filename:str input raster filename out_filename:str ouput raster filename, if None - input raster would be updated inplace shp_filename_or_ds: str input vector filename or dataset shp_layer_name: str name of the layer from the vector dataset to use shp_z_attribute: str='Height' name of the attribute to extract the z value from add: bool=True True to add to dtm values to shape values into the raster, False to burn shape values into the raster extent: GeoRectangle = ... None - use the extent of the input raster ... - use the extent of the input vector layer GeoRectangle - custom extent out_res: Real output resolution (if None then auto select) warp_srs: str=None output srs overwrite: bool=True what to do if the output exists (fail of overwrite) Returns ------- output raster dataset """ # shp = gdalos_util.open_ds(shp_filename, gdal.gdalconst.OF_VECTOR) if shp_filename_or_ds is None: shp = None elif isinstance(shp_filename_or_ds, (str, Path)): shp = gdal.OpenEx(str(shp_filename_or_ds), gdal.gdalconst.OF_VECTOR) else: shp = shp_filename_or_ds if out_filename is None: dstDs = gdalos_util.open_ds(in_filename, gdal.GA_Update) else: pj4326 = projdef.get_srs_pj(4326) if extent is ...: cov_pj_srs = projdef.get_srs_pj(shp) if not cov_pj_srs: cov_pj_srs = pj4326 cov_extent = ogr_get_layer_extent(shp.GetLayer()) transform = projdef.get_transform(cov_pj_srs, pj4326) extent = gdalos_extent.transform_extent(cov_extent, transform) dstDs = gdalos_trans(in_filename, out_filename, extent=extent, cog=False, ovr_type=OvrType.no_overviews, return_ds=True, **kwargs) if shp is not None: rasteize_options = gdal.RasterizeOptions(layers=shp_layer_name, add=add, attribute=shp_z_attribute) ret = gdal.Rasterize(dstDs, shp, options=rasteize_options) if ret != 1: raise Exception('Rasterize failed') return dstDs