def burn_vector_mask_into_raster(raster_path, vector_path, out_path,
vector_field=None):
"""Create a new raster based on the input raster with vector features
burned into the raster. To be used as a mask for pixels in the vector.
Parameters
----------
raster_path : str
vector_path : str
out_path : str
Path for output raster. Format and Datatype are the same as ``ras``.
vector_field : str or None
Name of a field in the vector to burn values from. If None, all vector
features are burned with a constant value of 1.
Returns
-------
gdal.Dataset
Single band raster with vector geometries burned.
"""
ras = open_raster(raster_path)
vec = open_vector(vector_path)
# Check EPSG are same, if not reproject vector.
if not same_epsg(ras, vec):
raise ValueError(
'Raster and vector are not the same EPSG.\n'
'{} != {}'.format(get_epsg(ras), get_epsg(vec))
)
# Create an empty for GDALRasterize to burn vector values to.
out_ds = _create_empty_raster(ras, out_path, n_bands=1, no_data_value=0)
if vector_field is None:
# Use a constant value for all features.
burn_values = [1]
attribute = None
else:
# Use the values given in the vector field.
burn_values = None
attribute = vector_field
# Options for Rasterize.
# note: burnValues and attribute are exclusive.
rasterize_opts = gdal.RasterizeOptions(
bands=[1],
burnValues=burn_values,
attribute=attribute,
allTouched=True)
_ = gdal.Rasterize(out_ds, vector_path, options=rasterize_opts)
# Explicitly close raster to ensure it is saved.
out_ds.FlushCache()
out_ds = None
return open_raster(out_path)
def shape_rasterize(shp_path, out_path, tif_path):
"""
统计矢量转换为栅格
:param shp_path: 转换矢量
:param out_path: 输出路径 若设置为空字符串则默认以MEM模式输出
:param tif_path: 为转栅格提供xy向分辨率
:return:
"""
tif_obj = GeoTiffFile(tif_path)
tif_obj.readTif()
x_res = tif_obj.getXRes()
y_res = tif_obj.getYRes()
shp_obj = ShapeFile(shp_path)
shp_obj.readShp()
shp_extend = shp_obj.getExtent()
out_bounds = (shp_extend["xMin"], shp_extend["yMin"],
shp_extend["xMax"], shp_extend["yMax"])
if out_path == '':
options = gdal.RasterizeOptions(outputBounds=out_bounds,
outputType=gdal.GDT_UInt16,
noData=65535,
attribute="obj_id",
useZ=False,
xRes=x_res,
yRes=y_res,
format="MEM")
ds = gdal.Rasterize(out_path, shp_path, options=options)
else:
options = gdal.RasterizeOptions(outputBounds=out_bounds,
outputType=gdal.GDT_UInt16,
noData=65535,
attribute="obj_id",
useZ=False,
xRes=x_res,
yRes=y_res,
format="GTiff")
ds = gdal.Rasterize(out_path, shp_path, options=options)
return ds
def PolygonToRaster(in_path,
out_path,
x_res,
y_res,
field_name,
out_bounds=None,
no_data=65535):
"""
面矢量转为栅格
:param in_path: str 输入矢量文件路径
:param out_path: str 输出栅格文件路径
:param x_res: float 输出栅格x向分辨率
:param y_res: float 输出栅格y向分辨率
:param field_name: str 栅格赋值的矢量字段名
:param out_bounds: tuple 输出栅格范围 [xmin, ymin, xmax, ymax]
:param no_data: int 无效值大小
:return:
"""
try:
if not out_bounds:
gdal.SetConfigOption("GDAL_FILENAME_IS_UTF8",
"NO") # 为了支持中文路径，请添加
gdal.SetConfigOption("SHAPE_ENCODING",
"GBK") # 为了使属性表字段支持中文，请添加
ogr.RegisterAll() # 注册所有的驱动
layer = ogr.Open(in_path, gdal.GA_ReadOnly).GetLayer(0)
shp_extent = layer.GetExtent()
out_bounds = (shp_extent[0], shp_extent[2], shp_extent[1],
shp_extent[3])
options = gdal.RasterizeOptions(outputBounds=out_bounds,
outputType=gdal.GDT_UInt16,
noData=no_data,
attribute=field_name,
useZ=False,
xRes=x_res,
yRes=y_res,
format="GTiff")
ds = gdal.Rasterize(out_path, in_path, options=options)
if not ds:
return False
del ds
return True
except Exception as e:
print(e)
return False
def shp_rasterize(in_path,
out_path,
xres,
yres,
field_name,
out_bounds=None,
nodataValue=65535):
"""
矢量文件栅格化
:param in_path: str: 输入矢量文件路径
:param out_path: str: 输出栅格文件路径
:param xres: float: x向分辨率
:param yres: float: y向分辨率
:param field_name: str: 栅格化字段
:return:
"""
try:
shp_obj = ShapeFile(in_path)
shp_obj.readShp()
if not out_bounds:
shp_extend = shp_obj.getExtent()
out_bounds = (shp_extend["xMin"], shp_extend["yMin"],
shp_extend["xMax"], shp_extend["yMax"])
# outputBounds [xmin, ymin, xmax, ymax]
options = gdal.RasterizeOptions(outputBounds=out_bounds,
outputType=gdal.GDT_UInt16,
noData=nodataValue,
attribute=field_name,
useZ=False,
xRes=xres,
yRes=yres,
format="GTiff")
ds = gdal.Rasterize(out_path, in_path, options=options)
if not ds:
return False
del ds
return True
except Exception as e:
print(e)
return False
def gdal_rasterize(src, dst, options):
"""
a simple wrapper for :osgeo:func:`gdal.Rasterize`
Parameters
----------
src: str or :osgeo:class:`ogr.DataSource`
the input data set
dst: str
the output data set
options: dict
additional parameters passed to :osgeo:func:`gdal.Rasterize`; see :osgeo:func:`gdal.RasterizeOptions`
Returns
-------
"""
out = gdal.Rasterize(dst, src, options=gdal.RasterizeOptions(**options))
out = None
def rasterize_feat_shp_ds(shp_ds,
layer_name,
feat_name,
feat_id,
raster_ds,
all_touched=False,
exclude=False):
if not exclude:
str_eq = "='"
else:
str_eq = "!='"
sql_stat = 'SELECT * FROM ' + layer_name + ' WHERE ' + feat_name + str_eq + feat_id + "'"
opts = gdal.RasterizeOptions(burnValues=[1],
bands=[1],
SQLStatement=sql_stat,
allTouched=all_touched)
gdal.Rasterize(raster_ds, shp_ds, options=opts)
def gdal_rasterize(src, dst, options):
"""
a simple wrapper for gdal.Rasterize
Parameters
----------
src: str or ogr.DataSource
the input data set
dst: str
the output data set
options: dict
additional parameters passed to gdal.Rasterize;
see http://gdal.org/python/osgeo.gdal-module.html#RasterizeOptions
Returns
-------
"""
out = gdal.Rasterize(dst, src, options=gdal.RasterizeOptions(**options))
out = None
def rasterize_shp(fn_shp, fn_raster_ref, all_touched=False):
#create memory raster based on reference raster
raster_ds = create_mem_raster_on_ref(fn_raster_ref)
#open .shp with GDAL
shp_ds = gdal.OpenEx(fn_shp, gdal.OF_VECTOR)
#rasterize
opts = gdal.RasterizeOptions(burnValues=[1],
bands=[1],
allTouched=all_touched)
gdal.Rasterize(raster_ds, shp_ds, options=opts)
#get rasterized array
rast_array = raster_ds.GetRasterBand(1).ReadAsArray()
rast_array = rast_array.astype(float)
rast_array[rast_array != 1] = np.nan
#close shp
shp_ds = None
raster_ds = None
return ~np.isnan(rast_array)
for region in REGIONS:
output_folder = f"data/tif/proximity/{region.get('name')}"
if not os.path.exists(output_folder):
os.makedirs(output_folder)
match_fn = f"data/nc/MODIS/MCD64A1/{region.get('name')}/MCD64A1_500m.nc"
match_ds = xarray.open_dataset(match_fn)
stack = []
for feature in ACCESSIBILITY_FEATURES:
# Create temporal raster
rasterize_options = gdal.RasterizeOptions(
xRes=xres, yRes=yres, allTouched=True, burnValues=[1]
)
temp1 = gdal.Rasterize(
"/vsimem/temp", feature.get("path"), options=rasterize_options
)
temp1_band = temp1.GetRasterBand(1)
# Create temporal proximity raster
driver = gdal.GetDriverByName("MEM")
temp2 = driver.Create("temp2", temp1.RasterXSize, temp1.RasterYSize)
temp2.SetGeoTransform(temp1.GetGeoTransform())
temp2.SetProjection(temp1.GetProjection())
temp2_band = temp2.GetRasterBand(1)
gdal.ComputeProximity(temp1_band, temp2_band, ["VALUES=1"])
temp2_band.FlushCache()
def gdal_rasterize(src, dst, options):
out = gdal.Rasterize(dst, src, options=gdal.RasterizeOptions(**options))
out = None
def prep_mask(product_dict,
maskfilename,
bbox_file,
prods_TOTbbox,
proj,
amp_thresh=None,
arrshape=None,
workdir='./',
outputFormat='ENVI'):
'''
Function to load and export mask file.
If "Download" flag is specified, GSHHS water mask will be donwloaded on the fly.
'''
# Import functions
from ARIAtools.vrtmanager import renderVRT
import glob
_world_watermask = [
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L1.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L2.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L3.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L4.shp',
' /vsizip/vsicurl/https://osmdata.openstreetmap.de/download/land-polygons-complete-4326.zip/land-polygons-complete-4326/land_polygons.shp'
]
# Get bounds of user bbox_file
bounds = open_shapefile(bbox_file, 0, 0).bounds
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT':
outputFormat = 'ENVI'
# Download mask
if maskfilename.lower() == 'download':
maskfilename = os.path.join(workdir, 'watermask' + '.msk')
###Make coastlines/islands union VRT
os.system('ogrmerge.py -o ' +
os.path.join(workdir, 'watermsk_shorelines.vrt') +
''.join(_world_watermask[::2]) +
' -field_strategy Union -f VRT -single')
###Make lakes/ponds union VRT
os.system('ogrmerge.py -o ' +
os.path.join(workdir, 'watermsk_lakes.vrt') +
''.join(_world_watermask[1::2]) +
' -field_strategy Union -f VRT -single')
###Initiate water-mask with coastlines/islands union VRT
gdal.Rasterize(maskfilename,
os.path.join(workdir, 'watermsk_shorelines.vrt'),
options=gdal.RasterizeOptions(format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged'))
gdal.Open(maskfilename, gdal.GA_Update).SetProjection(proj)
gdal.Translate(maskfilename + '.vrt',
maskfilename,
options=gdal.TranslateOptions(format="VRT"))
###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land)
lake_masks = gdal.Rasterize(
'',
os.path.join(workdir, 'watermsk_lakes.vrt'),
options=gdal.RasterizeOptions(format='MEM',
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged',
inverse=True))
lake_masks.SetProjection(proj)
lake_masks = lake_masks.ReadAsArray()
if amp_thresh:
###Make average amplitude mask
# Iterate through all IFGs
for i, j in enumerate(product_dict[0]):
amp_file = gdal.Warp('',
j,
options=gdal.WarpOptions(
format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds))
amp_file_arr = np.ma.masked_where(
amp_file.ReadAsArray() == amp_file.GetRasterBand(
1).GetNoDataValue(), amp_file.ReadAsArray())
# Iteratively update average amplitude file
# If looping through first amplitude file, nothing to sum so just save to file
if os.path.exists(os.path.join(workdir, 'avgamplitude.msk')):
amp_file = gdal.Open(
os.path.join(workdir, 'avgamplitude.msk'),
gdal.GA_Update)
amp_file = amp_file.GetRasterBand(1).WriteArray(
amp_file_arr + amp_file.ReadAsArray())
else:
renderVRT(
os.path.join(workdir, 'avgamplitude.msk'),
amp_file_arr,
geotrans=amp_file.GetGeoTransform(),
drivername=outputFormat,
gdal_fmt=amp_file_arr.dtype.name,
proj=amp_file.GetProjection(),
nodata=amp_file.GetRasterBand(1).GetNoDataValue())
amp_file = coh_val = amp_file_arr = None
# Take average of amplitude sum
amp_file = gdal.Open(os.path.join(workdir, 'avgamplitude.msk'),
gdal.GA_Update)
arr_mean = amp_file.ReadAsArray() / len(product_dict[0])
arr_mean = np.where(arr_mean < float(amp_thresh), 0, 1)
amp_file = amp_file.GetRasterBand(1).WriteArray(arr_mean)
amp_file = None
arr_mean = None
amp_file = gdal.Open(os.path.join(
workdir, 'avgamplitude.msk')).ReadAsArray()
else:
amp_file = np.ones((lake_masks.shape[0], lake_masks.shape[1]))
###Update water-mask with lakes/ponds union and average amplitude
update_file = gdal.Open(maskfilename, gdal.GA_Update)
update_file = update_file.GetRasterBand(1).WriteArray(
update_file.ReadAsArray() * lake_masks * amp_file)
print('Downloaded water-mask here: ' + maskfilename)
update_file = None
lake_masks = None
amp_file = None
#Delete temp files
os.remove(os.path.join(workdir, 'watermsk_shorelines.vrt'))
os.remove(os.path.join(workdir, 'watermsk_lakes.vrt'))
for i in glob.glob(os.path.join(workdir, 'avgamplitude.msk*')):
os.remove(i)
# Load mask
try:
mask = gdal.Warp('',
maskfilename,
options=gdal.WarpOptions(format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
dstNodata=0))
mask.SetProjection(proj)
# If no data value
if mask.GetRasterBand(1).GetNoDataValue():
mask = np.ma.masked_where(
mask.ReadAsArray() == mask.GetRasterBand(1).GetNoDataValue(),
mask.ReadAsArray())
else:
mask = mask.ReadAsArray()
except:
raise Exception('Failed to open user mask')
return mask
def rasterize_shp2raster_extent(ogr_ds,
gdal_ds,
attribute=None,
burnValues=None,
where=None,
write_rasterized=False,
out_path=None,
nodata_val=None):
"""
Rasterize a ogr datasource to the extent, projection, resolution of a given
gdal datasource object. Optionally write out the rasterized product.
ogr_ds : osgeo.ogr.DataSource OR os.path.abspath
gdal_ds : osgeo.gdal.Dataset
write_rasterised : True to write rasterized product, must provide out_path
out_path : Path to write rasterized product
Writes
Rasterized dataset to file.
Returns
osgeo.gdal.Dataset
or
None
"""
logger.debug('Rasterizing OGR DataSource: {}'.format(ogr_ds))
# If datasources are not open, open them
if isinstance(ogr_ds, ogr.DataSource):
pass
else:
ogr_ds = gdal.OpenEx(ogr_ds)
if isinstance(gdal_ds, gdal.Dataset):
pass
else:
gdal_ds = gdal.Open(gdal_ds)
# Get DEM attributes
if nodata_val is None:
nodata_val = gdal_ds.GetRasterBand(1).GetNoDataValue()
dem_sr = gdal_ds.GetProjection()
dem_gt = gdal_ds.GetGeoTransform()
x_min = dem_gt[0]
y_max = dem_gt[3]
x_res = dem_gt[1]
y_res = dem_gt[5]
x_sz = gdal_ds.RasterXSize
y_sz = gdal_ds.RasterYSize
x_max = x_min + x_res * x_sz
y_min = y_max + y_res * y_sz
gdal_ds = None
## Open shapefile
# ogr_lyr = ogr_ds.GetLayer()
# Create new raster in memory
if write_rasterized is False:
out_path = r'/vsimem/rasterized.tif'
if os.path.exists(out_path):
os.remove(out_path)
ro = gdal.RasterizeOptions(format='GTiff',
xRes=x_res,
yRes=y_res,
width=x_sz,
height=y_sz,
attribute=attribute,
burnValues=burnValues,
where=where)
# driver = gdal.GetDriverByName('GTiff')
# out_ds = driver.Create(out_path, x_sz, y_sz, 1, gdal.GDT_Float32)
# out_ds.SetGeoTransform((x_min, x_res, 0, y_min, 0, -y_res))
# out_ds.SetProjection(dem_sr)
# band = out_ds.GetRasterBand(1)
# band.SetNoDataValue(
# dem_no_data_val) # fix to get no_data_val before(?) clipping rasters
# band.FlushCache()
out_ds = gdal.Rasterize(out_path, ogr_ds, options=ro)
out_ds.GetRasterBand(1).SetNoDataValue(nodata_val)
# if write_rasterized is False:
# return out_ds
# else:
# out_ds = None
# return out_path
out_ds = None
return out_path
def image_analysis(f):
# for f in tqdm(glob.glob("../Data/LongTermStudy/Orthomosaics/Georeferenced/"+"*.tif")):
## Translating RGB image into 16-bit template
rgb_img = gdal.Open(f)
#Getting filenames
fullname = os.path.split(f)[1]
filename = os.path.splitext(fullname)[0]
#Output to new format
template = gdal.Translate(
"../Data/LongTermStudy/Templates/NewGeoref/" + filename +
"_ITC.tif",
rgb_img,
format="GTiff",
outputType=gdal.GDT_UInt16,
creationOptions=['COMPRESS=PACKBITS'])
#Properly close the datasets to flush to disk
rgb_img = None
template = None
##Rasterizing tree crown polygons onto template
#Open RGB image, raster template and polygons to burn
bgr_img = cv2.imread(f, cv2.IMREAD_COLOR)
#Getting dimension of rgb image
[height, width, dim] = bgr_img.shape
#Burn tree crown polygons onto template
if argv[2] == 'Manual':
#For manual delineation
rasterizeOptions = gdal.RasterizeOptions(
format="GTiff",
width=width,
height=height,
attribute="Tree_ID",
outputType=gdal.GDT_UInt16)
rasterpoly = gdal.Rasterize(
"../Data/LongTermStudy/Templates/NewGeoref/" + filename +
"_ITC.tif",
"../Data/LongTermStudy/ITCSegmentation/Manual/ManualDelineation.shp",
options=rasterizeOptions)
elif argv[2] == 'VTree':
#For VTrees
rasterizeOptions = gdal.RasterizeOptions(
format="GTiff",
width=width,
height=height,
attribute="id",
outputType=gdal.GDT_UInt16)
rasterpoly = gdal.Rasterize(
"../Data/LongTermStudy/Templates/NewGeoref/" + filename +
"_ITC.tif",
"../Data/LongTermStudy/ITCSegmentation/VTreeDelineation.shp",
options=rasterizeOptions)
elif argv[2] == '2DSFM':
#For 3D Point Cloud delineation
rasterizeOptions = gdal.RasterizeOptions(
format="GTiff",
width=width,
height=height,
attribute="ID",
outputType=gdal.GDT_UInt16)
rasterpoly = gdal.Rasterize(
"../Data/LongTermStudy/Templates/NewGeoref/" + filename +
"_ITC.tif",
"../Data/LongTermStudy/ITCSegmentation/itc2DSFM/2DSFMDelineation.shp",
options=rasterizeOptions)
elif argv[2] == 'LiDAR':
#For LiDAR delineation
rasterizeOptions = gdal.RasterizeOptions(
format="GTiff",
width=width,
height=height,
attribute="ID",
outputType=gdal.GDT_UInt16)
rasterpoly = gdal.Rasterize(
"../Data/LongTermStudy/Templates/NewGeoref/" + filename +
"_ITC.tif",
"../Data/LongTermStudy/ITCSegmentation/itcLiDAR/LiDARDelineation.shp",
options=rasterizeOptions)
#Properly close the datasets to flush to disk
rasterpoly = None
#~ rgb_img = None
bgr_img = None
print("Tree crown delineation successful.")
#----------------------------------------------------------------------#
#----------------------------------------------------------------------#
#####################################
## EXTRACTING FILENAME INFORMATION ##
#####################################
##Create empty arrays for the data
Date = []
Start_Time = []
FlightNo = []
Location = []
Software = []
Light = []
Cloud = []
Wind = []
Height = []
##Splitting path from filename
fullname = os.path.split(f)[1]
filename = os.path.splitext(fullname)[0]
##When the image was taken
#Date
find_date = re.compile(r"_([0-9]+-[0-9]+-[0-9]+)_")
date = find_date.search(filename).group(1)
Date.append(date)
#Start Time
find_time = re.compile(r"_([0-9]+.[0-9]+)_")
start_time = find_time.search(filename).group(1)
start_time.replace(".", ":")
Start_Time.append(start_time)
##Where the image was taken
find_where = re.compile(r"^([a-zA-Z]+\d+){1}_")
where = find_where.search(filename).group(1)
where = re.split('(\d+)', where)
FlightNo.append(where[1])
Location.append(where[0])
##Software used to stitch
find_software = re.compile(r"_([a-z]+)_")
software = find_software.search(filename).group(1)
if software[0] == "o":
Software.append("OpenDroneMap")
if software[0] == "p":
Software.append("Pix4D")
if software[0] == "d":
Software.append("DroneDeploy")
##Conditions it was taken in
find_cond = re.compile(r"_([A-Z]+)")
cond = find_cond.search(filename).group(1)
#Explaning conditions
if cond[0] == "D":
Light.append("Dull")
if cond[0] == "B":
Light.append("Bright")
if cond[0] == "S":
Light.append("Sunny")
if cond[1] == "N":
Cloud.append("None")
if cond[1] == "S":
Cloud.append("Some")
if cond[1] == "C":
Cloud.append("Cloudy")
if cond[1] == "O":
Cloud.append("Overcast")
if cond[2] == "N":
Wind.append("None")
if cond[2] == "L":
Wind.append("Light")
if cond[2] == "M":
Wind.append("Medium")
if cond[2] == "H":
Wind.append("High")
##() Height of flight in ft or m
find_height = re.compile(r"_(\d+[a-zA-Z]+)_")
height = find_height.search(filename).group(1)
Height.append(height)
print("Variables have been added.")
#----------------------------------------------------------------------#
#----------------------------------------------------------------------#
####################################
## PIXEL ANALYSIS OF ITC SEGMENTS ##
####################################
##Reading 16 bit ITC tiff file and bgr image
treecrowns = tiff.imread("../Data/LongTermStudy/Templates/NewGeoref/" +
filename + "_ITC.tif")
img = cv2.imread(f)
#Converting bgr to rgb
b, g, r = cv2.split(img)
img = cv2.merge([r, g, b])
##Empty lists to populate
Trees = []
R_mean = []
G_mean = []
B_mean = []
R_SD = []
G_SD = []
B_SD = []
RCC = []
GCC = []
BCC = []
ExG = []
##Pixel Analysis
print("Starting pixel analysis.")
def pixel_analysis(i):
# for i in tqdm(range(1, np.amax(treecrowns+1))):
#Individual Tree Number
Trees.append(i)
#Finding tree crowns in array
locations = np.where(treecrowns == i)
#Extract based on tree crown cells
values = img[locations]
values = np.ma.masked_equal(values, 0)
values_table = pd.DataFrame(values,
columns=["R", "G",
"B"]) #Edit to speed up
#Calculating mean for each colour channel
Rmean = values_table["R"].mean()
Gmean = values_table["G"].mean()
Bmean = values_table["B"].mean()
#Calculating standard deviation for each colour channel
Rsd = values_table["R"].std()
Gsd = values_table["G"].std()
Bsd = values_table["B"].std()
#Appending results
R_mean.append(Rmean)
G_mean.append(Gmean)
B_mean.append(Bmean)
R_SD.append(Rsd)
G_SD.append(Gsd)
B_SD.append(Bsd)
#Calculating overall brightness
rgb = (Rmean + Gmean + Bmean)
#Calculating chromatic coordinates for each channel
rcc = Rmean / rgb
gcc = Gmean / rgb
bcc = Bmean / rgb
exg = (2 * Gmean) / (Rmean + Bmean)
#Appending chromatic coordinates to lists
GCC.append(gcc)
RCC.append(rcc)
BCC.append(bcc)
ExG.append(exg)
# *map(pixel_analysis, tqdm(range(1, np.amax(treecrowns+1)))),
[pixel_analysis(i) for i in tqdm(range(1, np.amax(treecrowns + 1)))]
print("Pixel Analysis Completed.")
#~ #----------------------------------------------------------------------#
#~ #----------------------------------------------------------------------#
#########################
## CREATING DATAFRAMES ##
#########################
##Converting results table to dataframe
pixels_df = pd.DataFrame({
"Tree_Crown_ID": Trees,
"R_Mean": R_mean,
"G_Mean": G_mean,
"B_Mean": B_mean,
"R_StDev": R_SD,
"G_StDev": G_SD,
"B_StDev": B_SD,
"RCC": RCC,
"GCC": GCC,
"BCC": BCC,
"ExG": ExG
})
variables_df = pd.DataFrame({
"Date": Date,
"Start_Time": Start_Time,
"Flight_Number": FlightNo,
"Location": Location,
"Software": Software,
"Light": Light,
"Cloud": Cloud,
"Wind": Wind,
"Height": Height
})
##Matching dataframe lenghts
repeat_variables_df = pd.concat([variables_df] * len(pixels_df),
ignore_index=True)
##Combining dataframe
combined_df = pd.concat([repeat_variables_df, pixels_df], axis=1)
##Rearranging dataframe
#~ results_table = results_table[["Number", "Date", "Location", "Software", "Start_Time", "Height", "Light", "Cloud", "Wind", "Mean_Green", "GCC", "Mean_Red", "RCC", "Mean_Blue", "BCC", "ExG"]]
## Saving dataframe to new csv or existing csv
#Command line arguments to name the csv file
if not os.path.isfile(argv[1]):
combined_df.to_csv(argv[1], index=False)
else:
with open(argv[1], "a") as f:
combined_df.to_csv(f, header=False, index=False)
def prep_mask(product_dict,
maskfilename,
bbox_file,
prods_TOTbbox,
proj,
amp_thresh=None,
arrshape=None,
workdir='./',
outputFormat='ENVI',
num_threads='2'):
'''
Function to load and export mask file.
If "Download" flag is specified, GSHHS water mask will be donwloaded on the fly.
If the full resolution NLCD landcover data is given (NLCD...img) it will be cropped to match product
'''
# Import functions
from ARIAtools.vrtmanager import renderOGRVRT
_world_watermask = [
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L1.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L2.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L3.shp',
' /vsizip/vsicurl/http://www.soest.hawaii.edu/pwessel/gshhg/gshhg-shp-2.3.7.zip/GSHHS_shp/f/GSHHS_f_L4.shp',
' /vsizip/vsicurl/https://osmdata.openstreetmap.de/download/land-polygons-complete-4326.zip/land-polygons-complete-4326/land_polygons.shp'
]
# If specified DEM subdirectory exists, delete contents
workdir = os.path.join(workdir, 'mask')
if os.path.exists(
workdir) and os.path.abspath(maskfilename) != os.path.abspath(
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')
[0].split('uncropped')[0] + '.msk')) and os.path.abspath(
maskfilename) != os.path.abspath(
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] +
'.msk')) or maskfilename.lower() == 'download':
for i in glob.glob(os.path.join(workdir, '*.*')):
os.remove(i)
if not os.path.exists(workdir):
os.mkdir(workdir)
# Get bounds of user bbox_file
bounds = open_shapefile(bbox_file, 0, 0).bounds
# File must be physically extracted, cannot proceed with VRT format. Defaulting to ENVI format.
if outputFormat == 'VRT':
outputFormat = 'ENVI'
# Download mask
if maskfilename.lower() == 'download':
log.info("***Downloading water mask... ***")
maskfilename = os.path.join(workdir, 'watermask' + '.msk')
os.environ['CPL_ZIP_ENCODING'] = 'UTF-8'
###Make coastlines/islands union VRT
renderOGRVRT(os.path.join(workdir, 'watermsk_shorelines.vrt'),
_world_watermask[::2])
###Make lakes/ponds union VRT
renderOGRVRT(os.path.join(workdir, 'watermsk_lakes.vrt'),
_world_watermask[1::2])
###Initiate water-mask with coastlines/islands union VRT
# save uncropped mask
gdal.Rasterize(os.path.join(workdir, 'watermask_uncropped.msk'),
os.path.join(workdir, 'watermsk_shorelines.vrt'),
options=gdal.RasterizeOptions(format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged'))
gdal.Translate(os.path.join(workdir, 'watermask_uncropped.msk.vrt'),
os.path.join(workdir, 'watermask_uncropped.msk'),
options=gdal.TranslateOptions(format="VRT"))
# save cropped mask
gdal.Warp(maskfilename,
os.path.join(workdir, 'watermask_uncropped.msk.vrt'),
options=gdal.WarpOptions(
format=outputFormat,
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)]))
update_file = gdal.Open(maskfilename, gdal.GA_Update)
update_file.SetProjection(proj)
update_file.GetRasterBand(1).SetNoDataValue(0.)
del update_file
gdal.Translate(maskfilename + '.vrt',
maskfilename,
options=gdal.TranslateOptions(format="VRT"))
###Must take inverse of lakes/ponds union because of opposite designation (1 for water, 0 for land) as desired (0 for water, 1 for land)
lake_masks = gdal.Rasterize(
'',
os.path.join(workdir, 'watermsk_lakes.vrt'),
options=gdal.RasterizeOptions(format='MEM',
outputBounds=bounds,
outputType=gdal.GDT_Byte,
width=arrshape[1],
height=arrshape[0],
burnValues=[1],
layers='merged',
inverse=True))
lake_masks.SetProjection(proj)
lake_masks = lake_masks.ReadAsArray()
###Update water-mask with lakes/ponds union and average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
lake_masks * gdal.Open(maskfilename).ReadAsArray())
#Delete temp files
del lake_masks, mask_file
if os.path.basename(maskfilename).lower().startswith('nlcd'):
log.info("***Accessing and cropping the NLCD mask...***")
maskfilename = NLCDMasker(os.path.dirname(workdir))(proj, bounds,
arrshape,
outputFormat)
# Make sure to apply amplitude mask to downloaded products
if os.path.exists(os.path.join(workdir, 'watermsk_shorelines.vrt')
) or os.path.basename(maskfilename) == 'NLCD_crop.msk':
###Make average amplitude mask
if amp_thresh:
amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'),
product_dict[0],
bounds,
prods_TOTbbox,
outputFormat=outputFormat,
thresh=amp_thresh)
###Update mask with average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
gdal.Open(maskfilename).ReadAsArray() * amp_file)
#Delete temp files
del mask_file, amp_file
if os.path.exists(os.path.join(workdir, 'watermsk_shorelines.vrt')):
os.remove(os.path.join(workdir, 'watermsk_shorelines.vrt'))
if os.path.exists(os.path.join(workdir, 'watermsk_lakes.vrt')):
os.remove(os.path.join(workdir, 'watermsk_lakes.vrt'))
# Load mask
try:
# Check if uncropped/cropped maskfiles exist in 'mask' subdirectory
if not os.path.exists(
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] + '.msk')):
# save uncropped masfile
gdal.BuildVRT(os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] +
'_uncropped.msk.vrt'),
maskfilename,
options=gdal.BuildVRTOptions(outputBounds=bounds))
# update maskfilename
maskfilename = os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0].split('uncropped')
[0] + '.msk')
# save cropped maskfile
gdal.Warp(maskfilename,
os.path.join(
workdir,
os.path.basename(maskfilename).split('.')[0] +
'_uncropped.msk.vrt'),
options=gdal.WarpOptions(
format=outputFormat,
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)]))
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.SetProjection(proj)
del mask_file
gdal.Translate(maskfilename + '.vrt',
maskfilename,
options=gdal.TranslateOptions(format="VRT"))
###Make average amplitude mask
if amp_thresh:
amp_file = rasterAverage(os.path.join(workdir, 'avgamplitude'),
product_dict[0],
bounds,
prods_TOTbbox,
outputFormat=outputFormat,
thresh=amp_thresh)
###Update mask with average amplitude
mask_file = gdal.Open(maskfilename, gdal.GA_Update)
mask_file.GetRasterBand(1).WriteArray(
gdal.Open(maskfilename).ReadAsArray() * amp_file)
#Delete temp files
del mask_file, amp_file
#pass cropped DEM
mask = gdal.Warp('',
maskfilename,
options=gdal.WarpOptions(
format="MEM",
cutlineDSName=prods_TOTbbox,
outputBounds=bounds,
width=arrshape[1],
height=arrshape[0],
multithread=True,
options=['NUM_THREADS=%s' % (num_threads)]))
mask.SetProjection(proj)
mask.SetDescription(maskfilename)
except:
raise Exception('Failed to open user mask')
return mask
def proximity_shp(fn_shp, raster_ref, type_prox='both'):
# create memory raster based on reference raster
raster_ds = create_mem_raster_on_ref(raster_ref)
# open .shp with GDAL
shp_ds = gdal.OpenEx(fn_shp, gdal.OF_VECTOR)
# rasterize
opts = gdal.RasterizeOptions(burnValues=[1], bands=[1])
gdal.Rasterize(raster_ds, shp_ds, options=opts)
# close shp
shp_ds = None
drv = gdal.GetDriverByName('MEM')
proxy_ds = drv.Create('', raster_ds.RasterXSize, raster_ds.RasterYSize, 1,
gdal.GetDataTypeByName('Float32'))
if type_prox == 'exterior':
gdal.ComputeProximity(raster_ds.GetRasterBand(1),
proxy_ds.GetRasterBand(1),
["VALUES=1", "DISTUNITS=GEO"])
proxy_array = proxy_ds.GetRasterBand(1).ReadAsArray()
elif type_prox == 'interior':
raster_arr = raster_ds.GetRasterBand(1).ReadAsArray()
mask = (raster_arr == 1)
raster_arr[mask] = raster_ds.GetRasterBand(1).GetNoDataValue()
raster_arr[~mask] = 1
raster_ds.GetRasterBand(1).WriteArray(raster_arr)
gdal.ComputeProximity(raster_ds.GetRasterBand(1),
proxy_ds.GetRasterBand(1),
["VALUES=1", "DISTUNITS=GEO"])
proxy_array = proxy_ds.GetRasterBand(1).ReadAsArray()
elif type_prox == 'both':
gdal.ComputeProximity(raster_ds.GetRasterBand(1),
proxy_ds.GetRasterBand(1),
["VALUES=1", "DISTUNITS=GEO"])
proxy_ext = proxy_ds.GetRasterBand(1).ReadAsArray()
raster_arr = raster_ds.GetRasterBand(1).ReadAsArray()
mask = (raster_arr == 1)
raster_arr[mask] = raster_ds.GetRasterBand(1).GetNoDataValue()
raster_arr[~mask] = 1
raster_ds.GetRasterBand(1).WriteArray(raster_arr)
gdal.ComputeProximity(raster_ds.GetRasterBand(1),
proxy_ds.GetRasterBand(1),
["VALUES=1", "DISTUNITS=GEO"])
proxy_int = proxy_ds.GetRasterBand(1).ReadAsArray()
proxy_array = proxy_ext + proxy_int
else:
sys.exit('Type of proximity:' + type_prox +
' not recognized. Must be "interior", "exterior" or "both".')
return proxy_array
def get_training_data(image_path,
shape_path,
attribute="CODE",
shape_projection_id=4326):
"""
Given an image and a shapefile with categories, returns training data and features suitable
for fitting a scikit-learn classifier.
For full details of how to create an appropriate shapefile, see [here](../index.html#training_data).
Parameters
----------
image_path : str
The path to the raster image to extract signatures from
shape_path : str
The path to the shapefile containing labelled class polygons
attribute : str, optional
The shapefile field containing the class labels. Defaults to "CODE".
shape_projection_id : int, optional
The EPSG number of the projection of the shapefile. Defaults to EPSG 4326.
Returns
-------
training_data : array_like
A numpy array of shape (n_pixels, bands), where n_pixels is the number of pixels covered by the training polygons
features : array_like
A 1-d numpy array of length (n_pixels) containing the class labels for the corresponding pixel in training_data
Notes
-----
For performance, this uses scikit's sparse.nonzero() function to get the location of each training data pixel.
This means that this will ignore any classes with a label of '0'.
"""
# TODO: WRITE A TEST FOR THIS TOO; if this goes wrong, it'll go wrong
# quietly and in a way that'll cause the most issues further on down the line
FILL_VALUE = -9999
with TemporaryDirectory() as td:
# Step 1; rasterise shapefile into .tif of class values
shape_projection = osr.SpatialReference()
shape_projection.ImportFromEPSG(shape_projection_id)
image = gdal.Open(image_path)
image_gt = image.GetGeoTransform()
x_res, y_res = image_gt[1], image_gt[5]
ras_path = os.path.join(td, "poly_ras")
ras_params = gdal.RasterizeOptions(noData=0,
attribute=attribute,
xRes=x_res,
yRes=y_res,
outputType=gdal.GDT_Int16,
outputSRS=shape_projection)
# This produces a rasterised geotiff that's right, but not perfectly aligned to pixels.
# This can probably be fixed.
gdal.Rasterize(ras_path, shape_path, options=ras_params)
rasterised_shapefile = gdal.Open(ras_path)
shape_array = rasterised_shapefile.GetVirtualMemArray()
local_x, local_y = get_local_top_left(image, rasterised_shapefile)
shape_sparse = sp.coo_matrix(np.asarray(shape_array).squeeze())
y, x, features = sp.find(shape_sparse)
training_data = np.empty((len(features), image.RasterCount))
image_array = image.GetVirtualMemArray()
image_view = image_array[:, local_y:local_y +
rasterised_shapefile.RasterYSize,
local_x:local_x +
rasterised_shapefile.RasterXSize]
for index in range(len(features)):
training_data[index, :] = image_view[:, y[index], x[index]]
image_view = None
image_array = None
shape_array = None
rasterised_shapefile = None
return training_data, features
def cellneigh_ctry(raster=None, region=None, vector=None, csize=10, rank=1):
"""
Compute number of spatial cells and neighbours.
:param raster: Path to raster file to compute region.
:param vector: Path to vector file with country borders.
:param region: List/tuple of region coordinates (east, west, south, north).
:param csize: Spatial cell size (in km).
:param rank: Rank of the neighborhood (1 for chess king's move).
:return: List of length 2 with number of neighbours for each cell \
and adjacent cells.
"""
# Region
if raster is not None:
r = gdal.Open(raster)
ncol_r = r.RasterXSize
nrow_r = r.RasterYSize
gt = r.GetGeoTransform()
Xmin = gt[0]
Xmax = gt[0] + gt[1] * ncol_r
Ymin = gt[3] + gt[5] * nrow_r
Ymax = gt[3]
elif region is not None:
Xmin = region[0]
Xmax = region[1]
Ymin = region[2]
Ymax = region[3]
else:
print("raster or region must be specified")
sys.exit(1)
# Cell number from region
print("Compute number of {} x {} km spatial cells".format(csize, csize))
csize = csize * 1000 # Transform km in m
ncol = np.int(np.ceil((Xmax - Xmin) / csize))
nrow = np.int(np.ceil((Ymax - Ymin) / csize))
Xmax_new = Xmin + ncol * csize
Ymin_new = Ymax + nrow * (-csize)
ncell = ncol * nrow
print("... {} cells ({} x {})".format(ncell, nrow, ncol))
# Cells within country borders (rasterizing method)
cb_ds = gdal.OpenEx(vector, gdal.OF_VECTOR)
rOptions = gdal.RasterizeOptions(
xRes=csize,
yRes=-csize,
allTouched=True,
outputBounds=[Xmin, Ymin_new, Xmax_new, Ymax],
burnValues=1,
noData=0)
outfile = "/vsimem/tmpfile"
ds = gdal.Rasterize(outfile, cb_ds, options=rOptions)
mask = ds.ReadAsArray()
ds = None
gdal.Unlink(outfile)
y_in, x_in = np.where(mask == 1)
cell_in = y_in * ncol + x_in
# Adjacent cells and number of neighbors
print("Identify adjacent cells and compute number of neighbors")
nneigh = []
adj = []
adj_sort = []
around = np.arange(-rank, rank + 1)
for i in range(nrow):
for j in range(ncol):
if mask[i, j] == 1:
I = i + around
Iprim = I[(I >= 0) & (I < nrow)]
J = j + around
Jprim = J[(J >= 0) & (J < ncol)]
# Loop on potential neighbors
nneighbors = 0
for cy in Iprim:
for cx in Jprim:
if (not (cy == i and cx == j)) and (mask[cy, cx] == 1):
adj.append(cy * ncol + cx)
nneighbors += 1
nneigh.append(nneighbors)
nneigh = np.array(nneigh)
adj = np.array(adj)
for i in adj:
adj_sort.append(np.flatnonzero(cell_in == i)[0])
adj_sort = np.array(adj_sort)
return (nneigh, adj_sort, cell_in, ncell)
def gdalos_rasterize(in_filename: str,
shp_filename_or_ds: str,
out_filename: str = None,
shp_layer_name: str = None,
shp_z_attribute: str = 'Height',
add: bool = True,
extent: GeoRectangle = ...,
**kwargs):
"""rasterize a vector layer into a given raster layer, overriding or adding the values
Parameters
----------
in_filename:str
input raster filename
out_filename:str
ouput raster filename, if None - input raster would be updated inplace
shp_filename_or_ds: str
input vector filename or dataset
shp_layer_name: str
name of the layer from the vector dataset to use
shp_z_attribute: str='Height'
name of the attribute to extract the z value from
add: bool=True
True to add to dtm values to shape values into the raster, False to burn shape values into the raster
extent: GeoRectangle = ...
None - use the extent of the input raster
... - use the extent of the input vector layer
GeoRectangle - custom extent
out_res: Real
output resolution (if None then auto select)
warp_srs: str=None
output srs
overwrite: bool=True
what to do if the output exists (fail of overwrite)
Returns
-------
output raster dataset
"""
# shp = gdalos_util.open_ds(shp_filename, gdal.gdalconst.OF_VECTOR)
if shp_filename_or_ds is None:
shp = None
elif isinstance(shp_filename_or_ds, (str, Path)):
shp = gdal.OpenEx(str(shp_filename_or_ds), gdal.gdalconst.OF_VECTOR)
else:
shp = shp_filename_or_ds
if out_filename is None:
dstDs = gdalos_util.open_ds(in_filename, gdal.GA_Update)
else:
pj4326 = projdef.get_srs_pj(4326)
if extent is ...:
cov_pj_srs = projdef.get_srs_pj(shp)
if not cov_pj_srs:
cov_pj_srs = pj4326
cov_extent = ogr_get_layer_extent(shp.GetLayer())
transform = projdef.get_transform(cov_pj_srs, pj4326)
extent = gdalos_extent.transform_extent(cov_extent, transform)
dstDs = gdalos_trans(in_filename,
out_filename,
extent=extent,
cog=False,
ovr_type=OvrType.no_overviews,
return_ds=True,
**kwargs)
if shp is not None:
rasteize_options = gdal.RasterizeOptions(layers=shp_layer_name,
add=add,
attribute=shp_z_attribute)
ret = gdal.Rasterize(dstDs, shp, options=rasteize_options)
if ret != 1:
raise Exception('Rasterize failed')
return dstDs
