import vtk
from osgeo import gdal,ogr,osr
import math
import imp
import sys
import xml.etree.ElementTree as ET
import subprocess
import numpy as np
import os
from gdalconst import GA_ReadOnly
import resource
from rasterio.warp import transform
import xarray as xr
gdal.UseExceptions() # Enable errors
# Print iterations progress
# http://stackoverflow.com/a/34325723/410074
def printProgress(iteration, total, prefix='', suffix='', decimals=2, barLength=100):
"""
Call in a loop to create terminal progress bar
@params:
iterations - Required : current iteration (Int)
total - Required : total iterations (Int)
prefix - Optional : prefix string (Str)
suffix - Optional : suffix string (Str)
"""
filledLength = int(round(barLength * iteration / float(total)))
percents = round(100.00 * (iteration / float(total)), decimals)
bar = '#' * filledLength + '-' * (barLength - filledLength)
def __init__(self, source_path, dico_dataset, tipo, txt=""):
"""Uses OGR functions to extract basic informations about
geographic vector file (handles shapefile or MapInfo tables)
and store into dictionaries.
source_path = path to the DXF file
dico_dataset = dictionary for global informations
tipo = format
text = dictionary of text in the selected language
"""
# handling ogr specific exceptions
errhandler = gdal_err.handler
gdal.PushErrorHandler(errhandler)
gdal.UseExceptions()
self.alert = 0
# changing working directory to layer folder
chdir(path.dirname(source_path))
# opening DXF
try:
# driver_dxf = ogr.GetDriverByName(str("DXF"))
# dxf = driver_dxf.Open(source_path, 0)
src = gdal.OpenEx(source_path, 0)
except Exception as e:
logging.error(e)
youtils.erratum(dico_dataset, source_path, "err_corrupt")
self.alert = self.alert + 1
return None
# raising incompatible files
if not src:
""" if file is not compatible """
self.alert += 1
dico_dataset["err_gdal"] = gdal_err.err_type, gdal_err.err_msg
youtils.erratum(dico_dataset, source_path, "err_nobjet")
return None
else:
layer = src.GetLayer() # get the layer
pass
# DXF name and parent folder
try:
dico_dataset["name"] = path.basename(src.GetName())
dico_dataset["folder"] = path.dirname(src.GetName())
except AttributeError as err:
logger.warning(err)
dico_dataset["name"] = path.basename(source_path)
dico_dataset["folder"] = path.dirname(source_path)
# specific AutoDesk informations
douxef = dxfgrabber.readfile(source_path)
dico_dataset["version_code"] = douxef.dxfversion
# see: http://dxfgrabber.readthedocs.org/en/latest/#Drawing.dxfversion
if douxef.dxfversion == "AC1009":
dico_dataset["version_name"] = "AutoCAD R12"
elif douxef.dxfversion == "AC1015":
dico_dataset["version_name"] = "AutoCAD R2000"
elif douxef.dxfversion == "AC1018":
dico_dataset["version_name"] = "AutoCAD R2004"
elif douxef.dxfversion == "AC1021":
dico_dataset["version_name"] = "AutoCAD R2007"
elif douxef.dxfversion == "AC1024":
dico_dataset["version_name"] = "AutoCAD R2010"
elif douxef.dxfversion == "AC1027":
dico_dataset["version_name"] = "AutoCAD R2013"
else:
dico_dataset["version_name"] = "douxef.dxfversion"
# layers count and names
dico_dataset["layers_count"] = src.GetLayerCount()
li_layers_names = []
li_layers_idx = []
dico_dataset["layers_names"] = li_layers_names
dico_dataset["layers_idx"] = li_layers_idx
# dependencies and total size
dependencies = youtils.list_dependencies(source_path, "auto")
dico_dataset["dependencies"] = dependencies
dico_dataset["total_size"] = youtils.sizeof(source_path, dependencies)
# global dates
crea, up = path.getctime(source_path), path.getmtime(source_path)
dico_dataset["date_crea"] = strftime("%Y/%m/%d", localtime(crea))
dico_dataset["date_actu"] = strftime("%Y/%m/%d", localtime(up))
# total fields count
total_fields = 0
dico_dataset["total_fields"] = total_fields
# total objects count
total_objs = 0
dico_dataset["total_objs"] = total_objs
# parsing layers
for layer_idx in range(src.GetLayerCount()):
# dictionary where will be stored informations
dico_layer = OrderedDict()
dico_layer["src_name"] = dico_dataset.get("name")
# getting layer object
layer = src.GetLayerByIndex(layer_idx)
# layer globals
li_layers_names.append(layer.GetName())
dico_layer["title"] = georeader.get_title(layer)
li_layers_idx.append(layer_idx)
# features
layer_feat_count = layer.GetFeatureCount()
dico_layer["num_obj"] = layer_feat_count
if layer_feat_count == 0:
""" if layer doesn't have any object, return an error """
dico_layer["error"] = "err_nobjet"
self.alert = self.alert + 1
else:
pass
# fields
layer_def = layer.GetLayerDefn()
dico_layer["num_fields"] = layer_def.GetFieldCount()
dico_layer["fields"] = georeader.get_fields_details(layer_def)
# geometry type
dico_layer["type_geom"] = georeader.get_geometry_type(layer)
# SRS
srs_details = georeader.get_srs_details(layer, txt)
dico_layer["srs"] = srs_details[0]
dico_layer["epsg"] = srs_details[1]
dico_layer["srs_type"] = srs_details[2]
# spatial extent
extent = georeader.get_extent_as_tuple(layer)
dico_layer["xmin"] = extent[0]
dico_layer["xmax"] = extent[1]
dico_layer["ymin"] = extent[2]
dico_layer["ymax"] = extent[3]
# storing layer into the GDB dictionary
dico_dataset["{0}_{1}".format(
layer_idx, dico_layer.get("title"))] = dico_layer
# summing fields number
total_fields += dico_layer.get("num_fields", 0)
# summing objects number
total_objs += dico_layer.get("num_obj", 0)
# deleting dictionary to ensure having cleared space
del dico_layer
# storing fileds and objects sum
dico_dataset["total_fields"] = total_fields
dico_dataset["total_objs"] = total_objs
# warnings messages
if self.alert:
dico_dataset["err_gdal"] = gdal_err.err_type, gdal_err.err_msg
else:
pass
# clean exit
del src
import cartopy.crs as ccrs
from cartopy.io.srtm import add_shading
from osgeo import gdal, osr
from obspy.geodetics import gps2dist_azimuth
import utm
import time
import shutil
import os
import subprocess
import warnings
from .travel_time import celerity_travel_time, fdtd_travel_time
from .plotting import _plot_geographic_context
from .stack import calculate_semblance
from . import RTMWarning
gdal.UseExceptions() # Allows for more Pythonic errors from GDAL
MIN_CELERITY = 220 # [m/s] Used for travel time buffer calculation
OUTPUT_DIR = 'rtm_dem' # Name of directory to place rtm_dem_*.tif files into
# (created in same dir as where function is called)
TMP_TIFF = 'rtm_dem_tmp.tif' # Filename to use for temporary I/O
# Values in brackets below are latitude, longitude, x_radius, y_radius, spacing
TEMPLATE = 'rtm_dem_{:.4f}_{:.4f}_{}x_{}y_{}m.tif'
NODATA = -9999 # Nodata value to use for output rasters
RESAMPLE_ALG = 'cubicspline' # Algorithm to use for resampling
# See https://gdal.org/programs/gdalwarp.html#cmdoption-gdalwarp-r
def map(ctx, map_type, date, output, root, result, image, date_frmt, ndv,
gdal_frmt, warn_on_empty, band, coef, after, before, qa, refit_prefix,
amplitude, predict_proba):
"""
Map types: coef, predict, class, pheno
\b
Map QA flags:
- 0 => no values
- 1 => before
- 2 => after
- 3 => intersect
\b
Examples:
> yatsm map --coef intercept --coef slope
... --band 3 --band 4 --band 5 --ndv -9999
... coef 2000-01-01 coef_map.gtif
\b
> yatsm map -c intercept -c slope -b 3 -b 4 -b 5 --ndv -9999
... coef 2000-01-01 coef_map.gtif
\b
> yatsm map --date "%Y-%j" predict 2000-001 prediction.gtif
\b
> yatsm map --result "YATSM_new" --after class 2000-01-01 LCmap.gtif
\b
Notes:
- Image predictions will not use categorical information in timeseries
models.
"""
from osgeo import gdal
from ..mapping import (get_classification, get_phenology, get_coefficients,
get_prediction)
from ..utils import write_output
gdal.AllRegister()
gdal.UseExceptions()
if len(band) == 0:
band = 'all'
try:
image_ds = gdal.Open(image, gdal.GA_ReadOnly)
except RuntimeError as err:
raise click.ClickException('Could not open example image for reading '
'(%s)' % str(err))
date = date.toordinal()
# Append underscore to prefix if not included
if refit_prefix and not refit_prefix.endswith('_'):
refit_prefix += '_'
band_names = None
if map_type == 'class':
raster, band_names = get_classification(date,
result,
image_ds,
after=after,
before=before,
qa=qa,
pred_proba=predict_proba,
warn_on_empty=warn_on_empty)
elif map_type == 'coef':
raster, band_names = get_coefficients(date,
result,
image_ds,
band,
coef,
prefix=refit_prefix,
amplitude=amplitude,
after=after,
before=before,
qa=qa,
ndv=ndv,
warn_on_empty=warn_on_empty)
elif map_type == 'predict':
raster, band_names = get_prediction(date,
result,
image_ds,
band,
prefix=refit_prefix,
after=after,
before=before,
qa=qa,
ndv=ndv,
warn_on_empty=warn_on_empty)
elif map_type == 'pheno':
raster, band_names = get_phenology(date,
result,
image_ds,
after=after,
before=before,
qa=qa,
ndv=ndv,
warn_on_empty=warn_on_empty)
write_output(raster, output, image_ds, gdal_frmt, ndv, band_names)
image_ds = None
def basic_test_11():
ds = gdal.OpenEx('data/byte.tif')
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_VECTOR)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_ALL)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', gdal.OF_UPDATE)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx(
'data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR | gdal.OF_UPDATE
| gdal.OF_VERBOSE_ERROR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=[])
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=['GTiff'])
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=['PNG'])
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
with gdaltest.error_handler():
ds = gdal.OpenEx('data/byte.tif', open_options=['FOO'])
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ar_ds = [gdal.OpenEx('data/byte.tif', gdal.OF_SHARED) for i in range(1024)]
if ar_ds[1023] is None:
gdaltest.post_reason('fail')
return 'fail'
ar_ds = None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER)
if ds is not None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_VECTOR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayerCount() != 1:
gdaltest.post_reason('fail')
return 'fail'
if ds.GetLayer(0) is None:
gdaltest.post_reason('fail')
return 'fail'
ds.GetLayer(0).GetMetadata()
ds = gdal.OpenEx('../ogr/data/poly.shp',
allowed_drivers=['ESRI Shapefile'])
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER | gdal.OF_VECTOR)
if ds is None:
gdaltest.post_reason('fail')
return 'fail'
ds = gdal.OpenEx('non existing')
if ds is not None or gdal.GetLastErrorMsg() != '':
gdaltest.post_reason('fail')
return 'fail'
gdal.PushErrorHandler('CPLQuietErrorHandler')
ds = gdal.OpenEx('non existing', gdal.OF_VERBOSE_ERROR)
gdal.PopErrorHandler()
if ds is not None or gdal.GetLastErrorMsg() == '':
gdaltest.post_reason('fail')
return 'fail'
old_use_exceptions_status = gdal.GetUseExceptions()
gdal.UseExceptions()
got_exception = False
try:
ds = gdal.OpenEx('non existing')
except:
got_exception = True
if old_use_exceptions_status == 0:
gdal.DontUseExceptions()
if not got_exception:
gdaltest.post_reason('fail')
return 'fail'
return 'success'
def run(self):
logging.debug(f' Executing SHP2DB filter with params: {self.params}')
# In general, we always want errors as exceptions. Having to enable them by hand is a "Python Gotcha" in gdal
# see: https://trac.osgeo.org/gdal/wiki/PythonGotchas
gdal.UseExceptions()
shp_driver = ogr.GetDriverByName("ESRI Shapefile")
input_data_source = shp_driver.Open(self.get_inputs()[0], 0)
input_layer = input_data_source.GetLayer()
# Find out the geometry type of the input to use it in the output
# I am assuming that the input has at least one feature, and that all features are of the same
# type (with shapefiles this is always so)
geometry_type = input_layer.GetGeomType()
# Warning: the shp format advertises polygons (ogr.wkbPolygon) even when there are multipolygons, and that
# would make the insertion in PostGIS fail, so we will have to check and "upgrade" them to multipolygons when
# loading to postgis
if geometry_type == ogr.wkbPolygon:
geometry_type = ogr.wkbMultiPolygon
# If they are defined, they must be a string that can be processed with the Srs class
if 'input_srs' in self.params:
input_srs_string = self.params['input_srs']
else:
input_srs_string = None
if 'output_srs' in self.params:
output_srs_string = self.params['output_srs']
else:
output_srs_string = None
# If input_srs is given use that. Otherwise, take the SRS of the input dataset
if input_srs_string != None:
input_srs = osr.SpatialReference()
input_srs.ImportFromEPSG(
int(gdtcsrs.Srs(input_srs_string).as_epsg_number()))
else:
input_srs = input_layer.GetSpatialRef()
# If output_srs is given use that. Otherwise, take the same SRS used for the input
if output_srs_string != None:
output_srs = osr.SpatialReference()
output_srs.ImportFromEPSG(
int(gdtcsrs.Srs(output_srs_string).as_epsg_number()))
else:
output_srs = input_srs
db = gdtcdb.db_factory(self.get_outputs()[0])
db_connection_string = db.to_ogr_connection_string()
conn = ogr.Open(db_connection_string)
# TODO: Consider a mode where OVERWRITE is not always YES?
output_layer = conn.CreateLayer(self.get_outputs()[0]['db_table'],
output_srs, geometry_type,
['OVERWRITE=YES'])
# Create table fields from those in the shapefile
input_layer_defn = input_layer.GetLayerDefn()
for i in range(0, input_layer_defn.GetFieldCount()):
field_defn = input_layer_defn.GetFieldDefn(i)
output_layer.CreateField(field_defn)
# Add features from the input layer to the database table (output layer)
output_layer_defn = output_layer.GetLayerDefn()
for input_feature in input_layer:
# Create output Feature
output_feature = ogr.Feature(output_layer_defn)
# Add field values from input Layer
for i in range(0, output_layer_defn.GetFieldCount()):
output_feature.SetField(
output_layer_defn.GetFieldDefn(i).GetNameRef(),
input_feature.GetField(i))
# Set geometry
geom = input_feature.GetGeometryRef()
# Upgrade to multipolygon if polygon
if geom.GetGeometryType() == ogr.wkbPolygon:
geom = ogr.ForceToMultiPolygon(geom)
output_feature.SetGeometry(geom.Clone())
# Add new feature to output Layer
output_layer.CreateFeature(output_feature)
output_feature = None
# This is required to close, and for the output to save, the data sources
input_data_source = None
conn = None
import json
import logging
import pickle
import numpy as np
from osgeo import gdal
from gdalconst import GA_ReadOnly
from glob import glob
from datetime import datetime, timedelta
import isce
from iscesys.Component.ProductManager import ProductManager as PM
from .utils import get_bperp, gdal_translate, get_geocoded_coords
gdal.UseExceptions() # make GDAL raise python exceptions
log_format = "[%(asctime)s: %(levelname)s/%(name)s/%(funcName)s] %(message)s"
logging.basicConfig(format=log_format, level=logging.INFO)
logger = logging.getLogger(os.path.splitext(os.path.basename(__file__))[0])
DT_RE = re.compile(r'^(\d{4})-(\d{2})-(\d{2})')
S1_RE = re.compile(r'^S1\w$')
PLATFORMS = {
"S1A": "Sentinel-1A",
"S1B": "Sentinel-1B",
}
def loadImgButtClick(self):
print('loadImgButtClick button pressed')
self.outputText.insert(tk.END, '\nloadImgButtClick button pressed')
self.outputText.see(tk.END)
imgpath = self.requestImgEntryVar.get()
if len(self.requestImgEntry.get()) != 0:
# Get file size
filesize = os.path.getsize(imgpath)
print('The image filesize is: %f' % filesize)
self.outputText.insert(tk.END,
'\nThe image filesize is: %f' % filesize)
if filesize / 1e9 >= 4:
answer = messagebox.askquestion(
"Resize Image",
"The current image size is too large to display/process. Do you want to resize image?",
icon='warning')
if answer == 'yes':
imgpath = self.resizeImage(imgpath)
elif answer == 'no':
print('Did not compress and display image.')
self.outputText.insert(
tk.END, '\nDid not compress and display image.')
self.outputText.see(tk.END)
return
try:
if imgpath[-4:] == '.tif':
# this allows GDAL to throw Python Exceptions
gdal.UseExceptions()
try:
src_ds = gdal.Open(imgpath)
except RuntimeError as e:
print('Unable to open ( %s )' % imgpath)
mystring = '\nUnable to open ( %s )' % imgpath
self.outputText.insert(tk.END, mystring)
self.outputText.see(tk.END)
print(e)
sys.exit(1)
try:
redband = src_ds.GetRasterBand(
1) # Get the red band raster data out
redband = redband.ReadAsArray(
) # Convert the red band data into array
except RuntimeError as e:
print('Band 1 ( Red ) not found')
self.outputText.insert(tk.END,
'\nBand 1 ( Red ) not found')
self.outputText.see(tk.END)
print(e)
sys.exit(1)
try:
greenband = src_ds.GetRasterBand(
2) # Get the green band raster data out
greenband = greenband.ReadAsArray(
) # Convert the green band data into array
except RuntimeError as e:
print('Band 2 ( Green ) not found')
self.outputText.insert(tk.END,
'\nBand 2 ( Green ) not found')
self.outputText.see(tk.END)
print(e)
sys.exit(1)
try:
blueband = src_ds.GetRasterBand(
3) # Get the blue band raster data out
blueband = blueband.ReadAsArray(
) # Convert the blue band data into array
except RuntimeError as e:
print('Band 3 ( Blue ) not found')
self.outputText.insert(tk.END,
'\nBand 3 ( Blue ) not found')
self.outputText.see(tk.END)
print(e)
sys.exit(1)
# Show the image in GUI
img = Image.open(imgpath) # Convert the tiff image
self.img_copy = img.copy()
labelWidth = self.imgPanel.winfo_width(
) - 100 # Get the image panel Width
labelHeight = self.imgPanel.winfo_height(
) - 100 # Get the image panel Height
maxsize = (
labelWidth, labelHeight
) # Create a tuple for the image panel diamensions
img = img.resize(
maxsize) # Resize the image to fit the image panel
img = ImageTk.PhotoImage(
img) # Convert the image into a tkinker image
self.imgPanel.configure(
image=img) # Set the image into the image panel
self.imgPanel.image = img # Set the image into the image panel 2 (MUST DO)
self.update()
elif imgpath[-4:] == '.jpg' or imgpath[
-4:] == '.JPG' or imgpath[-4:] == '.png' or imgpath[
-4:] == '.PNG':
img = Image.open(imgpath) # Open the image
self.img_copy = img.copy()
labelWidth = self.imgPanel.winfo_width(
) # Get the image panel Width
labelHeight = self.imgPanel.winfo_height(
) # Get the image panel Height
maxsize = (
labelWidth, labelHeight
) # Create a tuple for the image panel diamensions
img = img.resize(
maxsize) # Resize the image to fit the image panel
img = ImageTk.PhotoImage(
img) # Convert the image into a tkinker image
self.imgPanel.configure(
image=img) # Set the image into the image panel
self.imgPanel.image = img # Set the image into the image panel 2 (MUST DO)
except Exception as e:
# logger.error('Unable to load image: ' + str(e))
print('Unable to load image: ' + str(e))
mystring = '\nUnable to load image: ' + str(e)
self.outputText.insert(tk.END, mystring)
self.outputText.see(tk.END)
return
else:
print('No Image to load')
def __init__(
self,
host="localhost",
port=5432,
db_name="postgis",
user="******",
password="******",
views_included=1,
dico_dataset=OrderedDict(),
txt=dict(),
):
"""Uses gdal/ogr functions to extract basic informations about
geographic file (handles shapefile or MapInfo tables)
and store into the dictionaries.
layer = path to the geographic file
dico_dataset = dictionary for global informations
dico_fields = dictionary for the fields' informations
tipo = feature type to read
text = dictionary of texts to display
"""
# handling GDAL/OGR specific exceptions
gdal.AllRegister()
ogr.UseExceptions()
gdal.UseExceptions()
# Creating variables
self.dico_dataset = dico_dataset
self.txt = txt
self.alert = 0
if views_included:
gdal.SetConfigOption(str("PG_LIST_ALL_TABLES"), str("YES"))
logger.info("PostgreSQL views enabled.")
else:
gdal.SetConfigOption(str("PG_LIST_ALL_TABLES"), str("NO"))
logger.info("PostgreSQL views disabled.")
# connection infos
self.host = host
self.port = port
self.db_name = db_name
self.user = user
self.password = password
self.conn_settings = "PG: host={} port={} dbname={} user={} password={}".format(
host, port, db_name, user, password)
# testing connection
self.conn = self.get_connection()
if not self.conn:
self.alert += 1
youtils.erratum(
ctner=dico_dataset,
mess_type=1,
ds_lyr=self.conn_settings,
mess="err_connection_failed",
)
dico_dataset["err_gdal"] = gdal_err.err_type, gdal_err.err_msg
return None
else:
pass
# sgbd info
dico_dataset["sgbd_version"] = self.get_version()
dico_dataset["sgbd_schemas"] = self.get_schemas()
def _lp_characteristics(self, lp_fn):
"""Get LP characteristics to check.
:param lp_fn: input lp raster file
:return dict: lp characteristics
"""
import numpy as np
from osgeo import gdal, gdal_array, osr
from osgeo import gdalconst
gdal.UseExceptions()
lp_characteristics = {}
try:
ids = gdal.Open(lp_fn, gdalconst.GA_ReadOnly)
lp_characteristics['read'] = True
except RuntimeError:
lp_characteristics['read'] = False
return lp_characteristics
# Spatial resolution
ids = gdal.Open(lp_fn, gdalconst.GA_ReadOnly)
img_array = ids.ReadAsArray()
geotransform = list(ids.GetGeoTransform())
lp_characteristics['xRes'] = abs(geotransform[1])
lp_characteristics['yRes'] = abs(geotransform[5])
# Projection
proj = osr.SpatialReference(wkt=ids.GetProjection())
map_epsg = (proj.GetAttrValue('AUTHORITY', 1))
lp_characteristics['epsg'] = map_epsg
# Coding data type
map_dtype = gdal_array.GDALTypeCodeToNumericTypeCode(
ids.GetRasterBand(1).DataType)
if (map_dtype == np.dtype('uint8')):
lp_characteristics['dataType'] = 'u8'
else:
lp_characteristics['dataType'] = str(ids.GetRasterBand(1).DataType)
# Map extent in the 'map_epsg'
ulx, xres, xskew, uly, yskew, yres = ids.GetGeoTransform()
lrx = ulx + (ids.RasterXSize * xres)
lry = uly + (ids.RasterYSize * yres)
lp_characteristics['extentUlLr'] = [ulx, uly, lrx, lry]
# Do spatial overlay
aoi_polygon = os.path.join(self.config['map_product']['path'],
self.config['map_product']['map_aoi'])
coding_val = []
for prod in self.config['land_product']['product_type']:
for val in self.config['land_product']['raster_coding'][prod]:
coding_val.append(
self.config['land_product']['raster_coding'][prod][val])
lp_min = min(coding_val)
lp_max = max(coding_val)
unclassifiable = self.config['land_product']['raster_coding'][
'unclassifiable']
out_of_aoi = self.config['land_product']['raster_coding']['out_of_aoi']
lp_characteristics['aoiCoveragePct'] = self._calc_aoiCoveragePct(
aoi_polygon, lp_fn, lp_min, lp_max, unclassifiable, out_of_aoi)
# Map format
raster_format = lp_fn.split('.')[-1]
if raster_format == 'tif':
raster_format = "GeoTIFF"
lp_characteristics['rasterFormat'] = raster_format
return lp_characteristics
def run():
"""
Run tests of a number of more common output format drivers
"""
riostestutils.reportStart(TESTNAME)
usingExceptions = gdal.GetUseExceptions()
gdal.UseExceptions()
driverTestList = [
('HFA', ['COMPRESS=YES'], '.img'),
('GTiff', ['COMPRESS=LZW', 'TILED=YES', 'INTERLEAVE=BAND'], '.tif'),
('ENVI', ['INTERLEAVE=BSQ'], ''), ('KEA', [], '.kea')
]
# Remove any which current GDAL not suporting
driverTestList = [(drvrName, options, suffix)
for (drvrName, options, suffix) in driverTestList
if gdal.GetDriverByName(drvrName) is not None]
riostestutils.report(
TESTNAME,
'Testing drivers {}'.format(str([d[0] for d in driverTestList])))
filename = 'test.img'
riostestutils.genRampImageFile(filename)
ok = True
outfileList = []
errorList = []
for (drvr, creationOptions, suffix) in driverTestList:
infiles = applier.FilenameAssociations()
outfiles = applier.FilenameAssociations()
infiles.inimg = filename
outfiles.outimg = "testout" + suffix
outfileList.append(outfiles.outimg)
controls = applier.ApplierControls()
controls.setOutputDriverName(drvr)
try:
applier.apply(copyImg, infiles, outfiles, controls=controls)
except Exception as e:
ok = False
errorList.append("{}:{}".format(drvr, str(e)))
if ok:
riostestutils.report(TESTNAME, "Passed")
else:
riostestutils.report(
TESTNAME, "Resulted in these apparent errors:\n {}".format(
'\n '.join(errorList)))
for fn in [filename] + outfileList:
if os.path.exists(fn):
drvr = gdal.Open(fn).GetDriver()
# drvr.Delete(fn)
if not usingExceptions:
gdal.DontUseExceptions()
return ok
def Write_Dict_To_Shapefile_osgeo(totalList, shapefileName, EPSG):
'''
Adapted from
https://github.com/GeoscienceAustralia/LidarProcessingScripts/RasterIndexTool_GDAL.py
This function takes a list of dictionaries where each row in the list
is a feature consisting of the X/Y geometries in the dictionary for each
item in the list.
Arguments:
totalList -- List of dictionaries
shapefileName -- Name of the shapefile to be created/overwritten
'''
gdal.UseExceptions()
shapePath = os.path.join(workspace, shapefileName)
# Get driver
driver = ogr.GetDriverByName('ESRI Shapefile')
# Create shapeData, overwrite the data if it exists
os.chdir(workspace)
if os.path.exists(shapefileName):
print 'Shapefile exists and will be deleted'
driver.DeleteDataSource(shapePath)
assert not os.path.exists(shapePath)
shapeData = driver.CreateDataSource(shapefileName)
# Create spatialReference for output
outputspatialRef = osr.SpatialReference()
# Set coordinate reference system to GDA94/MGA zone 56
outputspatialRef.ImportFromEPSG(EPSG)
# Create layer
layer = shapeData.CreateLayer(shapePath,
srs=outputspatialRef,
geom_type=ogr.wkbPolygon)
# add fields
fieldNames = ["Date", "Time"]
for n in range(0, len(fieldNames)):
# add short text fields - convoluted method but was more extensive
# in Jonah's script to capture more fields and various field types.
if fieldNames[n] in ["Date", "Time"]:
fieldstring = str(fieldNames[n])
field_name = ogr.FieldDefn(fieldstring, ogr.OFTString)
field_name.SetWidth(24)
layer.CreateField(field_name)
# Create polyline object
ring = ogr.Geometry(ogr.wkbLinearRing)
count = 0
for entry in totalList:
logAndprint('Row {0} being processed'.format(count))
dictCounter = 0
# while dictCounter < 10:
logAndprint('Dictionary count {0}'.format(len(totalList[count])))
for key, value in totalList[count].iteritems():
logAndprint('\n{0} row, {1} vertex being created'.format(
count, dictCounter))
logAndprint('\tKey: {0}, value: {1}'.format(key, value))
# print count
if dictCounter < 2:
logAndprint('\tRow passed')
pass
dictCounter += 1
else:
ring.AddPoint(float(key), float(value))
logAndprint('\t\t{0} and {1} vertex added to ring'.format(
key, value))
dictCounter += 1
poly = ogr.Geometry(ogr.wkbPolygon)
logAndprint('Adding geometry')
poly.AddGeometry(ring)
# Create feature
layerDefinition = layer.GetLayerDefn()
feature = ogr.Feature(layerDefinition)
feature.SetGeometry(poly)
# Set the FID field to the count
feature.SetFID(count)
# Calculate fields
logAndprint('Calculating "Date" & "Time" fields')
# Date is supplied in YYYY/MM/DD
date = str(totalList[count].keys()[1].split()[0])
# For animation in ArcGIS the date needs to be in the form
# DD/MM/YYYY
reformattedDate = (date.split('/')[2] + '/' + date.split('/')[1] +
'/' + date.split('/')[0])
logAndprint('reformatted date:' + reformattedDate)
feature.SetField('Date', str(reformattedDate))
feature.SetField('Time', str(totalList[count].keys()[1].split()[1]))
# Save feature
layer.CreateFeature(feature)
logAndprint('{0} rows processed'.format(count))
count += 1
# Empty the ring otherwise the vertices are accumulatied
ring.Empty()
# Cleanup
poly.Destroy()
feature.Destroy()
# Cleanup
shapeData.Destroy()
# Return
return shapePath
def __init__(self):
# this allows GDAL to throw Python Exceptions
gdal.UseExceptions()
def main():
gdal.UseExceptions() # Enable errors
##### load user configurable paramters here #######
# Check user defined configuraiton file
if len(sys.argv) == 1:
print('ERROR: main.py requires one argument [configuration file] (i.e. python main.py vtu2geo_config.py)')
return
# Get name of configuration file/module
configfile = sys.argv[1]
# Load in configuration file as module
X = imp.load_source('',configfile)
# if a 2nd command line argument is present, it is the input_path so use that, otherwise try to use the one from passed script
input_path = ''
if len(sys.argv) == 3: # we have a 2nd CLI arg
input_path = sys.argv[2]
if hasattr(X,'input_path'):
print 'Warning: Overwriting script defined input path with CL path'
elif hasattr(X,'input_path'):
input_path = X.input_path
else:
print('ERROR: No input path. A pvd or vtu file must be specified.')
exit(-1)
if os.path.isdir(input_path):
print('ERROR: Either a pvd or vtu file must be specified.')
exit(-1)
variables = X.variables
parameters = []
if hasattr(X,'parameters'):
parameters = X.parameters
# Check if we want to constrain output to a example geotif
constrain_flag = False
if hasattr(X,'constrain_tif_file'):
constrain_tif_file = X.constrain_tif_file
var_resample_method = X.var_resample_method
param_resample_method = X.param_resample_method
constrain_flag = True
output_path = input_path[:input_path.rfind('/')+1]
if hasattr(X,'output_path'):
output_path = X.output_path
pixel_size = 10
if hasattr(X,'pixel_size'):
pixel_size = X.pixel_size
else:
print 'Default pixel size of 10 mx10 m will be used.'
user_define_extent = False
if hasattr(X,'user_define_extent'):
user_define_extent = X.user_define_extent
#Produces a lat/long regular grid in CF netCDF format instead of the TIF files
nc_archive = False
if hasattr(X,'nc_archive'):
nc_archive = X.nc_archive
#user defined output EPSG to use instead of the proj4 as defined in the vtu
out_EPSG = None
if hasattr(X,'out_EPSG'):
out_EPSG = X.out_EPSG
if parameters is not None and nc_archive:
print('Parameters are ignored when writing the nc archive.')
parameters = []
all_touched = True
if hasattr(X,'all_touched'):
all_touched = X.all_touched
#####
reader = vtk.vtkXMLUnstructuredGridReader()
# see if we were given a single vtu file or a pvd xml file
filename, file_extension = os.path.splitext(input_path)
is_pvd = False
pvd = [input_path] # if not given a pvd file, make this iterable for the below code
timesteps=None
if file_extension == '.pvd':
print 'Detected pvd file, processing all linked vtu files'
is_pvd = True
parse = ET.parse(input_path)
pvd = parse.findall(".//*[@file]")
timesteps = parse.findall(".//*[@timestep]")
# Get info for constrained output extent/resolution if selected
if constrain_flag :
ex_ds = gdal.Open(constrain_tif_file,GA_ReadOnly)
gt = ex_ds.GetGeoTransform()
pixel_width = np.abs(gt[1])
pixel_height = np.abs(gt[5])
# Take extent from user input
if user_define_extent:
o_xmin = X.o_xmin
o_xmax = X.o_xmax
o_ymin = X.o_ymin
o_ymax = X.o_ymax
else: # Get extent for clipping from input tif
o_xmin = gt[0]
o_ymax = gt[3]
o_xmax = o_xmin + gt[1] * ex_ds.RasterXSize
o_ymin = o_ymax + gt[5] * ex_ds.RasterYSize
print "Output pixel size is " + str(pixel_width) + " by " + str(pixel_height)
ex_ds = None
if constrain_flag:
print(" Constrain flag currently not supported!")
return -1
files_processed=1 # this really should be 1 for useful output
#information to build up the nc meta data
nc_rasters = {}
for v in variables:
nc_rasters[v]=[]
nc_time_counter=0
tifs_to_remove = []
epoch=np.datetime64(0,'s')
# if we are loading a pvd, we have access to the timestep information if we want to build
if is_pvd and timesteps is not None:
epoch = np.datetime64(int(timesteps[0].get('timestep')),'s')
dt=1 # model timestep, in seconds
if timesteps is not None and len(timesteps) > 1:
dt = int(timesteps[1].get('timestep')) - int(timesteps[0].get('timestep'))
print('Start epoch: %s, model dt = %i (s)' %(epoch,dt))
#because of how the netcdf is built we hold a file of file handles before converting. ensure we can do so
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
total_output_files = len(pvd) * (len(variables)+len(parameters))
if soft < total_output_files or hard < total_output_files:
print('The users soft or hard file limit is less than the total number of tmp files to be created.')
print('The system ulimit should be raised to at least ' + total_output_files)
return -1
for vtu in pvd:
path = vtu
vtu_file = ''
if is_pvd:
vtu_file = vtu.get('file')
path = input_path[:input_path.rfind('/')+1] + vtu_file
else:
base = os.path.basename(path) # since we have a full path to vtu, we need to get just the vtu filename
vtu_file = os.path.splitext(base)[0] #we strip out vtu later so keep here
printProgress(files_processed,len(pvd),decimals=0)
reader.SetFileName(path)
#shut up a deprecated warning from vtk 8.1
with stdchannel_redirected(sys.stderr, os.devnull):
reader.Update()
mesh = reader.GetOutput()
#default the pixel size to (min+max)/2
if not pixel_size:
area_range = mesh.GetCellData().GetArray('Area').GetRange()
pixel_size = (math.sqrt(area_range[0]) + math.sqrt(area_range[1]))/2
pixel_size = int( math.ceil(pixel_size) )
driver = ogr.GetDriverByName('Memory')
output_usm = driver.CreateDataSource('out')
srsin = osr.SpatialReference()
if not mesh.GetFieldData().HasArray("proj4"):
print "VTU file does not contain a proj4 field"
return -1
vtu_proj4 = mesh.GetFieldData().GetAbstractArray("proj4").GetValue(0)
srsin.ImportFromProj4(vtu_proj4)
is_geographic = srsin.IsGeographic()
#output conic equal area for geotiff
srsout = osr.SpatialReference()
srsout.ImportFromProj4(vtu_proj4)
if out_EPSG:
srsout.ImportFromEPSG(out_EPSG)
trans = osr.CoordinateTransformation(srsin,srsout)
layer = output_usm.CreateLayer('poly', srsout, ogr.wkbPolygon)
cd = mesh.GetCellData()
for i in range(0,cd.GetNumberOfArrays()):
layer.CreateField(ogr.FieldDefn(cd.GetArrayName(i), ogr.OFTReal))
#build the triangulation geometery
for i in range(0, mesh.GetNumberOfCells()):
v0 = mesh.GetCell(i).GetPointId(0)
v1 = mesh.GetCell(i).GetPointId(1)
v2 = mesh.GetCell(i).GetPointId(2)
ring = ogr.Geometry(ogr.wkbLinearRing)
if is_geographic:
scale = 100000. #undo the scaling that CHM does for the paraview output
ring.AddPoint( mesh.GetPoint(v0)[0] / scale, mesh.GetPoint(v0)[1] / scale)
ring.AddPoint (mesh.GetPoint(v1)[0]/ scale, mesh.GetPoint(v1)[1]/ scale )
ring.AddPoint( mesh.GetPoint(v2)[0]/ scale, mesh.GetPoint(v2)[1]/ scale )
ring.AddPoint( mesh.GetPoint(v0)[0]/ scale, mesh.GetPoint(v0)[1]/ scale ) # add again to complete the ring.
else:
ring.AddPoint( mesh.GetPoint(v0)[0], mesh.GetPoint(v0)[1] )
ring.AddPoint (mesh.GetPoint(v1)[0], mesh.GetPoint(v1)[1] )
ring.AddPoint( mesh.GetPoint(v2)[0], mesh.GetPoint(v2)[1] )
ring.AddPoint( mesh.GetPoint(v0)[0], mesh.GetPoint(v0)[1] ) # add again to complete the ring.
ring.Transform(trans)
tpoly = ogr.Geometry(ogr.wkbPolygon)
tpoly.AddGeometry(ring)
feature = ogr.Feature( layer.GetLayerDefn() )
feature.SetGeometry(tpoly)
if variables is None:
for j in range(0, cd.GetNumberOfArrays()):
name = cd.GetArrayName(j)
variables.append(name)
for v in variables:
try:
data = cd.GetArray(v).GetTuple(i)
feature.SetField(str(v), float(data[0]))
except:
print "Variable %s not present in mesh" % v
return -1
if parameters is not None:
for p in parameters:
try:
data = cd.GetArray(p).GetTuple(i)
feature.SetField(str(p), float(data[0]))
except:
print "Parameter %s not present in mesh" % v
return -1
layer.CreateFeature(feature)
for var in variables:
target_fname = os.path.join(output_path,
vtu_file + '_' + var.replace(" ", "_") + str(pixel_size) + 'x' + str(
pixel_size) + '.tif')
rasterize(layer, srsout, target_fname, pixel_size, var,all_touched)
if nc_archive:
df = xr.open_rasterio(target_fname).sel(band=1).drop('band')
df = df.rename({'x':'lon','y':'lat'})
df.coords['time']=epoch + nc_time_counter*np.timedelta64(dt,'s') # this will automatically get converted to min or hours in the output nc
df.name=var
nc_rasters[var].append(df) # these are lazy loaded at the to netcdf call
# remove the tifs we produce
tifs_to_remove.append(target_fname)
if parameters is not None:
for p in parameters:
target_param_fname = os.path.join(output_path, vtu_file + '_'+ p.replace(" ","_") + str(pixel_size)+'x'+str(pixel_size)+'.tif')
rasterize(layer, srsout, target_fname, pixel_size, p,all_touched)
nc_time_counter += 1
# we don't need to dump parameters for each timestep as they are currently assumed invariant with time.
parameters = None
#no parameters and no variables, just exit at this point
if not variables and parameters is None:
break
files_processed += 1
if nc_archive:
datasets = []
for var, rasters in nc_rasters.iteritems():
a = xr.concat(rasters,dim='time')
datasets.append(a.to_dataset())
arr = xr.merge(datasets)
print('Writing netCDF file')
fname=os.path.join(os.path.splitext(input_path)[0]+'.nc')
arr.to_netcdf(fname,engine='netcdf4')
for f in tifs_to_remove:
try:
os.remove(f)
except:
pass
from math import pi
from itertools import count
import os
import re
from subprocess import CalledProcessError, check_output, Popen, PIPE
from tempfile import NamedTemporaryFile
from xml.etree import ElementTree
import numpy
from osgeo import gdal, gdalconst, osr
from osgeo.gdalconst import (GA_ReadOnly, GRA_Bilinear, GRA_Cubic,
GRA_CubicSpline, GRA_Lanczos,
GRA_NearestNeighbour)
gdal.UseExceptions() # Make GDAL throw exceptions on error
osr.UseExceptions() # And OSR as well.
from .constants import (EPSG_WEB_MERCATOR, ESRI_102113_PROJ, ESRI_102100_PROJ,
GDALTRANSLATE, GDALWARP, TILE_SIDE)
from .exceptions import (GdalError, CalledGdalError, UnalignedInputError,
UnknownResamplingMethodError)
from .types import Extents, GdalFormat, XY
from .utils import rmfile
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
RESAMPLING_METHODS = {
GRA_NearestNeighbour: 'near',
GRA_Bilinear: 'bilinear',
#OutputFolder
outFolder = 'layers'
#Check that the folder exists
if not os.path.exists(outFolder):
os.makedirs(outFolder)
#os.chdir(outFolder)
#Emtpy the folder
#files = glob.glob(TKoutFolder + '\*')
#for f in files:
# os.remove(f)
#Make error messages visible
gdal.UseExceptions() #Fail when can't open!
def gdal_error_handler(err_class, err_num, err_msg):
errtype = {
gdal.CE_None: 'None',
gdal.CE_Debug: 'Debug',
gdal.CE_Warning: 'Warning',
gdal.CE_Failure: 'Failure',
gdal.CE_Fatal: 'Fatal'
}
err_msg = err_msg.replace('\n', ' ')
err_class = errtype.get(err_class, 'None')
print('Error Number: %s' % (err_num))
print('Error Type: %s' % (err_class))
print('Error Message: %s' % (err_msg))
def changemap(ctx, map_type, start_date, end_date, output, root, result, image,
date_frmt, ndv, gdal_frmt, out_date_frmt, warn_on_empty,
magnitude):
"""
Examples: TODO
"""
from osgeo import gdal
from ..mapping import get_change_date, get_change_num
from ..utils import write_output
gdal.AllRegister()
gdal.UseExceptions()
gdal_frmt = str(gdal_frmt) # GDAL GetDriverByName doesn't work on Unicode
frmt = '%Y%j'
start_txt, end_txt = start_date.strftime(frmt), end_date.strftime(frmt)
start_date, end_date = start_date.toordinal(), end_date.toordinal()
try:
image_ds = gdal.Open(image, gdal.GA_ReadOnly)
except:
logger.error('Could not open example image for reading')
raise
if map_type in ('first', 'last'):
changemap, magnitudemap, magnitude_indices = get_change_date(
start_date,
end_date,
result,
image_ds,
first=map_type == 'first',
out_format=out_date_frmt,
magnitude=magnitude,
ndv=ndv,
pattern='yatsm_r*',
warn_on_empty=warn_on_empty)
band_names = ['ChangeDate_s{s}-e{e}'.format(s=start_txt, e=end_txt)]
write_output(changemap,
output,
image_ds,
gdal_frmt,
ndv,
band_names=band_names)
if magnitudemap is not None:
band_names = ([
'Magnitude Index {}'.format(i) for i in magnitude_indices
])
name, ext = os.path.splitext(output)
output = name + '_mag' + ext
write_output(magnitudemap,
output,
image_ds,
gdal_frmt,
ndv,
band_names=band_names)
elif map_type == 'num':
changemap = get_change_num(start_date,
end_date,
result,
image_ds,
ndv=ndv,
pattern='yatsm_r*',
warn_on_empty=warn_on_empty)
band_names = ['NumChanges_s{s}-e{e}'.format(s=start_txt, e=end_txt)]
write_output(changemap,
output,
image_ds,
gdal_frmt,
ndv,
band_names=band_names)
image_ds = None
def save_geopandas_tofile(inputGeoDataFrame, output_filename, overwrite=True, file_encoding='ascii'):
"""Save a geodataframe to file.
- adds functionality to asses and rename columns to ESRI compatible 10 alpha-numeric characters.
- Maps lists and boolean column types to string.
Args:
inputGeoDataFrame (geopandas.geodataframe.GeoDataFrame): The Geodataframe to save
output_filename (str): The output filename
overwrite (bool): Overwrite Existing file
file_encoding (str): encoding type for output file.
"""
if not isinstance(inputGeoDataFrame, GeoDataFrame):
raise TypeError('Invalid Type : inputGeodataFrame')
# if out_shapefilename doesn't include a path then add tempdir as well as overwriting it
if output_filename is not None and not os.path.isabs(output_filename):
output_filename = os.path.join(TEMPDIR, output_filename)
overwrite = True
if os.path.exists(output_filename) and not overwrite:
raise IOError('Output file ({}) already exists, and overwrite is false'.format(output_filename))
if os.path.splitext(output_filename)[-1] != '.shp':
raise NotImplementedError('Currently only support shapefiles.... ')
step_time = time.time()
driver = 'ESRI Shapefile'
if driver == 'ESRI Shapefile':
inputGeoDataFrame = inputGeoDataFrame.copy()
fldProp = get_column_properties(inputGeoDataFrame)
# get a list of either bool or list columns and convert to string.
fix_cols = [(key, val['type']) for key, val in fldProp.items() if val['type'] in ['bool', 'list']]
fix_cols += [(key, val['dtype']) for key, val in fldProp.items() if 'datetime' in val['dtype'].lower()]
# Convert them to Strings
for col, col_type in fix_cols:
LOGGER.info('Converting column {} datatype from {} to str'.format(col, col_type))
if col_type == 'list':
inputGeoDataFrame[col] = inputGeoDataFrame[col].apply(lambda x: ",".join(map(str, x)))
else:
inputGeoDataFrame[col] = inputGeoDataFrame[col].astype(str)
# rename columns to alias names. columns must be listed in the same order
inputGeoDataFrame.columns = [val['shapefile'] for key, val in fldProp.items()]
'''Saving to file sometimes throws an error similar to
CPLE_AppDefined in Value xxxx of field Timestamp of feature xxxx not successfully written. Possibly due to too
larger number with respect to field width. This is a known GDAL Error. The following two lines will hide this
from the user but may hide other message.
https://gis.stackexchange.com/a/68042 is also an option that works
'''
gdal.UseExceptions()
gdal.PushErrorHandler('CPLQuietErrorHandler')
if file_encoding == 'ascii':
inputGeoDataFrame.to_file(output_filename, driver=driver)
else:
inputGeoDataFrame.to_file(output_filename, driver=driver, encoding=file_encoding)
if config.get_debug_mode():
LOGGER.info('{:<30} {:<15} {dur}'.format('Saved to file',output_filename,
dur=datetime.timedelta(seconds=time.time() - step_time)))
from osgeo import gdal, ogr, osr
from pyproj import Proj, transform, Transformer
import numpy as np
import sys
import importlib
from functools import partial
import itertools
from scipy import ndimage
from os import environ
from concurrent import futures
from tqdm import tqdm
import random
import time
import rasterio as rio
gdal.UseExceptions() # Enable exception support
def main():
####### load user configurable paramters here #######
# Check user defined configuration file
if len(sys.argv) == 1:
print(
'ERROR: wind_mapper.py requires one argument [configuration file] (i.e. wind_mapper.py '
'param_existing_DEM.py)')
exit(-1)
# Get name of configuration file/module
configfile = sys.argv[-1]
def readSamples(DB, bands, Field="Class", NODATA=0):
"""
TBC
"""
# Initialization
X, Y, P = [], [], []
# Use gdal Exceptions
gdal.UseExceptions()
# Open DB with OGR
driverOgr = ogr.GetDriverByName('SQLite')
vectorIn = driverOgr.Open(DB, gdalconst.GA_ReadOnly)
layerIn = vectorIn.GetLayer()
# # Open raster with GDAL
# rasterIn = gdal.Open(raster,gdalconst.GA_ReadOnly)
# if rasterIn is None:
# print 'Impossible to open '+filename
# exit()
# W,H,D= rasterIn.RasterXSize,rasterIn.RasterYSize,rasterIn.RasterCount
# GeoTransform = rasterIn.GetGeoTransform()
# Projection = rasterIn.GetProjection()
# ox,oy = GeoTransform[0],GeoTransform[3]
# sx,sy = GeoTransform[1],GeoTransform[5]
# print "X={},Y={},D={}".format(W,H,D)
# Iterate over features and store pixels position
for feat in layerIn:
geom = feat.GetGeometryRef()
if feat.GetField("band_0") != NODATA:
P.append([geom.GetX(), geom.GetY()])
Y.append(feat.GetField(Field))
X.append([feat.GetField(b) for b in bands])
# pi_ = [sp.floor((Xc-ox)/sx).astype(int), sp.floor((Yc-oy)/sy).astype(int)]
# if (pi_[0]>=0) and (pi_[0]<W) and (pi_[1]>=0) and (pi_[1]<H): # Check if the point is in the image
# pi.append(pi_)
# y.append(feat.GetField(Field))
# # Filter NODATA values
# band = rasterIn.GetRasterBand(1).ReadAsArray()
# PI,Y=[],[]
# for y_,pi_ in zip(y,pi):x
# if band[pi_[1], pi_[0]] > 0:
# PI.append(pi_)
# Y.append(y_)
# del y,pi
# # Load samples
# ns = len(Y)
# X = sp.empty((ns,bands.size))
# print X.shape
# for d_ in bands:
# print "Bands number {}".format(d_)
# band = rasterIn.GetRasterBand(d_+1).ReadAsArray()
# for p, pi_ in enumerate(PI):
# X[p, d_] = band[pi_[1], pi_[0]]
return X, Y, P
def _create_base_map(self,):
'''
Deal with different types way to define the AOI, if none is specified, then the image bound is used.
'''
gdal.UseExceptions()
ogr.UseExceptions()
if self.aoi is not None:
if os.path.exists(self.aoi):
try:
g = gdal.Open(self.aoi)
#subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir + '/AOI.json', self.aoi])
geojson = get_boundary(self.aoi)[0]
with open(self.toa_dir + '/AOI.json', 'wb') as f:
f.write(geojson.encode())
except:
try:
gr = ogr.Open(self.aoi)
l = gr.GetLayer(0)
f = l.GetFeature(0)
g = f.GetGeometryRef()
except:
raise IOError('AOI file cannot be opened by gdal, please check it or transform into format can be opened by gdal')
else:
try:
g = ogr.CreateGeometryFromJson(self.aoi)
except:
try:
g = ogr.CreateGeometryFromGML(self.aoi)
except:
try:
g = ogr.CreateGeometryFromWkt(self.aoi)
except:
try:
g = ogr.CreateGeometryFromWkb(self.aoi)
except:
raise IOError('The AOI has to be one of GeoJSON, GML, Wkt or Wkb.')
gjson_str = '''{"type":"FeatureCollection","features":[{"type":"Feature","properties":{},"geometry":%s}]}'''% g.ExportToJson()
with open(self.toa_dir + '/AOI.json', 'wb') as f:
f.write(gjson_str.encode())
ogr.DontUseExceptions()
gdal.DontUseExceptions()
if not os.path.exists(self.toa_dir + '/AOI.json'):
#g = gdal.Open(self.toa_bands[0])
#proj = g.GetProjection()
#if 'WGS 84' in proj:
# subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]])
#else:
# subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir +'/AOI.json', self.toa_bands[0]])
if 'WGS 84' in proj:
#subprocess.call(['gdaltindex', '-f', 'GeoJSON', self.toa_dir +'/AOI.json', self.toa_bands[0]])
geojson = get_boundary(self.toa_bands[0], to_wgs84 = False)
with open(self.toa_dir + '/AOI.json', 'wb') as f:
f.write(geojson.encode())
else:
#subprocess.call(['gdaltindex', '-f', 'GeoJSON', '-t_srs', 'EPSG:4326', self.toa_dir +'/AOI.json', self.toa_bands[0]])
geojson = get_boundary(self.toa_bands[0])[0]
with open(self.toa_dir + '/AOI.json', 'wb') as f:
f.write(geojson.encode())
self.logger.warning('AOI is not created and full band extend is used')
self.aoi = self.toa_dir + '/AOI.json'
else:
self.aoi = self.toa_dir + '/AOI.json'
def main():
build_dirs()
loss_tile_list = [
'00N_000E', '00N_010E', '00N_010W', '00N_020E', '00N_020W', '00N_030E',
'00N_030W', '00N_040E', '00N_040W', '00N_050E', '00N_050W', '00N_060E',
'00N_060W', '00N_070E', '00N_070W', '00N_080E', '00N_080W', '00N_090E',
'00N_090W', '00N_100E', '00N_100W', '00N_110E', '00N_110W', '00N_120E',
'00N_120W', '00N_130E', '00N_130W', '00N_140E', '00N_140W', '00N_150E',
'00N_150W', '00N_160E', '00N_160W', '00N_170E', '00N_170W', '00N_180W',
'10N_000E', '10N_010E', '10N_010W', '10N_020E', '10N_020W', '10N_030E',
'10N_030W', '10N_040E', '10N_040W', '10N_050E', '10N_050W', '10N_060E',
'10N_060W', '10N_070E', '10N_070W', '10N_080E', '10N_080W', '10N_090E',
'10N_090W', '10N_100E', '10N_100W', '10N_110E', '10N_110W', '10N_120E',
'10N_120W', '10N_130E', '10N_130W', '10N_140E', '10N_140W', '10N_150E',
'10N_150W', '10N_160E', '10N_160W', '10N_170E', '10N_170W', '10N_180W',
'10S_000E', '10S_010E', '10S_010W', '10S_020E', '10S_020W', '10S_030E',
'10S_030W', '10S_040E', '10S_040W', '10S_050E', '10S_050W', '10S_060E',
'10S_060W', '10S_070E', '10S_070W', '10S_080E', '10S_080W', '10S_090E',
'10S_090W', '10S_100E', '10S_100W', '10S_110E', '10S_110W', '10S_120E',
'10S_120W', '10S_130E', '10S_130W', '10S_140E', '10S_140W', '10S_150E',
'10S_150W', '10S_160E', '10S_160W', '10S_170E', '10S_170W', '10S_180W',
'20N_000E', '20N_010E', '20N_010W', '20N_020E', '20N_020W', '20N_030E',
'20N_030W', '20N_040E', '20N_040W', '20N_050E', '20N_050W', '20N_060E',
'20N_060W', '20N_070E', '20N_070W', '20N_080E', '20N_080W', '20N_090E',
'20N_090W', '20N_100E', '20N_100W', '20N_110E', '20N_110W', '20N_120E',
'20N_120W', '20N_130E', '20N_130W', '20N_140E', '20N_140W', '20N_150E',
'20N_150W', '20N_160E', '20N_160W', '20N_170E', '20N_170W', '20N_180W',
'20S_000E', '20S_010E', '20S_010W', '20S_020E', '20S_020W', '20S_030E',
'20S_030W', '20S_040E', '20S_040W', '20S_050E', '20S_050W', '20S_060E',
'20S_060W', '20S_070E', '20S_070W', '20S_080E', '20S_080W', '20S_090E',
'20S_090W', '20S_100E', '20S_100W', '20S_110E', '20S_110W', '20S_120E',
'20S_120W', '20S_130E', '20S_130W', '20S_140E', '20S_140W', '20S_150E',
'20S_150W', '20S_160E', '20S_160W', '20S_170E', '20S_170W', '20S_180W',
'30N_000E', '30N_010E', '30N_010W', '30N_020E', '30N_020W', '30N_030E',
'30N_030W', '30N_040E', '30N_040W', '30N_050E', '30N_050W', '30N_060E',
'30N_060W', '30N_070E', '30N_070W', '30N_080E', '30N_080W', '30N_090E',
'30N_090W', '30N_100E', '30N_100W', '30N_110E', '30N_110W', '30N_120E',
'30N_120W', '30N_130E', '30N_130W', '30N_140E', '30N_140W', '30N_150E',
'30N_150W', '30N_160E', '30N_160W', '30N_170E', '30N_170W', '30N_180W',
'30S_000E', '30S_010E', '30S_010W', '30S_020E', '30S_020W', '30S_030E',
'30S_030W', '30S_040E', '30S_040W', '30S_050E', '30S_050W', '30S_060E',
'30S_060W', '30S_070E', '30S_070W', '30S_080E', '30S_080W', '30S_090E',
'30S_090W', '30S_100E', '30S_100W', '30S_110E', '30S_110W', '30S_120E',
'30S_120W', '30S_130E', '30S_130W', '30S_140E', '30S_140W', '30S_150E',
'30S_150W', '30S_160E', '30S_160W', '30S_170E', '30S_170W', '30S_180W',
'40N_000E', '40N_010E', '40N_010W', '40N_020E', '40N_020W', '40N_030E',
'40N_030W', '40N_040E', '40N_040W', '40N_050E', '40N_050W', '40N_060E',
'40N_060W', '40N_070E', '40N_070W', '40N_080E', '40N_080W', '40N_090E',
'40N_090W', '40N_100E', '40N_100W', '40N_110E', '40N_110W', '40N_120E',
'40N_120W', '40N_130E', '40N_130W', '40N_140E', '40N_140W', '40N_150E',
'40N_150W', '40N_160E', '40N_160W', '40N_170E', '40N_170W', '40N_180W',
'40S_000E', '40S_010E', '40S_010W', '40S_020E', '40S_020W', '40S_030E',
'40S_030W', '40S_040E', '40S_040W', '40S_050E', '40S_050W', '40S_060E',
'40S_060W', '40S_070E', '40S_070W', '40S_080E', '40S_080W', '40S_090E',
'40S_090W', '40S_100E', '40S_100W', '40S_110E', '40S_110W', '40S_120E',
'40S_120W', '40S_130E', '40S_130W', '40S_140E', '40S_140W', '40S_150E',
'40S_150W', '40S_160E', '40S_160W', '40S_170E', '40S_170W', '40S_180W',
'50N_000E', '50N_010E', '50N_010W', '50N_020E', '50N_020W', '50N_030E',
'50N_030W', '50N_040E', '50N_040W', '50N_050E', '50N_050W', '50N_060E',
'50N_060W', '50N_070E', '50N_070W', '50N_080E', '50N_080W', '50N_090E',
'50N_090W', '50N_100E', '50N_100W', '50N_110E', '50N_110W', '50N_120E',
'50N_120W', '50N_130E', '50N_130W', '50N_140E', '50N_140W', '50N_150E',
'50N_150W', '50N_160E', '50N_160W', '50N_170E', '50N_170W', '50N_180W',
'50S_000E', '50S_010E', '50S_010W', '50S_020E', '50S_020W', '50S_030E',
'50S_030W', '50S_040E', '50S_040W', '50S_050E', '50S_050W', '50S_060E',
'50S_060W', '50S_070E', '50S_070W', '50S_080E', '50S_080W', '50S_090E',
'50S_090W', '50S_100E', '50S_100W', '50S_110E', '50S_110W', '50S_120E',
'50S_120W', '50S_130E', '50S_130W', '50S_140E', '50S_140W', '50S_150E',
'50S_150W', '50S_160E', '50S_160W', '50S_170E', '50S_170W', '50S_180W',
'60N_000E', '60N_010E', '60N_010W', '60N_020E', '60N_020W', '60N_030E',
'60N_030W', '60N_040E', '60N_040W', '60N_050E', '60N_050W', '60N_060E',
'60N_060W', '60N_070E', '60N_070W', '60N_080E', '60N_080W', '60N_090E',
'60N_090W', '60N_100E', '60N_100W', '60N_110E', '60N_110W', '60N_120E',
'60N_120W', '60N_130E', '60N_130W', '60N_140E', '60N_140W', '60N_150E',
'60N_150W', '60N_160E', '60N_160W', '60N_170E', '60N_170W', '60N_180W',
'70N_000E', '70N_010E', '70N_010W', '70N_020E', '70N_020W', '70N_030E',
'70N_030W', '70N_040E', '70N_040W', '70N_050E', '70N_050W', '70N_060E',
'70N_060W', '70N_070E', '70N_070W', '70N_080E', '70N_080W', '70N_090E',
'70N_090W', '70N_100E', '70N_100W', '70N_110E', '70N_110W', '70N_120E',
'70N_120W', '70N_130E', '70N_130W', '70N_140E', '70N_140W', '70N_150E',
'70N_150W', '70N_160E', '70N_160W', '70N_170E', '70N_170W', '70N_180W',
'80N_000E', '80N_010E', '80N_010W', '80N_020E', '80N_020W', '80N_030E',
'80N_030W', '80N_040E', '80N_040W', '80N_050E', '80N_050W', '80N_060E',
'80N_060W', '80N_070E', '80N_070W', '80N_080E', '80N_080W', '80N_090E',
'80N_090W', '80N_100E', '80N_100W', '80N_110E', '80N_110W', '80N_120E',
'80N_120W', '80N_130E', '80N_130W', '80N_140E', '80N_140W', '80N_150E',
'80N_150W', '80N_160E', '80N_160W', '80N_170E', '80N_170W', '80N_180W'
]
template_url = r'http://glad.geog.umd.edu/Potapov/GFW_2017/tiles_2017/{}.tif'
s3_output_dir = r's3://gfw2-data/alerts-tsv/loss_2017/'
local_output_dir = r'/home/ubuntu/output/'
output = r'/home/ubuntu/data/umd/loss/source/{0}.tif'
nd_output = r'/home/ubuntu/data/umd/loss/processed/{0}.tif'
gdal_warp_cmd = ['gdalwarp', '-co', 'COMPRESS=LZW', '-srcnodata', '0']
gdal_warp_cmd += [
'-dstnodata', '255', '--config', 'GDAL_CACHEMAX', '5%', '-overwrite'
]
chunk_list = [x for x in chunks(loss_tile_list, 40)][list_index]
for tile_name in chunk_list:
url = template_url.format(tile_name)
outfile = output.format(tile_name)
cmd = ['wget', '-O', outfile, url]
subprocess.check_call(cmd)
gdal.UseExceptions()
# Source: http://gis.stackexchange.com/questions/90726
# open raster and choose band to find min, max
gtif = gdal.Open(outfile)
srcband = gtif.GetRasterBand(1)
stats = srcband.GetStatistics(False, True)
if len(set(stats)) == 1 and int(stats[0]) == 0:
print 'No Data found for {}, skipping'.format(tile_name)
else:
processed_file = nd_output.format(tile_name)
cmd2 = gdal_warp_cmd + [outfile, processed_file]
print ' '.join(cmd2)
subprocess.check_call(cmd2)
# For the update with 2017 Hansen data (summer 2018), these biomass tiles were used:
# s3://WHRC-carbon/global_27m_tiles/final_global_27m_tiles/biomass_10x10deg/{}_biomass.tif
# For the update with 2018 Hansen data, we should use the below biomass tiles, which are v4 from Woods Hole (delivered summer 2018)
biomass_s3 = r's3://WHRC-carbon/WHRC_V4/Processed/{}_biomass.tif'.format(
tile_name)
biomass_local = r'/home/ubuntu/data/emissions/{}.tif'.format(
tile_name)
biomass_cmd = ['aws', 's3', 'cp', biomass_s3, biomass_local]
subprocess.check_call(biomass_cmd)
extent_url = r'http://commondatastorage.googleapis.com/earthenginepartners-hansen/GFC2015/Hansen_GFC2015_treecover2000_{}.tif'.format(
tile_name)
extent_local = r'/home/ubuntu/data/extent2000/{}.tif'.format(
tile_name)
subprocess.check_call(['wget', '-O', extent_local, extent_url])
ras_cwd = r'/home/ubuntu/raster-to-tsv'
ras_to_vec_cmd = [
'python', 'write-tsv.py', '--datasets', processed_file,
extent_local, biomass_local, '--threads', '50', '--prefix',
tile_name, '--separate', '--local-output-dir',
local_output_dir, '--csv-process', 'area'
]
subprocess.check_call(ras_to_vec_cmd, cwd=ras_cwd)
for ras in [processed_file, extent_local, biomass_local]:
os.remove(ras)
os.remove(outfile)
# upload all files in the local output dir
# should be fast-- wait for all tsvs to finish, then use 64 threads to upload
# configured using aws configure set default.s3.max_concurrent_requests 64
cmd = [
'aws', 's3', 'cp', '--recursive', local_output_dir, s3_output_dir
]
subprocess.check_call(cmd)
# delete TSVs
files = glob.glob(local_output_dir + '*')
for f in files:
os.remove(f)
def resample_geotiff(geotiff, width, outFormat, outFile, use_nn=False):
# Check output format
formats = ['GEOTIFF', 'JPEG', 'JPG', 'PNG', 'KML']
if outFormat.upper() not in formats:
raise ValueError(
f'Unknown output format ({outFormat.upper()})! Accepted formats: {formats}'
)
if use_nn:
resampleMethod = GRIORA_NearestNeighbour
else:
resampleMethod = GRIORA_Cubic
# Suppress GDAL warnings
gdal.UseExceptions()
gdal.PushErrorHandler('CPLQuietErrorHandler')
# Extract information from GeoTIFF
raster = gdal.Open(geotiff)
bandCount = raster.RasterCount
band = raster.GetRasterBand(1)
colorTable = band.GetColorTable()
# Downsample by multiples of pixel size to avoid interpolation issues
# (if needed)
orgExt = os.path.splitext(outFile)[1]
resampleFile = None
resampleFile2 = None
gt = raster.GetGeoTransform()
cols = raster.RasterXSize
# rows = raster.RasterYSize
scale = cols / float(width)
mult = 2**(math.floor(math.log(scale, 2)) - 1)
if mult > 1:
pixelWidth = gt[1] * mult
pixelHeight = gt[5] * mult
if outFormat.upper() == 'PNG' and bandCount == 3:
tmpExt = ('_resamp{0}.png'.format(os.getpid()))
resampleFile = outFile.replace(orgExt, tmpExt)
tmpExt2 = ('_resamp2{0}.png'.format(os.getpid()))
resampleFile2 = outFile.replace(orgExt, tmpExt2)
gdal.Translate(resampleFile, raster, format='PNG', noData='0 0 0')
gdal.Translate(resampleFile2,
resampleFile,
resampleAlg=resampleMethod,
format='PNG',
xRes=pixelWidth,
yRes=pixelHeight,
noData="0 0 0")
raster = gdal.Open(resampleFile2)
else:
tmpExt = ('_resamp{0}.tif'.format(os.getpid()))
resampleFile = outFile.replace(orgExt, tmpExt)
gdal.Translate(resampleFile,
raster,
resampleAlg=resampleMethod,
xRes=pixelWidth,
yRes=pixelHeight,
noData="0")
raster = gdal.Open(resampleFile)
# Resample image using cubic interpolation
# Save it in the various image formats
if outFormat.upper() == 'GEOTIFF':
gdal.Translate(outFile,
raster,
resampleAlg=resampleMethod,
width=width)
elif outFormat.upper() == 'JPEG' or outFormat.upper() == 'JPG':
if colorTable is None:
gdal.Translate(outFile,
raster,
format='JPEG',
resampleAlg=resampleMethod,
width=width)
else:
gdal.Translate(outFile,
raster,
format='JPEG',
resampleAlg=resampleMethod,
width=width,
rgbExpand='RGB')
elif outFormat.upper() == 'PNG':
if bandCount == 1:
gdal.Translate(outFile,
raster,
format='PNG',
resampleAlg=resampleMethod,
width=width,
noData='0')
elif bandCount == 3:
gdal.Translate(outFile,
raster,
format='PNG',
resampleAlg=resampleMethod,
width=width,
noData='0 0 0')
elif outFormat.upper() == 'KML':
# Reproject to geographic coordinates first
tmpExt = ('_geo{0}.tif'.format(os.getpid()))
tmpFile = outFile.replace(orgExt, tmpExt)
rgbFile = None
if bandCount == 1:
if colorTable is None:
gdal.Warp(tmpFile,
raster,
resampleAlg=GRIORA_Cubic,
width=width,
srcNodata='0',
dstSRS='EPSG:4326',
dstAlpha=True)
else:
rgbExt = ('_rgb{0}.tif'.format(os.getpid()))
rgbFile = outFile.replace(orgExt, rgbExt)
gdal.Translate(rgbFile, raster, rgbExpand='RGBA')
raster = gdal.Open(rgbFile)
gdal.Warp(tmpFile,
raster,
resampleAlg=GRIORA_Cubic,
width=width,
srcNodata='0',
dstSRS='EPSG:4326',
dstAlpha=True)
elif bandCount == 3:
gdal.Warp(tmpFile,
raster,
resampleAlg=GRIORA_Cubic,
width=width,
srcNodata='0 0 0',
dstSRS='EPSG:4326',
dstAlpha=True)
raster = None
# Convert GeoTIFF to PNG - since warp cannot do that in one step
raster = gdal.Open(tmpFile)
pngFile = outFile.replace(orgExt, '.png')
gdal.Translate(pngFile,
raster,
format='PNG',
resampleAlg=resampleMethod)
# Extract metadata from GeoTIFF to fill into the KML
gt = raster.GetGeoTransform()
coordStr = (
'%.4f,%.4f %.4f,%.4f %.4f,%.4f %.4f,%.4f' %
(gt[0], gt[3] + raster.RasterYSize * gt[5], gt[0] +
raster.RasterXSize * gt[1], gt[3] + raster.RasterYSize * gt[5],
gt[0] + raster.RasterXSize * gt[1], gt[3], gt[0], gt[3]))
# Take care of namespaces
prefix = {}
gx = '{http://www.google.com/kml/ext/2.2}'
prefix['gx'] = gx
ns_gx = {'gx': 'http://www.google.com/kml/ext/2.2'}
ns_main = {None: 'http://www.opengis.net/kml/2.2'}
ns = dict(list(ns_main.items()) + list(ns_gx.items()))
# Fill in the tree structure
kmlFile = outFile.replace(orgExt, '.kml')
kml = et.Element('kml', nsmap=ns)
overlay = et.SubElement(kml, 'GroundOverlay')
et.SubElement(overlay, 'name').text = \
os.path.basename(kmlFile).replace('.kml', '') + ' overlay'
icon = et.SubElement(overlay, 'Icon')
et.SubElement(icon, 'href').text = os.path.basename(pngFile)
et.SubElement(icon, 'viewBoundScale').text = '0.75'
latLonQuad = et.SubElement(overlay, '{0}LatLonQuad'.format(gx))
et.SubElement(latLonQuad, 'coordinates').text = coordStr
with open(kmlFile, 'wb') as outF:
outF.write(
et.tostring(kml,
xml_declaration=True,
encoding='utf-8',
pretty_print=True))
# Zip PNG and KML together - need to be in the directory where the files are
# in order to remove any path issues...
back = os.getcwd()
path = os.path.dirname(outFile)
if path != '':
os.chdir(path)
zipFile = os.path.basename(outFile.replace(orgExt, '.kmz'))
zip = zipfile.ZipFile(zipFile, 'w', zipfile.ZIP_DEFLATED)
zip.write(os.path.basename(kmlFile))
zip.write(os.path.basename(pngFile))
zip.close()
os.chdir(back)
# Clean up - remove temporary GeoTIFF, KML and PNG
os.remove(tmpFile)
os.remove(pngFile)
os.remove(pngFile + '.aux.xml')
os.remove(kmlFile)
if rgbFile is not None:
os.remove(rgbFile)
if resampleFile is not None:
for myfile in glob.glob("{0}*".format(resampleFile)):
os.remove(myfile)
if resampleFile2 is not None:
for myfile in glob.glob("{0}*".format(resampleFile2)):
os.remove(myfile)
def main(image_ws, ini_path, blocksize=2048, smooth_flag=False,
stats_flag=False, overwrite_flag=False):
"""Prep a Landsat scene for METRIC
Parameters
----------
image_ws : str
Landsat scene folder that will be prepped.
ini_path : str
File path of the input parameters file.
blocksize : int, optional
Size of blocks to process (the default is 2048).
smooth_flag : bool, optional
If True, dilate/erode image to remove fringe/edge pixels
(the default is False).
stats_flag : bool, optional
If True, compute raster statistics (the default is True).
overwrite_flag : bool, optional
If True, overwrite existing files (the default is False).
Returns
-------
True is successful
"""
# Open config file
config = python_common.open_ini(ini_path)
# Get input parameters
logging.debug(' Reading Input File')
calc_refl_toa_flag = python_common.read_param(
'calc_refl_toa_flag', True, config, 'INPUTS')
calc_refl_toa_qa_flag = python_common.read_param(
'calc_refl_toa_qa_flag', True, config, 'INPUTS')
# calc_refl_sur_ledaps_flag = python_common.read_param(
# 'calc_refl_sur_ledaps_flag', False, config, 'INPUTS')
# calc_refl_sur_qa_flag = python_common.read_param(
# 'calc_refl_sur_qa_flag', False, config, 'INPUTS')
calc_ts_bt_flag = python_common.read_param(
'calc_ts_bt_flag', True, config, 'INPUTS')
# Use QA band to set common area
# Fmask cloud, shadow, & snow pixels will be removed from common area
calc_fmask_common_flag = python_common.read_param(
'calc_fmask_common_flag', True, config, 'INPUTS')
fmask_buffer_flag = python_common.read_param(
'fmask_buffer_flag', False, config, 'INPUTS')
fmask_erode_flag = python_common.read_param(
'fmask_erode_flag', False, config, 'INPUTS')
if fmask_erode_flag:
fmask_erode_cells = int(python_common.read_param(
'fmask_erode_cells', 10, config, 'INPUTS'))
if fmask_erode_cells == 0 and fmask_erode_flag:
fmask_erode_flag = False
# Number of cells to buffer Fmask clouds
# For now use the same buffer radius and apply to
if fmask_buffer_flag:
fmask_buffer_cells = int(python_common.read_param(
'fmask_buffer_cells', 25, config, 'INPUTS'))
if fmask_buffer_cells == 0 and fmask_buffer_flag:
fmask_buffer_flag = False
# Include hand made cloud masks
cloud_mask_flag = python_common.read_param(
'cloud_mask_flag', False, config, 'INPUTS')
cloud_mask_ws = ""
if cloud_mask_flag:
cloud_mask_ws = config.get('INPUTS', 'cloud_mask_ws')
# Extract separate Fmask rasters
calc_fmask_flag = python_common.read_param(
'calc_fmask_flag', True, config, 'INPUTS')
calc_fmask_cloud_flag = python_common.read_param(
'calc_fmask_cloud_flag', True, config, 'INPUTS')
calc_fmask_snow_flag = python_common.read_param(
'calc_fmask_snow_flag', True, config, 'INPUTS')
calc_fmask_water_flag = python_common.read_param(
'calc_fmask_water_flag', True, config, 'INPUTS')
# Keep Landsat DN, LEDAPS, and Fmask rasters
keep_dn_flag = python_common.read_param(
'keep_dn_flag', True, config, 'INPUTS')
# keep_sr_flag = python_common.read_param(
# 'keep_sr_flag', True, config, 'INPUTS')
# For this to work I would need to pass in the metric input file
# calc_elev_flag = python_common.read_param(
# 'calc_elev_flag', False, config, 'INPUTS')
# calc_landuse_flag = python_common.read_param(
# 'calc_landuse_flag', False, config, 'INPUTS')
# calc_acca_cloud_flag = python_common.read_param(
# 'calc_acca_cloud_flag', True, config, 'INPUTS')
# calc_acca_snow_flag = python_common.read_param(
# 'calc_acca_snow_flag', True, config, 'INPUTS')
# calc_ledaps_dem_land_flag = python_common.read_param(
# 'calc_ledaps_dem_land_flag', False, config, 'INPUTS')
# calc_ledaps_veg_flag = python_common.read_param(
# 'calc_ledaps_veg_flag', False, config, 'INPUTS')
# calc_ledaps_snow_flag = python_common.read_param(
# 'calc_ledaps_snow_flag', False, config, 'INPUTS')
# calc_ledaps_land_flag = python_common.read_param(
# 'calc_ledaps_land_flag', False, config, 'INPUTS')
# calc_ledaps_cloud_flag = python_common.read_param(
# 'calc_ledaps_cloud_flag', False, config, 'INPUTS')
# Interpolate/clip/project hourly rasters for each Landsat scene
# calc_metric_flag = python_common.read_param(
# 'calc_metric_flag', False, config, 'INPUTS')
calc_metric_ea_flag = python_common.read_param(
'calc_metric_ea_flag', False, config, 'INPUTS')
calc_metric_wind_flag = python_common.read_param(
'calc_metric_wind_flag', False, config, 'INPUTS')
calc_metric_etr_flag = python_common.read_param(
'calc_metric_etr_flag', False, config, 'INPUTS')
calc_metric_tair_flag = python_common.read_param(
'calc_metric_tair_flag', False, config, 'INPUTS')
# Interpolate/clip/project AWC and daily ETr/PPT rasters
# to compute SWB Ke for each Landsat scene
calc_swb_ke_flag = python_common.read_param(
'calc_swb_ke_flag', False, config, 'INPUTS')
if cloud_mask_flag:
spinup_days = python_common.read_param(
'swb_spinup_days', 30, config, 'INPUTS')
min_spinup_days = python_common.read_param(
'swb_min_spinup_days', 5, config, 'INPUTS')
# Round ea raster to N digits to save space
rounding_digits = python_common.read_param(
'rounding_digits', 3, config, 'INPUTS')
env = drigo.env
image = et_image.Image(image_ws, env)
np.seterr(invalid='ignore', divide='ignore')
gdal.UseExceptions()
# Input file paths
dn_image_dict = et_common.landsat_band_image_dict(
image.orig_data_ws, image.image_re)
# # Open METRIC config file
# if config_file:
# logging.info(
# log_f.format('METRIC INI File:', os.path.basename(config_file)))
# config = configparser.ConfigParser()
# try:
# config.read(config_file)
# except:
# logging.error('\nERROR: Config file could not be read, ' +
# 'is not an input file, or does not exist\n' +
# 'ERROR: config_file = {}\n').format(config_file)
# sys.exit()
# # Overwrite
# overwrite_flag = read_param('overwrite_flag', True, config)
#
# # Elevation and landuse parameters/flags from METRIC input file
# calc_elev_flag = read_param('save_dem_raster_flag', True, config)
# calc_landuse_flag = read_param(
# 'save_landuse_raster_flag', True, config)
# if calc_elev_flag:
# elev_pr_path = config.get('INPUTS','dem_raster')
# if calc_landuse_flag:
# landuse_pr_path = config.get('INPUTS', 'landuse_raster')
# else:
# overwrite_flag = False
# calc_elev_flag = False
# calc_landuse_flag = False
#
# Elev raster must exist
# if calc_elev_flag and not os.path.isfile(elev_pr_path):
# logging.error('\nERROR: Elevation raster {} does not exist\n'.format(
# elev_pr_path))
# return False
# Landuse raster must exist
# if calc_landuse_flag and not os.path.isfile(landuse_pr_path):
# logging.error('\nERROR: Landuse raster {} does not exist\n'.format(
# landuse_pr_path))
# return False
# Removing ancillary files before checking for inputs
if os.path.isdir(os.path.join(image.orig_data_ws, 'gap_mask')):
shutil.rmtree(os.path.join(image.orig_data_ws, 'gap_mask'))
for item in os.listdir(image.orig_data_ws):
if (image.type == 'Landsat7' and
(item.endswith('_B8.TIF') or
item.endswith('_B6_VCID_2.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (image.type == 'Landsat8' and
(item.endswith('_B1.TIF') or
item.endswith('_B8.TIF') or
item.endswith('_B9.TIF') or
item.endswith('_B11.TIF'))):
os.remove(os.path.join(image.orig_data_ws, item))
elif (item.endswith('_VER.jpg') or
item.endswith('_VER.txt') or
item.endswith('_GCP.txt') or
item == 'README.GTF'):
os.remove(os.path.join(image.orig_data_ws, item))
# Check correction level (image must be L1T to process)
if image.correction != 'L1TP':
logging.debug(' Image is not L1TP corrected, skipping')
return False
# calc_fmask_common_flag = False
# calc_refl_toa_flag = False
# calc_ts_bt_flag = False
# calc_metric_ea_flag = False
# calc_metric_wind_flag = False
# calc_metric_etr_flag = False
# overwrite_flag = False
# QA band must exist
if (calc_fmask_common_flag and image.qa_band not in dn_image_dict.keys()):
logging.warning(
('\nQA band does not exist but calc_fmask_common_flag=True' +
'\n Setting calc_fmask_common_flag=False\n {}').format(
os.path.basename(image.qa_input_raster)))
calc_fmask_common_flag = False
if cloud_mask_flag and not os.path.isdir(cloud_mask_ws):
logging.warning(
('\ncloud_mask_ws is not a directory but cloud_mask_flag=True.' +
'\n Setting cloud_mask_flag=False\n {}').format(cloud_mask_ws))
cloud_mask_flag = False
# Check for Landsat TOA images
if (calc_refl_toa_flag and
(set(list(image.band_toa_dict.keys()) + [image.thermal_band, image.qa_band]) !=
set(dn_image_dict.keys()))):
logging.warning(
'\nMissing Landsat images but calc_refl_toa_flag=True' +
'\n Setting calc_refl_toa_flag=False')
calc_refl_toa_flag = False
# Check for Landsat brightness temperature image
if calc_ts_bt_flag and image.thermal_band not in dn_image_dict.keys():
logging.warning(
'\nThermal band image does not exist but calc_ts_bt_flag=True' +
'\n Setting calc_ts_bt_flag=False')
calc_ts_bt_flag = False
# DEADBEEF - Should the function return False if Ts doesn't exist?
# return False
# Check for METRIC hourly/daily input folders
if calc_metric_ea_flag:
metric_ea_input_ws = config.get('INPUTS', 'metric_ea_input_folder')
if not os.path.isdir(metric_ea_input_ws):
logging.warning(
('\nHourly Ea folder does not exist but calc_metric_ea_flag=True' +
'\n Setting calc_metric_ea_flag=False\n {}').format(
metric_ea_input_ws))
calc_metric_ea_flag = False
if calc_metric_wind_flag:
metric_wind_input_ws = config.get('INPUTS', 'metric_wind_input_folder')
if not os.path.isdir(metric_wind_input_ws):
logging.warning(
('\nHourly wind folder does not exist but calc_metric_wind_flag=True' +
'\n Setting calc_metric_wind_flag=False\n {}').format(
metric_wind_input_ws))
calc_metric_wind_flag = False
if calc_metric_etr_flag:
metric_etr_input_ws = config.get('INPUTS', 'metric_etr_input_folder')
if not os.path.isdir(metric_etr_input_ws):
logging.warning(
('\nHourly ETr folder does not exist but calc_metric_etr_flag=True' +
'\n Setting calc_metric_etr_flag=False\n {}').format(
metric_etr_input_ws))
calc_metric_etr_flag = False
if calc_metric_tair_flag:
metric_tair_input_ws = config.get('INPUTS', 'metric_tair_input_folder')
if not os.path.isdir(metric_tair_input_ws):
logging.warning(
('\nHourly Tair folder does not exist but calc_metric_tair_flag=True' +
'\n Setting calc_metric_tair_flag=False\n {}').format(
metric_tair_input_ws))
calc_metric_tair_flag = False
if (calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag):
metric_hourly_re = re.compile(config.get('INPUTS', 'metric_hourly_re'))
metric_daily_re = re.compile(config.get('INPUTS', 'metric_daily_re'))
if calc_swb_ke_flag:
awc_input_path = config.get('INPUTS', 'awc_input_path')
etr_input_ws = config.get('INPUTS', 'etr_input_folder')
ppt_input_ws = config.get('INPUTS', 'ppt_input_folder')
etr_input_re = re.compile(config.get('INPUTS', 'etr_input_re'))
ppt_input_re = re.compile(config.get('INPUTS', 'ppt_input_re'))
if not os.path.isfile(awc_input_path):
logging.warning(
('\nAWC raster does not exist but calc_swb_ke_flag=True' +
'\n Setting calc_swb_ke_flag=False\n {}').format(
awc_input_path))
calc_swb_ke_flag = False
if not os.path.isdir(etr_input_ws):
logging.warning(
('\nDaily ETr folder does not exist but calc_swb_ke_flag=True' +
'\n Setting calc_swb_ke_flag=False\n {}').format(
etr_input_ws))
calc_swb_ke_flag = False
if not os.path.isdir(ppt_input_ws):
logging.warning(
('\nDaily PPT folder does not exist but calc_swb_ke_flag=True' +
'\n Setting calc_swb_ke_flag=False\n {}').format(
ppt_input_ws))
calc_swb_ke_flag = False
# Build folders for support rasters
if ((calc_fmask_common_flag or calc_refl_toa_flag or
# calc_refl_sur_ledaps_flag or
calc_ts_bt_flag or
calc_metric_ea_flag or calc_metric_wind_flag or
calc_metric_etr_flag or calc_metric_tair_flag or
calc_swb_ke_flag) and
not os.path.isdir(image.support_ws)):
os.makedirs(image.support_ws)
if calc_refl_toa_flag and not os.path.isdir(image.refl_toa_ws):
os.makedirs(image.refl_toa_ws)
# if calc_refl_sur_ledaps_flag and not os.path.isdir(image.refl_sur_ws):
# os.makedirs(image.refl_sur_ws)
# Apply overwrite flag
if overwrite_flag:
overwrite_list = [
image.fmask_cloud_raster, image.fmask_snow_raster,
image.fmask_water_raster
# image.elev_raster, image.landuse_raster
# image.common_area_raster
]
for overwrite_path in overwrite_list:
try:
python_common.remove_file(image.fmask_cloud_raster)
except:
pass
# Use QA band to build common area rasters
logging.info('\nCommon Area Raster')
qa_ds = gdal.Open(dn_image_dict[image.qa_band], 0)
common_geo = drigo.raster_ds_geo(qa_ds)
common_extent = drigo.raster_ds_extent(qa_ds)
common_proj = drigo.raster_ds_proj(qa_ds)
common_osr = drigo.raster_ds_osr(qa_ds)
# Initialize common_area as all non-fill QA values
qa_array = drigo.raster_ds_to_array(qa_ds, return_nodata=False)
common_array = qa_array != 1
common_rows, common_cols = common_array.shape
del qa_ds
# First try applying user defined cloud masks
cloud_mask_path = os.path.join(
cloud_mask_ws, image.folder_id + '_mask.shp')
if cloud_mask_flag and os.path.isfile(cloud_mask_path):
logging.info(' Applying cloud mask shapefile')
feature_path = os.path.join(
cloud_mask_ws, (image.folder_id + '_mask.shp'))
logging.info(' {}'.format(feature_path))
cloud_mask_memory_ds = drigo.polygon_to_raster_ds(
feature_path, nodata_value=0, burn_value=1,
output_osr=common_osr, output_cs=30,
output_extent=common_extent)
cloud_array = drigo.raster_ds_to_array(
cloud_mask_memory_ds, return_nodata=False)
# DEADBEEF - If user sets a cloud mask,
# it is probably better than Fmask
# Eventually change "if" calc_fmask_common_flag: to "elif"
common_array[cloud_array == 1] = 0
del cloud_mask_memory_ds, cloud_array
if calc_fmask_common_flag:
fmask_array = et_numpy.bqa_fmask_func(qa_array)
fmask_mask = (fmask_array >= 2) & (fmask_array <= 4)
if fmask_erode_flag:
logging.info(
(' Eroding and dilating Fmask clouds, shadows, and snow ' +
'{} cells\n to remove errantly masked pixels.').format(
fmask_erode_cells))
fmask_mask = ndimage.binary_erosion(
fmask_mask, iterations=fmask_erode_cells,
structure=ndimage.generate_binary_structure(2, 2))
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_erode_cells,
structure=ndimage.generate_binary_structure(2, 2))
if fmask_buffer_flag:
logging.info(
(' Buffering Fmask clouds, shadows, and snow ' +
'{} cells').format(fmask_buffer_cells))
# Only buffer clouds, shadow, and snow (not water or nodata)
if fmask_mask is None:
fmask_mask = (fmask_array >= 2) & (fmask_array <= 4)
fmask_mask = ndimage.binary_dilation(
fmask_mask, iterations=fmask_buffer_cells,
structure=ndimage.generate_binary_structure(2, 2))
# Reset common_array for buffered cells
common_array[fmask_mask] = 0
del fmask_array, fmask_mask
if common_array is not None:
# Erode and dilate to remove fringe on edge
# Default is to not smooth, but user can force smoothing
if smooth_flag:
common_array = smooth_func(common_array)
# Check that there are some cloud free pixels
if not np.any(common_array):
logging.error(' ERROR: There are no cloud/snow free pixels')
return False
# Always overwrite common area raster
# if not os.path.isfile(image.common_area_raster):
drigo.array_to_raster(
common_array, image.common_area_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Print common geo/extent
logging.debug(' Common geo: {}'.format(common_geo))
logging.debug(' Common extent: {}'.format(common_extent))
# Extract Fmask components as separate rasters
if (calc_fmask_flag or calc_fmask_cloud_flag or calc_fmask_snow_flag or
calc_fmask_water_flag):
logging.info('\nFmask')
fmask_array = et_numpy.bqa_fmask_func(qa_array)
# Save Fmask data as separate rasters
if (calc_fmask_flag and not os.path.isfile(image.fmask_output_raster)):
drigo.array_to_raster(
fmask_array.astype(np.uint8), image.fmask_output_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
if (calc_fmask_cloud_flag and
not os.path.isfile(image.fmask_cloud_raster)):
fmask_cloud_array = (fmask_array == 2) | (fmask_array == 4)
drigo.array_to_raster(
fmask_cloud_array.astype(np.uint8), image.fmask_cloud_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_cloud_array
if (calc_fmask_snow_flag and
not os.path.isfile(image.fmask_snow_raster)):
fmask_snow_array = (fmask_array == 3)
drigo.array_to_raster(
fmask_snow_array.astype(np.uint8), image.fmask_snow_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_snow_array
if (calc_fmask_water_flag and
not os.path.isfile(image.fmask_water_raster)):
fmask_water_array = (fmask_array == 1)
drigo.array_to_raster(
fmask_water_array.astype(np.uint8), image.fmask_water_raster,
output_geo=common_geo, output_proj=common_proj,
mask_array=None, output_nodata=255, stats_flag=stats_flag)
del fmask_water_array
del fmask_array
# # Calculate elevation
# if calc_elev_flag and not os.path.isfile(elev_path):
# logging.info('Elevation')
# elev_array, elev_nodata = drigo.raster_to_array(
# elev_pr_path, 1, common_extent)
# drigo.array_to_raster(
# elev_array, elev_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del elev_array, elev_nodata, elev_path
#
# # Calculate landuse
# if calc_landuse_flag and not os.path.isfile(landuse_raster):
# logging.info('Landuse')
# landuse_array, landuse_nodata = drigo.raster_to_array(
# landuse_pr_path, 1, common_extent)
# drigo.array_to_raster(
# landuse_array, landuse_raster,
# output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array, stats_flag=stats_flag)
# del landuse_array, landuse_nodata, landuse_raster
# Calculate toa reflectance
# f32_gtype, f32_nodata = numpy_to_gdal_type(np.float32)
if calc_refl_toa_flag:
logging.info('Top-of-Atmosphere Reflectance')
if os.path.isfile(image.refl_toa_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(
image.refl_toa_raster))
python_common.remove_file(image.refl_toa_raster)
if not os.path.isfile(image.refl_toa_raster):
# First build empty composite raster
drigo.build_empty_raster(
image.refl_toa_raster, image.band_toa_cnt, np.float32, None,
env.snap_proj, env.cellsize, common_extent)
# cos_theta_solar_flt = et_common.cos_theta_solar_func(
# image.sun_elevation)
# Process by block
logging.info('Processing by block')
logging.debug(' Mask cols/rows: {}/{}'.format(
common_cols, common_rows))
for b_i, b_j in drigo.block_gen(common_rows, common_cols, blocksize):
logging.debug(' Block y: {:5d} x: {:5d}'.format(b_i, b_j))
block_data_mask = drigo.array_to_block(
common_array, b_i, b_j, blocksize).astype(np.bool)
block_rows, block_cols = block_data_mask.shape
block_geo = drigo.array_offset_geo(common_geo, b_j, b_i)
block_extent = drigo.geo_extent(
block_geo, block_rows, block_cols)
logging.debug(' Block rows: {} cols: {}'.format(
block_rows, block_cols))
logging.debug(' Block extent: {}'.format(block_extent))
logging.debug(' Block geo: {}'.format(block_geo))
# Process each TOA band
# for band, band_i in sorted(image.band_toa_dict.items()):
for band, dn_image in sorted(dn_image_dict.items()):
if band not in image.band_toa_dict.keys():
continue
# thermal_band_flag = (band == image.thermal_band)
# Set 0 as nodata value
drigo.raster_path_set_nodata(dn_image, 0)
# Calculate TOA reflectance
dn_array, dn_nodata = drigo.raster_to_array(
dn_image, 1, block_extent)
dn_array = dn_array.astype(np.float64)
# dn_array = dn_array.astype(np.float32)
dn_array[dn_array == 0] = np.nan
#
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
refl_toa_array = et_numpy.l457_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.dr, image.esun_dict[band],
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band])
elif image.type in ['Landsat8']:
refl_toa_array = et_numpy.l8_refl_toa_band_func(
dn_array, image.cos_theta_solar,
image.refl_mult_dict[band],
image.refl_add_dict[band])
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band])
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band],
# image.refl_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array,
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band],
# image.qcalmax_dict[band],
# # image.rad_mult_dict[band],
# # image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band],
# image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# refl_toa_array = et_numpy.refl_toa_band_func(
# dn_array, cos_theta_solar_flt,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag)
drigo.block_to_raster(
refl_toa_array.astype(np.float32),
image.refl_toa_raster,
b_i, b_j, band=image.band_toa_dict[band])
# drigo.array_to_comp_raster(
# refl_toa_array.astype(np.float32),
# image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
del refl_toa_array, dn_array
if stats_flag:
drigo.raster_statistics(image.refl_toa_raster)
# # Process each TOA band
# # for band, band_i in sorted(image.band_toa_dict.items()):
# for band, dn_image in sorted(dn_image_dict.items()):
# thermal_band_flag = (band == image.thermal_band)
# # Set 0 as nodata value
# drigo.raster_path_set_nodata(dn_image, 0)
# # Calculate TOA reflectance
# dn_array, dn_nodata = drigo.raster_to_array(
# dn_image, 1, common_extent)
# dn_array = dn_array.astype(np.float64)
# # dn_array = dn_array.astype(np.float32)
# dn_array[dn_array == 0] = np.nan
# #
# if (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l457_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band])
# # image.rad_mult_dict[band], image.rad_add_dict[band])
# elif (image.type in ['Landsat4', 'Landsat5', 'Landsat7'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l457_ts_bt_band_func(
# dn_array, image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# # image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# elif (image.type in ['Landsat8'] and
# not thermal_band_flag):
# refl_toa_array = et_numpy.l8_refl_toa_band_func(
# dn_array, image.cos_theta_solar,
# image.refl_mult_dict[band], image.refl_add_dict[band])
# elif (image.type in ['Landsat8'] and
# thermal_band_flag):
# refl_toa_array = et_numpy.l8_ts_bt_band_func(
# dn_array,
# image.rad_mult_dict[band], image.rad_add_dict[band],
# image.k1_dict[band], image.k2_dict[band])
# # refl_toa_array = et_numpy.refl_toa_band_func(
# # dn_array, cos_theta_solar_flt,
# # image.dr, image.esun_dict[band],
# # image.lmin_dict[band], image.lmax_dict[band],
# # image.qcalmin_dict[band], image.qcalmax_dict[band],
# # thermal_band_flag)
# drigo.array_to_comp_raster(
# refl_toa_array.astype(np.float32), image.refl_toa_raster,
# image.band_toa_dict[band], common_array)
# del refl_toa_array, dn_array
# Calculate brightness temperature
if calc_ts_bt_flag:
logging.info('Brightness Temperature')
if os.path.isfile(image.ts_bt_raster) and overwrite_flag:
logging.debug(' Overwriting: {}'.format(image.ts_bt_raster))
python_common.remove_file(image.ts_bt_raster)
if not os.path.isfile(image.ts_bt_raster):
band = image.thermal_band
thermal_dn_path = dn_image_dict[band]
drigo.raster_path_set_nodata(thermal_dn_path, 0)
thermal_dn_array, thermal_dn_nodata = drigo.raster_to_array(
thermal_dn_path, 1, common_extent, return_nodata=True)
thermal_dn_mask = thermal_dn_array != thermal_dn_nodata
if image.type in ['Landsat4', 'Landsat5', 'Landsat7']:
ts_bt_array = et_numpy.l457_ts_bt_band_func(
thermal_dn_array,
image.lmin_dict[band], image.lmax_dict[band],
image.qcalmin_dict[band], image.qcalmax_dict[band],
# image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
elif image.type in ['Landsat8']:
ts_bt_array = et_numpy.l8_ts_bt_band_func(
thermal_dn_array,
image.rad_mult_dict[band], image.rad_add_dict[band],
image.k1_dict[band], image.k2_dict[band])
# thermal_rad_array = et_numpy.refl_toa_band_func(
# thermal_dn_array, image.cos_theta_solar,
# image.dr, image.esun_dict[band],
# image.lmin_dict[band], image.lmax_dict[band],
# image.qcalmin_dict[band], image.qcalmax_dict[band],
# thermal_band_flag=True)
# ts_bt_array = et_numpy.ts_bt_func(
# thermal_rad_array, image.k1_dict[image.thermal_band],
# image.k2_dict[image.thermal_band])
ts_bt_array[~thermal_dn_mask] = np.nan
drigo.array_to_raster(
ts_bt_array, image.ts_bt_raster,
output_geo=common_geo, output_proj=env.snap_proj,
# mask_array=common_array,
stats_flag=stats_flag)
# del thermal_dn_array, thermal_rad_array
del thermal_dn_path, thermal_dn_array, ts_bt_array
# Interpolate/project/clip METRIC hourly/daily rasters
if (calc_metric_ea_flag or
calc_metric_wind_flag or
calc_metric_etr_flag):
logging.info('METRIC hourly/daily rasters')
# Get bracketing hours from image acquisition time
image_prev_dt = image.acq_datetime.replace(
minute=0, second=0, microsecond=0)
image_next_dt = image_prev_dt + timedelta(seconds=3600)
# Get NLDAS properties from one of the images
input_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(input_ws, file_name)
for file_name in os.listdir(input_ws)
for match in [metric_hourly_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
except IndexError:
logging.error(' No hourly file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d %H00')))
return False
try:
input_ds = gdal.Open(input_path)
input_osr = drigo.raster_ds_osr(input_ds)
# input_proj = drigo.osr_proj(input_osr)
input_extent = drigo.raster_ds_extent(input_ds)
input_cs = drigo.raster_ds_cellsize(input_ds, x_only=True)
# input_geo = input_extent.geo(input_cs)
input_x, input_y = input_extent.origin()
input_ds = None
except:
logging.error(' Could not get default input image properties')
logging.error(' {}'.format(input_path))
return False
# Project Landsat scene extent to NLDAS GCS
common_gcs_osr = common_osr.CloneGeogCS()
common_gcs_extent = drigo.project_extent(
common_extent, common_osr, common_gcs_osr,
cellsize=env.cellsize)
common_gcs_extent.buffer_extent(0.1)
common_gcs_extent.adjust_to_snap(
'EXPAND', input_x, input_y, input_cs)
# common_gcs_geo = common_gcs_extent.geo(input_cs)
def metric_weather_func(output_raster, input_ws, input_re,
prev_dt, next_dt,
resample_method=gdal.GRA_NearestNeighbour,
rounding_flag=False):
"""Interpolate/project/clip METRIC hourly rasters"""
logging.debug(' Output: {}'.format(output_raster))
if os.path.isfile(output_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
python_common.remove_file(output_raster)
else:
logging.debug(' Skipping, file already exists ' +
'and overwrite is False')
return False
prev_ws = os.path.join(input_ws, str(prev_dt.year))
next_ws = os.path.join(input_ws, str(next_dt.year))
# Technically previous and next could come from different days
# or even years, although this won't happen in the U.S.
try:
prev_path = [
os.path.join(prev_ws, input_name)
for input_name in os.listdir(prev_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(prev_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input prev: {}'.format(prev_path))
except IndexError:
logging.error(' No previous hourly file')
logging.error(' {}'.format(prev_dt))
return False
try:
next_path = [
os.path.join(next_ws, input_name)
for input_name in os.listdir(next_ws)
for input_match in [input_re.match(input_name)]
if (input_match and
(next_dt.strftime('%Y%m%d') ==
input_match.group('YYYYMMDD')))][0]
logging.debug(' Input next: {}'.format(next_path))
except IndexError:
logging.error(' No next hourly file')
logging.error(' {}'.format(next_dt))
return False
# Band numbers are 1's based
prev_band = int(prev_dt.strftime('%H')) + 1
next_band = int(next_dt.strftime('%H')) + 1
logging.debug(' Input prev band: {}'.format(prev_band))
logging.debug(' Input next band: {}'.format(next_band))
# Read arrays
prev_array = drigo.raster_to_array(
prev_path, band=prev_band, mask_extent=common_gcs_extent,
return_nodata=False)
next_array = drigo.raster_to_array(
next_path, band=next_band, mask_extent=common_gcs_extent,
return_nodata=False)
if not np.any(prev_array) or not np.any(next_array):
logging.warning('\nWARNING: Input NLDAS array is all nodata\n')
return None
output_array = hourly_interpolate_func(
prev_array, next_array,
prev_dt, next_dt, image.acq_datetime)
output_array = drigo.project_array(
output_array, resample_method,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent, output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
if rounding_flag:
output_array = np.around(output_array, rounding_digits)
# Force output to 32-bit float
drigo.array_to_raster(
output_array.astype(np.float32), output_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
return True
# Ea - Project to Landsat scene after clipping
if calc_metric_ea_flag:
logging.info(' Hourly vapor pressure (Ea)')
metric_weather_func(
image.metric_ea_raster, metric_ea_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_Bilinear, rounding_flag=True)
# Wind - Project to Landsat scene after clipping
if calc_metric_wind_flag:
logging.info(' Hourly windspeed')
metric_weather_func(
image.metric_wind_raster, metric_wind_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Hourly reference ET (ETr)')
metric_weather_func(
image.metric_etr_raster, metric_etr_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# ETr 24hr - Project to Landsat scene after clipping
if calc_metric_etr_flag:
logging.info(' Daily reference ET (ETr)')
logging.debug(' Output: {}'.format(
image.metric_etr_24hr_raster))
if (os.path.isfile(image.metric_etr_24hr_raster) and
overwrite_flag):
logging.debug(' Overwriting output')
os.remove(image.metric_etr_24hr_raster)
if not os.path.isfile(image.metric_etr_24hr_raster):
etr_prev_ws = os.path.join(
metric_etr_input_ws, str(image_prev_dt.year))
try:
input_path = [
os.path.join(etr_prev_ws, file_name)
for file_name in os.listdir(etr_prev_ws)
for match in [metric_daily_re.match(file_name)]
if (match and
(image_prev_dt.strftime('%Y%m%d') ==
match.group('YYYYMMDD')))][0]
logging.debug(' Input: {}'.format(input_path))
except IndexError:
logging.error(' No daily file for {}'.format(
image_prev_dt.strftime('%Y-%m-%d')))
return False
output_array = drigo.raster_to_array(
input_path, mask_extent=common_gcs_extent,
return_nodata=False)
output_array = drigo.project_array(
output_array, gdal.GRA_NearestNeighbour,
input_osr, input_cs, common_gcs_extent,
common_osr, env.cellsize, common_extent,
output_nodata=None)
# Apply common area mask
output_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest n digits
# output_array = np.around(output_array, rounding_digits)
drigo.array_to_raster(
output_array, image.metric_etr_24hr_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
del output_array
del input_path
# Tair - Project to Landsat scene after clipping
if calc_metric_tair_flag:
logging.info(' Hourly air temperature (Tair)')
metric_weather_func(
image.metric_tair_raster, metric_tair_input_ws,
metric_hourly_re, image_prev_dt, image_next_dt,
gdal.GRA_NearestNeighbour, rounding_flag=False)
# Cleanup
del image_prev_dt, image_next_dt
# Soil Water Balance
if calc_swb_ke_flag:
logging.info('Daily soil water balance')
# Check if output file already exists
logging.debug(' Ke: {}'.format(image.ke_raster))
if os.path.isfile(image.ke_raster):
if overwrite_flag:
logging.debug(' Overwriting output')
python_common.remove_file(image.ke_raster)
else:
logging.debug(' Skipping, file already ' +
'exists and overwrite is False')
return False
ke_array = et_common.raster_swb_func(
image.acq_datetime, common_osr, env.cellsize, common_extent,
awc_input_path, etr_input_ws, etr_input_re,
ppt_input_ws, ppt_input_re,
spinup_days=spinup_days, min_spinup_days=min_spinup_days)
# Apply common area mask
ke_array[~common_array] = np.nan
# Reduce the file size by rounding to the nearest 2 digits
np.around(ke_array, 2, out=ke_array)
# Force output to 32-bit float
drigo.array_to_raster(
ke_array.astype(np.float32), image.ke_raster,
output_geo=common_geo, output_proj=common_proj,
stats_flag=stats_flag)
# Remove Landsat TOA rasters
if not keep_dn_flag:
for landsat_item in python_common.build_file_list(
image.orig_data_ws, image.image_re):
os.remove(os.path.join(image.orig_data_ws, landsat_item))
return True
def test_basic_test_11():
ds = gdal.OpenEx('data/byte.tif')
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_VECTOR)
assert ds is None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_ALL)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', gdal.OF_UPDATE)
assert ds is not None
ds = gdal.OpenEx(
'data/byte.tif', gdal.OF_RASTER | gdal.OF_VECTOR | gdal.OF_UPDATE
| gdal.OF_VERBOSE_ERROR)
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=[])
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=['GTiff'])
assert ds is not None
ds = gdal.OpenEx('data/byte.tif', allowed_drivers=['PNG'])
assert ds is None
with gdaltest.error_handler():
ds = gdal.OpenEx('data/byte.tif', open_options=['FOO'])
assert ds is not None
ar_ds = [gdal.OpenEx('data/byte.tif', gdal.OF_SHARED) for _ in range(1024)]
assert ar_ds[1023] is not None
ar_ds = None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER)
assert ds is None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_VECTOR)
assert ds is not None
assert ds.GetLayerCount() == 1
assert ds.GetLayer(0) is not None
ds.GetLayer(0).GetMetadata()
ds = gdal.OpenEx('../ogr/data/poly.shp',
allowed_drivers=['ESRI Shapefile'])
assert ds is not None
ds = gdal.OpenEx('../ogr/data/poly.shp', gdal.OF_RASTER | gdal.OF_VECTOR)
assert ds is not None
ds = gdal.OpenEx('non existing')
assert ds is None and gdal.GetLastErrorMsg() == ''
gdal.PushErrorHandler('CPLQuietErrorHandler')
ds = gdal.OpenEx('non existing', gdal.OF_VERBOSE_ERROR)
gdal.PopErrorHandler()
assert ds is None and gdal.GetLastErrorMsg() != ''
old_use_exceptions_status = gdal.GetUseExceptions()
gdal.UseExceptions()
got_exception = False
try:
ds = gdal.OpenEx('non existing')
except RuntimeError:
got_exception = True
if old_use_exceptions_status == 0:
gdal.DontUseExceptions()
assert got_exception
def convert_raster(
input_files,
output_file,
driver=None,
access_mode="overwrite",
creation_options=None,
band_type=None,
dst_alpha=None,
boundary=None,
src_srs=None,
dst_srs=None,
task_uid=None,
warp_params: dict = None,
translate_params: dict = None,
use_translate: bool = False,
config_options: List[Tuple[str]] = None,
):
"""
:param warp_params: A dict of options to pass to gdal warp (done first in conversion), overrides other settings.
:param translate_params: A dict of options to pass to gdal translate (done second in conversion),
overrides other settings.
:param input_files: A file or list of files to convert.
:param output_file: The file to convert.
:param driver: The file format to convert.
:param creation_options: Special GDAL options for conversion.
Search for "gdal driver <format> creation options" creation options for driver specific implementation.
:param band_type: The GDAL data type (e.g. gdal.GDT_BYTE).
:param dst_alpha: If including an alpha band in the destination file.
:param boundary: The boundary to be used for clipping, this must be a file.
:param src_srs: The srs of the source (e.g. "EPSG:4326")
:param dst_srs: The srs of the destination (e.g. "EPSG:3857")
:param task_uid: The eventkit task uid used for tracking the work.
:param use_translate: Make true if needing to use translate for conversion instead of warp.
:param config_options: A list of gdal configuration options as a tuple (option, value).
:return: The output file.
"""
if not driver:
raise Exception(
"Cannot use convert_raster without specififying a gdal driver.")
if isinstance(input_files, str) and not use_translate:
input_files = [input_files]
elif isinstance(input_files, list) and use_translate:
# If a single file is provided in an array, we can simply pull it out
if len(input_files) == 1:
input_files = input_files[0]
else:
raise Exception("Cannot use_translate with a list of files.")
gdal.UseExceptions()
subtask_percentage = 50 if driver.lower() == "gtiff" else 100
options = clean_options({
"callback": progress_callback,
"callback_data": {
"task_uid": task_uid,
"subtask_percentage": subtask_percentage
},
"creationOptions": creation_options,
"format": driver,
})
if not warp_params:
warp_params = clean_options({
"outputType": band_type,
"dstAlpha": dst_alpha,
"srcSRS": src_srs,
"dstSRS": dst_srs
})
if not translate_params:
translate_params = dict()
if boundary:
warp_params.update({"cutlineDSName": boundary, "cropToCutline": True})
# Keep the name imagery which is used when seeding the geopackages.
# Needed because arcpy can't change table names.
if driver.lower() == "gpkg":
options["creationOptions"] = options.get(
"creationOptions", []) + ["RASTER_TABLE=imagery"]
if use_translate:
logger.info(
f"calling gdal.Translate('{output_file}', {input_files}'),"
f"{stringify_params(options)}, {stringify_params(warp_params)},)")
options.update(translate_params)
gdal.Translate(output_file, input_files, **options)
else:
logger.info(
f"calling gdal.Warp('{output_file}', [{', '.join(input_files)}],"
f"{stringify_params(options)}, {stringify_params(warp_params)},)")
gdal.Warp(output_file, input_files, **options, **warp_params)
if driver.lower() == "gtiff" or translate_params:
# No need to compress in memory objects as they will be removed later.
if "vsimem" in output_file:
return output_file
input_file, output_file = get_dataset_names(output_file, output_file)
if translate_params:
options.update(translate_params)
else:
options.update({
"creationOptions":
["COMPRESS=LZW", "TILED=YES", "BIGTIFF=YES"]
})
logger.info(f"calling gdal.Translate('{output_file}', '{input_file}', "
f"{stringify_params(options)},)")
gdal.Translate(output_file, input_file, **options)
return output_file
from osgeo import gdal, ogr, osr, gdal_array # 地理及遥感数据运算的核心开源库。
from PyQt5 import QtCore, QtWidgets, QtGui
from PyQt5.QtCore import Qt, QPoint, QPointF, QLine, QLineF, QRect, QRectF, QTime, qrand
from PyQt5.QtGui import QImage, QPixmap, QPainter, QBrush, QPen, QColor, QRadialGradient, QPainterPath, QPicture, \
QPolygonF, QPolygon
from PyQt5.QtWidgets import QAction, QWidget, QPushButton, QApplication, QMessageBox, QFileDialog, QGraphicsScene, \
QGraphicsPixmapItem, QGraphicsRectItem, QMainWindow, QGraphicsView, QGraphicsItem, QSizePolicy
# 通过from…import…导入PyQt5中所需的模块，减轻脚本依赖。
from GUI.Main import Ui_MainWindow as GUI0 # 导入界面脚本。
from GUI.SFC1 import sFC
# from GUI.ShowGraphic import myGraphicsScene,myGraphicsView
from GUI.WindowMove import windowMove
from GUI.Size import size
gdal.UseExceptions() # 抛出gdal异常
gdal.AllRegister() # gdal库需要注册后使用。
ogr.UseExceptions() # 抛出异常
ogr.RegisterAll() # ogr库需要注册后使用。
wholefiletype = ['.jpg', '.jpeg', '.JPG', '.JPEG', '.tif',
'.TIF'] # 创建一个数组存储图象格式
currentfiletype = ['.shp', '.txt'] # 创建一个存储矢量文件的数组。
class myGraphicsView(QGraphicsView):
def __init__(self, parent=None):
super(myGraphicsView, self).__init__(parent)
self.setCacheMode(QGraphicsView.CacheBackground)
self.setViewportUpdateMode(QGraphicsView.BoundingRectViewportUpdate)
self.setRenderHint(QPainter.Antialiasing)
self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse)
def convert_vector(
input_file,
output_file,
driver=None,
access_mode="overwrite",
src_srs=None,
dst_srs=None,
task_uid=None,
layers=None,
layer_name=None,
boundary=None,
bbox=None,
dataset_creation_options=None,
layer_creation_options=None,
config_options: List[Tuple[str]] = None,
distinct_field=None,
):
"""
:param input_files: A file or list of files to convert.
:param output_file: The file to convert.
:param driver: The file format to convert.
:param creation_options: Special GDAL options for conversion.
Search for "gdal driver <format> creation options" creation options for driver specific implementation.
:param access_mode: The access mode for the file (e.g. "append" or "overwrite")
:param bbox: A bounding box as a list (w,s,e,n) to be used for limiting the AOI that is used during conversion.
:param boundary: The boundary to be used for clipping.
This must be a file (i.e. a path as a string) and cannot be used with bbox.
:param src_srs: The srs of the source (e.g. "EPSG:4326")
:param dst_srs: The srs of the destination (e.g. "EPSG:3857")
:param task_uid: The eventkit task uid used for tracking the work.
:param layers: A list of layers to include for translation.
:param layer_name: Table name in database for in_dataset
:param config_options: A list of gdal configuration options as a tuple (option, value).
:param distinct_field: A field for selecting distinct features to prevent duplicates.
:return: The output file.
"""
if isinstance(input_file, str) and access_mode == "append":
input_file = [input_file]
elif isinstance(input_file, list) and access_mode == "overwrite":
# If a single file is provided in an array, we can simply pull it out
if len(input_file) == 1:
input_file = input_file[0]
else:
raise Exception("Cannot overwrite with a list of files.")
gdal.UseExceptions()
options = clean_options({
"callback":
progress_callback,
"callback_data": {
"task_uid": task_uid
},
"datasetCreationOptions":
dataset_creation_options,
"layerCreationOptions":
layer_creation_options,
"format":
driver,
"layers":
layers,
"layerName":
layer_name,
"srcSRS":
src_srs,
"dstSRS":
dst_srs,
"accessMode":
access_mode,
"reproject":
src_srs != dst_srs,
"skipFailures":
True,
"spatFilter":
bbox,
"options": ["-clipSrc", boundary] if boundary and not bbox else None,
})
if "gpkg" in driver.lower():
options["geometryType"] = ["PROMOTE_TO_MULTI"]
if config_options:
for config_option in config_options:
gdal.SetConfigOption(*config_option)
if access_mode == "append":
for _input_file in input_file:
logger.info(
f"calling gdal.VectorTranslate('{output_file}', '{_input_file}', {stringify_params(options)})"
)
gdal.VectorTranslate(output_file, _input_file, **options)
else:
logger.info(
f"calling gdal.VectorTranslate('{output_file}', '{input_file}', {stringify_params(options)})"
)
gdal.VectorTranslate(output_file, input_file, **options)
if distinct_field:
logger.error(f"Normalizing features based on field: {distinct_field}")
table_name = layer_name or os.path.splitext(
os.path.basename(output_file))[0]
options[
"SQLStatement"] = f"SELECT * from '{table_name}' GROUP BY '{distinct_field}'"
options["SQLDialect"] = "sqlite"
logger.error(
f"calling gdal.VectorTranslate('{output_file}', '{output_file}', {stringify_params(options)})"
)
gdal.VectorTranslate(output_file, rename_duplicate(output_file),
**options)
return output_file
# ============= standard library imports ========================
# import os
# import fiona
# import rasterio
# import rasterio.tools.mask as maskit
# from rasterio import features
#from etrm.raster_tools import convert_raster_to_array
import operator
from osgeo import gdal, gdalnumeric, ogr, osr
from PIL import Image, ImageDraw
from numpy import array, asarray
import os, sys
gdal.UseExceptions()
# ============= local library imports ===========================
"""
This script will be used to:
1) separate out pixels of an image by land use by multiplying pixels by a mask array.
"""
def convert_raster_to_array(input_raster_path, raster=None, band=1):
"""
Convert .tif raster into a numpy numerical array.
def main():
parser = argparse.ArgumentParser()
parser.add_argument('dem_path', help='Input DEM file.')
parser.add_argument('output_path', help='Output path for pixel array.')
parser.add_argument('--decimation',
type=int,
help='Optional downsample factor.')
parser.add_argument('--trim')
flags = parser.parse_args()
gdal.UseExceptions()
print('loading DEM...')
dem = Dem(flags.dem_path)
print('x size: ' + str(dem.x_size()))
print('y size: ' + str(dem.y_size()))
print()
print('making master image...')
t0 = time.time()
heightmap_data = dem.get_raster_band_array()
print('assembled master image in {} seconds'.format(time.time() - t0))
print('filling nans to no-data regions...')
t0 = time.time()
heightmap_data[np.where(heightmap_data < -1e30)] = np.nan
print('filled nans in {} seconds'.format(time.time() - t0))
# convert to float
print('converting to 32 bit floats...')
t0 = time.time()
heightmap_data = heightmap_data.astype(np.float32)
print('converted to 32 bit floats in {} seconds'.format(time.time() - t0))
# trim
if flags.trim:
# Expect flags.trim to be a string like '22,2222,44,4444'
# Parse this into (min_x, max_x, min_y, max_y).
trim = eval(flags.trim)
assert len(trim) == 4
min_x, max_x, min_y, max_y = trim
heightmap_data[0:min_x, :] = np.nan
heightmap_data[max_x:-1, :] = np.nan
heightmap_data[:, 0:min_y] = np.nan
heightmap_data[:, max_y:-1] = np.nan
# trim leading and trailing rows/cols that are all nans
print('trimming nans...')
t0 = time.time()
heightmap_data = trim_nans(heightmap_data)
print('trimming nans took {} s'.format(time.time() - t0))
# downsample
if flags.decimation:
print('decimating...')
t0 = time.time()
heightmap_data = heightmap_data[::flags.decimation, ::flags.
decimation] / flags.decimation
print('decimated in {} seconds'.format(time.time() - t0))
# normalize image height
print("Normalizing heightmap...")
t0 = time.time()
heightmap_data -= np.nanmin(heightmap_data)
print("Normalized heightmap in {} seconds.".format(time.time() - t0))
# write blob
print(f"Writing image to {flags.output_path}...")
t0 = time.time()
_save_image(flags.output_path, heightmap_data)
print('Wrote heightmap in {} seconds to {}'.format(time.time() - t0,
flags.output_path))
print('Dimensions {}, num elements {}'.format(str(heightmap_data.shape),
heightmap_data.size))
