def __init__(self, scene_id):
"""
class constructor for the inversion class
Parameters
----------
scene_id : String
ID of the scene to be inverted -> links to the metadata
stored in the OBIA4RTM database
Returns
-------
None
"""
self.scene_id = scene_id
# get a logger
self.__logger = get_logger()
self.__sensor = None
self.__scene_id = None
self.acquisition_time, self.acquisition_date = None, None
# angles
self.__tts, self.__tto, self.__psi = None, None, None
# setup the DB connection
self.conn, self.cursor = connect_db.connect_db()
self.__logger.info('Connected to PostgreSQL engine sucessfully!')
# determine the directory the configuration files are located
obia4rtm_dir = os.path.dirname(OBIA4RTM.__file__)
fname = obia4rtm_dir + os.sep + 'OBIA4RTM_HOME'
with open(fname, 'r') as data:
self.__directory = data.readline()
def enable_extensions(self):
"""
enables PostGIS and HSTORE extension required for OBIA4RTM
"""
# connect to the created database
self.__con, self.__cursor = connect_db()
# enable the PostGIS extension
sql = "CREATE EXTENSION PostGIS;"
try:
self.__cursor.execute(sql)
self.__con.commit()
except (ProgrammingError, DatabaseError):
print("PostGIS setup failed!")
sys.exit(-1)
# enable the HSTORE extension
sql = "CREATE EXTENSION HSTORE;"
try:
self.__cursor.execute(sql)
self.__con.commit()
except (ProgrammingError, DatabaseError):
print("HSTORE setup failed!")
sys.exit(-1)
def get_scene_metadata(self):
"""
queries the relevant scene metadata from the database
"""
# check if cursor is closed -> if closed, reconnect to database
if self.cursor.closed:
self.conn, self.cursor = connect_db.connect_db()
# database query
query = "SELECT acquisition_time, "\
"scene_id, sun_zenith, obs_zenith, rel_azimuth, sensor "\
"FROM public.scene_metadata WHERE scene_id = '{}';".format(
self.scene_id)
try:
self.cursor.execute(query)
res = self.cursor.fetchall()[0]
except DatabaseError:
self.__logger.error("Querying scene meta for scene '{}' failed".format(
self.scene_id), exc_info=True)
close_logger(self.__logger)
sys.exit(error_message)
# make sure that res is not empty
try:
assert res is not None
except AssertionError:
self.__logger.error("Could not find metadata for scene '{}'".format(
self.scene_id))
close_logger(self.__logger)
sys.exit(error_message)
# extract the desired information
self.acquisition_time = res[0]
self.acquisition_date = self.acquisition_time[0:10]
self.scene_id = res[1]
self.__tts = res[2]
self.__tto = res[3]
self.__psi = res[4]
self.__sensor = res[5]
def get_mean_refl_ee(shp_file, img, acqui_date, scene_id, table_name):
"""
calculates mean reflectance per object in image. Uses GEE-Python bindings
for reading the shape and Sentinel-2 imagery data.
Parameters
----------
shp_file : String
file-path to ESRI shapefile with the image object boundaries
img : ee.image.Image
GEE imagery containing the atmospherically collected Sentinel-2 data
acqui_date : String
acquisition date of the imagery (used for linking to LUT and metadata)
scene_id : String
ID of the Sentinel-2 scene
table_name : String
Name of the table the object reflectance values should be written to
Returns
-------
None
"""
# open the database connection to OBIA4RTM's backend
conn, cursor = connect_db()
# get a logger
logger = get_logger()
# in case it isn't done yet:
ee.Initialize()
# iterate over the shapefile to get the metadata
# Shapefile handling
driver = ogr.GetDriverByName('ESRI Shapefile')
shpfile = driver.Open(shp_file)
# check if shapefile exists and could be opened
if shpfile is None:
raise TypeError(
"The provided File '{}' is invalid or blocked!".format(shp_file))
layer = shpfile.GetLayer(0)
num_objects = layer.GetFeatureCount()
logger.info(
"{0} image objects will be processed. This might take a while...".
format(num_objects))
# loop over single features
# get geometry of features and their ID as well as mean reflectane per band
# before that check the raster metadata from GEE
img_epsg = img.select('B2').projection().crs().getInfo()
img_epsg = int(img_epsg.split(':')[1])
# check with the epsg of the shapefile
ref = layer.GetSpatialRef()
if ref is None:
logger.warning('The layer has no projection info! Assume it is the same'\
'as for the imagery - but check results!')
shp_epsg = img_epsg
# asuming that the imagery is projected in UTM as it should
# the UTM-Zone is stored in the last two digits
utm = int(str(shp_epsg)[3::])
else:
code = ref.GetAuthorityCode(None)
shp_epsg = int(code)
utm = ref.GetUTMZone()
if img_epsg != shp_epsg:
logger.error('The projection of the imagery does not match the projection '\
'of the shapefile you provided!'\
'EPSG-Code of the Image: EPSG:{0}; '\
'EPSG-Code of the Shapefile: EPSG:{1}'.format(
img_epsg,
shp_epsg))
close_logger(logger)
sys.exit(
'An error occured while execute get_mean_refl. Check logfile!')
# determine the min area of an object (determined by S2 spatial resolution)
# use the "standard" resolution of 20 meters
# an object must be twice times larger
min_area = 20 * 60 * 2
# for requesting the landuse information
luc_field = 'LU' + acqui_date.replace('-', '')
# start iterating over features
# Get geometry and extent of feature
for ii in range(num_objects):
feature = layer.GetFeature(ii)
# extract the geometry
geom = feature.GetGeometryRef()
# get a well-know text representation -> required by PostGIS
wkt = geom.ExportToWkt()
# get the ID
# f_id = feature.GetFID() # depraceted
f_id = feature.GetField('id')
# get the land cover code
luc = feature.GetField(luc_field)
# convert to integer coding if luc is provided as text
try:
luc = int(luc)
except ValueError:
luc = luc.upper()
query = "SELECT landuse FROM public.s2_landuse WHERE landuse_semantic = "\
"'{0}';".format(
luc)
cursor.execute(query)
res = cursor.fetchall()
luc = int(res[0][0])
# end try-except
# get the area of the feature and check if it fits the image
# resolution -> if the object is to small skip it
area = geom.Area() # m2
# the area must be at least 2.5 times larger than the coarsest
# possible spatial resolution of Sentinel-2 (60 by 60 meters)
if area < min_area:
logger.warning('The object {0} was too small compared to the '\
'spatial resolution of Sentinel-2! '\
'Object area (m2): {1}; Minimum area required (m2): '\
'{2} -> skipping'.format(
f_id,
area,
min_area))
continue
# export the coordinates of the geometry temporarily to JSON dictionary
# for communicating with GEE
geom_json = ast.literal_eval(geom.ExportToJson())
# get the geometry type
# allowed values: Polygon and Multipolygon
geom_type = geom_json.get('type')
# get the coordinates
geom_coords = geom_json.get('coordinates')[0]
# must be converted to lon, lat for GEE
geo_coords = []
for geom_coord in geom_coords:
easting = geom_coord[0]
northing = geom_coord[1]
# call transform method
lon, lat, alt = transform_utm_to_wgs84(easting, northing, utm)
geo_coord = []
geo_coord.append(lon)
geo_coord.append(lat)
geo_coords.append(geo_coord)
if geom_type not in ['Polygon', 'Multipolygon']:
logger.warning('Object with ID {} is not of type Polygon or '\
'Multipolygon -> skipping'.format(f_id))
continue
# construct a GEE geometry
# TODO -> test what happens for Multipolygon!
geom_gee = ee.geometry.Geometry.Polygon(geo_coords)
# use the image reduce function to get the mean reflectance values
# for each of the nine bands used in GEE
meanDictionary = img.reduceRegion(reducer=ee.Reducer.mean(),
geometry=geom_gee)
# extract the computed mean values for the particular image
# only use those bands required for OBIA4RTM
# multiply with 100 to get % surface reflectance values
multiplier = 100
# surround with try-except in case only blackfill was found for a object
try:
B2 = meanDictionary.get('B2').getInfo() * multiplier
B3 = meanDictionary.get('B3').getInfo() * multiplier
B4 = meanDictionary.get('B4').getInfo() * multiplier
B5 = meanDictionary.get('B5').getInfo() * multiplier
B6 = meanDictionary.get('B6').getInfo() * multiplier
B7 = meanDictionary.get('B7').getInfo() * multiplier
B8A = meanDictionary.get('B8A').getInfo() * multiplier
B11 = meanDictionary.get('B11').getInfo() * multiplier
B12 = meanDictionary.get('B12').getInfo() * multiplier
except TypeError:
logger.info(
'No spectral information found for Object with ID {}'.format(
f_id))
continue
# check cloud and shadow mask
# the cloud and shadow masks are binary
# if the average is zero everything is OK (=no clouds, no shadows)
cm = meanDictionary.get('CloudMask').getInfo()
sm = meanDictionary.get('ShadowMask').getInfo()
# if the shadow and/ or the cloud mask is not zero on average
# -> skip the object as it is cloud covered or affected by
# cloud shadows
if cm > 0:
logger.info(
'Object with ID {} is coverd by clouds -> skipping'.format(
f_id))
continue
if sm > 0:
logger.info(
'Object with ID {} is coverd by cloud shadows -> skipping'.
format(f_id))
continue
# also make sure that the object really contains reflectance values
# checking the first band should be sufficient
if B2 is None:
logger.info(
'Object with ID {} contains only NaN values -> skipping'.
format(f_id))
continue
# otherwise insert the data into the PostgreSQL database
try:
query = "INSERT INTO {0} (object_id, acquisition_date, landuse, object_geom, "\
"b2, b3, b4, b5, b6, b7, b8a, b11, b12, scene_id) VALUES ( " \
"{1}, '{2}', {3}, ST_Multi(ST_GeometryFromText('{4}', {5})), " \
"{6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, '{15}') "\
" ON CONFLICT (object_id, scene_id) DO NOTHING;".format(
table_name,
f_id,
acqui_date,
luc,
wkt,
img_epsg,
np.round(B2, 4),
np.round(B3, 4),
np.round(B4, 4),
np.round(B5, 4),
np.round(B6, 4),
np.round(B7, 4),
np.round(B8A, 4),
np.round(B11, 4),
np.round(B12, 4),
scene_id
)
except ValueError:
logger.error("Invalid string syntax encountered when attempting"\
" to generate INSERT for field {0} on '{1}'".format(
f_id, acqui_date))
continue
# catch errors for single objects accordingly and continue with next
# object to avoid interrupts of whole workflow
try:
cursor.execute(query)
conn.commit()
except (DatabaseError, ProgrammingError):
logger.error(
"Could not insert image object with ID {0} into table '{1}'".
format(f_id, table_name),
exc_info=True)
conn.rollback()
continue
#endfor
# close the GDAL-bindings to the files
shpfile = None
layer = None
# close database connection
close_db_connection(conn, cursor)
# close the logger
close_logger(logger)
def create_schema():
"""
this function is used to generate a new schema in the OBIA4RTM database.
In case the schema already exists, nothing will happen.
The schema to be created is taken from the obia4rtm_backend.cfg file
Parameters
----------
None
Returns
-------
status : integer
zero if everything was OK
"""
status = 0
# connect to OBIA4RTM database
con, cursor = connect_db()
# open a logger
logger = get_logger()
logger.info('Trying to setup a new schema for the OBIA4RTM database')
# read in the obia4rtm_backend information to get the name of the schema
# therefore the obia4rtm_backend.cfg file must be read
install_dir = os.path.dirname(OBIA4RTM.__file__)
home_pointer = install_dir + os.sep + 'OBIA4RTM_HOME'
if not os.path.isfile(home_pointer):
logger.error('Cannot determine OBIA4RTM Home directory!')
close_logger(logger)
sys.exit(-1)
with open(home_pointer, "r") as data:
obia4rtm_home = data.read()
backend_cfg = obia4rtm_home + os.sep + 'obia4rtm_backend.cfg'
if not os.path.isfile(backend_cfg):
logger.error(
'Cannot read obia4rtm_backend.cfg from {}!'.format(obia4rtm_home))
close_logger(logger)
sys.exit(sys_exit_message)
# now, the cfg information can be read in using the configParser class
parser = ConfigParser()
try:
parser.read(backend_cfg)
except MissingSectionHeaderError:
logger.error(
'The obia4rtm_backend.cfg does not fulfil the formal requirements!',
exc_info=True)
close_logger(logger)
sys.exit(-1)
# no get the name of the schema
schema = parser.get('schema-setting', 'schema_obia4rtm')
try:
assert schema is not None and schema != ''
except AssertionError:
logger.error(
'The version of your obia4rtm_backend.cfg file seems to be corrupt!',
exc_info=True)
close_logger(logger)
sys.exit(sys_exit_message)
# if the schema name is OK, the schema can be created
# if the schema already exists in the current database, nothing will happen
sql = 'CREATE SCHEMA IF NOT EXISTS {};'.format(schema)
cursor.execute(sql)
con.commit()
# enable PostGIS and HSTORE extension
# enable the PostGIS extension
# in case it fails it is most likely because the extension was almost
# enabled as it should
sql = "CREATE EXTENSION PostGIS;"
try:
cursor.execute(sql)
con.commit()
except (ProgrammingError, DatabaseError):
logger.info("PostGIS already enabled!")
con.rollback()
pass
# enable the HSTORE extension
sql = "CREATE EXTENSION HSTORE;"
try:
cursor.execute(sql)
con.commit()
except (ProgrammingError, DatabaseError):
logger.error("HSTORE already enabled!")
con.rollback()
pass
logger.info(
"Successfully created schema '{}' in current OBIA4RTM database!".
format(schema))
# after that the schema-specific tables are created that are required
# in OBIA4RTM
sql_home = install_dir + os.sep + 'SQL' + os.sep + 'Tables'
# the tables 's2_inversion_results, s2_lookuptable, s2_objects and s2_inversion_mapping
# must be created within the schema
# check if the tables already exist before trying to create them
sql_scripts = [
's2_lookuptable.sql', 's2_inversion_results.sql', 's2_objects.sql',
'inversion_mapping.sql'
]
# go through the config file to get the table-names
table_names = []
table_names.append(parser.get('schema-setting', 'table_lookuptabe'))
table_names.append(parser.get('schema-setting', 'table_inv_results'))
table_names.append(parser.get('schema-setting', 'table_object_spectra'))
table_names.append(parser.get('schema-setting', 'table_inv_mapping'))
# the parser can be cleared now as all information is read
parser.clear()
# iterate through the 4 scripts to create the tables given they not exist
for index in range(len(sql_scripts)):
sql_script = sql_home + os.sep + sql_scripts[index]
table_name = table_names[index]
# check if the table already exists
exists = check_if_exists(schema, table_name, cursor)
# if already exists table log a warning and continue with the next table
if exists:
logger.warning(
"Table '{0}' already exists in schema '{1}' - skipping".format(
table_name, schema))
continue
# else create the table
# get the corresponding sql-statment and try to execute it
sql_statement = create_sql_statement(sql_script, schema, table_name,
logger)
try:
cursor.execute(sql_statement)
con.commit()
except (DatabaseError, ProgrammingError):
logger.error("Creating table '{0}' in schema '{1}' failed!".format(
table_name, schema),
exc_info=True)
close_logger(logger)
sys.exit(sys_exit_message)
# log success
logger.info("Successfully created table '{0}' in schema '{1}'".format(
table_name, schema))
# create the RMSE function required for inverting the spectra
fun_home = install_dir + os.sep + 'SQL' + os.sep + 'Queries_Functions'
rmse_fun = fun_home + os.sep + 'rmse_function.sql'
sql_statement = create_function_statement(rmse_fun, logger)
try:
cursor.execute(sql_statement)
con.commit()
except (DatabaseError, ProgrammingError):
logger.error("Creating function '{0}' failed!".format(rmse_fun),
exc_info=True)
close_logger(logger)
sys.exit(sys_exit_message)
# after iterating, the db connection and the logger can be close
close_db_connection(con, cursor)
close_logger(logger)
return status
def update_luc_table(landcover_table, landcover_cfg=None):
"""
updates the land-cover/ land use table in OBIA4RTM that is required for
performing land-cover class specific vegetation parameter retrieval
Make sure that the classes in the config file match the land cover classes
provided for the image objects and used for generating the lookup-table.
Otherwise bad things might happen.
NOTE: in case land cover classes that are about to be inserted are already
stored in the table, they will be overwritten!
Parameters
----------
landcover_table : String
name of the table with the land cover information (<schema.table>)
landcover_cfg : String
file-path to land cover configurations file
Returns
-------
None
"""
# open the logger
logger = get_logger()
# if no other file is specified the default file from the OBIA4RTM
# directory in the user profile will be used (landcover.cfg)
if landcover_cfg is None:
# determine the directory the configuration files are located
obia4rtm_dir = os.path.dirname(OBIA4RTM.__file__)
fname = obia4rtm_dir + os.sep + 'OBIA4RTM_HOME'
with open(fname, 'r') as data:
directory = data.readline()
landcover_cfg = directory + os.sep + 'landcover.cfg'
# check if specified file exists
if not os.path.isfile(landcover_cfg):
logger.error('The specified landcover.cfg cannot be found!',
exc_info=True)
close_logger(logger)
sys.exit('Error during inserting landcover information. Check log!')
# connect database
con, cursor = connect_db()
# read the landcover information
luc_classes = get_landcover_classes(landcover_cfg)
# now read in the actual data
n_classes = len(luc_classes) # number of land cover classes
try:
assert n_classes >= 1
except AssertionError:
logger.error('Error: >=1 land cover class must be provided!',
exc_info=True)
close_logger(logger)
sys.exit('Error while reading the landcover.cfg file. Check log.')
# now, iterate through the lines of the cfg files and insert it into
# the Postgres database
logger.info("Try to insert values into table '{0}' from landcover.cfg "\
"file ({1})".format(
landcover_table,
landcover_cfg))
for luc_class in luc_classes:
# the first item of the tuple must be an integer value
# the second one a string
try:
luc_code = int(luc_class[0])
except ValueError:
logger.error('Landcover.cfg file seems to be corrupt. '\
'Excepted integer for land cover code!',
exc_info=True)
close_logger(logger)
sys.exit('Error during inserting landcover.cfg. Check log!')
try:
luc_desc = luc_class[1]
except ValueError:
logger.error('Landcover.cfg file seems to be corrupt. '\
'Excepted string for land cover description!',
exc_info=True)
close_logger(logger)
sys.exit('Error during inserting landcover.cfg. Check log!')
# insert into database
# ON CONFLICT -> old values will be replaced
sql = "INSERT INTO {0} (landuse, landuse_semantic) VALUES ({1},'{2}')"\
" ON CONFLICT (landuse) DO UPDATE SET landuse = {1},"\
" landuse_semantic = '{2}';".format(
landcover_table,
luc_code,
luc_desc)
cursor.execute(sql)
con.commit()
# close the logger and database connection afterwards
logger.info("Updated land cover information in table '{}'".format(
landcover_table))
close_logger(logger)
close_db_connection(con, cursor)
def get_mean_refl(shp_file, raster_file, acqui_date, scene_id, table_name):
"""
calculates mean reflectance per object in image. Uses GDAL-Python bindings
for reading the shape and raster data.
Parameters
----------
shp_file : String
file-path to ESRI shapefile with the image object boundaries
raster_file : String
file-path to raster containing Sentinel-2 imagery as GeoTiff
it is assumed that clouds/ shadows etc have already been masked out
and these pixels are set to the according NoData value
acqui_date : String
acquisition date of the imagery (used for linking to LUT and metadata)
scene_id : String
ID of the Sentinel-2 scene
table_name : String
Name of the table the object reflectance values should be written to
Returns
-------
None
"""
# open the database connection to OBIA4RTM's backend
conn, cursor = connect_db()
# get a logger
logger = get_logger()
# iterate over the shapefile to get the metadata
# Shapefile handling
driver = ogr.GetDriverByName('ESRI Shapefile')
shpfile = driver.Open(shp_file)
layer = shpfile.GetLayer(0)
num_objects = layer.GetFeatureCount()
logger.info("{0} image objects will be processed".format(num_objects))
# loop over single features
# get geometry of features and their ID as well as mean reflectane per band
# open raster data value
raster = gdal.Open(raster_file)
# Get image raster georeference info
transform = raster.GetGeoTransform()
xOrigin = transform[0]
yOrigin = transform[3]
pixelWidth = transform[1]
pixelHeight = transform[5]
# extract the epsg-code
proj = osr.SpatialReference(wkt=raster.GetProjection())
epsg = int(proj.GetAttrValue('AUTHORITY', 1))
# check with the epsg of the shapefile
ref = layer.GetSpatialRef()
if ref is None:
logger.warning('The layer has no projection info! Assume it is the same'\
'as for the imagery - but check results!')
shp_epsg = epsg
else:
code = ref.GetAuthorityCode(None)
shp_epsg = int(code)
# check if the raster and the shapefile epsg match
if epsg != shp_epsg:
logger.error('The projection of the imagery does not match the projection '\
'of the shapefile you provided!'\
'EPSG-Code of the Image: EPSG:{0}; '\
'EPSG-Code of the Shapefile: EPSG:{1}'.format(
epsg,
shp_epsg))
close_logger(logger)
sys.exit(
'An error occured while execute get_mean_refl. Check logfile!')
# check the image raster
num_bands = 10 # Sentinel-2 bands: B2, B3, B4, B5, B6, B7, B8A, B11, B12 + SLC
if (raster.RasterCount != num_bands):
logger.error(
"The number of bands you provided does not match the image file!")
close_logger(logger)
sys.exit(-1)
# determine the min area of an object (determined by S2 spatial resolution)
# use the "standard" resolution of 20 meters
# an object must be twice times larger
min_area = 20 * 20 * 2 # 20 by 20 meters times two as the minimum size constraint
# for requesting the landuse information
luc_field = 'LU' + acqui_date.replace('-', '')
# Get geometry and extent of feature
for ii in range(num_objects):
feature = layer.GetFeature(ii)
# extract the geometry
geom = feature.GetGeometryRef()
# get well-known-text of feature geomtry
wkt = geom.ExportToWkt()
# extract feature ID
f_id = feature.GetFID()
# get the area of the current feature
area = geom.Area() # m2
# the area must be at least 2.5 times larger than the coarsest
# possible spatial resolution of Sentinel-2 (60 by 60 meters)
if area < min_area:
logger.warning('The object {0} was too small compared to the '\
'spatial resolution of Sentinel-2! '\
'Object area (m2): {1}; Minimum area required (m2): '\
'{2} -> skipping'.format(
f_id,
area,
min_area))
continue
luc = feature.GetField(luc_field)
# convert to integer coding if luc is provided as text
try:
luc = int(luc)
except ValueError:
luc = luc.upper()
query = "SELECT landuse FROM s2_landuse WHERE landuse_semantic = '{0}';".format(
luc)
cursor.execute(query)
res = cursor.fetchall()
luc = int(res[0][0])
# end try-except
# check for feature type -> could be either POLYGON or MULTIPOLYGON
if (geom.GetGeometryName() == 'MULTIPOLYGON'):
count = 0
pointsX = []
pointsY = []
for polygon in geom:
geomInner = geom.GetGeometryRef(count)
ring = geomInner.GetGeometryRef(0)
numpoints = ring.GetPointCount()
for p in range(numpoints):
lon, lat, z = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
count += 1
elif (geom.GetGeometryName() == 'POLYGON'):
ring = geom.GetGeometryRef(0)
numpoints = ring.GetPointCount()
pointsX = []
pointsY = []
values = []
for p in range(numpoints):
lon, lat, val = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
values.append(val)
else:
sys.exit(
"ERROR: Geometry needs to be either Polygon or Multipolygon")
#endif
#get exact extent of feature for masking
xmin = min(pointsX)
xmax = max(pointsX)
ymin = min(pointsY)
ymax = max(pointsY)
# Specify offset and rows and columns to read
# -> thus, only a part of the array must be read
# -> calculate the offset in rows in cols to go the specific part of the S2-raster
xoff = int((xmin - xOrigin) / pixelWidth)
yoff = int((yOrigin - ymax) / pixelWidth)
xcount = int((xmax - xmin) / pixelWidth) + 1
ycount = int((ymax - ymin) / pixelWidth) + 1
# temporary raster for masking the actual feature
target_ds = gdal.GetDriverByName('MEM').Create('', xcount, ycount, 1,
gdal.GDT_Byte)
target_ds.SetGeoTransform((
xmin,
pixelWidth,
0,
ymax,
0,
pixelHeight,
))
# Rasterize zone polygon to raster
gdal.RasterizeLayer(target_ds, [1], layer, burn_values=[1])
# the mask to be used for the calculation of the stats
bandmask = target_ds.GetRasterBand(1)
datamask = bandmask.ReadAsArray(0, 0, xcount, ycount).astype(np.float)
# Rasterize zone polygon to raster -> thus data is only read at the location of the
#actual feature
gdal.RasterizeLayer(target_ds, [1], layer, burn_values=[1])
# Read image raster as array
meanValues = []
# iterator variable for looping over Sentinel-2 bands
index = 1
# in case the object is cloud covered or of affected by cirrus
skip_flag = False
# iterate over the bands
for ii in range(raster.RasterCount):
banddataraster = raster.GetRasterBand(index)
# read image data at the specific extent covering the actual feature
dataraster = banddataraster.ReadAsArray(xoff, yoff, xcount,
ycount).astype(np.float)
# Mask zone of raster
zoneraster = np.ma.masked_array(dataraster,
np.logical_not(datamask))
# apply conversion factor of 0.01 to get the correct reflectance
# values for ProSAIL
if ii < raster.RasterCount - 1:
mean = np.nanmean(zoneraster) * 0.01
meanValues.append(mean)
# treat the SCL band with the pre-class info differently
else:
counts = np.bincount(zoneraster)
# get the most frequent value
argmax = np.argmax(counts)
# in case the value is greater than 4 (vegetation) skip the object
if argmax > 4.:
skip_flag = True
# increment index
index += 1
#endfor
# in case the skip flag was set -> skip
if skip_flag:
logger.info('The object is not vegetated -> skipping!')
continue
# check if the results are not nan -> if there are nans skip the object
# as the ProSAIL model inversion cannot deal with missing values
if any(np.isnan(meanValues)):
logger.warning('The object with ID {} contains NaNs -> skipping!')
continue
# insert the mean reflectane and the object geometry into DB
query = "INSERT INTO {0} (object_id, acquisition_date, landuse object_geom, "\
"b2, b3, b4, b5, b6, b7, b8a, b11, b12, scene_id) VALUES ( " \
"{1}, '{2}', {3}, ST_Multi(ST_GeometryFromText('{4}', {5})), " \
"{6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, '{15}'}) "\
" ON CONFLICT (object_id, scene_id) DO NOTHING;".format(
table_name,
f_id,
acqui_date,
luc,
wkt,
epsg,
np.round(meanValues[0], 4),
np.round(meanValues[1], 4),
np.round(meanValues[2], 4),
np.round(meanValues[3], 4),
np.round(meanValues[4], 4),
np.round(meanValues[5], 4),
np.round(meanValues[6], 4),
np.round(meanValues[7], 4),
np.round(meanValues[8], 4),
scene_id
)
# catch errors for single objects accordingly and continue with next
# object to avoid interrupts of whole workflow
try:
cursor.execute(query)
conn.commit()
except (DatabaseError, ProgrammingError):
logger.error(
"Could not insert image object with ID {0} into table '{1}'".
format(f_id, table_name),
exc_info=True)
conn.rollback()
continue
# endfor
# close the GDAL-bindings to the files
raster = None
shpfile = None
layer = None
# close database connection
close_db_connection(conn, cursor)
def do_inversion(self, land_use, num_solutions, res_table,
object_table, inv_mapping_table, lut_table,
return_specs=True):
"""
performs inversion on all objects for a given date.
NOTE: the object reflectance values must be already available in the data
base.
Run gen_lut therefore before!
Works as a wrapper around the do_object_inversion method
Parameters
----------
lande_use : Integer
land cover code for the specific object and date
num_solutions : Integer
how many solutions should be used for generating the inversion result
res_table : String
tablename where to store the results of the inversion
(<schema.table>)
object_table : String
tablename of table containing the object spectra (<schema.table>)
inv_mapping_table : String
tablename of the table containing the parameters to be inverted
per acqusition date (scene) and land use/ cover class
lut_table : String
table containing the ProSAIL lut on a per scene and landuse / cover
class base
return_specs : Boolean
determines whether inverted spectra should be returned (True; default)
Returns
-------
None
"""
# read in the scene metata
self.get_scene_metadata()
# get list of objects available for a given land use class at a given day
query = "SELECT DISTINCT object_id FROM {0} " \
" WHERE acquisition_date = '{1}'" \
" AND landuse = {2};".format(
object_table,
self.acquisition_date,
land_use)
try:
self.cursor.execute(query)
object_ids = self.cursor.fetchall()
object_ids = [item[0] for item in object_ids]
except Exception:
self.__logger.error("Could not query objects for acquistion date "\
"'{0}' and LUC {1}".format(
self.acquisition_date,
land_use),
exc_info=True)
close_logger(self.__logger)
sys.exit(-1)
# get the list of params to be inverted
query = "SELECT params_to_be_inverted FROM {0}" \
" WHERE scene_id = '{1}' AND landuse = {2};".format(
inv_mapping_table,
self.scene_id,
land_use
)
try:
self.cursor.execute(query)
params = self.cursor.fetchall()
params_dict = params[0][0]
# convert to list
params_list = []
for key, val in params_dict.items():
params_list.append(val)
# if inverted spectra should be returned add them to params_list
if (return_specs):
band_names = ["B2", "B3", "B4", "B5", "B6", "B7", "B8A", "B11", "B12"]
for band_name in band_names:
params_list.append(band_name)
# endfor
# endif
except Exception :
self.__logger.error("Retrieving inversion metadata for acquisition "\
"scene '{0}' and LUC {1} failed!".format(
self.scene_id,
land_use),
exc_info=True)
close_logger(self.__logger)
sys.exit(error_message)
# iterate over all objects to perform the inversion per object
for ii in range(len(object_ids)):
object_id = object_ids[ii]
resrun = self.do_obj_inversion(object_id,
self.acquisition_date,
land_use,
num_solutions,
params_list,
res_table,
object_table,
lut_table)
# in case an error happened continue with next object
if resrun != 0:
# reopen the database connection in case an error occured
self.conn, self.cursor = connect_db.connect_db()
continue
# endif
# endfor
# close database connection at the end
if self.conn is not None:
self.cursor.close()
self.conn.close()
def insert_scene_metadata(metadata, use_gee, raster=None):
"""
inserts the most important scene metadata before starting the inversion
procedure into the OBIA4RTM PostgreSQL database
Parameters
----------
metadata : Dictionary
Sentinel-2 scene metadata
use_gee : Boolean
true if GEE was used, false if Sen2Core was used
raster : String
File-path to the Sentinel-2 imagery in case Sen2core was used
Returns
-------
None
"""
# open database connection
conn, cursor = connect_db()
# get sensor and scene_id
sensor, scene_id = get_sensor_and_sceneid(metadata)
# get mean angles from scene-metadata
# tto -> sensor zenith angle
# psi -> relative azimuth angle between sensor and sun
tto, psi = get_mean_angles(metadata)
# sun zenith angle
tts = get_sun_zenith_angle(metadata)
# get the footprint already as PostGIS insert statment
footprint_statement = get_scene_footprint(metadata, gee=use_gee)
# full metadata as JSON
metadata_json = json.dumps(metadata)
# storage drive and filename of the image raster data
# this part only applies to Sen2Core preprocessing
if use_gee:
storage_drive = 'NA: Google Earth Engine'
filename = 'NA: Google Earth Engine'
else:
splitted = os.path.split(raster)
storage_drive = splitted[0]
filename = splitted[1]
# get acquisition time and date
acquisition_time, acquisition_date = get_acqusition_time(metadata)
# insert this basic metadata direclty into the OBIA4RTM database before
# continuing
statement = "INSERT INTO public.scene_metadata (acquisition_time, "\
"scene_id, sun_zenith, "\
"obs_zenith, rel_azimuth, sensor, footprint, full_description, "\
"storage_drive, filename) VALUES ('{0}','{1}',{2},{3},{4},"\
"'{5}',{6},'{7}','{8}','{9}') ON CONFLICT (scene_id) "\
"DO NOTHING;".format(
acquisition_time,
scene_id,
tts,
tto,
psi,
sensor,
footprint_statement,
metadata_json,
storage_drive,
filename
)
try:
cursor.execute(statement)
conn.commit()
except DatabaseError:
raise DatabaseError('Insert of metadata failed!')
sys.exit()
# close database connection
close_db_connection(conn, cursor)