def __init__(self, scene_id):
"""
class constructor for the inversion class
Parameters
----------
scene_id : String
ID of the scene to be inverted -> links to the metadata
stored in the OBIA4RTM database
Returns
-------
None
"""
self.scene_id = scene_id
# get a logger
self.__logger = get_logger()
self.__sensor = None
self.__scene_id = None
self.acquisition_time, self.acquisition_date = None, None
# angles
self.__tts, self.__tto, self.__psi = None, None, None
# setup the DB connection
self.conn, self.cursor = connect_db.connect_db()
self.__logger.info('Connected to PostgreSQL engine sucessfully!')
# determine the directory the configuration files are located
obia4rtm_dir = os.path.dirname(OBIA4RTM.__file__)
fname = obia4rtm_dir + os.sep + 'OBIA4RTM_HOME'
with open(fname, 'r') as data:
self.__directory = data.readline()
def __init__(self):
"""
class constructor and basic setup of processing environment
"""
# find the directory the 6S binary has been installed to
# this is a sub-directory of the OBIA4RTM_HOME
with open(os.path.dirname(OBIA4RTM.__file__) + os.sep + 'OBIA4RTM_HOME',
'r') as data:
obia4rtm_dir = data.readline()
self.sixS_install_dir = obia4rtm_dir + os.sep + 'sixS'+ os.sep + \
'src' + os.sep + '6SV1.1'
# make sure that 6S is installed
if not os.path.isdir(self.sixS_install_dir):
print("Error: 6S is not installed on your computer or cannot be found!\n"\
"Expected installation location: '{}'\n"\
"You might want to run OBIA4RTM.S2_PreProcessor.install_6S "\
"for installing 6S".format(self.sixS_install_dir))
# add this directory of 6S binary to system path temporally (does not work properly)
# sys.path.append(sixS_install_dir)
# get a logger for recording sucess and error messages
self.__logger = get_logger()
self.__logger.info('Setting up Google EE environment for Sentinel-2 '\
'atmospheric correction using the 6S algorithm')
try:
ee.Initialize()
except EEException:
self.__logger.error('No (valid) Earth-Engine credentials provided!',
exc_info=True)
close_logger(self.__logger)
sys.exit(-1)
# for storing the metadata
self.info = None
# for the image object
self.S2 = None
# for the solar zenith angle and the scene timestamp
self.solar_z = None
self.scene_date = None
def get_mean_refl_ee(shp_file, img, acqui_date, scene_id, table_name):
"""
calculates mean reflectance per object in image. Uses GEE-Python bindings
for reading the shape and Sentinel-2 imagery data.
Parameters
----------
shp_file : String
file-path to ESRI shapefile with the image object boundaries
img : ee.image.Image
GEE imagery containing the atmospherically collected Sentinel-2 data
acqui_date : String
acquisition date of the imagery (used for linking to LUT and metadata)
scene_id : String
ID of the Sentinel-2 scene
table_name : String
Name of the table the object reflectance values should be written to
Returns
-------
None
"""
# open the database connection to OBIA4RTM's backend
conn, cursor = connect_db()
# get a logger
logger = get_logger()
# in case it isn't done yet:
ee.Initialize()
# iterate over the shapefile to get the metadata
# Shapefile handling
driver = ogr.GetDriverByName('ESRI Shapefile')
shpfile = driver.Open(shp_file)
# check if shapefile exists and could be opened
if shpfile is None:
raise TypeError(
"The provided File '{}' is invalid or blocked!".format(shp_file))
layer = shpfile.GetLayer(0)
num_objects = layer.GetFeatureCount()
logger.info(
"{0} image objects will be processed. This might take a while...".
format(num_objects))
# loop over single features
# get geometry of features and their ID as well as mean reflectane per band
# before that check the raster metadata from GEE
img_epsg = img.select('B2').projection().crs().getInfo()
img_epsg = int(img_epsg.split(':')[1])
# check with the epsg of the shapefile
ref = layer.GetSpatialRef()
if ref is None:
logger.warning('The layer has no projection info! Assume it is the same'\
'as for the imagery - but check results!')
shp_epsg = img_epsg
# asuming that the imagery is projected in UTM as it should
# the UTM-Zone is stored in the last two digits
utm = int(str(shp_epsg)[3::])
else:
code = ref.GetAuthorityCode(None)
shp_epsg = int(code)
utm = ref.GetUTMZone()
if img_epsg != shp_epsg:
logger.error('The projection of the imagery does not match the projection '\
'of the shapefile you provided!'\
'EPSG-Code of the Image: EPSG:{0}; '\
'EPSG-Code of the Shapefile: EPSG:{1}'.format(
img_epsg,
shp_epsg))
close_logger(logger)
sys.exit(
'An error occured while execute get_mean_refl. Check logfile!')
# determine the min area of an object (determined by S2 spatial resolution)
# use the "standard" resolution of 20 meters
# an object must be twice times larger
min_area = 20 * 60 * 2
# for requesting the landuse information
luc_field = 'LU' + acqui_date.replace('-', '')
# start iterating over features
# Get geometry and extent of feature
for ii in range(num_objects):
feature = layer.GetFeature(ii)
# extract the geometry
geom = feature.GetGeometryRef()
# get a well-know text representation -> required by PostGIS
wkt = geom.ExportToWkt()
# get the ID
# f_id = feature.GetFID() # depraceted
f_id = feature.GetField('id')
# get the land cover code
luc = feature.GetField(luc_field)
# convert to integer coding if luc is provided as text
try:
luc = int(luc)
except ValueError:
luc = luc.upper()
query = "SELECT landuse FROM public.s2_landuse WHERE landuse_semantic = "\
"'{0}';".format(
luc)
cursor.execute(query)
res = cursor.fetchall()
luc = int(res[0][0])
# end try-except
# get the area of the feature and check if it fits the image
# resolution -> if the object is to small skip it
area = geom.Area() # m2
# the area must be at least 2.5 times larger than the coarsest
# possible spatial resolution of Sentinel-2 (60 by 60 meters)
if area < min_area:
logger.warning('The object {0} was too small compared to the '\
'spatial resolution of Sentinel-2! '\
'Object area (m2): {1}; Minimum area required (m2): '\
'{2} -> skipping'.format(
f_id,
area,
min_area))
continue
# export the coordinates of the geometry temporarily to JSON dictionary
# for communicating with GEE
geom_json = ast.literal_eval(geom.ExportToJson())
# get the geometry type
# allowed values: Polygon and Multipolygon
geom_type = geom_json.get('type')
# get the coordinates
geom_coords = geom_json.get('coordinates')[0]
# must be converted to lon, lat for GEE
geo_coords = []
for geom_coord in geom_coords:
easting = geom_coord[0]
northing = geom_coord[1]
# call transform method
lon, lat, alt = transform_utm_to_wgs84(easting, northing, utm)
geo_coord = []
geo_coord.append(lon)
geo_coord.append(lat)
geo_coords.append(geo_coord)
if geom_type not in ['Polygon', 'Multipolygon']:
logger.warning('Object with ID {} is not of type Polygon or '\
'Multipolygon -> skipping'.format(f_id))
continue
# construct a GEE geometry
# TODO -> test what happens for Multipolygon!
geom_gee = ee.geometry.Geometry.Polygon(geo_coords)
# use the image reduce function to get the mean reflectance values
# for each of the nine bands used in GEE
meanDictionary = img.reduceRegion(reducer=ee.Reducer.mean(),
geometry=geom_gee)
# extract the computed mean values for the particular image
# only use those bands required for OBIA4RTM
# multiply with 100 to get % surface reflectance values
multiplier = 100
# surround with try-except in case only blackfill was found for a object
try:
B2 = meanDictionary.get('B2').getInfo() * multiplier
B3 = meanDictionary.get('B3').getInfo() * multiplier
B4 = meanDictionary.get('B4').getInfo() * multiplier
B5 = meanDictionary.get('B5').getInfo() * multiplier
B6 = meanDictionary.get('B6').getInfo() * multiplier
B7 = meanDictionary.get('B7').getInfo() * multiplier
B8A = meanDictionary.get('B8A').getInfo() * multiplier
B11 = meanDictionary.get('B11').getInfo() * multiplier
B12 = meanDictionary.get('B12').getInfo() * multiplier
except TypeError:
logger.info(
'No spectral information found for Object with ID {}'.format(
f_id))
continue
# check cloud and shadow mask
# the cloud and shadow masks are binary
# if the average is zero everything is OK (=no clouds, no shadows)
cm = meanDictionary.get('CloudMask').getInfo()
sm = meanDictionary.get('ShadowMask').getInfo()
# if the shadow and/ or the cloud mask is not zero on average
# -> skip the object as it is cloud covered or affected by
# cloud shadows
if cm > 0:
logger.info(
'Object with ID {} is coverd by clouds -> skipping'.format(
f_id))
continue
if sm > 0:
logger.info(
'Object with ID {} is coverd by cloud shadows -> skipping'.
format(f_id))
continue
# also make sure that the object really contains reflectance values
# checking the first band should be sufficient
if B2 is None:
logger.info(
'Object with ID {} contains only NaN values -> skipping'.
format(f_id))
continue
# otherwise insert the data into the PostgreSQL database
try:
query = "INSERT INTO {0} (object_id, acquisition_date, landuse, object_geom, "\
"b2, b3, b4, b5, b6, b7, b8a, b11, b12, scene_id) VALUES ( " \
"{1}, '{2}', {3}, ST_Multi(ST_GeometryFromText('{4}', {5})), " \
"{6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, '{15}') "\
" ON CONFLICT (object_id, scene_id) DO NOTHING;".format(
table_name,
f_id,
acqui_date,
luc,
wkt,
img_epsg,
np.round(B2, 4),
np.round(B3, 4),
np.round(B4, 4),
np.round(B5, 4),
np.round(B6, 4),
np.round(B7, 4),
np.round(B8A, 4),
np.round(B11, 4),
np.round(B12, 4),
scene_id
)
except ValueError:
logger.error("Invalid string syntax encountered when attempting"\
" to generate INSERT for field {0} on '{1}'".format(
f_id, acqui_date))
continue
# catch errors for single objects accordingly and continue with next
# object to avoid interrupts of whole workflow
try:
cursor.execute(query)
conn.commit()
except (DatabaseError, ProgrammingError):
logger.error(
"Could not insert image object with ID {0} into table '{1}'".
format(f_id, table_name),
exc_info=True)
conn.rollback()
continue
#endfor
# close the GDAL-bindings to the files
shpfile = None
layer = None
# close database connection
close_db_connection(conn, cursor)
# close the logger
close_logger(logger)
def create_schema():
"""
this function is used to generate a new schema in the OBIA4RTM database.
In case the schema already exists, nothing will happen.
The schema to be created is taken from the obia4rtm_backend.cfg file
Parameters
----------
None
Returns
-------
status : integer
zero if everything was OK
"""
status = 0
# connect to OBIA4RTM database
con, cursor = connect_db()
# open a logger
logger = get_logger()
logger.info('Trying to setup a new schema for the OBIA4RTM database')
# read in the obia4rtm_backend information to get the name of the schema
# therefore the obia4rtm_backend.cfg file must be read
install_dir = os.path.dirname(OBIA4RTM.__file__)
home_pointer = install_dir + os.sep + 'OBIA4RTM_HOME'
if not os.path.isfile(home_pointer):
logger.error('Cannot determine OBIA4RTM Home directory!')
close_logger(logger)
sys.exit(-1)
with open(home_pointer, "r") as data:
obia4rtm_home = data.read()
backend_cfg = obia4rtm_home + os.sep + 'obia4rtm_backend.cfg'
if not os.path.isfile(backend_cfg):
logger.error(
'Cannot read obia4rtm_backend.cfg from {}!'.format(obia4rtm_home))
close_logger(logger)
sys.exit(sys_exit_message)
# now, the cfg information can be read in using the configParser class
parser = ConfigParser()
try:
parser.read(backend_cfg)
except MissingSectionHeaderError:
logger.error(
'The obia4rtm_backend.cfg does not fulfil the formal requirements!',
exc_info=True)
close_logger(logger)
sys.exit(-1)
# no get the name of the schema
schema = parser.get('schema-setting', 'schema_obia4rtm')
try:
assert schema is not None and schema != ''
except AssertionError:
logger.error(
'The version of your obia4rtm_backend.cfg file seems to be corrupt!',
exc_info=True)
close_logger(logger)
sys.exit(sys_exit_message)
# if the schema name is OK, the schema can be created
# if the schema already exists in the current database, nothing will happen
sql = 'CREATE SCHEMA IF NOT EXISTS {};'.format(schema)
cursor.execute(sql)
con.commit()
# enable PostGIS and HSTORE extension
# enable the PostGIS extension
# in case it fails it is most likely because the extension was almost
# enabled as it should
sql = "CREATE EXTENSION PostGIS;"
try:
cursor.execute(sql)
con.commit()
except (ProgrammingError, DatabaseError):
logger.info("PostGIS already enabled!")
con.rollback()
pass
# enable the HSTORE extension
sql = "CREATE EXTENSION HSTORE;"
try:
cursor.execute(sql)
con.commit()
except (ProgrammingError, DatabaseError):
logger.error("HSTORE already enabled!")
con.rollback()
pass
logger.info(
"Successfully created schema '{}' in current OBIA4RTM database!".
format(schema))
# after that the schema-specific tables are created that are required
# in OBIA4RTM
sql_home = install_dir + os.sep + 'SQL' + os.sep + 'Tables'
# the tables 's2_inversion_results, s2_lookuptable, s2_objects and s2_inversion_mapping
# must be created within the schema
# check if the tables already exist before trying to create them
sql_scripts = [
's2_lookuptable.sql', 's2_inversion_results.sql', 's2_objects.sql',
'inversion_mapping.sql'
]
# go through the config file to get the table-names
table_names = []
table_names.append(parser.get('schema-setting', 'table_lookuptabe'))
table_names.append(parser.get('schema-setting', 'table_inv_results'))
table_names.append(parser.get('schema-setting', 'table_object_spectra'))
table_names.append(parser.get('schema-setting', 'table_inv_mapping'))
# the parser can be cleared now as all information is read
parser.clear()
# iterate through the 4 scripts to create the tables given they not exist
for index in range(len(sql_scripts)):
sql_script = sql_home + os.sep + sql_scripts[index]
table_name = table_names[index]
# check if the table already exists
exists = check_if_exists(schema, table_name, cursor)
# if already exists table log a warning and continue with the next table
if exists:
logger.warning(
"Table '{0}' already exists in schema '{1}' - skipping".format(
table_name, schema))
continue
# else create the table
# get the corresponding sql-statment and try to execute it
sql_statement = create_sql_statement(sql_script, schema, table_name,
logger)
try:
cursor.execute(sql_statement)
con.commit()
except (DatabaseError, ProgrammingError):
logger.error("Creating table '{0}' in schema '{1}' failed!".format(
table_name, schema),
exc_info=True)
close_logger(logger)
sys.exit(sys_exit_message)
# log success
logger.info("Successfully created table '{0}' in schema '{1}'".format(
table_name, schema))
# create the RMSE function required for inverting the spectra
fun_home = install_dir + os.sep + 'SQL' + os.sep + 'Queries_Functions'
rmse_fun = fun_home + os.sep + 'rmse_function.sql'
sql_statement = create_function_statement(rmse_fun, logger)
try:
cursor.execute(sql_statement)
con.commit()
except (DatabaseError, ProgrammingError):
logger.error("Creating function '{0}' failed!".format(rmse_fun),
exc_info=True)
close_logger(logger)
sys.exit(sys_exit_message)
# after iterating, the db connection and the logger can be close
close_db_connection(con, cursor)
close_logger(logger)
return status
def update_luc_table(landcover_table, landcover_cfg=None):
"""
updates the land-cover/ land use table in OBIA4RTM that is required for
performing land-cover class specific vegetation parameter retrieval
Make sure that the classes in the config file match the land cover classes
provided for the image objects and used for generating the lookup-table.
Otherwise bad things might happen.
NOTE: in case land cover classes that are about to be inserted are already
stored in the table, they will be overwritten!
Parameters
----------
landcover_table : String
name of the table with the land cover information (<schema.table>)
landcover_cfg : String
file-path to land cover configurations file
Returns
-------
None
"""
# open the logger
logger = get_logger()
# if no other file is specified the default file from the OBIA4RTM
# directory in the user profile will be used (landcover.cfg)
if landcover_cfg is None:
# determine the directory the configuration files are located
obia4rtm_dir = os.path.dirname(OBIA4RTM.__file__)
fname = obia4rtm_dir + os.sep + 'OBIA4RTM_HOME'
with open(fname, 'r') as data:
directory = data.readline()
landcover_cfg = directory + os.sep + 'landcover.cfg'
# check if specified file exists
if not os.path.isfile(landcover_cfg):
logger.error('The specified landcover.cfg cannot be found!',
exc_info=True)
close_logger(logger)
sys.exit('Error during inserting landcover information. Check log!')
# connect database
con, cursor = connect_db()
# read the landcover information
luc_classes = get_landcover_classes(landcover_cfg)
# now read in the actual data
n_classes = len(luc_classes) # number of land cover classes
try:
assert n_classes >= 1
except AssertionError:
logger.error('Error: >=1 land cover class must be provided!',
exc_info=True)
close_logger(logger)
sys.exit('Error while reading the landcover.cfg file. Check log.')
# now, iterate through the lines of the cfg files and insert it into
# the Postgres database
logger.info("Try to insert values into table '{0}' from landcover.cfg "\
"file ({1})".format(
landcover_table,
landcover_cfg))
for luc_class in luc_classes:
# the first item of the tuple must be an integer value
# the second one a string
try:
luc_code = int(luc_class[0])
except ValueError:
logger.error('Landcover.cfg file seems to be corrupt. '\
'Excepted integer for land cover code!',
exc_info=True)
close_logger(logger)
sys.exit('Error during inserting landcover.cfg. Check log!')
try:
luc_desc = luc_class[1]
except ValueError:
logger.error('Landcover.cfg file seems to be corrupt. '\
'Excepted string for land cover description!',
exc_info=True)
close_logger(logger)
sys.exit('Error during inserting landcover.cfg. Check log!')
# insert into database
# ON CONFLICT -> old values will be replaced
sql = "INSERT INTO {0} (landuse, landuse_semantic) VALUES ({1},'{2}')"\
" ON CONFLICT (landuse) DO UPDATE SET landuse = {1},"\
" landuse_semantic = '{2}';".format(
landcover_table,
luc_code,
luc_desc)
cursor.execute(sql)
con.commit()
# close the logger and database connection afterwards
logger.info("Updated land cover information in table '{}'".format(
landcover_table))
close_logger(logger)
close_db_connection(con, cursor)
def call_sen2core(sentinel_data_dir, zipped, resolution, path_sen2core):
"""
calls Sen2Core and runs it on a downloaded Sentinel 1C dataset to
convert it to Level 2A.
The output spatial resolution must be provided (10, 20 or 60 meters)
NOTE: If you have already L2 imagery then only run the gdal_merge wrapper
and to not use this function
Parameters
----------
sentinel_data_dir : String
path to the directory that contains the Level-1C data. In case
the data is zipped (default when downloaded from Copernicus) specify
the file-path of the zip
zipped : Boolean
specifies if the directory with the Sat data is zipped
resolution : Integer
spatial resolution of the atmospherically corrected imagery
possible value: 10, 20, 60 meters
path_sen2core : String
directory containing Sen2Core software (top-most level; e.g.
/home/user/Sen2Core/). Must be the same directory as specified
during the Sen2Core installation process using the --target option
Returns
-------
sentinel_data_dir_l2
"""
# check inputs
if zipped:
if not os.path.isfile(sentinel_data_dir):
print("Error: '{}' does not exist!".format(sentinel_data_dir))
sys.exit(-1)
else:
if not os.path.isdir(sentinel_data_dir):
print("Error: '{}' does not exist!".format(sentinel_data_dir))
sys.exit(-1)
# check specified spatial resolution of the result
allowed_res = [10, 20, 60] # m
if resolution not in allowed_res:
print("Error: The specified spatial resolution of {0} is not allowed!\n"\
"Must be one of {1}!".format(resolution, allowed_res))
sys.exit(-1)
# check if Sen2Core is installed and working
runs, cmd = check_sen2core_installation(path_sen2core)
if not runs:
print('Error: Sen2Core seems not to work properly!')
sys.exit(-1)
# after that, enable logging
logger = get_logger()
logger.info('Setting up Sen2Core processing environment for processing '\
'{}'.format(sentinel_data_dir))
# if the data is still zipped unzip
if zipped:
# extract the parent directory
parent_dir = os.path.dirname(sentinel_data_dir)
# unzip
zip_file = ZipFile(sentinel_data_dir)
zip_file.extractall(parent_dir)
zip_file.close()
# check if everything is OK
# new name of sentinel_data_dir is now without ending .zip
sentinel_data_dir = sentinel_data_dir.replace('.zip', '')
# should be a directory
if not os.path.isdir(sentinel_data_dir):
logger.error('Unpacking of Sentinel-2 data failed!')
close_logger(logger)
sys.exit(error_message)
logger.info("Successfully extracted zipped data to {}".format(
sentinel_data_dir))
# endif
# now the data is ready for Sen2Core
# create the command to run Sen2Core
cmd = cmd.replace(' --help', '')
cmd = cmd + ' ' + sentinel_data_dir + ' --resolution=' + str(resolution)
# call Sen2Core
# try to execute the command
logger.info('Starting Sen2Core processing')
out = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
# read the communication of the process
res = out.communicate()
stdout = res[0].decode()
stderr = res[1].decode()
# write output to file
# determine the filename
sub_pos = [m.start() for m in re.finditer('_', sentinel_data_dir)]
fname = sentinel_data_dir[0:sub_pos[2]]
# parent directory
parent_dir = os.path.dirname(sentinel_data_dir)
# save output of Sen2Core
sen2core_stderr = parent_dir + os.sep + 'Sen2Core_' + os.path.basename(
fname) + '.stdout'
with open(sen2core_stderr, 'a') as out:
out.writelines(stdout)
sen2core_stdout = parent_dir + os.sep + 'Sen2Core_' + os.path.basename(
fname) + '.stderr'
with open(sen2core_stdout, 'a') as out:
if stderr == '':
stderr = 'Sen2Core terminated without errors'
out.writelines(stderr)
# determine the dir-name of the processed data
fname = fname.replace('MSIL1C', 'MSIL2A')
# get list of all directories
list_dirs = os.listdir(parent_dir)
# directory of the L2 data
sentinel_data_dir_l2 = ''
for directory in list_dirs:
sub_pos = [m.start() for m in re.finditer('_', directory)]
fname_act = parent_dir + os.sep + directory[0:sub_pos[2]]
if fname_act == fname:
sentinel_data_dir_l2 = parent_dir + os.sep + directory
break
if sentinel_data_dir_l2 == '':
logger.error(
'Seems as if Sen2Core failed. Check {} for more info.'.format(
sen2core_stdout))
close_logger(logger)
sys.exit(error_message)
logger.info("Success - Processed data is in '{0}. See also {1}'".format(
sentinel_data_dir_l2, sen2core_stdout))
# delete the L1C directory in case the data was available as zip to
# save disk storage
if zipped:
shutil.rmtree(sentinel_data_dir, ignore_errors=True)
close_logger(logger)
return sentinel_data_dir_l2
def call_gdal_merge(sentinel_data_dir_l2, resolution, storage_dir=None):
"""
calls the gdal_merge.py script to make an image layer stack and prepare
the imagery for usage in OBIA4RTM.
Outputs a GeoTiff with the nine Sentinel-2 bands used in OBIA4RTM
and the SCL band that contains the preclassification information.
You can use this function also if you L2 data
Parameters
----------
sentinel_data_dir_l2 : String
path of the directory containing the output of sen2core in L2 level
resolution : Integer
spatial resolution of the atmospherically corrected imagery
possible value: 10, 20, 60 meters
storage_dir : String
path to the directory the layer stack should be moved to. If None,
the layer stack will remain the sentinel_data_dir_l2 in the img folder
Returns
-------
fname_stack : String
file-path to the stacked imagery
metadata_xml : String
file-path to the metadata xml file
"""
# enable logging
logger = get_logger()
# check inputs
if not os.path.isdir(sentinel_data_dir_l2):
logger.error(
"Error: '{}' does not exist!".format(sentinel_data_dir_l2))
close_logger(logger)
sys.exit(error_message)
# resolution
allowed_res = [10, 20, 60] # m
if resolution not in allowed_res:
logger.error("Error: The specified spatial resolution of {0} is not allowed! "\
"Must be one of {1}!".format(resolution, allowed_res))
sys.exit(error_message)
# storage directory
if storage_dir is not None:
if not os.path.isdir(storage_dir):
try:
os.mkdir(storage_dir)
except PermissionError:
logger.error("Could not create directory '{}'".format(
sentinel_data_dir_l2),
exc_info=True)
logger.info('Formatting sen2core output for OBIA4RTM')
# change to the sentinel_data_dir_l2 to avoid endless path names
# jump directly into the 'Granule directory'
sentinel_data_dir_l2 = sentinel_data_dir_l2 + os.sep + 'GRANULE'
os.chdir(sentinel_data_dir_l2)
# path to the image bands
next_subdir = os.listdir()[0]
# now the full path can be constructed
sentinel_data_dir_l2 += os.sep + next_subdir
os.chdir(sentinel_data_dir_l2)
# check if the MTD_TL.xml metadata file can be found
if not os.path.isfile('MTD_TL.xml'):
logger.warning('No metadata-xml file could be found!')
metadata_xml = ''
else:
metadata_xml = sentinel_data_dir_l2 + os.sep + 'MTD_TL.xml'
sentinel_data_dir_l2 += os.sep + 'IMG_DATA' + os.sep + 'R' + str(
resolution) + 'm'
try:
os.chdir(sentinel_data_dir_l2)
except FileNotFoundError:
logger.error('Could not find {}'.format(sentinel_data_dir_l2),
exc_info=True)
close_logger(logger)
sys.exit(error_message)
# get the jp2 files containing the single bands and the SCL information
band_files = os.listdir()
band_names = [
'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B8A', 'B11', 'B12', 'SCL'
]
stack_files = []
# loop over the filenames to find the important ones
for band_file in band_files:
for band_name in band_names:
if band_file.find(band_name) != -1:
stack_files.append(band_file)
break
# now the list must be sorted to bring the bands in the correct order
stack_files.sort()
# check if 10 bands were found
try:
assert len(stack_files) == 10
except AssertionError:
logger.error('Expected 10 bands got {}'.format(len(stack_files)),
exc_info=True)
close_logger(logger)
sys.exit(error_message)
# convert list to string
stack_files = str(stack_files).replace('[',
'').replace(']',
'').replace(',', ' ')
# make name of stacked layer file
sub_pos = [m.start() for m in re.finditer('_', band_files[0])]
prefix = band_files[0][0:sub_pos[1]]
fname_stack = prefix + '_merged.tiff'
# now everything is ready for running the gdal_merge.py script
# output is GTiff and not Jpeg2000 as there are still some driver problems
cmd = 'gdal_merge.py -of GTiff -separate ' + stack_files + ' -o ' + fname_stack
logger.info('Running {}'.format(cmd))
out = subprocess.Popen(cmd,
shell=True,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE)
res = out.communicate()
stderr = res[1].decode()
# check if the command worked
if stderr != '':
logger.error('gdal_merge failed: {}'.format(stderr))
close_logger(logger)
sys.exit(error_message)
logger.info(
'Successfully stacked Sentinel-2 bands ({})'.format(fname_stack))
# give the full path
fname_stack_short = fname_stack
fname_stack = sentinel_data_dir_l2 + os.sep + fname_stack
# in case an alternative directory was specified move the metadate file and
# the stacked layer file to this directory
if storage_dir is not None:
# rename the metadata-file and copy it to the storage drive
copy = shutil.copy2(metadata_xml,
storage_dir + os.sep + prefix + '_MTD_TL.xml')
try:
assert copy != ''
assert copy is not None
except AssertionError:
logger.error('Copying of metafile failed!', exc_info=True)
close_logger(logger)
sys.exit(error_message)
# move the stacked image
os.rename(fname_stack, storage_dir + os.sep + fname_stack_short)
logger.info("Moved the imagery and the metadata xml to '{}'".format(
storage_dir))
# close the logger
close_logger(logger)
# return the file-paths of the imagery and the metadata
return fname_stack, metadata_xml
def get_mean_refl(shp_file, raster_file, acqui_date, scene_id, table_name):
"""
calculates mean reflectance per object in image. Uses GDAL-Python bindings
for reading the shape and raster data.
Parameters
----------
shp_file : String
file-path to ESRI shapefile with the image object boundaries
raster_file : String
file-path to raster containing Sentinel-2 imagery as GeoTiff
it is assumed that clouds/ shadows etc have already been masked out
and these pixels are set to the according NoData value
acqui_date : String
acquisition date of the imagery (used for linking to LUT and metadata)
scene_id : String
ID of the Sentinel-2 scene
table_name : String
Name of the table the object reflectance values should be written to
Returns
-------
None
"""
# open the database connection to OBIA4RTM's backend
conn, cursor = connect_db()
# get a logger
logger = get_logger()
# iterate over the shapefile to get the metadata
# Shapefile handling
driver = ogr.GetDriverByName('ESRI Shapefile')
shpfile = driver.Open(shp_file)
layer = shpfile.GetLayer(0)
num_objects = layer.GetFeatureCount()
logger.info("{0} image objects will be processed".format(num_objects))
# loop over single features
# get geometry of features and their ID as well as mean reflectane per band
# open raster data value
raster = gdal.Open(raster_file)
# Get image raster georeference info
transform = raster.GetGeoTransform()
xOrigin = transform[0]
yOrigin = transform[3]
pixelWidth = transform[1]
pixelHeight = transform[5]
# extract the epsg-code
proj = osr.SpatialReference(wkt=raster.GetProjection())
epsg = int(proj.GetAttrValue('AUTHORITY', 1))
# check with the epsg of the shapefile
ref = layer.GetSpatialRef()
if ref is None:
logger.warning('The layer has no projection info! Assume it is the same'\
'as for the imagery - but check results!')
shp_epsg = epsg
else:
code = ref.GetAuthorityCode(None)
shp_epsg = int(code)
# check if the raster and the shapefile epsg match
if epsg != shp_epsg:
logger.error('The projection of the imagery does not match the projection '\
'of the shapefile you provided!'\
'EPSG-Code of the Image: EPSG:{0}; '\
'EPSG-Code of the Shapefile: EPSG:{1}'.format(
epsg,
shp_epsg))
close_logger(logger)
sys.exit(
'An error occured while execute get_mean_refl. Check logfile!')
# check the image raster
num_bands = 10 # Sentinel-2 bands: B2, B3, B4, B5, B6, B7, B8A, B11, B12 + SLC
if (raster.RasterCount != num_bands):
logger.error(
"The number of bands you provided does not match the image file!")
close_logger(logger)
sys.exit(-1)
# determine the min area of an object (determined by S2 spatial resolution)
# use the "standard" resolution of 20 meters
# an object must be twice times larger
min_area = 20 * 20 * 2 # 20 by 20 meters times two as the minimum size constraint
# for requesting the landuse information
luc_field = 'LU' + acqui_date.replace('-', '')
# Get geometry and extent of feature
for ii in range(num_objects):
feature = layer.GetFeature(ii)
# extract the geometry
geom = feature.GetGeometryRef()
# get well-known-text of feature geomtry
wkt = geom.ExportToWkt()
# extract feature ID
f_id = feature.GetFID()
# get the area of the current feature
area = geom.Area() # m2
# the area must be at least 2.5 times larger than the coarsest
# possible spatial resolution of Sentinel-2 (60 by 60 meters)
if area < min_area:
logger.warning('The object {0} was too small compared to the '\
'spatial resolution of Sentinel-2! '\
'Object area (m2): {1}; Minimum area required (m2): '\
'{2} -> skipping'.format(
f_id,
area,
min_area))
continue
luc = feature.GetField(luc_field)
# convert to integer coding if luc is provided as text
try:
luc = int(luc)
except ValueError:
luc = luc.upper()
query = "SELECT landuse FROM s2_landuse WHERE landuse_semantic = '{0}';".format(
luc)
cursor.execute(query)
res = cursor.fetchall()
luc = int(res[0][0])
# end try-except
# check for feature type -> could be either POLYGON or MULTIPOLYGON
if (geom.GetGeometryName() == 'MULTIPOLYGON'):
count = 0
pointsX = []
pointsY = []
for polygon in geom:
geomInner = geom.GetGeometryRef(count)
ring = geomInner.GetGeometryRef(0)
numpoints = ring.GetPointCount()
for p in range(numpoints):
lon, lat, z = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
count += 1
elif (geom.GetGeometryName() == 'POLYGON'):
ring = geom.GetGeometryRef(0)
numpoints = ring.GetPointCount()
pointsX = []
pointsY = []
values = []
for p in range(numpoints):
lon, lat, val = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
values.append(val)
else:
sys.exit(
"ERROR: Geometry needs to be either Polygon or Multipolygon")
#endif
#get exact extent of feature for masking
xmin = min(pointsX)
xmax = max(pointsX)
ymin = min(pointsY)
ymax = max(pointsY)
# Specify offset and rows and columns to read
# -> thus, only a part of the array must be read
# -> calculate the offset in rows in cols to go the specific part of the S2-raster
xoff = int((xmin - xOrigin) / pixelWidth)
yoff = int((yOrigin - ymax) / pixelWidth)
xcount = int((xmax - xmin) / pixelWidth) + 1
ycount = int((ymax - ymin) / pixelWidth) + 1
# temporary raster for masking the actual feature
target_ds = gdal.GetDriverByName('MEM').Create('', xcount, ycount, 1,
gdal.GDT_Byte)
target_ds.SetGeoTransform((
xmin,
pixelWidth,
0,
ymax,
0,
pixelHeight,
))
# Rasterize zone polygon to raster
gdal.RasterizeLayer(target_ds, [1], layer, burn_values=[1])
# the mask to be used for the calculation of the stats
bandmask = target_ds.GetRasterBand(1)
datamask = bandmask.ReadAsArray(0, 0, xcount, ycount).astype(np.float)
# Rasterize zone polygon to raster -> thus data is only read at the location of the
#actual feature
gdal.RasterizeLayer(target_ds, [1], layer, burn_values=[1])
# Read image raster as array
meanValues = []
# iterator variable for looping over Sentinel-2 bands
index = 1
# in case the object is cloud covered or of affected by cirrus
skip_flag = False
# iterate over the bands
for ii in range(raster.RasterCount):
banddataraster = raster.GetRasterBand(index)
# read image data at the specific extent covering the actual feature
dataraster = banddataraster.ReadAsArray(xoff, yoff, xcount,
ycount).astype(np.float)
# Mask zone of raster
zoneraster = np.ma.masked_array(dataraster,
np.logical_not(datamask))
# apply conversion factor of 0.01 to get the correct reflectance
# values for ProSAIL
if ii < raster.RasterCount - 1:
mean = np.nanmean(zoneraster) * 0.01
meanValues.append(mean)
# treat the SCL band with the pre-class info differently
else:
counts = np.bincount(zoneraster)
# get the most frequent value
argmax = np.argmax(counts)
# in case the value is greater than 4 (vegetation) skip the object
if argmax > 4.:
skip_flag = True
# increment index
index += 1
#endfor
# in case the skip flag was set -> skip
if skip_flag:
logger.info('The object is not vegetated -> skipping!')
continue
# check if the results are not nan -> if there are nans skip the object
# as the ProSAIL model inversion cannot deal with missing values
if any(np.isnan(meanValues)):
logger.warning('The object with ID {} contains NaNs -> skipping!')
continue
# insert the mean reflectane and the object geometry into DB
query = "INSERT INTO {0} (object_id, acquisition_date, landuse object_geom, "\
"b2, b3, b4, b5, b6, b7, b8a, b11, b12, scene_id) VALUES ( " \
"{1}, '{2}', {3}, ST_Multi(ST_GeometryFromText('{4}', {5})), " \
"{6}, {7}, {8}, {9}, {10}, {11}, {12}, {13}, {14}, '{15}'}) "\
" ON CONFLICT (object_id, scene_id) DO NOTHING;".format(
table_name,
f_id,
acqui_date,
luc,
wkt,
epsg,
np.round(meanValues[0], 4),
np.round(meanValues[1], 4),
np.round(meanValues[2], 4),
np.round(meanValues[3], 4),
np.round(meanValues[4], 4),
np.round(meanValues[5], 4),
np.round(meanValues[6], 4),
np.round(meanValues[7], 4),
np.round(meanValues[8], 4),
scene_id
)
# catch errors for single objects accordingly and continue with next
# object to avoid interrupts of whole workflow
try:
cursor.execute(query)
conn.commit()
except (DatabaseError, ProgrammingError):
logger.error(
"Could not insert image object with ID {0} into table '{1}'".
format(f_id, table_name),
exc_info=True)
conn.rollback()
continue
# endfor
# close the GDAL-bindings to the files
raster = None
shpfile = None
layer = None
# close database connection
close_db_connection(conn, cursor)