def __init__(self, analysis_name, req_keys=None):
"""
A class to do some analysis defined by the user. Implemented
as a plugin provided through the sensor configuration.
Any params inputted via the EODataDown configuration script for the plugin are passed
to the class using the set_users_param function.
This function initialises an attribute self.analysis_name to None unless an value is passed. However,
this attribute requires a value which must not be None. When creating an instance of this function you
must either pass a value to this function or define this value in your constructor after this parent
constructor has been called.
:param analysis_name: A name for the analysis, must be unique for sensor. Must not be None. This is
expected to be passed to the parent from the child constructor. No value will
be passed through from the EODataDown sensor when the plugin is instantiated.
:param req_keys: A list of required keys within the parameters (optionally) provided from the EODataDown
configuration file. If None then it will be assume no parameters are being provided to
the class.
"""
if analysis_name is None:
raise EODataDownException(
"The analysis name was None, a value is required.")
self.analysis_name = analysis_name
self.req_keys = req_keys
self.params = None
def parse_sensor_config(self, config_file, first_parse=False):
"""
Parse the JSON configuration file. If first_parse=True then a signature file will be created
which will be checked each time the system runs to ensure changes are not back to the
configuration file. If the signature does not match the input file then an expection will be
thrown. To update the configuration (e.g., extent date range or spatial area) run with first_parse=True.
:param config_file: string with the path to the JSON file.
:param first_parse: boolean as to whether the file has been previously parsed.
"""
edd_file_checker = eodatadown.eodatadownutils.EDDCheckFileHash()
# If it is the first time the config_file is parsed then create the signature file.
if first_parse:
edd_file_checker.createFileSig(config_file)
logger.debug("Created signature file for config file.")
if not edd_file_checker.checkFileSig(config_file):
raise EODataDownException("Input config did not match the file signature.")
with open(config_file) as f:
config_data = json.load(f)
json_parse_helper = eodatadown.eodatadownutils.EDDJSONParseHelper()
self.scn_rept_image_dir = json_parse_helper.getStrValue(config_data, ["eodatadown", "report",
"scn_rept_image_dir"])
if json_parse_helper.doesPathExist(config_data, ['eodatadown', 'report', 'overview_size']):
self.overview_size = json_parse_helper.getStrValue(config_data, ["eodatadown", "report",
"overview_size"])
self.scn_tmp_dir = json_parse_helper.getStrValue(config_data, ["eodatadown", "report", "tmp_dir"])
if json_parse_helper.doesPathExist(config_data, ['eodatadown', 'report', 'vec_overlay_file']):
self.scn_overlay_vec_file = json_parse_helper.getStrValue(config_data, ["eodatadown", "report",
"vec_overlay_file"])
self.scn_overlay_vec_lyr = json_parse_helper.getStrValue(config_data, ["eodatadown", "report",
"vec_overlay_lyr"])
def create_sensor_obj(self, sensor):
"""
Get an instance of an object for the sensor specified.
:param sensor:
:return:
"""
sensor_obj = None
if sensor == "LandsatGOOG":
logger.debug("Found sensor LandsatGOOG")
from eodatadown.eodatadownlandsatgoogsensor import EODataDownLandsatGoogSensor
sensor_obj = EODataDownLandsatGoogSensor(self.db_info_obj)
elif sensor == "Sentinel2GOOG":
logger.debug("Found sensor Sentinel2GOOG")
from eodatadown.eodatadownsentinel2googsensor import EODataDownSentinel2GoogSensor
sensor_obj = EODataDownSentinel2GoogSensor(self.db_info_obj)
elif sensor == "Sentinel1ASF":
logger.debug("Found sensor Sentinel1ASF")
from eodatadown.eodatadownsentinel1asf import EODataDownSentinel1ASFProcessorSensor
sensor_obj = EODataDownSentinel1ASFProcessorSensor(
self.db_info_obj)
elif sensor == "GEDI":
logger.debug("Found sensor GEDI")
from eodatadown.eodatadownGEDIsensor import EODataDownGEDISensor
sensor_obj = EODataDownGEDISensor(self.db_info_obj)
elif sensor == "ICESAT2":
logger.debug("Found sensor ICESAT2")
from eodatadown.eodatadownICESAT2sensor import EODataDownICESAT2Sensor
sensor_obj = EODataDownICESAT2Sensor(self.db_info_obj)
else:
raise EODataDownException(
"Do not know of an object for sensor: '" + sensor + "'")
return sensor_obj
def get_usage_db_obj(self):
logger.debug("Creating Usage database object.")
if self.db_info_obj is None:
raise EODataDownException(
"Need to parse the configuration file to find database information."
)
edd_usage_db = EODataDownUpdateUsageLogDB(self.db_info_obj)
return edd_usage_db
def run_user_plugins_scene(config_file, sensor, scene_id):
"""
A function which runs the process of generating a tilecache for an input scene.
:param config_file: The EODataDown configuration file path.
:param sensor: the string name of the sensor
:param scene_id:
:return:
"""
# Create the System 'Main' object and parse the configuration file.
sys_main_obj = eodatadown.eodatadownsystemmain.EODataDownSystemMain()
sys_main_obj.parse_config(config_file)
logger.debug("Parsed the system configuration.")
edd_usage_db = sys_main_obj.get_usage_db_obj()
edd_usage_db.add_entry(
"Started: Running User Plugins for specified scene ({0}: {1}).".format(
sensor, scene_id),
start_block=True)
sensor_objs = sys_main_obj.get_sensors()
sensor_obj_to_process = None
for sensor_obj in sensor_objs:
logger.debug("Testing Sensor Object: '{}'.".format(
sensor_obj.get_sensor_name()))
if sensor_obj.get_sensor_name() == sensor:
sensor_obj_to_process = sensor_obj
logger.debug("Found Sensor Object.")
break
if sensor_obj_to_process is None:
logger.error(
"Error occurred could not find sensor object for '{}'".format(
sensor))
raise EODataDownException(
"Could not find sensor object for '{}'".format(sensor))
try:
logger.debug(
"Going to try running the user analysis plugins for scene '{}'".
format(scene_id))
if not sensor_obj_to_process.is_scn_invalid(scene_id):
sensor_obj_to_process.run_usr_analysis(scene_id)
else:
logger.info(
"Scene '{}' is an invalid scene and therefore analysis is not executed."
.format(scene_id))
logger.debug(
"Finished to try running the user analysis plugins for scene '{}'".
format(scene_id))
except Exception as e:
logger.error(
"Error occurred while running user plugins for scene ({0}) from sensor: ({1})"
.format(scene_id, sensor_obj_to_process.get_sensor_name()))
logger.debug(e.__str__(), exc_info=True)
edd_usage_db.add_entry(
"Finished: Running User Plugins for specified scene ({0}: {1}).".
format(sensor, scene_id),
end_block=True)
def run_arcsi_landsat(input_mtl, dem_file, output_dir, tmp_dir, spacecraft_str,
sensor_str, reproj_outputs, proj_wkt_file, projabbv):
"""
A function to run ARCSI for a landsat scene using python rather than
the command line interface.
:param spacecraft_str:
:param sensor_str:
:param reproj_outputs:
:param proj_wkt_file:
:param projabbv:
:param input_mtl:
:param dem_file:
:param output_dir:
:param tmp_dir:
:return:
"""
import arcsilib.arcsirun
if (spacecraft_str == "LANDSAT_8") and (sensor_str == "OLI_TIRS"):
arcsi_sensor_str = "ls8"
elif (spacecraft_str == "LANDSAT_7") and (sensor_str == "ETM"):
arcsi_sensor_str = "ls7"
elif (spacecraft_str == "LANDSAT_5") and (sensor_str == "TM"):
arcsi_sensor_str = "ls5tm"
elif (spacecraft_str == "LANDSAT_4") and (sensor_str == "TM"):
arcsi_sensor_str = "ls4tm"
else:
logger.error(
"Did not recognise the spacecraft and sensor combination. (" +
spacecraft_str + ", " + sensor_str + ")")
raise EODataDownException(
"Did not recognise the spacecraft and sensor combination.")
if not reproj_outputs:
proj_wkt_file = None
projabbv = None
logger.info("Starting to run ARCSI for: " + input_mtl)
arcsilib.arcsirun.runARCSI(
input_mtl, None, None, arcsi_sensor_str, None, "KEA", output_dir, None,
proj_wkt_file, None, projabbv, None, None, [
"CLOUDS", "DOSAOTSGL", "STDSREF", "SATURATE", "TOPOSHADOW",
"FOOTPRINT", "METADATA"
], True, None, None, arcsilib.DEFAULT_ARCSI_AEROIMG_PATH,
arcsilib.DEFAULT_ARCSI_ATMOSIMG_PATH, "GreenVegetation", 0, None, None,
False, None, None, None, None, False, None, None, tmp_dir, 0.05, 0.5,
0.1, 0.4, dem_file, None, None, True, 20, False, False, 1000, "cubic",
"near", 3000, 3000, 1000, 21, True, False, False, None, None, False,
None, 'LSMSK')
logger.info("Finished running ARCSI for: " + input_mtl)
def create_obsdate_visual(self, sys_main_obj, sensor):
"""
A single threaded function to create the overview images for all the scenes which return it.
:param sys_main_obj: a EODataDownSystemMain instance.
:param sensor: Optionally a sensor can be specified, in which case the result will just be for that sensor.
"""
if not sys_main_obj.has_parsed_config():
raise EODataDownException(
"The EODataDownSystemMain instance has parsed a "
"config file so it not ready to use.")
gen_visuals_lst = self.get_lst_obsdates_need_processing(sensor)
for obs in gen_visuals_lst:
self.process_obsdata(sys_main_obj, obs[0], obs[1], obs[2])
def gen_scene_tilecache(config_file, sensor, scene_id):
"""
A function which runs the process of generating a tilecache for an input scene.
:param config_file: The EODataDown configuration file path.
:param sensor: the string name of the sensor
:param scene_id:
:return:
"""
# Create the System 'Main' object and parse the configuration file.
sys_main_obj = eodatadown.eodatadownsystemmain.EODataDownSystemMain()
sys_main_obj.parse_config(config_file)
logger.debug("Parsed the system configuration.")
edd_usage_db = sys_main_obj.get_usage_db_obj()
edd_usage_db.add_entry(
"Started: Generate tilecache for specified scene ({0}: {1}).".format(
sensor, scene_id),
start_block=True)
sensor_objs = sys_main_obj.get_sensors()
sensor_obj_to_process = None
for sensor_obj in sensor_objs:
if sensor_obj.get_sensor_name() == sensor:
sensor_obj_to_process = sensor_obj
break
if sensor_obj_to_process is None:
logger.error(
"Error occurred could not find sensor object for '{}'".format(
sensor))
raise EODataDownException(
"Could not find sensor object for '{}'".format(sensor))
try:
sensor_obj_to_process.scn2tilecache(scene_id)
except Exception as e:
logger.error(
"Error occurred while generating tilecache for scene ({0}) from sensor: ({1})"
.format(scene_id, sensor_obj_to_process.get_sensor_name()))
logger.debug(e.__str__(), exc_info=True)
edd_usage_db.add_entry(
"Finished: Generate tilecache for specified scene ({0}: {1}).".format(
sensor, scene_id),
end_block=True)
def datacube_load_scene(config_file, sensor, scene_id):
"""
A function which runs the process of converting the specified scene to an ARD product.
:param config_file: The EODataDown configuration file path.
:param sensor: List of sensor names.
:param scene_id:
:return:
"""
# Create the System 'Main' object and parse the configuration file.
sys_main_obj = eodatadown.eodatadownsystemmain.EODataDownSystemMain()
sys_main_obj.parse_config(config_file)
logger.debug("Parsed the system configuration.")
edd_usage_db = sys_main_obj.get_usage_db_obj()
edd_usage_db.add_entry(
"Started: Load Specified Scene into DataCube ({0}: {1}).".format(
sensor, scene_id),
start_block=True)
sensor_objs = sys_main_obj.get_sensors()
sensor_obj_to_process = None
for sensor_obj in sensor_objs:
if sensor_obj.get_sensor_name() == sensor:
sensor_obj_to_process = sensor_obj
break
if sensor_obj_to_process is None:
logger.error(
"Error occurred could not find sensor object for '{}'".format(
sensor))
raise EODataDownException(
"Could not find sensor object for '{}'".format(sensor))
try:
sensor_obj_to_process.scn2datacube(scene_id)
except Exception as e:
logger.error(
"Error occurred while loading scene ({0}) into DataCube for sensor: ({1})"
.format(scene_id, sensor_obj_to_process.get_sensor_name()))
logger.debug(e.__str__(), exc_info=True)
edd_usage_db.add_entry(
"Finished: Load Specified Scene into DataCube ({0}: {1}).".format(
sensor, scene_id),
end_block=True)
def get_sensor_obj(self, sensor):
"""
A function to get a sensor object.
:param sensor:
:return:
"""
sensor_obj_to_process = None
for sensor_obj in self.sensors:
if sensor_obj.get_sensor_name() == sensor:
sensor_obj_to_process = sensor_obj
break
if sensor_obj_to_process is None:
logger.error(
"Error occurred could not find sensor object for '{}'".format(
sensor))
raise EODataDownException(
"Could not find sensor object for '{}'".format(sensor))
return sensor_obj_to_process
def check_param_keys(self, req_keys=None, raise_except=False):
"""
Check that the parameters have the req_keys for the analysis
:param req_keys: list of keys, if None then the class version (defined in constructor) will be used.
:return: boolean (True if all present)
"""
keys_present = True
if req_keys is None:
req_keys = self.req_keys
for key in req_keys:
if key not in self.params:
keys_present = False
logger.debug("'{}' key NOT present.".format(key))
if raise_except:
raise EODataDownException(
"Parameters did not have expected key '{}'".format(
key))
break
else:
logger.debug("'{}' key present.".format(key))
return keys_present
def parse_config(self, config_file, first_parse=False):
"""
Parse the inputted JSON configuration file
:param first_parse:
:param config_file:
:return:
"""
edd_file_checker = eodatadown.eodatadownutils.EDDCheckFileHash()
if not edd_file_checker.checkFileSig(config_file):
raise EODataDownException(
"Input config did not match the file signature.")
with open(config_file) as f:
config_data = json.load(f)
json_parse_helper = eodatadown.eodatadownutils.EDDJSONParseHelper()
eodd_utils = eodatadown.eodatadownutils.EODataDownUtils()
# Get Basic System Info.
self.name = json_parse_helper.getStrValue(
config_data, ['eodatadown', 'details', 'name'])
self.description = json_parse_helper.getStrValue(
config_data, ['eodatadown', 'details', 'description'])
# Get Database Information
edd_pass_encoder = eodatadown.eodatadownutils.EDDPasswordTools()
if json_parse_helper.doesPathExist(
config_data, ['eodatadown', 'database', 'connection']):
db_conn_str = json_parse_helper.getStrValue(
config_data, ['eodatadown', 'database', 'connection'])
elif json_parse_helper.doesPathExist(
config_data,
['eodatadown', 'database', 'connection_file']):
connection_file = json_parse_helper.getStrValue(
config_data, ['eodatadown', 'database', 'connection_file'])
logger.debug("Using connection database file: '{}'".format(
connection_file))
if os.path.exists(connection_file):
logger.debug(
"Database connection file exists and being read")
db_conn_str = eodd_utils.readTextFileNoNewLines(
connection_file)
else:
raise EODataDownException(
"The database connection file specified was not present."
)
else:
raise EODataDownException(
"A database connection is required. You must provided via either"
" a 'connection' or 'connection_file' key.")
self.db_info_obj = eodatadown.eodatadownutils.EODataDownDatabaseInfo(
db_conn_str)
if json_parse_helper.doesPathExist(
config_data,
['eodatadown', 'reports', 'date_report_config']):
self.date_report_config_file = json_parse_helper.getStrValue(
config_data,
['eodatadown', 'reports', 'date_report_config'])
report_obj = EODataDownDateReports(self.db_info_obj)
report_obj.parse_sensor_config(self.date_report_config_file,
first_parse)
if json_parse_helper.doesPathExist(config_data,
['eodatadown', 'obsdates']):
self.obsdates_config_file = json_parse_helper.getStrValue(
config_data, ['eodatadown', 'obsdates'])
obsdates_obj = EODataDownObsDates(self.db_info_obj)
obsdates_obj.parse_sensor_config(self.obsdates_config_file,
first_parse)
# Get Sensor Configuration File List
for sensor in config_data['eodatadown']['sensors']:
self.sensorConfigFiles[sensor] = json_parse_helper.getStrValue(
config_data, ['eodatadown', 'sensors', sensor, 'config'])
logger.debug("Getting sensor object: '" + sensor + "'")
sensor_obj = self.create_sensor_obj(sensor)
logger.debug("Parse sensor config file: '" + sensor + "'")
sensor_obj.parse_sensor_config(self.sensorConfigFiles[sensor],
first_parse)
self.sensors.append(sensor_obj)
logger.debug("Parsed sensor config file: '" + sensor + "'")
self.parsed_config = True
def process_obsdata(self, sys_main_obj, sensor_id, platform_id, obs_date):
"""
A function to process a single scene - i.e., produce the over image.
:param sys_main_obj: a EODataDownSystemMain instance.
:param sensor_id: specify the sensor
:param platform_id: specify the platform (i.e., Sentinel-1A, Sentinel-2B, or Landsat-4)
:param obs_date: specify the date of interest, as a datetime.date object.
"""
if not sys_main_obj.has_parsed_config():
raise EODataDownException(
"The EODataDownSystemMain instance has parsed a "
"config file so it not ready to use.")
db_engine = sqlalchemy.create_engine(self.db_info_obj.dbConn)
session_sqlalc = sqlalchemy.orm.sessionmaker(bind=db_engine)
ses = session_sqlalc()
obs_qry = ses.query(EDDObsDates).filter(
EDDObsDates.SensorID == sensor_id,
EDDObsDates.PlatformID == platform_id,
EDDObsDates.ObsDate == obs_date).one_or_none()
if obs_qry is not None:
obsdate_scns_qry = ses.query(EDDObsDatesScns).filter(
EDDObsDatesScns.SensorID == obs_qry.SensorID,
EDDObsDatesScns.PlatformID == obs_qry.PlatformID,
EDDObsDatesScns.ObsDate == obs_qry.ObsDate).all()
scns_lst = list()
for scn in obsdate_scns_qry:
scns_lst.append(scn.Scene_PID)
print("\t {}".format(scns_lst))
sensor_obj = sys_main_obj.get_sensor_obj(obs_qry.SensorID)
obsdate_basename = "{}_{}_{}".format(
obs_qry.ObsDate.strftime('%Y%m%d'), obs_qry.SensorID,
obs_qry.PlatformID)
obsdate_dir = os.path.join(self.overview_img_base_dir,
obsdate_basename)
if not os.path.exists(obsdate_dir):
os.mkdir(obsdate_dir)
out_imgs_dict = dict()
out_img_files = list()
for out_img_size in self.overview_img_sizes:
out_img = os.path.join(
obsdate_dir, "{}_{}px.tif".format(obsdate_basename,
out_img_size))
out_img_files.append(out_img)
out_imgs_dict[out_img_size] = out_img
# Pass all to sensor function...
success = sensor_obj.create_multi_scn_visual(
scns_lst, out_img_files, self.overview_img_sizes,
self.overview_extent_vec_file, self.overview_extent_vec_lyr,
'GTIFF', self.overview_tmp_dir)
if success:
obs_qry.OverviewCreated = True
obs_qry.Overviews = out_imgs_dict
else:
obs_qry.OverviewCreated = False
obs_qry.Invalid = True
ses.commit()
ses.close()
def parse_sensor_config(self, config_file, first_parse=False):
"""
Parse the JSON configuration file. If first_parse=True then a signature file will be created
which will be checked each time the system runs to ensure changes are not back to the
configuration file. If the signature does not match the input file then an expection will be
thrown. To update the configuration (e.g., extent date range or spatial area) run with first_parse=True.
:param config_file: string with the path to the JSON file.
:param first_parse: boolean as to whether the file has been previously parsed.
"""
edd_file_checker = eodatadown.eodatadownutils.EDDCheckFileHash()
# If it is the first time the config_file is parsed then create the signature file.
if first_parse:
edd_file_checker.createFileSig(config_file)
logger.debug("Created signature file for config file.")
if not edd_file_checker.checkFileSig(config_file):
raise EODataDownException(
"Input config did not match the file signature.")
with open(config_file) as f:
config_data = json.load(f)
json_parse_helper = eodatadown.eodatadownutils.EDDJSONParseHelper()
eoddutils = eodatadown.eodatadownutils.EODataDownUtils()
logger.debug("Testing config file is for 'obsdates'")
if not json_parse_helper.doesPathExist(config_data,
["eodatadown", "obsdates"]):
raise EODataDownException(
"Config file should have top level eodatadown > obsdates.")
logger.debug("Have the correct config file for 'obsdates'")
if json_parse_helper.doesPathExist(
config_data, ["eodatadown", "obsdates", "overviews"]):
self.overview_proj_epsg = int(
json_parse_helper.getNumericValue(
config_data,
["eodatadown", "obsdates", "overviews", "epsg"], 0,
1000000000))
self.overview_img_base_dir = json_parse_helper.getStrValue(
config_data,
["eodatadown", "obsdates", "overviews", "scn_image_dir"])
self.overview_tmp_dir = json_parse_helper.getStrValue(
config_data,
["eodatadown", "obsdates", "overviews", "tmp_dir"])
tmp_overview_img_sizes = json_parse_helper.getListValue(
config_data,
["eodatadown", "obsdates", "overviews", "overviewsizes"])
self.overview_img_sizes = list()
for overview_size in tmp_overview_img_sizes:
if eoddutils.isNumber(overview_size):
self.overview_img_sizes.append(int(overview_size))
else:
raise EODataDownException(
"overviewsizes contained a value which was not a number."
)
if json_parse_helper.doesPathExist(
config_data,
["eodatadown", "obsdates", "overviews", "extent"]):
self.overview_extent_vec_file = json_parse_helper.getStrValue(
config_data, [
"eodatadown", "obsdates", "overviews", "extent",
"vec_file"
])
self.overview_extent_vec_lyr = json_parse_helper.getStrValue(
config_data, [
"eodatadown", "obsdates", "overviews", "extent",
"vec_lyr"
])
else:
raise EODataDownException(
"No information on eodatadown > obsdates > overviews.")