def initialize_list_of_ozone_parameter_value(pListOfOzone):
l_ListOfOzoneParameterValue = []
# Loops on the ozone list
for f in pListOfOzone:
# Normaly, the prodcut is uncompress and XML file checked in the PreProcessing method
l_HDRozoneFilename = os.path.splitext(f)[0] + ".HDR"
l_OzoneParameterValueType = L2BackwardProcessor.OzoneParameterValue(
)
# Load the EarthExplorer XML file
l_Handler = EarthExplorerXMLFileHandler(l_HDRozoneFilename)
# Converts these dates in Julian day
l_OzoneParameterValueType.JulianDayDateStart = date_utils.get_julianday_as_double(
date_utils.get_datetime_from_utc(
l_Handler.get_string_value_of("ValidityStart")))
l_OzoneParameterValueType.JulianDayDateStop = date_utils.get_julianday_as_double(
date_utils.get_datetime_from_utc(
l_Handler.get_string_value_of("ValidityStop")))
l_OzoneParameterValueType.Filename = os.path.splitext(
f)[0] + ".DBL"
LOGGER.debug("Add Ozone Value for date [" +
str(l_OzoneParameterValueType.JulianDayDateStart) +
"" +
str(l_OzoneParameterValueType.JulianDayDateStop) +
"] -> file " + l_OzoneParameterValueType.Filename)
l_ListOfOzoneParameterValue.append(l_OzoneParameterValueType)
return l_ListOfOzoneParameterValue
def get_list_of_input_l1_image_product_sorted(dir,tile_id=None):
list_of_product = L1ImageInformationsProvider.get_list_of_l1products(dir,tile_id=tile_id)
LOGGER.info("Nb L1 product found : " + str(list_of_product.__len__()))
if list_of_product.__len__() == 0:
raise MajaDataException("No L1 product found")
dictofproduct = {}
for pr in list_of_product:
l1product = L1ImageInformationsProvider.create(pr,tile_id=tile_id)
l1jdate = get_julianday_as_double(l1product.ProductDate)
dictofproduct[l1jdate] = l1product
dictofproduct_sorted = sorted(list(dictofproduct.items()), key=operator.itemgetter(0))
list_of_product_sorted = list()
for prs in dictofproduct_sorted:
list_of_product_sorted.append(prs[1])
return list_of_product_sorted
def initializelistofcamsfiledescription(self, list_of_cam_files):
# ---------------------------------------------------------------------------------------------
self._list_of_cams_description_file = []
# Loops on the cams list
for cam in list_of_cam_files:
# Normaly, the prodcut is uncompress and XML file checked in the PreProcessing method
l_hdrcamsfilename = os.path.splitext(cam)[0] + ".HDR"
l_camsfiledescription = {}
# Load the EarthExplorer XML file
l_handler = GippCAMSEarthExplorerXMLFileHandler(l_hdrcamsfilename)
# Converts these dates in Julian day
l_camsfiledescription["date_utc"] = l_handler.acquisition_date_time
l_camsfiledescription["date_jd"] = date_utils.get_julianday_as_double(
date_utils.get_datetime_from_utc(l_handler.acquisition_date_time))
l_camsfiledescription["filename"] = l_hdrcamsfilename
# Read the files to get the various CAMS part
l_list_of_date_filenames = l_handler.get_list_of_packaged_dbl_files(True, True)
# For each file, search the type
for f in l_list_of_date_filenames:
l_filenamename = os.path.basename(f)
# AOT File ?
if "CAMS_AOT" in l_filenamename:
l_camsfiledescription["aot_file"] = f
# MR File ?
if "CAMS_MR" in l_filenamename:
l_camsfiledescription["mr_file"] = f
# RH File ?
if "CAMS_RH" in l_filenamename:
l_camsfiledescription["rh_file"] = f
if "aot_file" not in l_camsfiledescription or "mr_file" not in l_camsfiledescription \
or "aot_file" not in l_camsfiledescription:
LOGGER.info(l_camsfiledescription)
raise MajaIOError("Missing one CAMS file")
if xml_tools.get_only_value(l_handler.root, "//Earth_Explorer_Header/Variable_Header/Specific_Product_Header/NumberOfNonInterpolableValues", check=True) is not None:
l_camsfiledescription["NbNonInterpolate"] = xml_tools.get_xml_int_value(l_handler.root, "//Earth_Explorer_Header/Variable_Header/Specific_Product_Header/NumberOfNonInterpolableValues")
LOGGER.debug(
"Add CAMS file for date [" +
l_camsfiledescription["date_utc"] +
";" + str(l_camsfiledescription["date_jd"]) +
"] -> file " +
l_camsfiledescription["filename"])
l_camsfiledescription["model_levels"] = [str(f) for f in l_handler.model_levels]
self._list_of_cams_description_file.append(l_camsfiledescription)
def get_list_of_input_l2_image_product_sorted(dir, app_handler, enable_public_data):
list_of_product = L2ImageReaderProvider.get_list_of_l2_products(dir)
LOGGER.info("Nb L2 product found : " + str(list_of_product.__len__()))
if list_of_product.__len__() == 0:
raise MajaDataException("No L2 product found")
dictofproduct = {}
for pr in list_of_product:
l2product = L2ImageReaderProvider.create(pr, app_handler, enable_public_data)
l2jdate = get_julianday_as_double(l2product.Date)
dictofproduct[l2jdate] = l2product
dictofproduct_sorted = sorted(list(dictofproduct.items()), key=operator.itemgetter(0))
list_of_product_sorted = list()
for prs in dictofproduct_sorted:
list_of_product_sorted.append(prs[1])
return list_of_product_sorted
def scientific_processing(self):
LOGGER.info("Starting ScientifiqProcessing")
l_InitMode = False
l_BackwardMode = False
l_Sat = self._l1product.Satellite
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Get and registers the L2COMM GIPP file L2COMM
parameters have been read in the L1ImaegFileReader !!
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2COMM_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2COMM", l_Sat)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
l_GIP_L2COMM_Filename = file_utils.copy_file_to_directory(
l_GIP_L2COMM_Filename,
self._apphandler.get_working_directory(),
notestmode=True)
translate_xsl(l_GIP_L2COMM_Filename, self._apphandler._stylesheet)
LOGGER.info("The GIP_L2COMM file detected for the satellite '" +
l_Sat + "' is <" + l_GIP_L2COMM_Filename + ">.")
l_GIPPL2COMMHandler = GippL2COMMEarthExplorerXMLFileHandler(
l_GIP_L2COMM_Filename)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Register the GIPP file "GIP_L2SITE" */
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2SITE_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2SITE",
self._l1product.UniqueSatellite)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2SITE parameters...")
l_GIP_L2SITE_Filename = file_utils.copy_file_to_directory(
l_GIP_L2SITE_Filename,
self._apphandler.get_working_directory(),
notestmode=True)
translate_xsl(l_GIP_L2SITE_Filename, self._apphandler._stylesheet)
LOGGER.info("The GIP_L2SITE file detected for the satellite '" +
self._l1product.UniqueSatellite + "' is <" +
l_GIP_L2SITE_Filename + ">.")
# Get the athmospheric/CAMS
# ---------------------------------------------------------------------------------------------
if l_GIPPL2COMMHandler.get_value_b("UseCamsData"):
l_modelList = l_GIPPL2COMMHandler.get_value("ListOfModels").split()
l_avgTime = date_utils.get_julianday_as_double(
self._l1product.ProductDate)
LOGGER.info("Cams data requested")
# Verify if cams file are available before anything
if self._CAMS_Files_HandlersMAP[l_Sat].has_cams_data(l_avgTime):
self._CAMS_Files_HandlersMAP[l_Sat].extract_cams_datas(
self._l1product.CenterCorner.latitude,
self._l1product.CenterCorner.longitude, l_avgTime)
if self._CAMS_Files_HandlersMAP[l_Sat].valid:
self._AthmosphericLutHandlerMap[l_Sat].set_proportions(
self._CAMS_Files_HandlersMAP[l_Sat].proportions)
LOGGER.info("CAMS found, sampling : " + str(
self._CAMS_Files_HandlersMAP[l_Sat].out_rh_sampling))
for m in l_modelList:
l_use_rh = self._CAMS_Files_HandlersMAP[
l_Sat].get_extinction_map()[m]["rh_dep"]
self._AthmosphericLutHandlerMap[
l_Sat].add_lut_for_model(
self._apphandler.get_input_directory(),
self._apphandler.get_working_directory(),
l_Sat,
m,
self._validate_schemas,
os.path.join(
self._apphandler.
get_schemas_root_install_dir(),
self.plugin.MAJA_INSTALL_SCHEMAS_DIR),
p_userh=l_use_rh,
p_rh=self._CAMS_Files_HandlersMAP[l_Sat].
out_rh_sampling)
tmp_atmolut = self._apphandler.get_directory_manager(
).get_temporary_directory("AtmoLut_",
do_always_remove=True)
self._AthmosphericLutHandlerMap[
l_Sat].gen_synthetised_luts(tmp_atmolut)
self._cams_status = CAMSStatus.ACTIVATED_OK
else:
LOGGER.warn(
"No valid cam data found for product, using constant model"
)
self._cams_status = CAMSStatus.ACTIVATED_NOAVAILABLEDATA
else:
LOGGER.warn("No valid cam data found for product at time " +
date_utils.get_date_yyyymmdd_from_tm(
self._l1product.ProductDate) +
", using constant model")
self._cams_status = CAMSStatus.ACTIVATED_NOAVAILABLEDATA
# ---------------------------------------------------------------------------------------------
# Initialize L1 image reader
l_UniqueSatelliteD = self._l1product.UniqueSatellite
l1_image_reader = L1ImageReaderProvider.create(
self._l1product.PluginName, self._l1product, self._apphandler,
l_GIPPL2COMMHandler, self.DataDEMMap.get(l_UniqueSatelliteD),
constants.ReadL1Mode.READ_L1_MODE_FOR_ALGORITHMS)
# ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
# Initialize the L2 Image file reader
# Only in Nominal and Backward mode
# ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
LOGGER.debug(
"Starting L2NominalProcessor::ScientificProcessing(): get the L2 product filename in the input directory..."
)
l2_image_reader = product_utils.get_input_l2_image_product(
self._apphandler.get_input_directory(), self._apphandler, False,
self.DataDEMMap.get(l_UniqueSatelliteD))
LOGGER.info("The L2 product found is <" +
l2_image_reader.ProductFilename + ">.")
# ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
# Initialize the Output L2 Image product filename
if self._apphandler.get_output_plugin() is not None:
l2_image_file_writer = L2ImageWriterProvider.create(
self._apphandler.get_output_plugin())
else:
l2_image_file_writer = L2ImageWriterProvider.create(
MAJAPluginProvider.auto_tm(self._l1product.PluginName))
if not l2_image_file_writer.can_write(self._l1product.PluginName):
raise MajaChainException("Plugin '" +
self._apphandler.get_output_plugin() +
"' can not write product from '" +
self._l1product.PluginName +
"' products ")
# ---------------------------------------------------------------------------------------------
# Get the ozone value
l_UseDefaultConstantOzoneAmount = self.plugin.ConfigUserCamera.get_Algorithms(). \
get_Atmospheric_Absorption_Correction().get_Use_Default_Constant_Ozone_Amount()
# l_UseDefaultConstantOzoneAmount = True
l_OzoneAmountValue = 0.0
if not l_UseDefaultConstantOzoneAmount:
l_OzoneAmountValue = get_ozone_amount(
gipp_utils.get_gipp_filename(
self._apphandler.get_input_directory(), "EXO_METDTA"),
self._l1product.CenterCorner)
else:
l_OzoneAmountValue = float(
l_GIPPL2COMMHandler.l2_comm_values["OzoneAmountDefaultValue"])
LOGGER.info("The 'OzoneAmount' used is " + str(l_OzoneAmountValue))
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
# check conditional clouds ?
self._checking_conditional_clouds = True
LOGGER.info(
"Starting from L2InitProcessor, the main ScientificSingleProductProcessing() method..."
)
l_enableL2resolution = self._apphandler.get_user_conf().get_Computing(
).get_EnableL2ResolutionProcessing()
super(L2NominalProcessor,
self).process_one_product(l_InitMode, l_BackwardMode,
l_enableL2resolution, False, True,
True, self._l1product, l1_image_reader,
l_OzoneAmountValue, self._cams_status,
l2_image_reader, l2_image_file_writer)
LOGGER.info("Ending ScientifiqProcessing")
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
def single_product_processing(
self, initmode, backwardmode, p_finalize_backward,
p_L1ImageInformationsProvider, p_InputL1ImageFileReader,
p_write_earthexplorer_headers, p_write_temporary_l2_product,
p_write_l2_product_to_l2resolution, p_InputL2ImageFileReader,
p_OutputL2ImageFileWriter, p_GIPPL2COMMHandler):
l_Sat = p_L1ImageInformationsProvider.Satellite
# ---------------------------------------------------------------------------------------------
# Get the ozone value
l_UseDefaultConstantOzoneAmount = self.plugin.ConfigUserCamera.get_Algorithms(). \
get_Atmospheric_Absorption_Correction().get_Use_Default_Constant_Ozone_Amount()
# l_UseDefaultConstantOzoneAmount = True
l_OzoneAmountValue = 0.0
if not l_UseDefaultConstantOzoneAmount:
LOGGER.debug("Find Ozone Value for the product date [" +
p_L1ImageInformationsProvider.UTCValidityStart + "" +
p_L1ImageInformationsProvider.UTCValidityStop +
"]...")
l_OzoneFileName = L2BackwardProcessor.get_valid_ozone_filename(
self._listofozoneparametervalue,
L2BackwardProcessor.get_average_product_time(
p_L1ImageInformationsProvider.UTCValidityStart,
p_L1ImageInformationsProvider.UTCValidityStop))
l_OzoneAmountValue = get_ozone_amount(
l_OzoneFileName, p_L1ImageInformationsProvider.CenterCorner)
else:
l_OzoneAmountValue = float(
p_GIPPL2COMMHandler.l2_comm_values["OzoneAmountDefaultValue"])
LOGGER.info("The 'OzoneAmount' used is " + str(l_OzoneAmountValue))
# Get the athmospheric/CAMS
# ---------------------------------------------------------------------------------------------
if p_GIPPL2COMMHandler.get_value_b("UseCamsData"):
l_modelList = p_GIPPL2COMMHandler.get_value("ListOfModels").split()
l_avgTime = date_utils.get_julianday_as_double(
p_L1ImageInformationsProvider.ProductDate)
LOGGER.info("Cams data requested")
# Verify if cams file are available before anything
if self._CAMS_Files_HandlersMAP[l_Sat].has_cams_data(l_avgTime):
self._CAMS_Files_HandlersMAP[l_Sat].extract_cams_datas(
p_L1ImageInformationsProvider.CenterCorner.latitude,
p_L1ImageInformationsProvider.CenterCorner.longitude,
l_avgTime)
if self._CAMS_Files_HandlersMAP[l_Sat].valid:
self._AthmosphericLutHandlerMap[l_Sat].set_proportions(
self._CAMS_Files_HandlersMAP[l_Sat].proportions)
LOGGER.info("CAMS found, sampling : " + str(
self._CAMS_Files_HandlersMAP[l_Sat].out_rh_sampling))
for m in l_modelList:
l_use_rh = self._CAMS_Files_HandlersMAP[
l_Sat].get_extinction_map()[m]["rh_dep"]
self._AthmosphericLutHandlerMap[
l_Sat].add_lut_for_model(
self._apphandler.get_input_directory(),
self._apphandler.get_working_directory(),
l_Sat,
m,
self._validate_schemas,
os.path.join(
self._apphandler.
get_schemas_root_install_dir(),
self.plugin.MAJA_INSTALL_SCHEMAS_DIR),
p_userh=l_use_rh,
p_rh=self._CAMS_Files_HandlersMAP[l_Sat].
out_rh_sampling)
tmp_atmolut = self._apphandler.get_directory_manager(
).get_temporary_directory("AtmoLut_",
do_always_remove=True)
self._AthmosphericLutHandlerMap[
l_Sat].gen_synthetised_luts(tmp_atmolut)
else:
LOGGER.warn(
"No valid cam data found for product, using constant model"
)
self.update_cam_status(
self._CAMS_Files_HandlersMAP[l_Sat].valid)
else:
LOGGER.warn("No valid cam data found for product date : " +
date_utils.get_date_yyyymmdd_from_tm(
p_L1ImageInformationsProvider.ProductDate) +
", using constant model")
self.update_cam_status(False)
# *************************************************************************************************************
# Call the Scientifics single product processing
# *************************************************************************************************************
# V 4-1-0 THOMAS erreur dans l'ordre des parametres
super(L2BackwardProcessor, self).process_one_product(
initmode, backwardmode, p_write_l2_product_to_l2resolution,
p_write_temporary_l2_product, p_write_earthexplorer_headers,
p_finalize_backward, p_L1ImageInformationsProvider,
p_InputL1ImageFileReader, l_OzoneAmountValue, self._cams_status,
p_InputL2ImageFileReader, p_OutputL2ImageFileWriter)