def get_user_conf_camera_filename(self, plugin_name):
conf_file = self._userconf_directory + os.path.sep + "MAJAUserConfig_" + plugin_name + ".xml"
if not os.path.exists(conf_file):
raise MajaDataException("The admin configuration file <" + conf_file +
"> (for the plugin <" + plugin_name + ">) doesn't exist!")
userconfigcamerafilename = file_utils.copy_file_to_directory(conf_file, self._workingDirectory, notestmode=True)
if self._stylesheet is not None:
translate_xsl(userconfigcamerafilename, self._stylesheet)
return userconfigcamerafilename
def scientific_processing(self):
LOGGER.info("Starting ScientifiqProcessing")
l_InitMode = False
l_BackwardMode = False
l_Sat = self._l1product.Satellite
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Get and registers the L2COMM GIPP file L2COMM
parameters have been read in the L1ImaegFileReader !!
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2COMM_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2COMM", l_Sat)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
l_GIP_L2COMM_Filename = file_utils.copy_file_to_directory(
l_GIP_L2COMM_Filename,
self._apphandler.get_working_directory(),
notestmode=True)
translate_xsl(l_GIP_L2COMM_Filename, self._apphandler._stylesheet)
LOGGER.info("The GIP_L2COMM file detected for the satellite '" +
l_Sat + "' is <" + l_GIP_L2COMM_Filename + ">.")
l_GIPPL2COMMHandler = GippL2COMMEarthExplorerXMLFileHandler(
l_GIP_L2COMM_Filename)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Register the GIPP file "GIP_L2SITE" */
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2SITE_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2SITE",
self._l1product.UniqueSatellite)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2SITE parameters...")
l_GIP_L2SITE_Filename = file_utils.copy_file_to_directory(
l_GIP_L2SITE_Filename,
self._apphandler.get_working_directory(),
notestmode=True)
translate_xsl(l_GIP_L2SITE_Filename, self._apphandler._stylesheet)
LOGGER.info("The GIP_L2SITE file detected for the satellite '" +
self._l1product.UniqueSatellite + "' is <" +
l_GIP_L2SITE_Filename + ">.")
# Get the athmospheric/CAMS
# ---------------------------------------------------------------------------------------------
if l_GIPPL2COMMHandler.get_value_b("UseCamsData"):
l_modelList = l_GIPPL2COMMHandler.get_value("ListOfModels").split()
l_avgTime = date_utils.get_julianday_as_double(
self._l1product.ProductDate)
LOGGER.info("Cams data requested")
# Verify if cams file are available before anything
if self._CAMS_Files_HandlersMAP[l_Sat].has_cams_data(l_avgTime):
self._CAMS_Files_HandlersMAP[l_Sat].extract_cams_datas(
self._l1product.CenterCorner.latitude,
self._l1product.CenterCorner.longitude, l_avgTime)
if self._CAMS_Files_HandlersMAP[l_Sat].valid:
self._AthmosphericLutHandlerMap[l_Sat].set_proportions(
self._CAMS_Files_HandlersMAP[l_Sat].proportions)
LOGGER.info("CAMS found, sampling : " + str(
self._CAMS_Files_HandlersMAP[l_Sat].out_rh_sampling))
for m in l_modelList:
l_use_rh = self._CAMS_Files_HandlersMAP[
l_Sat].get_extinction_map()[m]["rh_dep"]
self._AthmosphericLutHandlerMap[
l_Sat].add_lut_for_model(
self._apphandler.get_input_directory(),
self._apphandler.get_working_directory(),
l_Sat,
m,
self._validate_schemas,
os.path.join(
self._apphandler.
get_schemas_root_install_dir(),
self.plugin.MAJA_INSTALL_SCHEMAS_DIR),
p_userh=l_use_rh,
p_rh=self._CAMS_Files_HandlersMAP[l_Sat].
out_rh_sampling)
tmp_atmolut = self._apphandler.get_directory_manager(
).get_temporary_directory("AtmoLut_",
do_always_remove=True)
self._AthmosphericLutHandlerMap[
l_Sat].gen_synthetised_luts(tmp_atmolut)
self._cams_status = CAMSStatus.ACTIVATED_OK
else:
LOGGER.warn(
"No valid cam data found for product, using constant model"
)
self._cams_status = CAMSStatus.ACTIVATED_NOAVAILABLEDATA
else:
LOGGER.warn("No valid cam data found for product at time " +
date_utils.get_date_yyyymmdd_from_tm(
self._l1product.ProductDate) +
", using constant model")
self._cams_status = CAMSStatus.ACTIVATED_NOAVAILABLEDATA
# ---------------------------------------------------------------------------------------------
# Initialize L1 image reader
l_UniqueSatelliteD = self._l1product.UniqueSatellite
l1_image_reader = L1ImageReaderProvider.create(
self._l1product.PluginName, self._l1product, self._apphandler,
l_GIPPL2COMMHandler, self.DataDEMMap.get(l_UniqueSatelliteD),
constants.ReadL1Mode.READ_L1_MODE_FOR_ALGORITHMS)
# ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
# Initialize the L2 Image file reader
# Only in Nominal and Backward mode
# ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
LOGGER.debug(
"Starting L2NominalProcessor::ScientificProcessing(): get the L2 product filename in the input directory..."
)
l2_image_reader = product_utils.get_input_l2_image_product(
self._apphandler.get_input_directory(), self._apphandler, False,
self.DataDEMMap.get(l_UniqueSatelliteD))
LOGGER.info("The L2 product found is <" +
l2_image_reader.ProductFilename + ">.")
# ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
# Initialize the Output L2 Image product filename
if self._apphandler.get_output_plugin() is not None:
l2_image_file_writer = L2ImageWriterProvider.create(
self._apphandler.get_output_plugin())
else:
l2_image_file_writer = L2ImageWriterProvider.create(
MAJAPluginProvider.auto_tm(self._l1product.PluginName))
if not l2_image_file_writer.can_write(self._l1product.PluginName):
raise MajaChainException("Plugin '" +
self._apphandler.get_output_plugin() +
"' can not write product from '" +
self._l1product.PluginName +
"' products ")
# ---------------------------------------------------------------------------------------------
# Get the ozone value
l_UseDefaultConstantOzoneAmount = self.plugin.ConfigUserCamera.get_Algorithms(). \
get_Atmospheric_Absorption_Correction().get_Use_Default_Constant_Ozone_Amount()
# l_UseDefaultConstantOzoneAmount = True
l_OzoneAmountValue = 0.0
if not l_UseDefaultConstantOzoneAmount:
l_OzoneAmountValue = get_ozone_amount(
gipp_utils.get_gipp_filename(
self._apphandler.get_input_directory(), "EXO_METDTA"),
self._l1product.CenterCorner)
else:
l_OzoneAmountValue = float(
l_GIPPL2COMMHandler.l2_comm_values["OzoneAmountDefaultValue"])
LOGGER.info("The 'OzoneAmount' used is " + str(l_OzoneAmountValue))
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
# check conditional clouds ?
self._checking_conditional_clouds = True
LOGGER.info(
"Starting from L2InitProcessor, the main ScientificSingleProductProcessing() method..."
)
l_enableL2resolution = self._apphandler.get_user_conf().get_Computing(
).get_EnableL2ResolutionProcessing()
super(L2NominalProcessor,
self).process_one_product(l_InitMode, l_BackwardMode,
l_enableL2resolution, False, True,
True, self._l1product, l1_image_reader,
l_OzoneAmountValue, self._cams_status,
l2_image_reader, l2_image_file_writer)
LOGGER.info("Ending ScientifiqProcessing")
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
def scientific_processing(self):
LOGGER.info("Starting ScientifiqProcessing")
# Scientific processing mode. Set to INIT, NOMINAL or BACKWARD value
l_NumberOfL1Product = len(self._listofl1product)
LOGGER.debug(self._listofl1product)
# Check if more than one valid input is in the list
if l_NumberOfL1Product == 1:
raise MajaDataException(
"L2BackwardProcessing need more than one input L1 image product."
)
# Process the last product in the list with INIT MODE
# Initialize the mode
l_InitMode = True
l_BackwardMode = True
l_IndexOfL1Product = l_NumberOfL1Product
l_finalize_backward = False
l_EnableL2ResolutionProcessing = self._apphandler.get_user_conf(
).get_Computing().get_EnableL2ResolutionProcessing()
if not l_EnableL2ResolutionProcessing:
LOGGER.info(
"*************************************************************************************************"
)
LOGGER.info(
"ATTENTION: L2BACKWARD PROCESSING: The option to Write product to L2 resolution has been DISABLE !!!"
)
LOGGER.info(
"**************************************************************************************************"
)
# ----------------------------------------------------------------
# Initialize the flag to validate the clouds probably
# ----------------------------------------------------------------
self._checking_conditional_clouds = False
l_conditional_loop = False
l_write_earthexplorer_headers = False
l_write_temporary_l2_product = True
l_write_l2_product_to_l2resolution = False
l_InputL2ImageFilename = ""
while not l_conditional_loop:
l_InputL1ProductInfo = self._listofl1product[l_IndexOfL1Product -
1]
l_Sat = l_InputL1ProductInfo.Satellite
# If last L1 product, write headers and write in the output directory
# ----------------------------------------------------------------
if l_IndexOfL1Product == 1:
l_write_earthexplorer_headers = True
l_write_l2_product_to_l2resolution = l_EnableL2ResolutionProcessing
l_write_temporary_l2_product = False
l_finalize_backward = True
LOGGER.warn(
"L2Backward processing: the input L1 product valid for the Init mode is the last product. The Standard mode is disable !"
)
# *************************************************************************************************************
# Init the current PluginBase pointer for the satellite in the MAIN global DataApplicationHandler Instance
# *************************************************************************************************************
self.plugin = MAJAPluginProvider.create(
l_InputL1ProductInfo.PluginName, self._apphandler)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Get and registers the L2COMM GIPP file L2COMM
parameters have been read in the L1ImaegFileReader !!
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2COMM_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2COMM", l_Sat)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
l_GIP_L2COMM_Filename = file_utils.copy_file_to_directory(
l_GIP_L2COMM_Filename,
self._apphandler.get_working_directory(),
notestmode=True)
translate_xsl(l_GIP_L2COMM_Filename,
self._apphandler._stylesheet)
LOGGER.info("The GIP_L2COMM file detected for the satellite '" +
l_Sat + "' is <" + l_GIP_L2COMM_Filename + ">.")
l_GIPPL2COMMHandler = GippL2COMMEarthExplorerXMLFileHandler(
l_GIP_L2COMM_Filename)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Register the GIPP file "GIP_L2SITE" */
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2SITE_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2SITE",
l_InputL1ProductInfo.UniqueSatellite)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2SITE parameters...")
l_GIP_L2SITE_Filename = file_utils.copy_file_to_directory(
l_GIP_L2SITE_Filename,
self._apphandler.get_working_directory(),
notestmode=True)
translate_xsl(l_GIP_L2SITE_Filename,
self._apphandler._stylesheet)
LOGGER.info("The GIP_L2SITE file detected for the satellite '" +
l_InputL1ProductInfo.UniqueSatellite + "' is <" +
l_GIP_L2SITE_Filename + ">.")
# *************************************************************************************************************
# Initialize the L1 Image file reader
# *************************************************************************************************************
l_UniqueSatelliteD = l_InputL1ProductInfo.UniqueSatellite
l1_image_reader = L1ImageReaderProvider.create(
l_InputL1ProductInfo.PluginName, l_InputL1ProductInfo,
self._apphandler, l_GIPPL2COMMHandler,
self.DataDEMMap.get(l_UniqueSatelliteD),
constants.ReadL1Mode.READ_L1_MODE_FOR_ALGORITHMS)
# *************************************************************************************************************
# Initialize the L2 Image file writer
# *************************************************************************************************************
if self._apphandler.get_output_plugin() is not None:
l2_image_file_writer = L2ImageWriterProvider.create(
self._apphandler.get_output_plugin())
else:
l2_image_file_writer = L2ImageWriterProvider.create(
MAJAPluginProvider.auto_tm(
l_InputL1ProductInfo.PluginName))
if not l2_image_file_writer.can_write(
l_InputL1ProductInfo.PluginName):
raise MajaDriverException(
"Plugin '" + self._apphandler.get_output_plugin() +
"' can not write product from '" +
l_InputL1ProductInfo.PluginName + "' products ")
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
# Compute the Single product
# ----------------------------------------------------------------
self.single_product_processing(
l_InitMode, l_BackwardMode, l_finalize_backward,
l_InputL1ProductInfo, l1_image_reader,
l_write_earthexplorer_headers, l_write_temporary_l2_product,
l_write_l2_product_to_l2resolution, None, l2_image_file_writer,
l_GIPPL2COMMHandler)
LOGGER.debug(" -> self._productIsValid = " +
str(self._productIsValid))
LOGGER.debug(" -> p_checking_conditional_clouds = " +
str(self._checking_conditional_clouds))
if (self._checking_conditional_clouds is
True) and (self._productIsValid is True):
l_conditional_loop = True
elif l_IndexOfL1Product == 1:
l_conditional_loop = True
else:
l_conditional_loop = False
# Store the output file name for the following loop
# ----------------------------------------------------------------
l_InputL2ImageFilename = l2_image_file_writer.get_global_product_filename(
)
# Decrement the input L1 index product
l_IndexOfL1Product = l_IndexOfL1Product - 1
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
# Check the Init mode
# ----------------------------------------------------------------
# LAIG-DM-MAC-143599-CS
# LAIG-FA-MAC-1658-CNES
if (self._checking_conditional_clouds
== False) or (self._productIsValid == False):
LOGGER.warn(
"The INIT mode in Backward processing fail: all L1 products are not valid (too cloudy or too cirrus)"
" or the output L2 product generated contains too NoData values! "
"See previous [W] messages for more details.")
# *************************************************************************************************************
# *************************************************************************************************************
# Process the Nominal mode
# *************************************************************************************************************
# *************************************************************************************************************
if l_IndexOfL1Product > 0:
l_InitMode = False
l_write_earthexplorer_headers = False
l_write_l2_product_to_l2resolution = False
l_write_temporary_l2_product = True
while l_IndexOfL1Product > 1:
l_InputL1ProductInfo = self._listofl1product[l_IndexOfL1Product
- 1]
l_Sat = l_InputL1ProductInfo.Satellite
# *************************************************************************************************************
# Init the current PluginBase pointer for the satellite in the MAIN global DataApplicationHandler Instance
# *************************************************************************************************************
plugin = MAJAPluginProvider.create(
l_InputL1ProductInfo.PluginName, self._apphandler)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Get and registers the L2COMM GIPP file L2COMM
parameters have been read in the L1ImaegFileReader !!
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2COMM_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2COMM",
l_Sat)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
translate_xsl(l_GIP_L2COMM_Filename,
self._apphandler._stylesheet)
LOGGER.info(
"The GIP_L2COMM file detected for the satellite '" +
l_Sat + "' is <" + l_GIP_L2COMM_Filename + ">.")
l_GIPPL2COMMHandler = GippL2COMMEarthExplorerXMLFileHandler(
l_GIP_L2COMM_Filename)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Register the GIPP file "GIP_L2SITE" */
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2SITE_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2SITE",
l_InputL1ProductInfo.UniqueSatellite)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
translate_xsl(l_GIP_L2SITE_Filename,
self._apphandler._stylesheet)
LOGGER.info(
"The GIP_L2SITE file detected for the satellite '" +
l_InputL1ProductInfo.UniqueSatellite + "' is <" +
l_GIP_L2SITE_Filename + ">.")
# *************************************************************************************************************
# Initialize the L1 Image file reader
# *************************************************************************************************************
l_UniqueSatelliteD = l_InputL1ProductInfo.UniqueSatellite
l1_image_reader = L1ImageReaderProvider.create(
l_InputL1ProductInfo.PluginName, l_InputL1ProductInfo,
self._apphandler, l_GIPPL2COMMHandler,
self.DataDEMMap.get(l_UniqueSatelliteD),
constants.ReadL1Mode.READ_L1_MODE_FOR_ALGORITHMS)
# *************************************************************************************************************
# Initialize the L2 Image file reader
# *************************************************************************************************************
LOGGER.debug(
"Starting L2NominalProcessor::ScientificProcessing(): get the L2 product filename in the input directory..."
)
l2_image_reader = L2ImageReaderProvider.create(
l_InputL2ImageFilename, self._apphandler, False,
self.DataDEMMap.get(l_UniqueSatelliteD))
LOGGER.info("The L2 product found is <" +
l2_image_reader.ProductFilename + ">.")
# *************************************************************************************************************
# Initialize the L2 Image file writer
# *************************************************************************************************************
if self._apphandler.get_output_plugin() is not None:
l2_image_file_writer = L2ImageWriterProvider.create(
self._apphandler.get_output_plugin())
else:
l2_image_file_writer = L2ImageWriterProvider.create(
MAJAPluginProvider.auto_tm(
l_InputL1ProductInfo.PluginName))
if not l2_image_file_writer.can_write(
l_InputL1ProductInfo.PluginName):
raise MajaDriverException(
"Plugin '" + self._apphandler.get_output_plugin() +
"' can not write product from '" +
l_InputL1ProductInfo.PluginName + "' products ")
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
# Compute the L2 temporary product
# ----------------------------------------------------------------
self.single_product_processing(
l_InitMode, l_BackwardMode, l_finalize_backward,
l_InputL1ProductInfo, l1_image_reader,
l_write_earthexplorer_headers,
l_write_temporary_l2_product,
l_write_l2_product_to_l2resolution, l2_image_reader,
l2_image_file_writer, l_GIPPL2COMMHandler)
# Store the output file name for the following loop
# ----------------------------------------------------------------
l_InputL2ImageFilename = l2_image_file_writer.get_global_product_filename(
)
# Decrement Index of L1 image input
# ----------------------------------------------------------------
l_IndexOfL1Product = l_IndexOfL1Product - 1
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
# Finalize the product
# Compute the last L2 product
# ----------------------------------------------------------------
l_write_earthexplorer_headers = True
l_write_l2_product_to_l2resolution = self._apphandler.get_user_conf(
).get_Computing().get_EnableL2ResolutionProcessing()
l_write_temporary_l2_product = False
l_finalize_backward = True
l_InputL1ProductInfo = self._listofl1product[l_IndexOfL1Product -
1]
l_Sat = l_InputL1ProductInfo.Satellite
# *************************************************************************************************************
# Init the current PluginBase pointer for the satellite in the MAIN global DataApplicationHandler Instance
# *************************************************************************************************************
plugin = MAJAPluginProvider.create(l_InputL1ProductInfo.PluginName,
self._apphandler)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Get and registers the L2COMM GIPP file L2COMM
parameters have been read in the L1ImaegFileReader !!
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2COMM_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2COMM", l_Sat)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
translate_xsl(l_GIP_L2COMM_Filename,
self._apphandler._stylesheet)
LOGGER.info("The GIP_L2COMM file detected for the satellite '" +
l_Sat + "' is <" + l_GIP_L2COMM_Filename + ">.")
l_GIPPL2COMMHandler = GippL2COMMEarthExplorerXMLFileHandler(
l_GIP_L2COMM_Filename)
""" ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** **
Register the GIPP file "GIP_L2SITE" */
** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *"""
l_GIP_L2SITE_Filename = gipp_utils.get_gipp_filename_with_mission(
self._apphandler.get_input_directory(), "GIP_L2SITE",
l_InputL1ProductInfo.UniqueSatellite)
if self._apphandler._stylesheet is not None:
LOGGER.debug("Overloading L2COMM parameters...")
translate_xsl(l_GIP_L2SITE_Filename,
self._apphandler._stylesheet)
LOGGER.info("The GIP_L2SITE file detected for the satellite '" +
l_InputL1ProductInfo.UniqueSatellite + "' is <" +
l_GIP_L2SITE_Filename + ">.")
# *************************************************************************************************************
# Initialize the L1 Image file reader
# *************************************************************************************************************
l_UniqueSatelliteD = l_InputL1ProductInfo.UniqueSatellite
l1_image_reader = L1ImageReaderProvider.create(
l_InputL1ProductInfo.PluginName, l_InputL1ProductInfo,
self._apphandler, l_GIPPL2COMMHandler,
self.DataDEMMap.get(l_UniqueSatelliteD),
constants.ReadL1Mode.READ_L1_MODE_FOR_ALGORITHMS)
# *************************************************************************************************************
# Initialize the L2 Image file reader
# *************************************************************************************************************
LOGGER.debug(
"Starting L2NominalProcessor::ScientificProcessing(): get the L2 product filename in the input directory..."
)
l2_image_reader = L2ImageReaderProvider.create(
l_InputL2ImageFilename, self._apphandler, False,
self.DataDEMMap.get(l_UniqueSatelliteD))
LOGGER.info("The L2 product found is <" +
l2_image_reader.ProductFilename + ">.")
# *************************************************************************************************************
# Initialize the L2 Image file writer
# *************************************************************************************************************
if self._apphandler.get_output_plugin() is not None:
l2_image_file_writer = L2ImageWriterProvider.create(
self._apphandler.get_output_plugin())
else:
l2_image_file_writer = L2ImageWriterProvider.create(
MAJAPluginProvider.auto_tm(
l_InputL1ProductInfo.PluginName))
if not l2_image_file_writer.can_write(
l_InputL1ProductInfo.PluginName):
raise MajaDriverException(
"Plugin '" + self._apphandler.get_output_plugin() +
"' can not write product from '" +
l_InputL1ProductInfo.PluginName + "' products ")
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
self.single_product_processing(
l_InitMode, l_BackwardMode, l_finalize_backward,
l_InputL1ProductInfo, l1_image_reader,
l_write_earthexplorer_headers, l_write_temporary_l2_product,
l_write_l2_product_to_l2resolution, l2_image_reader,
l2_image_file_writer, l_GIPPL2COMMHandler)
# end if it is not the last product
else:
LOGGER.debug(
"L1ImageInformationProvider initialized during backward but no associated valid L2 product. No more products."
)
LOGGER.info("L2BackwardProcessor::ScientificProcessing() done.")
# Log system infos
LOGGER.info(self._apphandler.get_system_infos())
def initialize(self):
maja_description = """ ./maja [options] \n\n
MAJA Chains \n\n
CNES All rights reserved. For more details, see Copyright.txt file.\n\n
Description: \n
------------\n\n
The L2 processor offers advanced atmospheric correction algorithms including\n
water vapour and aerosol estimates based on multitemporal data analysis.\n
It also provides cloud mask generation.\n
- Cloud masking\n
* Cloud detection\n
* Shadow detection\n
- Atmospheric correction\n
* Gaseous absorption correction\n
* Scattering correction\n
- Environment and slope correction\n
* Environment effects\n
* Slope correction\n
- Composite image update\n
The data and GIPPs files mandatory for MAJA are:\n
- For L2 processing:\n
* GIP_L2COMM\n
* GIP_L2DIRT\n
* GIP_L2DIFT\n
* GIP_L2SMAC\n
* GIP_L2WATV\n
* GIP_L2TOCR\n
* GIP_L2ALBD\n
* GIP_L2SITE\n
* EXO_METDTA\n
* AUX_REFDE2\n
Processing description:\n
-----------------------\n
The maja launches the following processes:\n
- launches the pre-processing treatment\n
* Uncompresses all data (DBL package files and BZ2 images files)\n
* Check input data with the schemas\n
* Deletes all tarballs (if option is enable in the Configuration file)\n
* Applies a specific stylesheet on GIPPs files\n
- launches the scientific-processing treatment\n
* Reads image products\n
* Applies algorithms\n
* Formats EE and writes datas\n
- launches the post-processing treatment\n
* Check output data with the schemas\n
* Compress BZ2 all .TIF images data files\n
* Generates the .DBL image product data (L2/L3)\n\n
For more details, report to the SUM (Software User Manual, ref. LAIG-MU-MAC-010-CS)\n\n
Author: CS Systemes d'Information (France)\n\n
User cases:\n
-----------\n
1. First user case: Use only a JobOrder file to launch maja processing. \n
-> use the '--jobOrder' option.\n
Note: the possible values for the <Processor_Name> field are:\n
* m_MAJA_L2_INIT_CHAIN \n
* m_MAJA_L2_NOMINAL_CHAIN \n
* m_MAJA_L2_BACKWARD_CHAIN \n
2. Second user case: Use command line parameters to launch maja processing\n
-> for example, use at least the '--mode' and '--input' options\n\n
"""
parser = argparse.ArgumentParser(
description=maja_description, # main description for help
epilog='Beta', formatter_class=argparse.RawTextHelpFormatter) # displayed after help
parser.add_argument("-j", "--jobOrder", help="Specify the JobOrder file (xml file)")
parser.add_argument(
"-l",
"--loglevel",
help="Log level use and set to the JobOrder generated.",
choices=[
'INFO',
'PROGRESS',
'WARNING',
'DEBUG',
'ERROR'],
default='INFO')
parser.add_argument(
"-m",
"--mode",
help="Processing mode.",
choices=[
'L2INIT',
'L2NOMINAL',
'L2BACKWARD'
],
default='L2INIT',
required=False)
parser.add_argument(
"-t",
"--enableTest",
help="Enable/Disable the field value 'Test' set in the JobOrder generated.",
action="store_true")
parser.add_argument(
"--stylesheet",
help="XML Stylesheet filename, used to overloads parameters in the XML configuration files and GIPP files. "
"See the [MU] for an example of StyleSheet.")
parser.add_argument(
"-acs",
"--adminconf",
help="Administration Configuration directory (contains for example the MAJAAdminConfigSystem.xml)")
inputDescr = """Input data directory: must be contain images, all GIPPs files, the DTM, etc.).
The directory must be contain only one L1 product for the 'L2INIT' mode, a list of L1 products
for the 'L2BACKWARD' mode, one
L1 product and one L2 product for the 'L2NOMINAL' mode and a list of L2 products
"""
parser.add_argument("-i", "--input",
help=inputDescr,
required=False)
parser.add_argument(
"-o",
"--output",
help="Output data directory (product directory). Default value: '.'",
required=False)
parser.add_argument(
"-w",
"--workingdir",
help="Working directory (working 'temporary' directory). Default value: the '--output parameter value'",
required=False)
plugin_choices = "Available plugins : "
creators = FactoryBase.get_registered("PluginBase")
for crea in creators:
plugin_choices = plugin_choices + crea().PluginName + " , "
parser.add_argument(
"-p",
"--plugin",
help="Output plugin to use to write L2 product. "+plugin_choices)
parser.add_argument(
"-ucs",
"--conf",
help="User Configuration directory (contains for example MAJAUserConfigSystem.xml)")
parser.add_argument(
"--NbThreads",
type=int,
help="UserConfigSystem overloads value for the parameter 'NbThreads'")
parser.add_argument(
"--CheckXMLFilesWithSchema",
help="UserConfigSystem overloads value for the parameter 'CheckXMLFilesWithSchema'",
action="store_true")
parser.add_argument(
"--CleanInputZipFiles",
help="UserConfigSystem overloads value for the parameter 'CleanInputZipFiles'",
action="store_true")
parser.add_argument(
"--CleanFiles",
help="UserConfigSystem overloads value for the parameter 'CleanFiles'",
action="store_true")
parser.add_argument(
"--ZipFiles",
help="UserConfigSystem overloads value for the parameter 'ZipFiles'",
action="store_true")
parser.add_argument(
"--EnableCleaningCachingDirectoryBeforeProcessing",
help="UserConfigSystem overloads value for the parameter 'EnableCleaningCachingDirectoryBeforeProcessing'",
action="store_true")
parser.add_argument(
"--EnableCleaningCachingDirectoryAfterProcessing",
help="UserConfigSystem overloads value for the parameter 'EnableCleaningCachingDirectoryAfterProcessing'",
action="store_true")
parser.add_argument(
"--EnableCleaningTemporaryDirectory",
help="UserConfigSystem overloads value for the parameter 'EnableCleaningTemporaryDirectory'",
action="store_true")
parser.add_argument(
"--TileId",
help="Set the Tile id of the Sentinel2 L1 product (Only necessary for SENTINEL2 plug-in and only for "
"the L1C product with PSD version < PSD 14)",
required=False)
parser.add_argument(
"--perfos-log",
help="Enable performance measurement in log",
action="store_true")
parser.add_argument(
"--perfos-report",
help="Enable synthesize performance measurement in log and xml file",
action="store_true")
parser.add_argument(
"-v",
"--version",
help="Display version information and exit",
action="store_true"
)
args = parser.parse_args()
if args.version :
LOGGER.info("Maja Software Version: "+version.MAJA_VERSION)
exit(0)
# Set the log level
self._logLevel = args.loglevel
os.environ["MAJA_LOGGER_LEVEL"] = args.loglevel
if args.loglevel == "INFO":
LOGGER.setLevel(logging.INFO)
elif args.loglevel == "PROGRESS":
LOGGER.setLevel(logging.PROGRESS)
elif args.loglevel == "WARNING":
LOGGER.setLevel(logging.WARNING)
elif args.loglevel == "DEBUG":
LOGGER.setLevel(logging.DEBUG)
os.environ["OTB_LOGGER_LEVEL"] = "DEBUG"
elif args.loglevel == "ERROR":
LOGGER.setLevel(logging.ERROR)
LOGGER.info("Logger in %s mode ( %s )", args.loglevel, LOGGER.getEffectiveLevel())
if args.output is not None:
self._outputDirectory = args.output + os.path.sep
else:
raise MajaDataException("Output option -o/--output must be specified")
LOGGER.info("Output Directory : %s ", self._outputDirectory)
if not os.path.exists(self._outputDirectory):
LOGGER.debug("OUTPUT_DIR must be created in %s", self._outputDirectory)
file_utils.create_directory(self._outputDirectory)
if args.input is not None:
self._inputDirectory = args.input + os.path.sep
else:
raise MajaDataException("Input option -i/--input must be specified")
LOGGER.info("Input Directory : %s ", self._inputDirectory)
if args.workingdir is not None:
self._workingDirectory = args.workingdir
else:
self._workingDirectory = os.path.join(self._outputDirectory, ".maja-working-directory")
LOGGER.info("Working Directory : %s ", self._workingDirectory)
self._directory_manager = DirectoryManager(self._workingDirectory)
if not os.path.exists(self._workingDirectory):
LOGGER.debug("WORKING_DIR must be created in %s", self._workingDirectory)
file_utils.create_directory(self._workingDirectory)
if args.plugin is not None:
self._output_Plugin = args.plugin
if args.TileId is not None:
self._tile_id = args.TileId
if os.getenv("MAJA_INSTALL_DIR") is None :
raise MajaProcessingException("MAJA_INSTALL_DIR is not Set")
if args.stylesheet is not None:
self._stylesheet = args.stylesheet
# User config
if args.conf is not None:
self._userconf_directory = args.conf
self.luserConfigSystemFileName = os.path.join(self._userconf_directory, "MAJAUserConfigSystem.xml")
if not os.path.exists(self.luserConfigSystemFileName):
raise MajaDataException(
"The User configuration system file <" +
self.luserConfigSystemFileName +
"> doesn't exist!")
# Load the file
LOGGER.info("Using "+self.luserConfigSystemFileName+" as userConfig file")
self._userConfigSystemFileName = file_utils.copy_file_to_directory(self.luserConfigSystemFileName, self._workingDirectory,
notestmode=True)
if self._stylesheet is not None:
if not os.path.exists(self._stylesheet):
raise MajaDataException(str(self._stylesheet) + " doesn't exist !!!!")
translate_xsl(self._userConfigSystemFileName, self._stylesheet)
self._userConf = user_conf.parse(self._userConfigSystemFileName, True)
# overload values
if args.CheckXMLFilesWithSchema:
self._validate_schemas = True
else:
self._validate_schemas = self._userConf.get_Processing().get_CheckXMLFilesWithSchema()
if args.CleanFiles:
self._userConf.get_Processing().get_PostProcessing().set_CleanFiles(True)
if args.ZipFiles:
self._userConf.get_Processing().get_PostProcessing().set_ZipFiles(True)
if args.CleanInputZipFiles:
self._userConf.get_Processing().get_PreProcessing().set_CleanInputZipFiles(True)
if args.EnableCleaningTemporaryDirectory:
self._userConf.get_Computing().set_EnableCleaningTemporaryDirectory(True)
if args.NbThreads is not None:
self._nbThreads = args.NbThreads
else:
self._nbThreads = self._userConf.get_Computing().get_NbThreads()
# Admin config
if args.adminconf is not None:
self._adminconf_directory = args.adminconf
luseradminsystemfilename = os.path.join(self._adminconf_directory, "MAJAAdminConfigSystem.xml")
if not os.path.exists(luseradminsystemfilename):
raise MajaDataException("The Admin configuration system file <" + luseradminsystemfilename + "> doesn't exist!")
self._adminConfigSystemFileName = file_utils.copy_file_to_directory(
luseradminsystemfilename, self._workingDirectory, notestmode=True)
if self._stylesheet is not None:
translate_xsl(self._adminConfigSystemFileName, self._stylesheet)
# Load the file
self._adminConf = admin_conf.parse(self._adminConfigSystemFileName, True)
if args.mode is not None:
lProcessorName = args.mode
else:
raise MajaDataException("Mode must be specified : -m/--mode")
self._processorName = AppHandler.MAJA_L2_NOMINAL_CHAIN
if lProcessorName == AppHandler.L2INIT:
self._processorName = AppHandler.MAJA_L2_INIT_CHAIN
elif lProcessorName == AppHandler.L2NOMINAL:
self._processorName = AppHandler.MAJA_L2_NOMINAL_CHAIN
elif lProcessorName == AppHandler.L2BACKWARD:
self._processorName = AppHandler.MAJA_L2_BACKWARD_CHAIN
else:
raise MajaDataException("Unknown mode in parameters")
LOGGER.info("Processor is %s", self._processorName)
LOGGER.progress("Starting " + self._processorName)
LOGGER.info("Number of theads %i", self._nbThreads)