def initialize_list_of_ozone_parameter_value(pListOfOzone): l_ListOfOzoneParameterValue = [] # Loops on the ozone list for f in pListOfOzone: # Normaly, the prodcut is uncompress and XML file checked in the PreProcessing method l_HDRozoneFilename = os.path.splitext(f)[0] + ".HDR" l_OzoneParameterValueType = L2BackwardProcessor.OzoneParameterValue( ) # Load the EarthExplorer XML file l_Handler = EarthExplorerXMLFileHandler(l_HDRozoneFilename) # Converts these dates in Julian day l_OzoneParameterValueType.JulianDayDateStart = date_utils.get_julianday_as_double( date_utils.get_datetime_from_utc( l_Handler.get_string_value_of("ValidityStart"))) l_OzoneParameterValueType.JulianDayDateStop = date_utils.get_julianday_as_double( date_utils.get_datetime_from_utc( l_Handler.get_string_value_of("ValidityStop"))) l_OzoneParameterValueType.Filename = os.path.splitext( f)[0] + ".DBL" LOGGER.debug("Add Ozone Value for date [" + str(l_OzoneParameterValueType.JulianDayDateStart) + "" + str(l_OzoneParameterValueType.JulianDayDateStop) + "] -> file " + l_OzoneParameterValueType.Filename) l_ListOfOzoneParameterValue.append(l_OzoneParameterValueType) return l_ListOfOzoneParameterValue
def get_list_of_input_l1_image_product_sorted(dir,tile_id=None): list_of_product = L1ImageInformationsProvider.get_list_of_l1products(dir,tile_id=tile_id) LOGGER.info("Nb L1 product found : " + str(list_of_product.__len__())) if list_of_product.__len__() == 0: raise MajaDataException("No L1 product found") dictofproduct = {} for pr in list_of_product: l1product = L1ImageInformationsProvider.create(pr,tile_id=tile_id) l1jdate = get_julianday_as_double(l1product.ProductDate) dictofproduct[l1jdate] = l1product dictofproduct_sorted = sorted(list(dictofproduct.items()), key=operator.itemgetter(0)) list_of_product_sorted = list() for prs in dictofproduct_sorted: list_of_product_sorted.append(prs[1]) return list_of_product_sorted
def initializelistofcamsfiledescription(self, list_of_cam_files): # --------------------------------------------------------------------------------------------- self._list_of_cams_description_file = [] # Loops on the cams list for cam in list_of_cam_files: # Normaly, the prodcut is uncompress and XML file checked in the PreProcessing method l_hdrcamsfilename = os.path.splitext(cam)[0] + ".HDR" l_camsfiledescription = {} # Load the EarthExplorer XML file l_handler = GippCAMSEarthExplorerXMLFileHandler(l_hdrcamsfilename) # Converts these dates in Julian day l_camsfiledescription["date_utc"] = l_handler.acquisition_date_time l_camsfiledescription["date_jd"] = date_utils.get_julianday_as_double( date_utils.get_datetime_from_utc(l_handler.acquisition_date_time)) l_camsfiledescription["filename"] = l_hdrcamsfilename # Read the files to get the various CAMS part l_list_of_date_filenames = l_handler.get_list_of_packaged_dbl_files(True, True) # For each file, search the type for f in l_list_of_date_filenames: l_filenamename = os.path.basename(f) # AOT File ? if "CAMS_AOT" in l_filenamename: l_camsfiledescription["aot_file"] = f # MR File ? if "CAMS_MR" in l_filenamename: l_camsfiledescription["mr_file"] = f # RH File ? if "CAMS_RH" in l_filenamename: l_camsfiledescription["rh_file"] = f if "aot_file" not in l_camsfiledescription or "mr_file" not in l_camsfiledescription \ or "aot_file" not in l_camsfiledescription: LOGGER.info(l_camsfiledescription) raise MajaIOError("Missing one CAMS file") if xml_tools.get_only_value(l_handler.root, "//Earth_Explorer_Header/Variable_Header/Specific_Product_Header/NumberOfNonInterpolableValues", check=True) is not None: l_camsfiledescription["NbNonInterpolate"] = xml_tools.get_xml_int_value(l_handler.root, "//Earth_Explorer_Header/Variable_Header/Specific_Product_Header/NumberOfNonInterpolableValues") LOGGER.debug( "Add CAMS file for date [" + l_camsfiledescription["date_utc"] + ";" + str(l_camsfiledescription["date_jd"]) + "] -> file " + l_camsfiledescription["filename"]) l_camsfiledescription["model_levels"] = [str(f) for f in l_handler.model_levels] self._list_of_cams_description_file.append(l_camsfiledescription)
def get_list_of_input_l2_image_product_sorted(dir, app_handler, enable_public_data): list_of_product = L2ImageReaderProvider.get_list_of_l2_products(dir) LOGGER.info("Nb L2 product found : " + str(list_of_product.__len__())) if list_of_product.__len__() == 0: raise MajaDataException("No L2 product found") dictofproduct = {} for pr in list_of_product: l2product = L2ImageReaderProvider.create(pr, app_handler, enable_public_data) l2jdate = get_julianday_as_double(l2product.Date) dictofproduct[l2jdate] = l2product dictofproduct_sorted = sorted(list(dictofproduct.items()), key=operator.itemgetter(0)) list_of_product_sorted = list() for prs in dictofproduct_sorted: list_of_product_sorted.append(prs[1]) return list_of_product_sorted
def scientific_processing(self): LOGGER.info("Starting ScientifiqProcessing") l_InitMode = False l_BackwardMode = False l_Sat = self._l1product.Satellite """ ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** Get and registers the L2COMM GIPP file L2COMM parameters have been read in the L1ImaegFileReader !! ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *""" l_GIP_L2COMM_Filename = gipp_utils.get_gipp_filename_with_mission( self._apphandler.get_input_directory(), "GIP_L2COMM", l_Sat) if self._apphandler._stylesheet is not None: LOGGER.debug("Overloading L2COMM parameters...") l_GIP_L2COMM_Filename = file_utils.copy_file_to_directory( l_GIP_L2COMM_Filename, self._apphandler.get_working_directory(), notestmode=True) translate_xsl(l_GIP_L2COMM_Filename, self._apphandler._stylesheet) LOGGER.info("The GIP_L2COMM file detected for the satellite '" + l_Sat + "' is <" + l_GIP_L2COMM_Filename + ">.") l_GIPPL2COMMHandler = GippL2COMMEarthExplorerXMLFileHandler( l_GIP_L2COMM_Filename) """ ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** Register the GIPP file "GIP_L2SITE" */ ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** *""" l_GIP_L2SITE_Filename = gipp_utils.get_gipp_filename_with_mission( self._apphandler.get_input_directory(), "GIP_L2SITE", self._l1product.UniqueSatellite) if self._apphandler._stylesheet is not None: LOGGER.debug("Overloading L2SITE parameters...") l_GIP_L2SITE_Filename = file_utils.copy_file_to_directory( l_GIP_L2SITE_Filename, self._apphandler.get_working_directory(), notestmode=True) translate_xsl(l_GIP_L2SITE_Filename, self._apphandler._stylesheet) LOGGER.info("The GIP_L2SITE file detected for the satellite '" + self._l1product.UniqueSatellite + "' is <" + l_GIP_L2SITE_Filename + ">.") # Get the athmospheric/CAMS # --------------------------------------------------------------------------------------------- if l_GIPPL2COMMHandler.get_value_b("UseCamsData"): l_modelList = l_GIPPL2COMMHandler.get_value("ListOfModels").split() l_avgTime = date_utils.get_julianday_as_double( self._l1product.ProductDate) LOGGER.info("Cams data requested") # Verify if cams file are available before anything if self._CAMS_Files_HandlersMAP[l_Sat].has_cams_data(l_avgTime): self._CAMS_Files_HandlersMAP[l_Sat].extract_cams_datas( self._l1product.CenterCorner.latitude, self._l1product.CenterCorner.longitude, l_avgTime) if self._CAMS_Files_HandlersMAP[l_Sat].valid: self._AthmosphericLutHandlerMap[l_Sat].set_proportions( self._CAMS_Files_HandlersMAP[l_Sat].proportions) LOGGER.info("CAMS found, sampling : " + str( self._CAMS_Files_HandlersMAP[l_Sat].out_rh_sampling)) for m in l_modelList: l_use_rh = self._CAMS_Files_HandlersMAP[ l_Sat].get_extinction_map()[m]["rh_dep"] self._AthmosphericLutHandlerMap[ l_Sat].add_lut_for_model( self._apphandler.get_input_directory(), self._apphandler.get_working_directory(), l_Sat, m, self._validate_schemas, os.path.join( self._apphandler. get_schemas_root_install_dir(), self.plugin.MAJA_INSTALL_SCHEMAS_DIR), p_userh=l_use_rh, p_rh=self._CAMS_Files_HandlersMAP[l_Sat]. out_rh_sampling) tmp_atmolut = self._apphandler.get_directory_manager( ).get_temporary_directory("AtmoLut_", do_always_remove=True) self._AthmosphericLutHandlerMap[ l_Sat].gen_synthetised_luts(tmp_atmolut) self._cams_status = CAMSStatus.ACTIVATED_OK else: LOGGER.warn( "No valid cam data found for product, using constant model" ) self._cams_status = CAMSStatus.ACTIVATED_NOAVAILABLEDATA else: LOGGER.warn("No valid cam data found for product at time " + date_utils.get_date_yyyymmdd_from_tm( self._l1product.ProductDate) + ", using constant model") self._cams_status = CAMSStatus.ACTIVATED_NOAVAILABLEDATA # --------------------------------------------------------------------------------------------- # Initialize L1 image reader l_UniqueSatelliteD = self._l1product.UniqueSatellite l1_image_reader = L1ImageReaderProvider.create( self._l1product.PluginName, self._l1product, self._apphandler, l_GIPPL2COMMHandler, self.DataDEMMap.get(l_UniqueSatelliteD), constants.ReadL1Mode.READ_L1_MODE_FOR_ALGORITHMS) # ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** # Initialize the L2 Image file reader # Only in Nominal and Backward mode # ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** LOGGER.debug( "Starting L2NominalProcessor::ScientificProcessing(): get the L2 product filename in the input directory..." ) l2_image_reader = product_utils.get_input_l2_image_product( self._apphandler.get_input_directory(), self._apphandler, False, self.DataDEMMap.get(l_UniqueSatelliteD)) LOGGER.info("The L2 product found is <" + l2_image_reader.ProductFilename + ">.") # ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** ** # Initialize the Output L2 Image product filename if self._apphandler.get_output_plugin() is not None: l2_image_file_writer = L2ImageWriterProvider.create( self._apphandler.get_output_plugin()) else: l2_image_file_writer = L2ImageWriterProvider.create( MAJAPluginProvider.auto_tm(self._l1product.PluginName)) if not l2_image_file_writer.can_write(self._l1product.PluginName): raise MajaChainException("Plugin '" + self._apphandler.get_output_plugin() + "' can not write product from '" + self._l1product.PluginName + "' products ") # --------------------------------------------------------------------------------------------- # Get the ozone value l_UseDefaultConstantOzoneAmount = self.plugin.ConfigUserCamera.get_Algorithms(). \ get_Atmospheric_Absorption_Correction().get_Use_Default_Constant_Ozone_Amount() # l_UseDefaultConstantOzoneAmount = True l_OzoneAmountValue = 0.0 if not l_UseDefaultConstantOzoneAmount: l_OzoneAmountValue = get_ozone_amount( gipp_utils.get_gipp_filename( self._apphandler.get_input_directory(), "EXO_METDTA"), self._l1product.CenterCorner) else: l_OzoneAmountValue = float( l_GIPPL2COMMHandler.l2_comm_values["OzoneAmountDefaultValue"]) LOGGER.info("The 'OzoneAmount' used is " + str(l_OzoneAmountValue)) # Log system infos LOGGER.info(self._apphandler.get_system_infos()) # check conditional clouds ? self._checking_conditional_clouds = True LOGGER.info( "Starting from L2InitProcessor, the main ScientificSingleProductProcessing() method..." ) l_enableL2resolution = self._apphandler.get_user_conf().get_Computing( ).get_EnableL2ResolutionProcessing() super(L2NominalProcessor, self).process_one_product(l_InitMode, l_BackwardMode, l_enableL2resolution, False, True, True, self._l1product, l1_image_reader, l_OzoneAmountValue, self._cams_status, l2_image_reader, l2_image_file_writer) LOGGER.info("Ending ScientifiqProcessing") # Log system infos LOGGER.info(self._apphandler.get_system_infos())
def single_product_processing( self, initmode, backwardmode, p_finalize_backward, p_L1ImageInformationsProvider, p_InputL1ImageFileReader, p_write_earthexplorer_headers, p_write_temporary_l2_product, p_write_l2_product_to_l2resolution, p_InputL2ImageFileReader, p_OutputL2ImageFileWriter, p_GIPPL2COMMHandler): l_Sat = p_L1ImageInformationsProvider.Satellite # --------------------------------------------------------------------------------------------- # Get the ozone value l_UseDefaultConstantOzoneAmount = self.plugin.ConfigUserCamera.get_Algorithms(). \ get_Atmospheric_Absorption_Correction().get_Use_Default_Constant_Ozone_Amount() # l_UseDefaultConstantOzoneAmount = True l_OzoneAmountValue = 0.0 if not l_UseDefaultConstantOzoneAmount: LOGGER.debug("Find Ozone Value for the product date [" + p_L1ImageInformationsProvider.UTCValidityStart + "" + p_L1ImageInformationsProvider.UTCValidityStop + "]...") l_OzoneFileName = L2BackwardProcessor.get_valid_ozone_filename( self._listofozoneparametervalue, L2BackwardProcessor.get_average_product_time( p_L1ImageInformationsProvider.UTCValidityStart, p_L1ImageInformationsProvider.UTCValidityStop)) l_OzoneAmountValue = get_ozone_amount( l_OzoneFileName, p_L1ImageInformationsProvider.CenterCorner) else: l_OzoneAmountValue = float( p_GIPPL2COMMHandler.l2_comm_values["OzoneAmountDefaultValue"]) LOGGER.info("The 'OzoneAmount' used is " + str(l_OzoneAmountValue)) # Get the athmospheric/CAMS # --------------------------------------------------------------------------------------------- if p_GIPPL2COMMHandler.get_value_b("UseCamsData"): l_modelList = p_GIPPL2COMMHandler.get_value("ListOfModels").split() l_avgTime = date_utils.get_julianday_as_double( p_L1ImageInformationsProvider.ProductDate) LOGGER.info("Cams data requested") # Verify if cams file are available before anything if self._CAMS_Files_HandlersMAP[l_Sat].has_cams_data(l_avgTime): self._CAMS_Files_HandlersMAP[l_Sat].extract_cams_datas( p_L1ImageInformationsProvider.CenterCorner.latitude, p_L1ImageInformationsProvider.CenterCorner.longitude, l_avgTime) if self._CAMS_Files_HandlersMAP[l_Sat].valid: self._AthmosphericLutHandlerMap[l_Sat].set_proportions( self._CAMS_Files_HandlersMAP[l_Sat].proportions) LOGGER.info("CAMS found, sampling : " + str( self._CAMS_Files_HandlersMAP[l_Sat].out_rh_sampling)) for m in l_modelList: l_use_rh = self._CAMS_Files_HandlersMAP[ l_Sat].get_extinction_map()[m]["rh_dep"] self._AthmosphericLutHandlerMap[ l_Sat].add_lut_for_model( self._apphandler.get_input_directory(), self._apphandler.get_working_directory(), l_Sat, m, self._validate_schemas, os.path.join( self._apphandler. get_schemas_root_install_dir(), self.plugin.MAJA_INSTALL_SCHEMAS_DIR), p_userh=l_use_rh, p_rh=self._CAMS_Files_HandlersMAP[l_Sat]. out_rh_sampling) tmp_atmolut = self._apphandler.get_directory_manager( ).get_temporary_directory("AtmoLut_", do_always_remove=True) self._AthmosphericLutHandlerMap[ l_Sat].gen_synthetised_luts(tmp_atmolut) else: LOGGER.warn( "No valid cam data found for product, using constant model" ) self.update_cam_status( self._CAMS_Files_HandlersMAP[l_Sat].valid) else: LOGGER.warn("No valid cam data found for product date : " + date_utils.get_date_yyyymmdd_from_tm( p_L1ImageInformationsProvider.ProductDate) + ", using constant model") self.update_cam_status(False) # ************************************************************************************************************* # Call the Scientifics single product processing # ************************************************************************************************************* # V 4-1-0 THOMAS erreur dans l'ordre des parametres super(L2BackwardProcessor, self).process_one_product( initmode, backwardmode, p_write_l2_product_to_l2resolution, p_write_temporary_l2_product, p_write_earthexplorer_headers, p_finalize_backward, p_L1ImageInformationsProvider, p_InputL1ImageFileReader, l_OzoneAmountValue, self._cams_status, p_InputL2ImageFileReader, p_OutputL2ImageFileWriter)