def updatePixcVec(self, IN_write_to_shp=False): """ This function updates PIXCVec netcdf files obtained in output of PGE_LakeTile with improved longitude, latitude, height and lake_tile_id or prior_id if exists. :param IN_write_to_shp: =True to also write the shapefile of the PIXCVec product (default=False) :type IN_write_to_shp: boolean """ my_api.printInfo("[PixelCloudVecEdge] == updatePixcVec ==") for tile_idx in range(len(self.tile_ref_list)): # Loop over tiles tile_ref = self.tile_ref_list[tile_idx] # 1 - Get input and output files info # 1.1 - LakeTile_pixcvec path and filename lake_tile_pixcvec_file = self.lake_tile_pixcvec_file_list[tile_idx] # 1.2 - Get PIXCVec associated path and filename pixc_vec_file = self.pixc_vec_file_list[tile_idx] # 2 - Init proc_pixc_vec.PixelCloudVec object objPixCVec = proc_pixc_vec.PixelCloudVec("SP", lake_tile_pixcvec_file) objPixCVec.setContinent(self.continent) # 3 - Get corresponding objPixcEdgeSP tile_idx objPixcEdgeSP_tile_idx = self.objPixcEdgeSP.tile_ref.index( str(tile_ref)) # 4 - Update PIXCVec info if (self.objPixcEdgeSP.tile_idx == objPixcEdgeSP_tile_idx ).any(): # Only when pixels need to be updated # 4.1 - Retrieve corresponding indices in PixC_SP object pixc_sp_idx = np.where( self.objPixcEdgeSP.tile_idx == objPixcEdgeSP_tile_idx)[0] # 4.2 - Retrieve corresponding indices in original PIXC pixc_tile_idx = self.objPixcEdgeSP.edge_idx[pixc_sp_idx] # 4.3 - Update geolocation information if computed if my_var.IMP_GEOLOC: objPixCVec.longitude_vectorproc[ pixc_tile_idx] = self.longitude_vectorproc[pixc_sp_idx] objPixCVec.latitude_vectorproc[ pixc_tile_idx] = self.latitude_vectorproc[pixc_sp_idx] objPixCVec.height_vectorproc[ pixc_tile_idx] = self.height_vectorproc[pixc_sp_idx] # 4.4 - Update tag objPixCVec.tag[pixc_tile_idx] = str(self.tag[pixc_sp_idx]) # 5 - Write PIXCVec file objPixCVec.write_file(pixc_vec_file, None) # 6 - Write associated shapefile if asked if IN_write_to_shp: objPixCVec.write_file_asShp( pixc_vec_file.replace('.nc', '.shp')) my_api.printInfo("")
def getValue(self, IN_key): """ Get the value associated to the key named IN_key :param IN_key: name of the key :type IN_key: string :return: value associated to the key, None if doesn't exist :rtype: string """ if IN_key in self.parameters: return self.parameters.get(IN_key) else: for key in self.parameters.keys(): if IN_key in key: return self.parameters.get(key) # Key doesn't exist my_api.printInfo("[my_rdf_file/getValue] Unkown key = %s" % IN_key) return None
def gatherEdgeEntities(self, IN_tile_idx1, IN_tile_idx2): """ This function gives new labels for pixels at current tile edge. 1. Pixels within a buffer around tile edge are selected. 2. A water mask is computed 3. The mask is labeled :param IN_tile_idx1: indices of pixels edge of tile 1 :type IN_tile_idx1: 1D-array of int :param IN_tile_idx2: indices of pixels edge of tile 2 :type IN_tile_idx2: 1D-array of int :return: OUT_new_labels_subset = new labels given to pixels of edge entities :rtype: 1D-array of int """ my_api.printDebug("[PixelCloudSP] == gatherEdgeEntities ==") # 1 - Pixels at top / bottom of tile are loaded rg, az = self.selectEdgePixels(IN_tile_idx1, IN_tile_idx2) # 2 - Equivalent matrix size in azimuth and range # 2.1 - Equivalent matrix size in azimuth and range nb_pix_range = max(rg) + 1 nb_pix_azimuth = max(az) + 1 # 2.2 - Compute water mask over the subset waterMask = my_tools.computeBinMat(nb_pix_range, nb_pix_azimuth, rg, az) # 3 - Label entities over the subset of PixC at the edge tile sepEntities, nb_obj = my_tools.labelRegion(waterMask) # 4 - Convert into 1D-array OUT_new_labels_subset = my_tools.convert2dMatIn1dVec( rg, az, sepEntities) my_api.printInfo( "[PixelCloudSP] %d separate entities located at the edge tile" % np.unique(OUT_new_labels_subset).size) return OUT_new_labels_subset
def __init__(self, IN_laketile_dir, IN_output_dir, IN_cycle_num, IN_pass_num, IN_shp_option=False): """ Constructor: initialize variables :param IN_laketile_dir: full path of directory containing LakeTile products :type IN_laketile_dir: string :param IN_output_dir: output directory full path :type IN_output_dir: string :param IN_cycle_num: cycle number :type IN_cycle_num: int :param IN_pass_num: pass number :type IN_pass_num: int :param IN_shp_option: to also produce PIXCVec as shapefile (=True); else=False (default) :type IN_shp_option: boolean """ my_api.printInfo("[multiLakeSpProcessing] == INIT ==") # Input paths self.lake_tile_dir = IN_laketile_dir # Full path of directory containing LakeTile products self.output_dir = IN_output_dir # Output directory # Pass infos self.cycle_num = IN_cycle_num # Cycle number self.pass_num = IN_pass_num # Pass number # Flag to produce PIXCVec shapefile self.flag_prod_shp = False if IN_shp_option: self.flag_prod_shp = True # Init variables self.list_cycle = [] # List of cycle numbers to deal with self.list_pass = [] # List of pass numbers to deal with
def edgeGlobalRelabeling(self): """ This function gives new labels to entities gathered at tile edges. """ # 1 - Deal with all edges for i_edge in range(len(self.tile_ref) - 1): # Loop over tile edges my_api.printInfo( "[PixelCloudSP] ***** Processing edge of tiles %s and %s *****" % (self.tile_ref[i_edge], self.tile_ref[i_edge + 1])) # 1.1 - Get indices of pixels processed at the current edge tile_idx1 = np.where(self.tile_idx == i_edge)[0] tile_idx2 = np.where(self.tile_idx == i_edge + 1)[0] # 1.2 - If one tile does not have pixel to process, continue to next iteration if (tile_idx1.size == 0) or (tile_idx2.size == 0): my_api.printInfo("") continue # 1.3 - New labels for pixels at the edge of current edge new_labels_subset = self.gatherEdgeEntities(tile_idx1, tile_idx2) # 1.4 - Link old labels to new ones self.labelMatching(tile_idx1, tile_idx2, new_labels_subset) my_api.printInfo("") # 2 - Deal with edge at beginning or end of pass if (self.labels == 0).any(): # 2.1 - Get tiles with unprocessed labels (still initialized to zero) tiles_to_process = np.unique( self.tile_idx[np.where(self.labels == 0)]) for tile in tiles_to_process: # 2.2 - Get associated tile index tile_idx = np.where(self.tile_idx == tile)[0] # 2.3 - Get old labels from PGE_LakeTile old_labels = np.unique(self.edge_label[tile_idx][np.where( self.labels[tile_idx] == 0)]) # 2.4 - Compute a global new label new_labels = np.arange(old_labels.size) + np.max( self.labels) + 1 # 2.5 - Update global labels for idx in range(old_labels.size): self.labels[tile_idx[np.where( self.edge_label[tile_idx] == old_labels[idx])]] = new_labels[idx]
def run_postprocessing(self): """ Process PGE_L2_HR_LakeTile, i.e. convert output data in L2_HR_LakeTile product """ my_api.printInfo("") my_api.printInfo("") my_api.printInfo("[lakeSPProcessing] POST-PROCESSING...") my_api.printInfo("") # 1 - Close lake database if self.objLakeDb is not None: my_api.printInfo("[lakeTileProcessing] 1 - Closing lake database...") self.objLakeDb.close_db() my_api.printInfo("")
def __init__(self, IN_laketile_dir, IN_output_dir, IN_cycle_num, IN_pass_num, IN_shp_option=False): """ Constructor: initialize variables :param IN_laketile_dir: full path of directory containing LakeTile products :type IN_laketile_dir: string :param IN_output_dir: output directory full path :type IN_output_dir: string :param IN_cycle_num: cycle number :type IN_cycle_num: int :param IN_pass_num: pass number :type IN_pass_num: int :param IN_shp_option: to also produce PIXCVec as shapefile (=True); else=False (default) :type IN_shp_option: boolean """ my_api.printInfo("[lakeSPProcessing] == INIT ==") # Input paths self.lake_tile_dir = IN_laketile_dir # Full path of directory containing LakeTile products self.output_dir = IN_output_dir # Output directory # Pass infos self.cycle_num = IN_cycle_num # Cycle number self.pass_num = IN_pass_num # Pass number self.ascending = True # Orientation: True=ascending False=descending self.asc_dict = {True: "Ascending", False: "Descending"} # Associated dictionary # Flag to produce PIXCVec shapefile self.flag_prod_shp = False if IN_shp_option: self.flag_prod_shp = True # Number of tiles to process self.nb_input_tiles = 0 # List of continent(s) processed for the pass self.list_continent = [] # List of all LakeTile shapefiles to merge, organized by continent self.lake_tile_shp_file_path_list = {} # List of files to process self.lake_tile_pixcvec_path_list_R = {} # List of _pixcvec files for right swath, organized by continent self.lake_tile_edge_path_list_R = {} # List of _edge files for right swath, organized by continent self.lake_tile_pixcvec_path_list_L = {} # List of _pixcvec files for left swath, organized by continent self.lake_tile_edge_path_list_L = {} # List of _edge files for left swath, organized by continent # LakeSP filenames self.lake_sp_filenames = None # Objects self.objLakeDb = None # Lake DB object self.objPixc = None # PIXC object self.objPixcVec = None # PIXCVecRiver object self.objLake = None # LakeTile object # SP objects # Objects related to edge PixC for right (R) and left (L) swaths self.objPixc_SP_L = None self.objPixc_SP_R = None # Objects related to PIXCVec for right (R) and left (L) swaths self.objPixc_Vec_SP_L = None self.objPixc_Vec_SP_R = None # Objects related to LakeSP for right (R) and left (L) swaths self.objLake_SP_L = None self.objLake_SP_R = None
def run_processing(self): """ Process SAS_L2_HR_LakeTile """ my_api.printInfo("") my_api.printInfo("") my_api.printInfo("[lakeSPProcessing] PROCESSING tiles of cycle %03d and pass %03d (%s)..." % (self.cycle_num, self.pass_num, self.asc_dict[self.ascending])) my_api.printInfo("") for curContinent in self.list_continent: my_api.printInfo("") my_api.printInfo("==========================================") my_api.printInfo("[lakeSPProcessing] Processing continent %s" % curContinent) my_api.printInfo("==========================================") my_api.printInfo("") # 1 - Compute output filenames my_api.printInfo("[lakeTileProcessing] 1 - Computing LakeSP filenames...") self.lake_sp_filenames = my_names.lakeSPFilenames(self.lake_tile_shp_file_path_list[curContinent], curContinent, self.output_dir) my_api.printInfo("") # 2 - Objects initialisation my_api.printInfo("[lakeSPProcessing] 2 - Init objects...") my_api.printInfo("") # 2.1 - Right swath my_api.printInfo("[lakeSPProcessing] 2a - Right swath") # PIXC_edge self.objPixc_SP_R = proc_pixc_sp.PixC_Edge_SP(self.ascending, self.lake_tile_edge_path_list_R[curContinent]) # PIXCVec self.objPixc_Vec_SP_R = proc_pixc_vec_sp.PixC_Vec_SP(self.lake_tile_pixcvec_path_list_R[curContinent], self.objPixc_SP_R, self.output_dir, curContinent) # LakeSP self.objLake_SP_R = proc_lake.LakeProduct("SP", self.objPixc_SP_R, self.objPixc_Vec_SP_R, self.objLakeDb, "tmp_R", in_id_prefix=self.lake_sp_filenames.lake_id_prefix) my_api.printInfo("") # 2.2 - Left swath my_api.printInfo("[lakeSPProcessing] 2b - Left swath") # PIXC_edge self.objPixc_SP_L = proc_pixc_sp.PixC_Edge_SP(self.ascending, self.lake_tile_edge_path_list_L[curContinent]) # PIXCVec self.objPixc_Vec_SP_L = proc_pixc_vec_sp.PixC_Vec_SP(self.lake_tile_pixcvec_path_list_L[curContinent], self.objPixc_SP_L, self.output_dir, curContinent) # LakeSP self.objLake_SP_L = proc_lake.LakeProduct("SP", self.objPixc_SP_L, self.objPixc_Vec_SP_L, self.objLakeDb, "tmp_L", in_id_prefix=self.lake_sp_filenames.lake_id_prefix) my_api.printInfo("") # 3 - Compute lake products my_api.printInfo("[lakeSPProcessing] 3 - Computing lake products...") # 3.1 - Processing swath R if there are pixels to process my_api.printInfo("") my_api.printInfo(">>> RIGHT SWATH <<<") if self.objPixc_SP_R.nb_pixels > 0: my_api.printInfo("") # 3.1.1 - Gather pixels by entities for all the tiles of this swath self.objPixc_SP_R.edgeGlobalRelabeling() # 3.1.2 - Compute lake products self.objLake_SP_R.computeLakeProducts(np.unique(self.objPixc_SP_R.labels)) else: my_api.printInfo("No pixel to process") # 3.2 - Processing swath L if there are pixels to process my_api.printInfo("") my_api.printInfo(">>> LEFT SWATH <<<") if self.objPixc_SP_L.nb_pixels > 0: my_api.printInfo("") # 3.2.1 - Gather pixels by entities for all the tiles of this swath self.objPixc_SP_L.edgeGlobalRelabeling() # 3.2.2 - Compute lake products self.objLake_SP_L.computeLakeProducts(np.unique(self.objPixc_SP_L.labels)) else: my_api.printInfo("No pixel to process") my_api.printInfo("") # 4 - Merge shapefiles to get LakeSP product my_api.printInfo("[lakeSPProcessing] 4 - Merging shapefiles to get LakeSP product %s..." % os.path.basename(self.lake_sp_filenames.lake_sp_file)) # 4.1 - Merging Right and Left SP layers my_api.printDebug("[lakeSPProcessing] > Merging right and left SP layers...") dataSource_sp1, layer_sp = my_shp.merge_2_layers(self.objLake_SP_R.layer, self.objLake_SP_L.layer) # 4.2 - Merge SP layer with shapefiles retrieved from PGE_LakeTile my_api.printDebug("[lakeSPProcessing] > Merging SP layer with shapefiles retrieved from PGE_LakeTile...") dataSource_sp2, layer_sp = my_shp.merge_mem_layer_with_shp(self.lake_tile_shp_file_path_list[curContinent], layer_sp) # 4.3 - Write LakeSP shapefile product my_api.printDebug("[lakeSPProcessing] > Writing L2_HR_LakeSP shapefile = %s" % os.path.basename(self.lake_sp_filenames.lake_sp_file)) my_shp.write_mem_layer_as_shp(layer_sp, self.lake_sp_filenames.lake_sp_file) # 4.4 - Write XML metadatafile for shapefile if self.objPixc_SP_R.pass_num != 0: self.objLake_SP_R.writeMetadataFile("%s.xml" % self.lake_sp_filenames.lake_sp_file) elif self.objPixc_SP_L.pass_num != 0: self.objLake_SP_L.writeMetadataFile("%s.xml" % self.lake_sp_filenames.lake_sp_file) # 4.5 - Close dataSources dataSource_sp1.Destroy() dataSource_sp2.Destroy() self.objLake_SP_R.free_product() self.objLake_SP_L.free_product() my_api.printInfo("") # 5 - Update PIXCVec files my_api.printInfo("[lakeSPProcessing] 5 - Updating L2_HR_PIXCVec files...") self.objPixc_Vec_SP_R.updatePixcVec(self.flag_prod_shp) self.objPixc_Vec_SP_L.updatePixcVec(self.flag_prod_shp) my_api.printInfo("") my_api.printInfo("")
def run_preprocessing(self): """ Retrieve the list of input files, i.e. L2_HR_LakeTile products for wanted pass = shapefile + PIXC_edge + PIXCVec files """ my_api.printInfo("") my_api.printInfo("") my_api.printInfo("[lakeSPProcessing] PRE-PROCESSING...") my_api.printInfo("") # 1 - Test existence of directories my_api.printInfo("[lakeSPProcessing] > 1 - Testing existence of working directories ...") # 1.1 - LakeTile directory my_api.printInfo("[lakeSPProcessing] INPUT LakeTile DIR = %s" % self.lake_tile_dir) my_tools.testDir(self.lake_tile_dir) # 1.2 - Output directory my_api.printInfo("[lakeSPProcessing] OUTPUT DIR = %s" % self.output_dir) my_tools.testDir(self.output_dir) my_api.printInfo("") # 2 - Get list of input files my_api.printInfo("[lakeSPProcessing] > 2 - Retrieving input files ...") # 2.1 - Get ascending or descending orientation self.ascending = (self.pass_num%2 == 0) # TODO: replace when new orbits # NB: Ascending if pass_num is odd, descending if pass_num is pair. # self.ascending = ((self.pass_num%2 - 1) == 0) # 2.2 - Compute file prefix regarding cycle and pass conditions cond_prefix = my_var.LAKE_TILE_PREFIX # Add cycle number condition cond_prefix += "%03d" % self.cycle_num # Add orbit number condition cond_prefix += "_%03d" % self.pass_num my_api.printInfo("[lakeSPProcessing] LakeTile files with cycle=%03d and orbit=%03d (%s)" % (self.cycle_num, self.pass_num, self.asc_dict[self.ascending])) # 2.3 - List all files in LakeTile directory lake_tile_list = os.listdir(self.lake_tile_dir) # 2.4 - For each listed file, if it's a metadata file related to LakeTile_shp, get related _edge and _pixcvec files if they exist # Init tile_ref_list_R = {} # List of tile reference for right swath, organized by continent tile_ref_list_L = {} # List of tile reference for left swath, organized by continent # Loop on LakeTile_pixcvec files flag_first = True # Flag if it's the first tile to deal with for curFile in lake_tile_list: # Test if file meets the condition if curFile.startswith(cond_prefix) and curFile.endswith(my_var.LAKE_TILE_SHP_META_SUFFIX): # Test if it's a wanted LakeTile_shp file (NB: .shp.xml used instead of .shp because _edge and _pixcvec may also have associated .shp file) # Init add_tuple flag to True; set to False if 1 file of LakeTile product is missing add_tuple = True # Shapefile cur_shp = curFile.replace(my_var.LAKE_TILE_SHP_META_SUFFIX, my_var.LAKE_TILE_SHP_SUFFIX) if not os.path.exists(os.path.join(self.lake_tile_dir, cur_shp)): # Test if associated _shp file exists add_tuple = False # Shapefile cur_pixcvec = curFile.replace(my_var.LAKE_TILE_SHP_META_SUFFIX, my_var.LAKE_TILE_PIXCVEC_SUFFIX) if not os.path.exists(os.path.join(self.lake_tile_dir, cur_shp)): # Test if associated _shp file exists add_tuple = False # Edge file cur_edge = curFile.replace(my_var.LAKE_TILE_SHP_META_SUFFIX, my_var.LAKE_TILE_EDGE_SUFFIX) if not os.path.exists(os.path.join(self.lake_tile_dir, cur_edge)): # Test if associated _edge file exists add_tuple = False # Add tuple if exists if add_tuple: self.nb_input_tiles += 1 # Get metadata metadata = ET.parse(os.path.join(self.lake_tile_dir, curFile)) try: cur_continent = metadata.xpath("//LakeTile_shp/tile_info/continent")[0].text except: cur_continent = "WORLD" if flag_first: flag_first = False # Init list of continents for the pass self.list_continent = [cur_continent] # Init lists for continent self.lake_tile_shp_file_path_list[cur_continent] = [] self.lake_tile_pixcvec_path_list_R[cur_continent] = [] self.lake_tile_edge_path_list_R[cur_continent] = [] tile_ref_list_R[cur_continent] = [] self.lake_tile_pixcvec_path_list_L[cur_continent] = [] self.lake_tile_edge_path_list_L[cur_continent] = [] tile_ref_list_L[cur_continent] = [] # Overwrite metadata if 1st file processed print() print("WORKING VARIABLES retrieved from LakeTile processing") my_var.overwriteConfig_from_xml(metadata) print() else: # Test if new continent if not cur_continent in self.list_continent: # Add new continent to the list self.list_continent.append(cur_continent) # Init lists for new continent self.lake_tile_shp_file_path_list[cur_continent] = [] self.lake_tile_pixcvec_path_list_R[cur_continent] = [] self.lake_tile_edge_path_list_R[cur_continent] = [] tile_ref_list_R[cur_continent] = [] self.lake_tile_pixcvec_path_list_L[cur_continent] = [] self.lake_tile_edge_path_list_L[cur_continent] = [] tile_ref_list_L[cur_continent] = [] # Metadata should be the same as the others my_var.compareConfig_to_xml(metadata) # Add LakeTile_shp to list self.lake_tile_shp_file_path_list[cur_continent].append(os.path.join(self.lake_tile_dir, cur_shp)) # Get latitude from filename # TODO: change when tile numbering is fixed TMP_infos = my_names.getInfoFromFilename(curFile, "LakeTile") TMP_tile = TMP_infos["tile_ref"].split("-")[0] TMP_lat = int(TMP_tile[:-1]) if TMP_tile.endswith("S"): TMP_lat = -TMP_lat # In Right swath list if "-R" in curFile: self.lake_tile_pixcvec_path_list_R[cur_continent].append(os.path.join(self.lake_tile_dir, cur_pixcvec)) self.lake_tile_edge_path_list_R[cur_continent].append(os.path.join(self.lake_tile_dir, cur_edge)) tile_ref_list_R[cur_continent].append(TMP_lat) # In Left swath list elif "-L" in curFile: self.lake_tile_pixcvec_path_list_L[cur_continent].append(os.path.join(self.lake_tile_dir, cur_pixcvec)) self.lake_tile_edge_path_list_L[cur_continent].append(os.path.join(self.lake_tile_dir, cur_edge)) tile_ref_list_L[cur_continent].append(TMP_lat) # 2.5 - Test list of continents if ("WORLD" in self.list_continent) and (len(self.list_continent) > 1): my_api.exitWithError("[lakeSPProcessing] Mix of continent and no continent split; look at tiles process") # 2.6 - Sort files from south to north, continent per continent for curContinent in self.list_continent: sorted_idx_R = np.argsort(tile_ref_list_R[curContinent]) self.lake_tile_pixcvec_path_list_R[curContinent] = [self.lake_tile_pixcvec_path_list_R[curContinent][ind] for ind in sorted_idx_R] self.lake_tile_edge_path_list_R[curContinent] = [self.lake_tile_edge_path_list_R[curContinent][ind] for ind in sorted_idx_R] sorted_idx_L = np.argsort(tile_ref_list_L[curContinent]) self.lake_tile_pixcvec_path_list_L[curContinent] = [self.lake_tile_pixcvec_path_list_L[curContinent][ind] for ind in sorted_idx_L] self.lake_tile_edge_path_list_L[curContinent] = [self.lake_tile_edge_path_list_L[curContinent][ind] for ind in sorted_idx_L] # 2.7 - Print list of files, per continent for curContinent in self.list_continent: my_api.printInfo("[lakeSPProcessing] > Continent %s --> %d tile(s) to deal with" % (curContinent, len(self.lake_tile_shp_file_path_list[curContinent]))) for curFile in self.lake_tile_shp_file_path_list[curContinent]: my_api.printInfo("[lakeSPProcessing] %s" % os.path.basename(curFile)) my_api.printInfo("") my_api.printInfo("[lakeSPProcessing] --> %d tile(s) to deal with, over %d continent(s)" % (self.nb_input_tiles, len(self.list_continent))) my_api.printInfo("") # 3 - Retrieve lake Db layer my_api.printInfo("[lakeTileProcessing] > 3 - Retrieving lake database layer...") if my_var.LAKE_DB == "": my_api.printInfo("[lakeTileProcessing] NO database specified -> NO link of SWOT obs with a priori lake") else: if os.path.exists(my_var.LAKE_DB): type_db = my_var.LAKE_DB.split('.')[-1] # Type of database if type_db == "shp": # Shapefile format self.objLakeDb = lake_db.LakeDb_shp(my_var.LAKE_DB) elif type_db == "sqlite": # SGLite format self.objLakeDb = lake_db.LakeDb_sqlite(my_var.LAKE_DB) else: my_api.exitWithError("[lakeTileProcessing] Lake a priori database format (%s) is unknown: must be .shp or .sqlite" % type_db) else: my_api.exitWithError("[lakeTileProcessing] ERROR = %s doesn't exist" % my_var.LAKE_DB) my_api.printInfo("")
verbose_level = my_api.setVerbose(args.verbose) print("> Verbose level = %s" % verbose_level) # 1.5 - Init environment for log if args.logfile: logFile = os.path.join(output_dir, "pge_lake_sp_" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + ".log") my_api.initLogger(logFile, verbose_level) print("> Log file = %s" % logFile) else: print("> No log file ; print info on screen") print() print() # 2 - Initialization myLakeSP = Processing(laketile_dir, output_dir, cycle_num, pass_num, IN_shp_option=shp_option) my_api.printInfo(timer.info(0)) # 3 - Run pre-processing myLakeSP.run_preprocessing() my_api.printInfo(timer.info(0)) # 4 - Run processing myLakeSP.run_processing() my_api.printInfo(timer.info(0)) # 5 - Run post-processing myLakeSP.run_postprocessing() my_api.printInfo(timer.info(0)) my_api.printInfo("") my_api.printInfo(timer.stop())
def __init__(self, IN_lake_tile_pixcvec_file_list, IN_objPixcEdgeSP, IN_lake_sp_dir, IN_continent): """ This class is designed to process L2_HR_PIXCVec product . All pixels involved in entities covering more than one tile are processed here. The geolocation of pixels is improved for those pixels and PIXCVec NetCDF file is updated. The initialization of a PixC_Vec_SP consists in: - set class attributes - copy input LakeTile_pixcvec files into PIXCVec files NP: this object is related to one swath :param IN_lake_tile_pixcvec_file_list: list of LakeTile_pixcvec files full path concerning current swath :type IN_lake_tile_pixcvec_file_list:list of string :param IN_objPixcEdgeSP: list of subset of PixC for edge objects current swath :type IN_objPixcEdgeSP: list of PixC_Edge_SP.PixC_Edge_SP objects :param IN_lake_sp_dir: output LakeSP directory :type IN_lake_sp_dir: string :param IN_continent: continent covered by the tile (if global var CONTINENT_FILE exists) :type IN_continent: string Variables of the object: - From LakeTile_pixcvec file longitude_vectorproc / 1D-array of float: improved longitude of water pixels (= variable named longitude_vectorproc in LakeTile_pixcvec file) latitude_vectorproc / 1D-array of float: improved latitude of water pixels (= variable named latitude_vectorproc in LakeTile_pixcvec file) height_vectorproc / 1D-array of float: improved height of water pixels (= variable named height_vectorproc in LakeTile_pixcvec file) river_lake_other_tag / 1D-array of float: tag associated to river and lake databases (= variable named river_lake_other_tag in LakeTile_pixcvec file) - From process: lake_tile_pixcvec_file_list / list of str : list of input LakeTile_pixcvec files pixc_vec_file_list / list of str : list of output PIXCVec files objPixcEdgeSP / PixC_Edge_SP.PixC_Edge_SP object : subset of PixC related to pixels of objects at top/bottom edge of a PixC tile (output of PGE_LakeTile) nb_water_pix / int : number of pixels to process """ my_api.printInfo("[PixelCloudVecEdge] == INIT ==") # 1 - Init variables # List of LakeTile_pixcvec files concerning current swath self.lake_tile_pixcvec_file_list = IN_lake_tile_pixcvec_file_list # List of output PIXCVec files self.pixc_vec_file_list = [] # List of PixC_SP objects of current swath self.objPixcEdgeSP = IN_objPixcEdgeSP # Continent processed self.continent = IN_continent # Initialize PIXCVec variables to 0 self.longitude_vectorproc = np.zeros(self.objPixcEdgeSP.nb_pixels) self.latitude_vectorproc = np.zeros(self.objPixcEdgeSP.nb_pixels) self.height_vectorproc = np.zeros(self.objPixcEdgeSP.nb_pixels) self.tag = np.empty(self.objPixcEdgeSP.nb_pixels, dtype=object) self.tag[:] = "" # Init a list of tiles ref processed in thios class self.tile_ref_list = [] # 2 - List of output files computation for lake_tile_pixcvec_file in self.lake_tile_pixcvec_file_list: # 2.1 - Compute output PIXCVec file full path pixc_vec_file = my_names.computePixcvecFilename( lake_tile_pixcvec_file, IN_lake_sp_dir) # 2.2 - Remove if exists if os.path.exists(pixc_vec_file): os.remove(pixc_vec_file) # 2.3 - Copy LakeTile_pixcvec file full path to PIXCVec file list self.pixc_vec_file_list.append(pixc_vec_file) # 2.3 - Extact tile ref from PixC Vec file name tile_ref = my_names.getInfoFromFilename(lake_tile_pixcvec_file, "LakeTile")["tile_ref"] self.tile_ref_list.append(tile_ref)
def run_preprocessing(self): """ Retrieve the list of input files, i.e. L2_HR_LakeTile products for wanted passes = shapefile + PIXC_edge + PIXCVec files """ my_api.printInfo("") my_api.printInfo("") my_api.printInfo("[multiLakeSPProcessing] PRE-PROCESSING...") my_api.printInfo("") # 1 - Test existence of directories my_api.printInfo( "[multiLakeSPProcessing] > 1 - Testing existence of working directories..." ) # 1.1 - LakeTile directory my_api.printInfo("[multiLakeSPProcessing] INPUT LakeTile DIR = %s" % self.lake_tile_dir) my_tools.testDir(self.lake_tile_dir) # 1.2 - Output directory my_api.printInfo("[multiLakeSPProcessing] OUTPUT DIR = %s" % self.output_dir) my_tools.testDir(self.output_dir) my_api.printInfo("") # 2 - Get input files my_api.printInfo( "[multiLakeSPProcessing] > 2 - Retrieving input files...") # 2.1 - Compute file prefix regarding cycle / pass / tile conditions cond_prefix = my_var.LAKE_TILE_PREFIX # Deal with all LakeTile products in self.lake_tile_dir if (self.cycle_num is None) or (self.cycle_num == "-1"): my_api.printInfo( "[multiLakeSPProcessing] All LakeTile files in the input directory" ) else: # Deal with LakeTile files with cycle = self.cycle_num cond_prefix += "%03d" % self.cycle_num if (self.pass_num is None) or (self.pass_num == "-1"): my_api.printInfo( "[multiLakeSPProcessing] LakeTile files with cycle=%03d" % self.cycle_num) else: # Deal with LakeTile files with cycle = self.cycle_num and pass = self.pass_num cond_prefix += "_%03d" % self.pass_num my_api.printInfo( "[multiLakeSPProcessing] LakeTile files with cycle=%03d and pass=%03d" % (self.cycle_num, self.pass_num)) # 2.2 - List all files in input directory TMP_list = os.listdir(self.lake_tile_dir) # 2.3 - For each listed file, get (cycle, pass) pair for curFile in TMP_list: # Test if file meets the condition if curFile.startswith(cond_prefix) and curFile.endswith( my_var.LAKE_TILE_SHP_META_SUFFIX ): # Test if it's a wanted LakeTile_shp file TMP_infos = my_names.getInfoFromFilename(curFile, "LakeTile") if int(TMP_infos["pass"]) in self.list_pass: TMP_ind = [ indice for indice, valeur in enumerate(self.list_pass) if valeur == int(TMP_infos["pass"]) ] # All occurrences of current pass number in the list of passes TMP_subset_cycle = [ self.list_cycle[ind] for ind in TMP_ind ] # Subset of cycle numbers related to current pass if int( TMP_infos["cycle"] ) not in TMP_subset_cycle: # If current cycle not in subset = (cycle, pass) pair not listed self.list_cycle.append(int(TMP_infos["cycle"])) self.list_pass.append(int(TMP_infos["pass"])) else: self.list_cycle.append(int(TMP_infos["cycle"])) self.list_pass.append(int(TMP_infos["pass"])) my_api.printInfo( "[multiLakeSPProcessing] --> %d (cycle, pass) pair(s) to deal with" % len(self.list_cycle)) my_api.printInfo("")
def run_processing(self): """ Process SAS_L2_HR_LakeSP for each (cycle, pass) pair """ my_api.printInfo("") my_api.printInfo("") my_api.printInfo("[multiLakeSPProcessing] PROCESSING...") my_api.printInfo("") my_api.printInfo("") timer_proc = my_timer.Timer() timer_proc.start() if len(self.list_pass) != 0: for cur_cycle, cur_pass in zip( self.list_cycle, self.list_pass): # Deal with all selected pairs my_api.printInfo( "*******************************************************************" ) my_api.printInfo( "***** Dealing with cycle %03d and pass %03d *****" % (cur_cycle, cur_pass)) if my_api.GEN_ENV != 2: print() print("> Dealing with cycle %03d and pass %03d" % (cur_cycle, cur_pass)) my_api.printInfo( "*******************************************************************" ) my_api.printInfo("") my_api.printInfo("") # 1 - Initialization myLakeSP = pge_lake_sp.Processing( self.lake_tile_dir, self.output_dir, cur_cycle, cur_pass, IN_shp_option=self.flag_prod_shp) my_api.printInfo(timer.info(0)) # 2 - Run pre-processing myLakeSP.run_preprocessing() my_api.printInfo(timer.info(0)) # 3 - Run processing myLakeSP.run_processing() my_api.printInfo(timer.info(0)) # 4 - Run post-processing myLakeSP.run_postprocessing() my_api.printInfo(timer.info(0)) my_api.printInfo("") my_api.printInfo(timer.stop()) my_api.printInfo("") my_api.printInfo("") my_api.printInfo( "*******************************************************************" ) my_api.printInfo("") my_api.printInfo("")
def __init__(self, IN_ascending, IN_lake_tile_edge_path_list): """ This class is designed to process all L2_HR_LakeTile edge files of one swath. LakeTile edge files contain pixel cloud information (from L2_HR_PIXC) for pixels involved in lakes located at the top/bottom edges of tiles. The LakeTile edge file contains only PixC variables needed information, but also additional information like: - edge_loc field, the edge location of the lake : top of the tile, bottom of the tile or both top and bottom (0=bottom 1=top 2=both) - edge_label contains the object labels retrieved from PGE_LakeTile labeling process. - edge_idx contains the L2_HR_PIXC initial pixels indices. This information is needed to update the improved geoloc and tag fields of LakeTile pixcvec files. This class processes all LakeTile edge files of a single swath to gather all entities at the edge of tile into lake entities. :param IN_ascending: orbit orientation (True = ascending, False = descending) :type IN_ascending: boolean :param IN_lake_tile_edge_path_list: list of LakeTile edge files full path :type IN_lake_tile_edge_path_list: list of string Variables of the object: - From L2_HR_LakeTile edge file: Global attributes : nb_pix_range / int: number of pixels in range dimension (= global attribute named nb_pix_range in LakeTile_edge) nb_pix_azimuth / int: number of pixels in azimuth dimension (= global attribute named nb_pix_azimuth in LakeTile_edge) cycle_num / int: cycle number (= global attribute named cycle_number in LakeTile_edge) pass_num / int: pass number (= global attribute named pass_number in LakeTile_edge) Variables : range_idx / 1D-array of int: range indices of water pixels (= variable named range_index in LakeTile_edge) azimuth_idx / 1D-array of int: azimuth indices of water pixels (= variable named azimuth_index in LakeTile_edge) pixel_area / 1D-array of int: area of water pixels (= variable named pixel_area in LakeTile_edge) height / 1D-array of float: height of water pixels (= variable named height_medium in L2_HR_PIXC_main and LakeTile_edge) crosstrack / 1D-array of float: cross-track distance from nadir to center of water pixels (= variable named crosstrack_medium in LakeTile_edge) nadir_time / 1D-array of float: observation time of each nadir pixel (= variable named time in LakeTile_edge) nadir_longitude / 1D-array of float: longitude of each nadir pixel (= variable named nadir_lon in LakeTile_edge) nadir_lattitude / 1D-array of float: latitude of each nadir pixel (= variable named latitude in LakeTile_edge) nadir_[x|y|z] / 1D-array of float: [x|y|z] cartesian coordinates of each nadir pixel (= variables named nadir_[x|y|z] in LakeTile_edge) nadir_[vx|vy|vz] / 1D-array of float: velocity vector of each nadir pixel in cartesian coordinates (= variables named nadir_[vx|vy|vz] in LakeTile_edge) nadir_alt / 1D-array of float: satellite altitude at each nadir point (= variable named nadir_alt in LakeTile_edge) near_range / 1D-array of float: near range distance for each nadir point (= variable named nadir_near_range in LakeTile_edge) latitude / 1D-array of float: latitude of water pixels (= variable named latitude_medium in L2_HR_PIXC_main and LakeTile_edge) longitude / 1D-array of float: longitude of water pixels (= variable named longitude_medium in L2_HR_PIXC_main and LakeTile_edge) - From process: tile_ref / list of string : list of tile references to process ex: ['42N-R', '43N-R', '44N-R', '45N-R'] nb_tiles / int : number of tiles to process nb_pixels / int : number of pixels to process tile_idx / tuple of ndarrays : indices of tile for each pixel ascending / bool : orbit orientation (True = ascending, False=descending) labels / 1D-array of int : arrays of new labels """ my_api.printInfo("[PixelCloudSP] == INIT ==") # List of tile references to process ex: ['42N-R', '43N-R', '44N-R', '45N-R'] self.tile_ref = [] # Number of pixels to process self.nb_pixels = 0 # Tile reference of each pixel self.tile_idx = [] # Cycle number self.cycle_num = 0 # Orbit number self.pass_num = 0 # Continent self.continent = None # Number of pixel in range self.nb_pix_range = 0 # Orbit orientation (True = ascending, False=descending) self.ascending = IN_ascending # Merging of all PIXC_edge info into 1D vectors for lake_tile_edge_path in IN_lake_tile_edge_path_list: # For each LakeTile edge file my_api.printInfo( "[PixelCloudSP] Loading L2_HR_LakeTile edge file = %s" % lake_tile_edge_path) # Load data nb_pix_loaded, tile_ref = self.loadData(lake_tile_edge_path) my_api.printInfo("[PixelCloudSP] --> %d pixels loaded" % nb_pix_loaded) # Update nb_pixel self.nb_pixels += nb_pix_loaded # check if tile is neighboring the previus tile if len(self.tile_ref) >= 1: current_tile_number = int(tile_ref[:-3]) previus_tile_number = int(self.tile_ref[-1][:-3]) if current_tile_number != previus_tile_number + 1: # if current tile is not adjacent to previous tile, add an empty tile to tile_ref self.tile_ref.append("") # self.tile_idx must be set before incrementation of self.tile_ref to start by 0 self.tile_idx += [len(self.tile_ref)] * nb_pix_loaded self.tile_ref.append(tile_ref) # Convert list to numpy array self.tile_idx = np.array(self.tile_idx) my_api.printInfo("[PixelCloudSP] %d PixC loaded for current swath" % self.nb_pixels) # Init labels to 0 self.labels = np.zeros(self.nb_pixels)