def updatePixcVec(self, IN_write_to_shp=False):
"""
This function updates PIXCVec netcdf files obtained in output of PGE_LakeTile with improved longitude, latitude, height and lake_tile_id or prior_id if exists.
:param IN_write_to_shp: =True to also write the shapefile of the PIXCVec product (default=False)
:type IN_write_to_shp: boolean
"""
my_api.printInfo("[PixelCloudVecEdge] == updatePixcVec ==")
for tile_idx in range(len(self.tile_ref_list)): # Loop over tiles
tile_ref = self.tile_ref_list[tile_idx]
# 1 - Get input and output files info
# 1.1 - LakeTile_pixcvec path and filename
lake_tile_pixcvec_file = self.lake_tile_pixcvec_file_list[tile_idx]
# 1.2 - Get PIXCVec associated path and filename
pixc_vec_file = self.pixc_vec_file_list[tile_idx]
# 2 - Init proc_pixc_vec.PixelCloudVec object
objPixCVec = proc_pixc_vec.PixelCloudVec("SP",
lake_tile_pixcvec_file)
objPixCVec.setContinent(self.continent)
# 3 - Get corresponding objPixcEdgeSP tile_idx
objPixcEdgeSP_tile_idx = self.objPixcEdgeSP.tile_ref.index(
str(tile_ref))
# 4 - Update PIXCVec info
if (self.objPixcEdgeSP.tile_idx == objPixcEdgeSP_tile_idx
).any(): # Only when pixels need to be updated
# 4.1 - Retrieve corresponding indices in PixC_SP object
pixc_sp_idx = np.where(
self.objPixcEdgeSP.tile_idx == objPixcEdgeSP_tile_idx)[0]
# 4.2 - Retrieve corresponding indices in original PIXC
pixc_tile_idx = self.objPixcEdgeSP.edge_idx[pixc_sp_idx]
# 4.3 - Update geolocation information if computed
if my_var.IMP_GEOLOC:
objPixCVec.longitude_vectorproc[
pixc_tile_idx] = self.longitude_vectorproc[pixc_sp_idx]
objPixCVec.latitude_vectorproc[
pixc_tile_idx] = self.latitude_vectorproc[pixc_sp_idx]
objPixCVec.height_vectorproc[
pixc_tile_idx] = self.height_vectorproc[pixc_sp_idx]
# 4.4 - Update tag
objPixCVec.tag[pixc_tile_idx] = str(self.tag[pixc_sp_idx])
# 5 - Write PIXCVec file
objPixCVec.write_file(pixc_vec_file, None)
# 6 - Write associated shapefile if asked
if IN_write_to_shp:
objPixCVec.write_file_asShp(
pixc_vec_file.replace('.nc', '.shp'))
my_api.printInfo("")
def getValue(self, IN_key):
"""
Get the value associated to the key named IN_key
:param IN_key: name of the key
:type IN_key: string
:return: value associated to the key, None if doesn't exist
:rtype: string
"""
if IN_key in self.parameters:
return self.parameters.get(IN_key)
else:
for key in self.parameters.keys():
if IN_key in key:
return self.parameters.get(key)
# Key doesn't exist
my_api.printInfo("[my_rdf_file/getValue] Unkown key = %s" % IN_key)
return None
def gatherEdgeEntities(self, IN_tile_idx1, IN_tile_idx2):
"""
This function gives new labels for pixels at current tile edge.
1. Pixels within a buffer around tile edge are selected.
2. A water mask is computed
3. The mask is labeled
:param IN_tile_idx1: indices of pixels edge of tile 1
:type IN_tile_idx1: 1D-array of int
:param IN_tile_idx2: indices of pixels edge of tile 2
:type IN_tile_idx2: 1D-array of int
:return: OUT_new_labels_subset = new labels given to pixels of edge entities
:rtype: 1D-array of int
"""
my_api.printDebug("[PixelCloudSP] == gatherEdgeEntities ==")
# 1 - Pixels at top / bottom of tile are loaded
rg, az = self.selectEdgePixels(IN_tile_idx1, IN_tile_idx2)
# 2 - Equivalent matrix size in azimuth and range
# 2.1 - Equivalent matrix size in azimuth and range
nb_pix_range = max(rg) + 1
nb_pix_azimuth = max(az) + 1
# 2.2 - Compute water mask over the subset
waterMask = my_tools.computeBinMat(nb_pix_range, nb_pix_azimuth, rg,
az)
# 3 - Label entities over the subset of PixC at the edge tile
sepEntities, nb_obj = my_tools.labelRegion(waterMask)
# 4 - Convert into 1D-array
OUT_new_labels_subset = my_tools.convert2dMatIn1dVec(
rg, az, sepEntities)
my_api.printInfo(
"[PixelCloudSP] %d separate entities located at the edge tile" %
np.unique(OUT_new_labels_subset).size)
return OUT_new_labels_subset
def __init__(self,
IN_laketile_dir,
IN_output_dir,
IN_cycle_num,
IN_pass_num,
IN_shp_option=False):
"""
Constructor: initialize variables
:param IN_laketile_dir: full path of directory containing LakeTile products
:type IN_laketile_dir: string
:param IN_output_dir: output directory full path
:type IN_output_dir: string
:param IN_cycle_num: cycle number
:type IN_cycle_num: int
:param IN_pass_num: pass number
:type IN_pass_num: int
:param IN_shp_option: to also produce PIXCVec as shapefile (=True); else=False (default)
:type IN_shp_option: boolean
"""
my_api.printInfo("[multiLakeSpProcessing] == INIT ==")
# Input paths
self.lake_tile_dir = IN_laketile_dir # Full path of directory containing LakeTile products
self.output_dir = IN_output_dir # Output directory
# Pass infos
self.cycle_num = IN_cycle_num # Cycle number
self.pass_num = IN_pass_num # Pass number
# Flag to produce PIXCVec shapefile
self.flag_prod_shp = False
if IN_shp_option:
self.flag_prod_shp = True
# Init variables
self.list_cycle = [] # List of cycle numbers to deal with
self.list_pass = [] # List of pass numbers to deal with
def edgeGlobalRelabeling(self):
"""
This function gives new labels to entities gathered at tile edges.
"""
# 1 - Deal with all edges
for i_edge in range(len(self.tile_ref) - 1): # Loop over tile edges
my_api.printInfo(
"[PixelCloudSP] ***** Processing edge of tiles %s and %s *****"
% (self.tile_ref[i_edge], self.tile_ref[i_edge + 1]))
# 1.1 - Get indices of pixels processed at the current edge
tile_idx1 = np.where(self.tile_idx == i_edge)[0]
tile_idx2 = np.where(self.tile_idx == i_edge + 1)[0]
# 1.2 - If one tile does not have pixel to process, continue to next iteration
if (tile_idx1.size == 0) or (tile_idx2.size == 0):
my_api.printInfo("")
continue
# 1.3 - New labels for pixels at the edge of current edge
new_labels_subset = self.gatherEdgeEntities(tile_idx1, tile_idx2)
# 1.4 - Link old labels to new ones
self.labelMatching(tile_idx1, tile_idx2, new_labels_subset)
my_api.printInfo("")
# 2 - Deal with edge at beginning or end of pass
if (self.labels == 0).any():
# 2.1 - Get tiles with unprocessed labels (still initialized to zero)
tiles_to_process = np.unique(
self.tile_idx[np.where(self.labels == 0)])
for tile in tiles_to_process:
# 2.2 - Get associated tile index
tile_idx = np.where(self.tile_idx == tile)[0]
# 2.3 - Get old labels from PGE_LakeTile
old_labels = np.unique(self.edge_label[tile_idx][np.where(
self.labels[tile_idx] == 0)])
# 2.4 - Compute a global new label
new_labels = np.arange(old_labels.size) + np.max(
self.labels) + 1
# 2.5 - Update global labels
for idx in range(old_labels.size):
self.labels[tile_idx[np.where(
self.edge_label[tile_idx] ==
old_labels[idx])]] = new_labels[idx]
def run_postprocessing(self):
"""
Process PGE_L2_HR_LakeTile, i.e. convert output data in L2_HR_LakeTile product
"""
my_api.printInfo("")
my_api.printInfo("")
my_api.printInfo("[lakeSPProcessing] POST-PROCESSING...")
my_api.printInfo("")
# 1 - Close lake database
if self.objLakeDb is not None:
my_api.printInfo("[lakeTileProcessing] 1 - Closing lake database...")
self.objLakeDb.close_db()
my_api.printInfo("")
def __init__(self, IN_laketile_dir, IN_output_dir, IN_cycle_num, IN_pass_num, IN_shp_option=False):
"""
Constructor: initialize variables
:param IN_laketile_dir: full path of directory containing LakeTile products
:type IN_laketile_dir: string
:param IN_output_dir: output directory full path
:type IN_output_dir: string
:param IN_cycle_num: cycle number
:type IN_cycle_num: int
:param IN_pass_num: pass number
:type IN_pass_num: int
:param IN_shp_option: to also produce PIXCVec as shapefile (=True); else=False (default)
:type IN_shp_option: boolean
"""
my_api.printInfo("[lakeSPProcessing] == INIT ==")
# Input paths
self.lake_tile_dir = IN_laketile_dir # Full path of directory containing LakeTile products
self.output_dir = IN_output_dir # Output directory
# Pass infos
self.cycle_num = IN_cycle_num # Cycle number
self.pass_num = IN_pass_num # Pass number
self.ascending = True # Orientation: True=ascending False=descending
self.asc_dict = {True: "Ascending", False: "Descending"} # Associated dictionary
# Flag to produce PIXCVec shapefile
self.flag_prod_shp = False
if IN_shp_option:
self.flag_prod_shp = True
# Number of tiles to process
self.nb_input_tiles = 0
# List of continent(s) processed for the pass
self.list_continent = []
# List of all LakeTile shapefiles to merge, organized by continent
self.lake_tile_shp_file_path_list = {}
# List of files to process
self.lake_tile_pixcvec_path_list_R = {} # List of _pixcvec files for right swath, organized by continent
self.lake_tile_edge_path_list_R = {} # List of _edge files for right swath, organized by continent
self.lake_tile_pixcvec_path_list_L = {} # List of _pixcvec files for left swath, organized by continent
self.lake_tile_edge_path_list_L = {} # List of _edge files for left swath, organized by continent
# LakeSP filenames
self.lake_sp_filenames = None
# Objects
self.objLakeDb = None # Lake DB object
self.objPixc = None # PIXC object
self.objPixcVec = None # PIXCVecRiver object
self.objLake = None # LakeTile object
# SP objects
# Objects related to edge PixC for right (R) and left (L) swaths
self.objPixc_SP_L = None
self.objPixc_SP_R = None
# Objects related to PIXCVec for right (R) and left (L) swaths
self.objPixc_Vec_SP_L = None
self.objPixc_Vec_SP_R = None
# Objects related to LakeSP for right (R) and left (L) swaths
self.objLake_SP_L = None
self.objLake_SP_R = None
def run_processing(self):
"""
Process SAS_L2_HR_LakeTile
"""
my_api.printInfo("")
my_api.printInfo("")
my_api.printInfo("[lakeSPProcessing] PROCESSING tiles of cycle %03d and pass %03d (%s)..." % (self.cycle_num, self.pass_num, self.asc_dict[self.ascending]))
my_api.printInfo("")
for curContinent in self.list_continent:
my_api.printInfo("")
my_api.printInfo("==========================================")
my_api.printInfo("[lakeSPProcessing] Processing continent %s" % curContinent)
my_api.printInfo("==========================================")
my_api.printInfo("")
# 1 - Compute output filenames
my_api.printInfo("[lakeTileProcessing] 1 - Computing LakeSP filenames...")
self.lake_sp_filenames = my_names.lakeSPFilenames(self.lake_tile_shp_file_path_list[curContinent], curContinent, self.output_dir)
my_api.printInfo("")
# 2 - Objects initialisation
my_api.printInfo("[lakeSPProcessing] 2 - Init objects...")
my_api.printInfo("")
# 2.1 - Right swath
my_api.printInfo("[lakeSPProcessing] 2a - Right swath")
# PIXC_edge
self.objPixc_SP_R = proc_pixc_sp.PixC_Edge_SP(self.ascending, self.lake_tile_edge_path_list_R[curContinent])
# PIXCVec
self.objPixc_Vec_SP_R = proc_pixc_vec_sp.PixC_Vec_SP(self.lake_tile_pixcvec_path_list_R[curContinent], self.objPixc_SP_R, self.output_dir, curContinent)
# LakeSP
self.objLake_SP_R = proc_lake.LakeProduct("SP",
self.objPixc_SP_R,
self.objPixc_Vec_SP_R,
self.objLakeDb,
"tmp_R",
in_id_prefix=self.lake_sp_filenames.lake_id_prefix)
my_api.printInfo("")
# 2.2 - Left swath
my_api.printInfo("[lakeSPProcessing] 2b - Left swath")
# PIXC_edge
self.objPixc_SP_L = proc_pixc_sp.PixC_Edge_SP(self.ascending, self.lake_tile_edge_path_list_L[curContinent])
# PIXCVec
self.objPixc_Vec_SP_L = proc_pixc_vec_sp.PixC_Vec_SP(self.lake_tile_pixcvec_path_list_L[curContinent], self.objPixc_SP_L, self.output_dir, curContinent)
# LakeSP
self.objLake_SP_L = proc_lake.LakeProduct("SP",
self.objPixc_SP_L,
self.objPixc_Vec_SP_L,
self.objLakeDb,
"tmp_L",
in_id_prefix=self.lake_sp_filenames.lake_id_prefix)
my_api.printInfo("")
# 3 - Compute lake products
my_api.printInfo("[lakeSPProcessing] 3 - Computing lake products...")
# 3.1 - Processing swath R if there are pixels to process
my_api.printInfo("")
my_api.printInfo(">>> RIGHT SWATH <<<")
if self.objPixc_SP_R.nb_pixels > 0:
my_api.printInfo("")
# 3.1.1 - Gather pixels by entities for all the tiles of this swath
self.objPixc_SP_R.edgeGlobalRelabeling()
# 3.1.2 - Compute lake products
self.objLake_SP_R.computeLakeProducts(np.unique(self.objPixc_SP_R.labels))
else:
my_api.printInfo("No pixel to process")
# 3.2 - Processing swath L if there are pixels to process
my_api.printInfo("")
my_api.printInfo(">>> LEFT SWATH <<<")
if self.objPixc_SP_L.nb_pixels > 0:
my_api.printInfo("")
# 3.2.1 - Gather pixels by entities for all the tiles of this swath
self.objPixc_SP_L.edgeGlobalRelabeling()
# 3.2.2 - Compute lake products
self.objLake_SP_L.computeLakeProducts(np.unique(self.objPixc_SP_L.labels))
else:
my_api.printInfo("No pixel to process")
my_api.printInfo("")
# 4 - Merge shapefiles to get LakeSP product
my_api.printInfo("[lakeSPProcessing] 4 - Merging shapefiles to get LakeSP product %s..." % os.path.basename(self.lake_sp_filenames.lake_sp_file))
# 4.1 - Merging Right and Left SP layers
my_api.printDebug("[lakeSPProcessing] > Merging right and left SP layers...")
dataSource_sp1, layer_sp = my_shp.merge_2_layers(self.objLake_SP_R.layer, self.objLake_SP_L.layer)
# 4.2 - Merge SP layer with shapefiles retrieved from PGE_LakeTile
my_api.printDebug("[lakeSPProcessing] > Merging SP layer with shapefiles retrieved from PGE_LakeTile...")
dataSource_sp2, layer_sp = my_shp.merge_mem_layer_with_shp(self.lake_tile_shp_file_path_list[curContinent], layer_sp)
# 4.3 - Write LakeSP shapefile product
my_api.printDebug("[lakeSPProcessing] > Writing L2_HR_LakeSP shapefile = %s" % os.path.basename(self.lake_sp_filenames.lake_sp_file))
my_shp.write_mem_layer_as_shp(layer_sp, self.lake_sp_filenames.lake_sp_file)
# 4.4 - Write XML metadatafile for shapefile
if self.objPixc_SP_R.pass_num != 0:
self.objLake_SP_R.writeMetadataFile("%s.xml" % self.lake_sp_filenames.lake_sp_file)
elif self.objPixc_SP_L.pass_num != 0:
self.objLake_SP_L.writeMetadataFile("%s.xml" % self.lake_sp_filenames.lake_sp_file)
# 4.5 - Close dataSources
dataSource_sp1.Destroy()
dataSource_sp2.Destroy()
self.objLake_SP_R.free_product()
self.objLake_SP_L.free_product()
my_api.printInfo("")
# 5 - Update PIXCVec files
my_api.printInfo("[lakeSPProcessing] 5 - Updating L2_HR_PIXCVec files...")
self.objPixc_Vec_SP_R.updatePixcVec(self.flag_prod_shp)
self.objPixc_Vec_SP_L.updatePixcVec(self.flag_prod_shp)
my_api.printInfo("")
my_api.printInfo("")
def run_preprocessing(self):
"""
Retrieve the list of input files, i.e. L2_HR_LakeTile products for wanted pass = shapefile + PIXC_edge + PIXCVec files
"""
my_api.printInfo("")
my_api.printInfo("")
my_api.printInfo("[lakeSPProcessing] PRE-PROCESSING...")
my_api.printInfo("")
# 1 - Test existence of directories
my_api.printInfo("[lakeSPProcessing] > 1 - Testing existence of working directories ...")
# 1.1 - LakeTile directory
my_api.printInfo("[lakeSPProcessing] INPUT LakeTile DIR = %s" % self.lake_tile_dir)
my_tools.testDir(self.lake_tile_dir)
# 1.2 - Output directory
my_api.printInfo("[lakeSPProcessing] OUTPUT DIR = %s" % self.output_dir)
my_tools.testDir(self.output_dir)
my_api.printInfo("")
# 2 - Get list of input files
my_api.printInfo("[lakeSPProcessing] > 2 - Retrieving input files ...")
# 2.1 - Get ascending or descending orientation
self.ascending = (self.pass_num%2 == 0)
# TODO: replace when new orbits
# NB: Ascending if pass_num is odd, descending if pass_num is pair.
# self.ascending = ((self.pass_num%2 - 1) == 0)
# 2.2 - Compute file prefix regarding cycle and pass conditions
cond_prefix = my_var.LAKE_TILE_PREFIX
# Add cycle number condition
cond_prefix += "%03d" % self.cycle_num
# Add orbit number condition
cond_prefix += "_%03d" % self.pass_num
my_api.printInfo("[lakeSPProcessing] LakeTile files with cycle=%03d and orbit=%03d (%s)" % (self.cycle_num, self.pass_num, self.asc_dict[self.ascending]))
# 2.3 - List all files in LakeTile directory
lake_tile_list = os.listdir(self.lake_tile_dir)
# 2.4 - For each listed file, if it's a metadata file related to LakeTile_shp, get related _edge and _pixcvec files if they exist
# Init
tile_ref_list_R = {} # List of tile reference for right swath, organized by continent
tile_ref_list_L = {} # List of tile reference for left swath, organized by continent
# Loop on LakeTile_pixcvec files
flag_first = True # Flag if it's the first tile to deal with
for curFile in lake_tile_list:
# Test if file meets the condition
if curFile.startswith(cond_prefix) and curFile.endswith(my_var.LAKE_TILE_SHP_META_SUFFIX): # Test if it's a wanted LakeTile_shp file (NB: .shp.xml used instead of .shp because _edge and _pixcvec may also have associated .shp file)
# Init add_tuple flag to True; set to False if 1 file of LakeTile product is missing
add_tuple = True
# Shapefile
cur_shp = curFile.replace(my_var.LAKE_TILE_SHP_META_SUFFIX, my_var.LAKE_TILE_SHP_SUFFIX)
if not os.path.exists(os.path.join(self.lake_tile_dir, cur_shp)): # Test if associated _shp file exists
add_tuple = False
# Shapefile
cur_pixcvec = curFile.replace(my_var.LAKE_TILE_SHP_META_SUFFIX, my_var.LAKE_TILE_PIXCVEC_SUFFIX)
if not os.path.exists(os.path.join(self.lake_tile_dir, cur_shp)): # Test if associated _shp file exists
add_tuple = False
# Edge file
cur_edge = curFile.replace(my_var.LAKE_TILE_SHP_META_SUFFIX, my_var.LAKE_TILE_EDGE_SUFFIX)
if not os.path.exists(os.path.join(self.lake_tile_dir, cur_edge)): # Test if associated _edge file exists
add_tuple = False
# Add tuple if exists
if add_tuple:
self.nb_input_tiles += 1
# Get metadata
metadata = ET.parse(os.path.join(self.lake_tile_dir, curFile))
try:
cur_continent = metadata.xpath("//LakeTile_shp/tile_info/continent")[0].text
except:
cur_continent = "WORLD"
if flag_first:
flag_first = False
# Init list of continents for the pass
self.list_continent = [cur_continent]
# Init lists for continent
self.lake_tile_shp_file_path_list[cur_continent] = []
self.lake_tile_pixcvec_path_list_R[cur_continent] = []
self.lake_tile_edge_path_list_R[cur_continent] = []
tile_ref_list_R[cur_continent] = []
self.lake_tile_pixcvec_path_list_L[cur_continent] = []
self.lake_tile_edge_path_list_L[cur_continent] = []
tile_ref_list_L[cur_continent] = []
# Overwrite metadata if 1st file processed
print()
print("WORKING VARIABLES retrieved from LakeTile processing")
my_var.overwriteConfig_from_xml(metadata)
print()
else:
# Test if new continent
if not cur_continent in self.list_continent:
# Add new continent to the list
self.list_continent.append(cur_continent)
# Init lists for new continent
self.lake_tile_shp_file_path_list[cur_continent] = []
self.lake_tile_pixcvec_path_list_R[cur_continent] = []
self.lake_tile_edge_path_list_R[cur_continent] = []
tile_ref_list_R[cur_continent] = []
self.lake_tile_pixcvec_path_list_L[cur_continent] = []
self.lake_tile_edge_path_list_L[cur_continent] = []
tile_ref_list_L[cur_continent] = []
# Metadata should be the same as the others
my_var.compareConfig_to_xml(metadata)
# Add LakeTile_shp to list
self.lake_tile_shp_file_path_list[cur_continent].append(os.path.join(self.lake_tile_dir, cur_shp))
# Get latitude from filename
# TODO: change when tile numbering is fixed
TMP_infos = my_names.getInfoFromFilename(curFile, "LakeTile")
TMP_tile = TMP_infos["tile_ref"].split("-")[0]
TMP_lat = int(TMP_tile[:-1])
if TMP_tile.endswith("S"):
TMP_lat = -TMP_lat
# In Right swath list
if "-R" in curFile:
self.lake_tile_pixcvec_path_list_R[cur_continent].append(os.path.join(self.lake_tile_dir, cur_pixcvec))
self.lake_tile_edge_path_list_R[cur_continent].append(os.path.join(self.lake_tile_dir, cur_edge))
tile_ref_list_R[cur_continent].append(TMP_lat)
# In Left swath list
elif "-L" in curFile:
self.lake_tile_pixcvec_path_list_L[cur_continent].append(os.path.join(self.lake_tile_dir, cur_pixcvec))
self.lake_tile_edge_path_list_L[cur_continent].append(os.path.join(self.lake_tile_dir, cur_edge))
tile_ref_list_L[cur_continent].append(TMP_lat)
# 2.5 - Test list of continents
if ("WORLD" in self.list_continent) and (len(self.list_continent) > 1):
my_api.exitWithError("[lakeSPProcessing] Mix of continent and no continent split; look at tiles process")
# 2.6 - Sort files from south to north, continent per continent
for curContinent in self.list_continent:
sorted_idx_R = np.argsort(tile_ref_list_R[curContinent])
self.lake_tile_pixcvec_path_list_R[curContinent] = [self.lake_tile_pixcvec_path_list_R[curContinent][ind] for ind in sorted_idx_R]
self.lake_tile_edge_path_list_R[curContinent] = [self.lake_tile_edge_path_list_R[curContinent][ind] for ind in sorted_idx_R]
sorted_idx_L = np.argsort(tile_ref_list_L[curContinent])
self.lake_tile_pixcvec_path_list_L[curContinent] = [self.lake_tile_pixcvec_path_list_L[curContinent][ind] for ind in sorted_idx_L]
self.lake_tile_edge_path_list_L[curContinent] = [self.lake_tile_edge_path_list_L[curContinent][ind] for ind in sorted_idx_L]
# 2.7 - Print list of files, per continent
for curContinent in self.list_continent:
my_api.printInfo("[lakeSPProcessing] > Continent %s --> %d tile(s) to deal with" % (curContinent, len(self.lake_tile_shp_file_path_list[curContinent])))
for curFile in self.lake_tile_shp_file_path_list[curContinent]:
my_api.printInfo("[lakeSPProcessing] %s" % os.path.basename(curFile))
my_api.printInfo("")
my_api.printInfo("[lakeSPProcessing] --> %d tile(s) to deal with, over %d continent(s)" % (self.nb_input_tiles, len(self.list_continent)))
my_api.printInfo("")
# 3 - Retrieve lake Db layer
my_api.printInfo("[lakeTileProcessing] > 3 - Retrieving lake database layer...")
if my_var.LAKE_DB == "":
my_api.printInfo("[lakeTileProcessing] NO database specified -> NO link of SWOT obs with a priori lake")
else:
if os.path.exists(my_var.LAKE_DB):
type_db = my_var.LAKE_DB.split('.')[-1] # Type of database
if type_db == "shp": # Shapefile format
self.objLakeDb = lake_db.LakeDb_shp(my_var.LAKE_DB)
elif type_db == "sqlite": # SGLite format
self.objLakeDb = lake_db.LakeDb_sqlite(my_var.LAKE_DB)
else:
my_api.exitWithError("[lakeTileProcessing] Lake a priori database format (%s) is unknown: must be .shp or .sqlite" % type_db)
else:
my_api.exitWithError("[lakeTileProcessing] ERROR = %s doesn't exist" % my_var.LAKE_DB)
my_api.printInfo("")
verbose_level = my_api.setVerbose(args.verbose)
print("> Verbose level = %s" % verbose_level)
# 1.5 - Init environment for log
if args.logfile:
logFile = os.path.join(output_dir, "pge_lake_sp_" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + ".log")
my_api.initLogger(logFile, verbose_level)
print("> Log file = %s" % logFile)
else:
print("> No log file ; print info on screen")
print()
print()
# 2 - Initialization
myLakeSP = Processing(laketile_dir, output_dir, cycle_num, pass_num, IN_shp_option=shp_option)
my_api.printInfo(timer.info(0))
# 3 - Run pre-processing
myLakeSP.run_preprocessing()
my_api.printInfo(timer.info(0))
# 4 - Run processing
myLakeSP.run_processing()
my_api.printInfo(timer.info(0))
# 5 - Run post-processing
myLakeSP.run_postprocessing()
my_api.printInfo(timer.info(0))
my_api.printInfo("")
my_api.printInfo(timer.stop())
def __init__(self, IN_lake_tile_pixcvec_file_list, IN_objPixcEdgeSP,
IN_lake_sp_dir, IN_continent):
"""
This class is designed to process L2_HR_PIXCVec product .
All pixels involved in entities covering more than one tile are processed here.
The geolocation of pixels is improved for those pixels and PIXCVec NetCDF file is updated.
The initialization of a PixC_Vec_SP consists in:
- set class attributes
- copy input LakeTile_pixcvec files into PIXCVec files
NP: this object is related to one swath
:param IN_lake_tile_pixcvec_file_list: list of LakeTile_pixcvec files full path concerning current swath
:type IN_lake_tile_pixcvec_file_list:list of string
:param IN_objPixcEdgeSP: list of subset of PixC for edge objects current swath
:type IN_objPixcEdgeSP: list of PixC_Edge_SP.PixC_Edge_SP objects
:param IN_lake_sp_dir: output LakeSP directory
:type IN_lake_sp_dir: string
:param IN_continent: continent covered by the tile (if global var CONTINENT_FILE exists)
:type IN_continent: string
Variables of the object:
- From LakeTile_pixcvec file
longitude_vectorproc / 1D-array of float: improved longitude of water pixels (= variable named longitude_vectorproc in LakeTile_pixcvec file)
latitude_vectorproc / 1D-array of float: improved latitude of water pixels (= variable named latitude_vectorproc in LakeTile_pixcvec file)
height_vectorproc / 1D-array of float: improved height of water pixels (= variable named height_vectorproc in LakeTile_pixcvec file)
river_lake_other_tag / 1D-array of float: tag associated to river and lake databases (= variable named river_lake_other_tag in LakeTile_pixcvec file)
- From process:
lake_tile_pixcvec_file_list / list of str : list of input LakeTile_pixcvec files
pixc_vec_file_list / list of str : list of output PIXCVec files
objPixcEdgeSP / PixC_Edge_SP.PixC_Edge_SP object : subset of PixC related to pixels of objects at top/bottom edge of a PixC tile (output of PGE_LakeTile)
nb_water_pix / int : number of pixels to process
"""
my_api.printInfo("[PixelCloudVecEdge] == INIT ==")
# 1 - Init variables
# List of LakeTile_pixcvec files concerning current swath
self.lake_tile_pixcvec_file_list = IN_lake_tile_pixcvec_file_list
# List of output PIXCVec files
self.pixc_vec_file_list = []
# List of PixC_SP objects of current swath
self.objPixcEdgeSP = IN_objPixcEdgeSP
# Continent processed
self.continent = IN_continent
# Initialize PIXCVec variables to 0
self.longitude_vectorproc = np.zeros(self.objPixcEdgeSP.nb_pixels)
self.latitude_vectorproc = np.zeros(self.objPixcEdgeSP.nb_pixels)
self.height_vectorproc = np.zeros(self.objPixcEdgeSP.nb_pixels)
self.tag = np.empty(self.objPixcEdgeSP.nb_pixels, dtype=object)
self.tag[:] = ""
# Init a list of tiles ref processed in thios class
self.tile_ref_list = []
# 2 - List of output files computation
for lake_tile_pixcvec_file in self.lake_tile_pixcvec_file_list:
# 2.1 - Compute output PIXCVec file full path
pixc_vec_file = my_names.computePixcvecFilename(
lake_tile_pixcvec_file, IN_lake_sp_dir)
# 2.2 - Remove if exists
if os.path.exists(pixc_vec_file):
os.remove(pixc_vec_file)
# 2.3 - Copy LakeTile_pixcvec file full path to PIXCVec file list
self.pixc_vec_file_list.append(pixc_vec_file)
# 2.3 - Extact tile ref from PixC Vec file name
tile_ref = my_names.getInfoFromFilename(lake_tile_pixcvec_file,
"LakeTile")["tile_ref"]
self.tile_ref_list.append(tile_ref)
def run_preprocessing(self):
"""
Retrieve the list of input files, i.e. L2_HR_LakeTile products for wanted passes = shapefile + PIXC_edge + PIXCVec files
"""
my_api.printInfo("")
my_api.printInfo("")
my_api.printInfo("[multiLakeSPProcessing] PRE-PROCESSING...")
my_api.printInfo("")
# 1 - Test existence of directories
my_api.printInfo(
"[multiLakeSPProcessing] > 1 - Testing existence of working directories..."
)
# 1.1 - LakeTile directory
my_api.printInfo("[multiLakeSPProcessing] INPUT LakeTile DIR = %s" %
self.lake_tile_dir)
my_tools.testDir(self.lake_tile_dir)
# 1.2 - Output directory
my_api.printInfo("[multiLakeSPProcessing] OUTPUT DIR = %s" %
self.output_dir)
my_tools.testDir(self.output_dir)
my_api.printInfo("")
# 2 - Get input files
my_api.printInfo(
"[multiLakeSPProcessing] > 2 - Retrieving input files...")
# 2.1 - Compute file prefix regarding cycle / pass / tile conditions
cond_prefix = my_var.LAKE_TILE_PREFIX # Deal with all LakeTile products in self.lake_tile_dir
if (self.cycle_num is None) or (self.cycle_num == "-1"):
my_api.printInfo(
"[multiLakeSPProcessing] All LakeTile files in the input directory"
)
else: # Deal with LakeTile files with cycle = self.cycle_num
cond_prefix += "%03d" % self.cycle_num
if (self.pass_num is None) or (self.pass_num == "-1"):
my_api.printInfo(
"[multiLakeSPProcessing] LakeTile files with cycle=%03d"
% self.cycle_num)
else: # Deal with LakeTile files with cycle = self.cycle_num and pass = self.pass_num
cond_prefix += "_%03d" % self.pass_num
my_api.printInfo(
"[multiLakeSPProcessing] LakeTile files with cycle=%03d and pass=%03d"
% (self.cycle_num, self.pass_num))
# 2.2 - List all files in input directory
TMP_list = os.listdir(self.lake_tile_dir)
# 2.3 - For each listed file, get (cycle, pass) pair
for curFile in TMP_list:
# Test if file meets the condition
if curFile.startswith(cond_prefix) and curFile.endswith(
my_var.LAKE_TILE_SHP_META_SUFFIX
): # Test if it's a wanted LakeTile_shp file
TMP_infos = my_names.getInfoFromFilename(curFile, "LakeTile")
if int(TMP_infos["pass"]) in self.list_pass:
TMP_ind = [
indice for indice, valeur in enumerate(self.list_pass)
if valeur == int(TMP_infos["pass"])
] # All occurrences of current pass number in the list of passes
TMP_subset_cycle = [
self.list_cycle[ind] for ind in TMP_ind
] # Subset of cycle numbers related to current pass
if int(
TMP_infos["cycle"]
) not in TMP_subset_cycle: # If current cycle not in subset = (cycle, pass) pair not listed
self.list_cycle.append(int(TMP_infos["cycle"]))
self.list_pass.append(int(TMP_infos["pass"]))
else:
self.list_cycle.append(int(TMP_infos["cycle"]))
self.list_pass.append(int(TMP_infos["pass"]))
my_api.printInfo(
"[multiLakeSPProcessing] --> %d (cycle, pass) pair(s) to deal with"
% len(self.list_cycle))
my_api.printInfo("")
def run_processing(self):
"""
Process SAS_L2_HR_LakeSP for each (cycle, pass) pair
"""
my_api.printInfo("")
my_api.printInfo("")
my_api.printInfo("[multiLakeSPProcessing] PROCESSING...")
my_api.printInfo("")
my_api.printInfo("")
timer_proc = my_timer.Timer()
timer_proc.start()
if len(self.list_pass) != 0:
for cur_cycle, cur_pass in zip(
self.list_cycle,
self.list_pass): # Deal with all selected pairs
my_api.printInfo(
"*******************************************************************"
)
my_api.printInfo(
"***** Dealing with cycle %03d and pass %03d *****"
% (cur_cycle, cur_pass))
if my_api.GEN_ENV != 2:
print()
print("> Dealing with cycle %03d and pass %03d" %
(cur_cycle, cur_pass))
my_api.printInfo(
"*******************************************************************"
)
my_api.printInfo("")
my_api.printInfo("")
# 1 - Initialization
myLakeSP = pge_lake_sp.Processing(
self.lake_tile_dir,
self.output_dir,
cur_cycle,
cur_pass,
IN_shp_option=self.flag_prod_shp)
my_api.printInfo(timer.info(0))
# 2 - Run pre-processing
myLakeSP.run_preprocessing()
my_api.printInfo(timer.info(0))
# 3 - Run processing
myLakeSP.run_processing()
my_api.printInfo(timer.info(0))
# 4 - Run post-processing
myLakeSP.run_postprocessing()
my_api.printInfo(timer.info(0))
my_api.printInfo("")
my_api.printInfo(timer.stop())
my_api.printInfo("")
my_api.printInfo("")
my_api.printInfo(
"*******************************************************************"
)
my_api.printInfo("")
my_api.printInfo("")
def __init__(self, IN_ascending, IN_lake_tile_edge_path_list):
"""
This class is designed to process all L2_HR_LakeTile edge files of one swath. LakeTile edge files contain pixel cloud information (from L2_HR_PIXC) for pixels involved in lakes located at the top/bottom edges of tiles.
The LakeTile edge file contains only PixC variables needed information, but also additional information like:
- edge_loc field, the edge location of the lake : top of the tile, bottom of the tile or both top and bottom (0=bottom 1=top 2=both)
- edge_label contains the object labels retrieved from PGE_LakeTile labeling process.
- edge_idx contains the L2_HR_PIXC initial pixels indices. This information is needed to update the improved geoloc and tag fields of LakeTile pixcvec files.
This class processes all LakeTile edge files of a single swath to gather all entities at the edge of tile into lake entities.
:param IN_ascending: orbit orientation (True = ascending, False = descending)
:type IN_ascending: boolean
:param IN_lake_tile_edge_path_list: list of LakeTile edge files full path
:type IN_lake_tile_edge_path_list: list of string
Variables of the object:
- From L2_HR_LakeTile edge file:
Global attributes :
nb_pix_range / int: number of pixels in range dimension (= global attribute named nb_pix_range in LakeTile_edge)
nb_pix_azimuth / int: number of pixels in azimuth dimension (= global attribute named nb_pix_azimuth in LakeTile_edge)
cycle_num / int: cycle number (= global attribute named cycle_number in LakeTile_edge)
pass_num / int: pass number (= global attribute named pass_number in LakeTile_edge)
Variables :
range_idx / 1D-array of int: range indices of water pixels (= variable named range_index in LakeTile_edge)
azimuth_idx / 1D-array of int: azimuth indices of water pixels (= variable named azimuth_index in LakeTile_edge)
pixel_area / 1D-array of int: area of water pixels (= variable named pixel_area in LakeTile_edge)
height / 1D-array of float: height of water pixels (= variable named height_medium in L2_HR_PIXC_main and LakeTile_edge)
crosstrack / 1D-array of float: cross-track distance from nadir to center of water pixels (= variable named crosstrack_medium in LakeTile_edge)
nadir_time / 1D-array of float: observation time of each nadir pixel (= variable named time in LakeTile_edge)
nadir_longitude / 1D-array of float: longitude of each nadir pixel (= variable named nadir_lon in LakeTile_edge)
nadir_lattitude / 1D-array of float: latitude of each nadir pixel (= variable named latitude in LakeTile_edge)
nadir_[x|y|z] / 1D-array of float: [x|y|z] cartesian coordinates of each nadir pixel (= variables named nadir_[x|y|z] in LakeTile_edge)
nadir_[vx|vy|vz] / 1D-array of float: velocity vector of each nadir pixel in cartesian coordinates (= variables named nadir_[vx|vy|vz] in LakeTile_edge)
nadir_alt / 1D-array of float: satellite altitude at each nadir point (= variable named nadir_alt in LakeTile_edge)
near_range / 1D-array of float: near range distance for each nadir point (= variable named nadir_near_range in LakeTile_edge)
latitude / 1D-array of float: latitude of water pixels (= variable named latitude_medium in L2_HR_PIXC_main and LakeTile_edge)
longitude / 1D-array of float: longitude of water pixels (= variable named longitude_medium in L2_HR_PIXC_main and LakeTile_edge)
- From process:
tile_ref / list of string : list of tile references to process ex: ['42N-R', '43N-R', '44N-R', '45N-R']
nb_tiles / int : number of tiles to process
nb_pixels / int : number of pixels to process
tile_idx / tuple of ndarrays : indices of tile for each pixel
ascending / bool : orbit orientation (True = ascending, False=descending)
labels / 1D-array of int : arrays of new labels
"""
my_api.printInfo("[PixelCloudSP] == INIT ==")
# List of tile references to process ex: ['42N-R', '43N-R', '44N-R', '45N-R']
self.tile_ref = []
# Number of pixels to process
self.nb_pixels = 0
# Tile reference of each pixel
self.tile_idx = []
# Cycle number
self.cycle_num = 0
# Orbit number
self.pass_num = 0
# Continent
self.continent = None
# Number of pixel in range
self.nb_pix_range = 0
# Orbit orientation (True = ascending, False=descending)
self.ascending = IN_ascending
# Merging of all PIXC_edge info into 1D vectors
for lake_tile_edge_path in IN_lake_tile_edge_path_list: # For each LakeTile edge file
my_api.printInfo(
"[PixelCloudSP] Loading L2_HR_LakeTile edge file = %s" %
lake_tile_edge_path)
# Load data
nb_pix_loaded, tile_ref = self.loadData(lake_tile_edge_path)
my_api.printInfo("[PixelCloudSP] --> %d pixels loaded" %
nb_pix_loaded)
# Update nb_pixel
self.nb_pixels += nb_pix_loaded
# check if tile is neighboring the previus tile
if len(self.tile_ref) >= 1:
current_tile_number = int(tile_ref[:-3])
previus_tile_number = int(self.tile_ref[-1][:-3])
if current_tile_number != previus_tile_number + 1:
# if current tile is not adjacent to previous tile, add an empty tile to tile_ref
self.tile_ref.append("")
# self.tile_idx must be set before incrementation of self.tile_ref to start by 0
self.tile_idx += [len(self.tile_ref)] * nb_pix_loaded
self.tile_ref.append(tile_ref)
# Convert list to numpy array
self.tile_idx = np.array(self.tile_idx)
my_api.printInfo("[PixelCloudSP] %d PixC loaded for current swath" %
self.nb_pixels)
# Init labels to 0
self.labels = np.zeros(self.nb_pixels)