def get_steps(config_file):
'''
Read steps with a '1' at config file and create a ordered Top-Down list with steps to do.
Return list with steps.
Write Warning error log in case:
- A step is not correctly separatec by '='
'''
step_to_do = []
recording = 0
for line in config_file:
if re.match("~STEPS~", line) is not None:
recording = 1
i = 0 # We use this to not include 'STEPS' line
elif re.match("~END_STEPS~", line) is not None:
recording = 0
if (recording == 1) and (i > 0):
try:
if line.split('=')[1].strip() == '1':
step_to_do.append(line.split('=')[0].strip())
except IndexError:
msg_err_get_step = "WARNING: Processes to do should be separated " \
"by a '='. This process '" + line + "' is not properly introduced."
log.write_error(msg_err_get_step)
elif recording == 1:
i += 1 # We use this to not include 'STEPS' line
return step_to_do
def check_subset(key_list, value_list):
'''
Verification values that are necessary to do Subset process.
We check that all necessary values are on .par file.
Also we check that the values are properly setted.
This function does not specify a return value. Return value always will be None.
Write Error error log in case:
- Parameter has a wrong value. Feedback about parameter and current value.
- A necessary parameter is missing.
- Coordinates format has a incorrect value.
'''
index = key_list.index('~COORDINATES_FORMAT')
values_coord_format = ['FileXML', 'Geographics', 'Pixel']
value = value_list[index]
if value == 'Pixel':
correct_pixel = ['~PIXEL_X', '~PIXEL_Y', '~PIXEL_WIDTH', '~PIXEL_HEIGHT']
for i in correct_pixel:
check_numerical_value(i, key_list, value_list)
if value == 'Geographics':
correct_geo = ['~LATMAX', '~LONMAX', '~LATMIN', '~LONMIN']
for i in correct_geo:
check_numerical_value(i, key_list, value_list)
if value == 'FileXML':
index_path = key_list.index('~KML_FILE_ROUTE')
path_file = value_list[index_path]
if not os.path.isfile(path_file):
msg_err_xmlfile = "ERROR: XML file does not exist at path: " + path_file + "."
log.write_error(msg_err_xmlfile)
if not value in values_coord_format:
msg_err_cf = "ERROR: Value of parameter '~COORDINATES_FORMAT' is wrong. "\
"Should be 'FileXML', 'Geographics', 'Pixel'. Current value: " + value
log.write_error(msg_err_cf)
def get_parents_files(path_img, name_img):
'''
Clasifica las imagenes en función de si tienen fechas coincidentes o no
Return: img_par_out, path_par_out, par_out_join, img_uni_out, path_uni_out
Donde:
- img_par_out = Name images with same date
- path_par_out = Path images with same date
- par_out_join = Route to two images same date, Assembly format
- img_uni_out = Name iamges with unique date
- path_uni_out = Path images with unique date
'''
dates = []
img_par_out = [] # Name images with same data
path_par_out = [] # Path images with same data
par_out_join = [] # Route to two images same data, Assembly format
img_uni_out = name_img.copy() # Name iamges with unique data
path_uni_out = path_img.copy() # Path images with unique data
for image in name_img:
# Mofification, now we check if image has between 17:25 a date, if not, we maintain full name
string_date = image[17:25]
if string_date == '':
try:
datetime.strptime(image, '%Y%m%d')
dates.append(image)
except ValueError:
dates.append(image)
msg_err_par_files = "This image: '" + image + "' doesn't conform date format. Will be saved with full name"
log.write_error(msg_err_par_files)
else:
try:
datetime.strptime(image[17:25], '%Y%m%d')
dates.append(image[17:25])
except ValueError:
dates.append(image)
msg_err_par_files = "This image: '" + image + "' doesn't conform date format. Will be saved with full name"
log.write_error(msg_err_par_files)
# dates.append(image[17:25])
for xpos, _ in enumerate(dates):
for wpos, _ in enumerate(dates):
if xpos != wpos:
if wpos > xpos:
if dates[xpos] == dates[wpos]:
par_out_join.append(path_img[xpos] + ',' +
path_img[wpos])
path_par_out.append(path_img[xpos])
path_par_out.append(path_img[wpos])
img_par_out.append(name_img[xpos])
img_par_out.append(name_img[wpos])
path_uni_out.remove(path_img[xpos])
img_uni_out.remove(name_img[xpos])
path_uni_out.remove(path_img[wpos])
img_uni_out.remove(name_img[wpos])
return img_par_out, path_par_out, par_out_join, img_uni_out, path_uni_out
def check_parameter_key(key_list):
'''
Check that parameters key name start with '~'.
This function does not specify a return value. Return value always will be None.
Write Error error log in case:
- A parameter do not start with proper symbol.
'''
for i, _ in enumerate(key_list):
first_character = key_list[i][:1]
if first_character != '~':
msg_err_key = "ERROR: Introduced parameter name is wrong. The parameter '"\
+ key_list[i] \
+ "' has incorrect key name. All parameters keys should start with '~' symbol."
log.write_error(msg_err_key)
def get_polygon_from_kml(path_kml_file):
'''
Generate polygon from kml file at configuration file.
Return string with subset polygon format.
Write Error error log in case:
- The kml document does not exist at setted path.
'''
try:
polygon = ''
with open(path_kml_file, 'rt', encoding="utf-8") as file:
doc = parser.parse(file).getroot()
file.close()
coordinates = doc.Document.Placemark.Polygon.outerBoundaryIs.LinearRing.coordinates.text
coordinates = coordinates.replace(',0', '')
coordinates = coordinates.replace(' ', ',')
coordinates = re.sub(r'\s+', '', coordinates)
coordinates = coordinates.split(',')
coordinates = list(filter(None, coordinates))
len_coordinates = len(coordinates)
longitud = []
for i in range(0, len_coordinates, 2):
longitud.append(float(coordinates[i]))
longitud_max = str(max(longitud))
longitud_min = str(min(longitud))
latitud = []
for i in range(1, len_coordinates, 2):
latitud.append(float(coordinates[i]))
latitud_max = str(max(latitud))
latitud_min = str(min(latitud))
# Create polygon in SNAP format
polygon = 'POLYGON ((' + longitud_min + ' ' + latitud_max + ', ' \
+ longitud_max + ' ' + latitud_max + ', ' + longitud_max + ' ' \
+ latitud_min + ', ' + longitud_min + ' ' + latitud_min + ', ' \
+ longitud_min + ' ' + latitud_max + ', ' + longitud_min + ' ' + latitud_max + '))'
return polygon
except FileNotFoundError:
msg_err_file = "ERROR: The .kml file doesn't exist\n"
print(msg_err_file)
log.write_error(msg_err_file)
def check_value_from_list(value_to_check, list_to_check, key_list, value_list):
'''
Checks values from a list of correct values and in case that a value is wrong write error logs.
This function does not specify a return value. Return value always will be None.
'''
try:
index = key_list.index(value_to_check)
val = value_list[index]
try:
list_to_check.index(val)
except ValueError:
msg_err = "ERROR: Parameter '" + value_to_check \
+ "'. It has a wrong value: " + val
log.write_error(msg_err)
except ValueError:
msg_err = "ERROR: Parameter needed '" + value_to_check \
+ "'. It is not defined on parameter list."
log.write_error(msg_err)
def check_numerical_value(value_to_check, key_list, value_list):
'''
Check if value of a parameter is numeric.
If not write at error log.
If value to check does not find on key list, write at error log.
This function does not specify a return value. Return value always will be None.
'''
try:
index_tc = key_list.index(value_to_check)
val_tc = value_list[index_tc]
try:
float(val_tc)
except ValueError:
msg_err_num = "ERROR: Parameter '" + value_to_check \
+ "' should has a numeric value. It has a wrong value: " + val_tc
log.write_error(msg_err_num)
except ValueError:
msg_err_num = "ERROR: Parameter '" + value_to_check \
+"' it is not defined on parameter list."
log.write_error(msg_err_num)
def set_images_date(start_date, end_date, path_img, name_img):
'''
Complementing 'set_images' function. In case of set images by date, return
images within configured dates.
Set images to process in case that date is equal or between specified dates.
This function maintain order, so path_img[i] has her name at name_img[i] and so on.
Return a pair (path_img, name_img)
Write Error error log in case:
- Date format doesn't conform format YYYYMMDD
- Date has not introduced at configuration file.
'''
path_img_out = list()
name_img_out = list()
try:
start_date_ = datetime.strptime(start_date, '%Y%m%d')
end_date_ = datetime.strptime(end_date, '%Y%m%d')
except ValueError:
msg_err_date = "ERROR: Date format at images file is wrong"
log.write_error(msg_err_date)
except AttributeError:
msg_err_date = "ERROR: Dates are not introduced at images file."
log.write_error(msg_err_date)
for i, img in enumerate(name_img):
try:
img_date = datetime.strptime(img[17:25], '%Y%m%d')
except ValueError:
pass
try:
img_date = datetime.strptime(img, '%Y%m%d')
except ValueError:
pass
if start_date_ <= img_date <= end_date_:
path_img_out.append(path_img[i])
name_img_out.append(name_img[i])
return path_img_out, name_img_out
def check_images(path_img, name_img):
'''
Check if we have two images with same image name. Remove one of these duplicated.
Check if a image is not from Sentinel-1 mission. Remove from list and print a Warning error log.
This function maintain order, so path_img[i] has her name at name_img[i] and so on.
Return a pair (path_img, name_img).
'''
name_img_out = name_img
path_img_out = path_img
# Check for duplicates images
name_img_uniq = list(dict.fromkeys(name_img))
if not len(name_img_uniq) == len(path_img):
msg_err_len = "WARNING: We found two images with same name."
log.write_error(msg_err_len)
for i, image in enumerate(name_img):
for i_2, image_2 in enumerate(name_img):
if (i != i_2) and (image == image_2):
del name_img_out[i_2]
del path_img_out[i_2]
# Delete files who doesn't start with 'S1'
to_del_name = list()
to_del_path = list()
n_pos = 0
for i in name_img_out:
if name_img_out[n_pos][:2] != 'S1':
msg_err_mission = "WARNING: File is not a Sentinel-1 image. File: '" + i \
+ "'. This image are not going to be processed."
log.write_error(msg_err_mission)
print(msg_err_mission)
to_del_name.append(name_img_out[n_pos])
to_del_path.append(path_img_out[n_pos])
n_pos += 1
for i in to_del_name:
name_img_out.remove(i)
for i in to_del_path:
path_img_out.remove(i)
return path_img_out, name_img_out
def check_steps(steps):
'''
Verification at step to do values.
We check that values on .par file are properly setted.
This function does not specify a return value. Return value always will be None.
Write Warning error log in case:
- Steps to do has duplicates values.
- Step to do has a incorrect value.
'''
correct_values = ['Read', 'Apply-Orbit-File', 'Assembly_Orbit', 'Subset', \
'ThermalNoiseRemoval', 'Calibration', 'Speckle-Filter', \
'LinearToFromdB', 'Terrain-Correction', 'CreateStack', 'Write']
if len(steps) != len(set(steps)):
log.write_error('WARNING: Steps to do has duplicates values.')
for step in steps:
flag = 0
for value in correct_values:
if value == step:
flag += 1
if flag == 0:
msg_err_inc_val = "WARNING: Step to do has a incorrect value. Value: '" + step + "'."
log.write_error(msg_err_inc_val)
def get_parameters(config_file):
'''
Read content at configuration file 'parameter.par' and extract parameters.
Extract parameter onto a pair '~PARAMETER = VALUE' and return diferents list.
Return a pair of list (key, value).
- key: It is a list with parameter name.
- value: It is a list with parameter value.
Both list has items ordered, so key[i] has the value of value[i] and so on.
Write Error error log in case:
- A key-value is not correctly separatec by '='
'''
parameters = []
keys = []
values = []
recording = 0
for line in config_file:
if re.match("~PARAMETERS~", line) is not None:
recording = 1
i = 0 # We use this to not include 'STEPS' line
elif re.match("~END_PARAMETERS~", line) is not None:
recording = 0
if (recording == 1) and (i > 0):
parameters.append(line)
try:
values.append(line.split('=')[1].strip())
keys.append(line.split('=')[0].strip())
except IndexError:
log.write_error("ERROR: Parameters should be separated by a '='. " \
"This pair key = value: '" \
+ line + "' is not properly introduced.")
elif recording == 1:
i += 1 # We use this to not include 'STEPS' line
if len(keys) != len(values):
msg_err_leng = "ERROR: Parameters key and value list has differents lengs."
log.write_error(msg_err_leng)
return keys, values
def set_images(path_img_txt, path_img, name_img):
'''
Read images document and set images within configured parameters.
This function is complemented by 'set_images_date' and 'set_images_list'.
This function maintain order, so path_img[i] has her name at name_img[i] and so on.
Return a pair (path_img, name_img)
Write Error error log in case:
- Type parameter is wrong. Should be 'LIST' or 'DATE'.
'''
with open(path_img_txt, 'r') as file:
img_txt = clean_par_file(file.readlines())
file.close()
path_img_out = list()
name_img_out = list()
for line in img_txt:
if 'TYPE' in line:
type_restriction = line.split('=')[1].strip()
if 'START_DATE' in line:
start_date = line.split('=')[1].strip()
if 'END_DATE' in line:
end_date = line.split('=')[1].strip()
if type_restriction == 'DATE':
path_img_out, name_img_out = set_images_date(start_date, end_date,
path_img, name_img)
elif type_restriction == 'LIST':
path_img_out, name_img_out = set_images_list(img_txt, path_img,
name_img)
else:
msg_err_type = "ERROR: At images text file parameter TYPE it is wrong. " \
"Should be 'LIST' or 'DATE'. All images at setted folder will be processed"
log.write_error(msg_err_type)
path_img_out = path_img
name_img_out = name_img
return path_img_out, name_img_out
PATH_BIN_FLR = os.path.dirname(os.path.realpath(__file__))
PATH_MAIN_FLR = (PATH_BIN_FLR)[:-4]
#Create logs
PATH_LOG_FLR = PATH_MAIN_FLR + '/logs'
log.create(PATH_LOG_FLR)
#Start message
msg_init = "Execution launched: " + str(time.strftime('%c')) + "."
log.write_output(msg_init)
#Read parameter.par file
try:
with open(PATH_CONFIG_FILE, 'r') as file:
file_parameters = func.clean_par_file(file.readlines())
file.close()
except FileNotFoundError:
msg_err_fnf = "ERROR: File .par could not be founded at path: " + PATH_CONFIG_FILE
log.write_error(msg_err_fnf)
#Set configuration parameters at parameter.par
for x in file_parameters:
if "dir_slc" in x:
path_img_flr = x.split('=')[1].strip()
if path_img_flr != '':
PATH_IMG_FLR = path_img_flr
else:
PATH_IMG_FLR = PATH_MAIN_FLR + '/images'
msg_path_img = "Images directory is setted to: " + PATH_IMG_FLR
if not os.path.exists(PATH_IMG_FLR):
os.makedirs(PATH_IMG_FLR)
if not os.listdir(PATH_IMG_FLR):
msg_err_img = "WARNING: Images folder is empty"
log.write_error(msg_err_img)
if "dir_out" in x:
def set_images_list(img_txt, path_img, name_img):
'''
Complementing 'set_images' function. In case of set images by list, return
images that are at list.
Set images to process in case:
- Name conform with Sentinel-1 name convention without product extension.
- Name conform with product unique ID.
This function maintain order, so path_img[i] has her name at name_img[i] and so on.
Return a pair (path_img, name_img)
Write Warning error log in case:
- Image list value doesn't match with any image at images folder.
- Image list values doesn't conform with name format.
'''
input_list = list()
recording = 0
path_img_out = list()
name_img_out = list()
for line in img_txt:
if re.match("~LIST~", line) is not None:
recording = 1
i = 0
elif re.match("~END~", line) is not None:
recording = 0
if (recording == 1) and (i > 0):
input_list.append(line.strip())
elif recording == 1:
i += 1
img_code = list()
path_code = list()
date_code = list()
# Checks input list are dates
input_list_date = list()
for i in input_list:
if len(i) == 8:
try:
datetime.strptime(i, '%Y%m%d')
input_list_date.append(i)
except ValueError:
msg_err_list_format = "WARNING: At images date list, line content '" + i + "' is not with correct date format 'YYYYMMDD'. This date will not be processed."
log.write_error(msg_err_list_format)
else:
msg_err_list_leng = "WARNING: At images date list, line content '" + i + "' is not with correct date format 'YYYYMMDD'. This date will not be processed."
log.write_error(msg_err_list_leng)
# for num, img in enumerate(name_img):
num = 0
for img in name_img:
try:
datetime.strptime(img, '%Y%m%d')
date_code.append(img)
img_code.append(name_img[num])
path_code.append(path_img[num])
except ValueError:
pass
try:
datetime.strptime(img[17:25], '%Y%m%d')
date_code.append(img[17:25])
img_code.append(name_img[num])
path_code.append(path_img[num])
except ValueError:
pass
num += 1
# We search array with dates list and if match
for i in input_list_date:
flagg = 0
pos_img = 0
for w in date_code:
if i == w:
flagg = 1
path_img_out.append(path_img[pos_img])
name_img_out.append(name_img[pos_img])
pos_img += 1
if flagg == 0:
msg_err = "WARNING: Images list value '" + i + "' couldn't " \
"find at images directory. This image will be not processed."
log.write_error(msg_err)
return path_img_out, name_img_out