def generate_files_csv(slc_dir, custom_template_file):
""" Generates a csv file of the files to download serially.
Uses the `awk` command to generate a csv file containing the data files to be download
serially. The output csv file is then sent through the `sed` command to remove the first five
empty values to eliminate errors in download_ASF_serial.py.
"""
dataset_template = Template(custom_template_file)
dataset_template.options.update(
PathFind.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(),
ssaraopt.copy(),
delta_lat=0.0)
filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [
'--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>',
os.path.join(slc_dir, 'files.csv')
]
csv_command = ' '.join(filecsv_options)
message_rsmas.log(slc_dir, csv_command)
subprocess.Popen(csv_command, shell=True).wait()
# FA 8/2019: replaced new_files.csv by files.csv as infile argument
sed_command = "sed 's/^.\{5\}//' " + os.path.join(slc_dir, 'files.csv') + \
">" + os.path.join(slc_dir, 'new_files.csv')
message_rsmas.log(slc_dir, sed_command)
subprocess.Popen(sed_command, shell=True).wait()
def create_default_template(temp_inps):
"""
:param temp_inps: input parsed arguments
:return Updated template file added to temp_inps.
"""
inps = temp_inps
inps.customTemplateFile = os.path.abspath(inps.customTemplateFile)
inps.template_file = os.path.join(inps.work_dir, os.path.basename(inps.customTemplateFile))
# read custom template from file
custom_tempObj = Template(os.path.abspath(inps.customTemplateFile))
# check for required options
required_template_keys = pathObj.required_template_options
for template_key in required_template_keys:
if not template_key in custom_tempObj.options:
raise Exception('ERROR: {0} is required'.format(template_key))
# find default values from template_defaults.cfg to assign to default_tempObj
default_tempObj = Template(pathObj.auto_template)
config_template = get_config_defaults(config_file='template_defaults.cfg')
for each_section in config_template.sections():
for (each_key, each_val) in config_template.items(each_section):
default_tempObj.options.update({each_key: os.path.expandvars(each_val.strip("'"))})
inps.template = default_tempObj.options
pathObj.set_isce_defaults(inps)
# update default_temObj with custom_tempObj
for key, value in custom_tempObj.options.items():
if not value in [None, 'auto']:
inps.template.update({key: os.path.expandvars(value.strip("'"))})
if os.path.exists(inps.template_file):
if not os.path.samefile(inps.customTemplateFile, inps.template_file):
print('generate template file: {}'.format(inps.template_file))
shutil.copyfile(inps.customTemplateFile, inps.template_file)
else:
print('template file exists: {}'.format(inps.template_file))
else:
print('generate template file: {}'.format(inps.template_file))
shutil.copyfile(inps.customTemplateFile, inps.template_file)
# updates tempDefault dictionary with the given templateObj adding new keys
new_file = update_template_file(inps.template_file, custom_tempObj)
with open(inps.template_file, 'w') as file:
file.write(new_file)
inps.cropbox = pathObj.grab_cropbox(inps)
# build ssaraopt string from ssara options
custom_tempObj.options.update(pathObj.correct_for_ssara_date_format(custom_tempObj.options))
inps.ssaraopt = custom_tempObj.generate_ssaraopt_string()
return inps
def run_ssara(download_dir, template, delta_lat, logger, run_number=1):
""" Runs ssara_federated_query-cj.py and checks for download issues.
Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without
comleting or exited with an error code. If either of the above occur, the function is run again, for a
maxiumum of 10 times.
Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting)
Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success)
"""
# Compute SSARA options to use
dataset_template = Template(template)
dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat)
# get kml file and create listing
get_ssara_kml(download_dir, ssaraopt=ssaraopt)
# Runs ssara_federated_query.bash with proper options
ssara_call = ['ssara_federated_query.bash'] + ssaraopt + ['--print', '--download']
#FA 9/20: I could not figure out how to get the string into a bash shell variable, that is why writing a file
#print( ' '.join(ssara_call) )
with open('../ssara_command.txt', 'w') as f:
f.write(' '.join(ssara_call) + '\n')
return
def create_default_template(temp_inps):
"""
:param temp_inps: input parsed arguments
:return Updated template file added to temp_inps.
"""
inps = temp_inps
inps.custom_template_file = os.path.abspath(inps.custom_template_file)
inps.template_file = os.path.join(inps.work_dir, os.path.basename(inps.custom_template_file))
# read custom template from file
custom_tempObj = Template(inps.custom_template_file)
if not 'acquisition_mode' in custom_tempObj.options:
print('WARNING: "acquisition_mode" is not given --> default: tops (available options: tops, stripmap)')
inps.prefix = 'tops'
else:
inps.prefix = custom_tempObj.options['acquisition_mode']
# check for required options
required_template_keys = pathObj.required_template_options(inps.prefix)
for template_key in required_template_keys:
if template_key not in custom_tempObj.options:
raise Exception('ERROR: {0} is required'.format(template_key))
# find default values from minsar_template_defaults.cfg to assign to default_tempObj
default_tempObj = Template(pathObj.auto_template)
config_template = get_config_defaults(config_file='minsar_template_defaults.cfg')
for each_section in config_template.sections():
for (each_key, each_val) in config_template.items(each_section):
status = (inps.prefix == 'tops' and each_key.startswith('stripmap')) or \
(inps.prefix == 'stripmap' and each_key.startswith('tops'))
if status:
default_tempObj.options.pop(each_key)
else:
default_tempObj.options.update({each_key: os.path.expandvars(each_val.strip("'"))})
inps.template = default_tempObj.options
pathObj.set_isce_defaults(inps)
# update default_temObj with custom_tempObj
for key, value in custom_tempObj.options.items():
if value not in [None, 'auto']:
inps.template.update({key: os.path.expandvars(value.strip("'"))})
# update template file if necessary
if not os.path.exists(inps.template_file):
shutil.copyfile(inps.custom_template_file, inps.template_file)
else:
update_template_file(inps.template_file, custom_tempObj)
inps.cropbox = pathObj.grab_cropbox(inps)
# build ssaraopt string from ssara options
custom_tempObj.options.update(pathObj.correct_for_ssara_date_format(custom_tempObj.options))
inps.ssaraopt = custom_tempObj.generate_ssaraopt_string()
return inps
def run_ssara(work_dir, template, delta_lat):
""" Runs ssara_federated_query.py and checks for differences
"""
# Compute SSARA options to use
dataset_template = Template(template)
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string
ssaraopt_polygon = add_polygon_to_ssaraopt(dataset_template, ssaraopt.copy(), delta_lat)
# get kml file and create listing
compare_ssara_listings(work_dir, ssaraopt, ssaraopt_polygon)
return 0
def run_ssara(slc_dir, template, delta_lat, logger, run_number=1):
""" Runs ssara_federated_query-cj.py and checks for download issues.
Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without
comleting or exited with an error code. If either of the above occur, the function is run again, for a
maxiumum of 10 times.
Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting)
Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success)
"""
logger.log(loglevel.INFO, "RUN NUMBER: %s", str(run_number))
if run_number > 10:
return 0
logger.log(loglevel.INFO, "PASSED RUN NUMBER > 10")
# Compute SSARA options to use
dataset_template = Template(template)
dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
logger.log(loglevel.INFO, "GENERATED SSARAOPT STRING")
# add intersectWith to ssaraopt string
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat)
# get kml file and create listing
get_ssara_kml_and_listing(slc_dir, ssaraopt=ssaraopt)
# Runs ssara_federated_query-cj.py with proper options
ssara_call = ['ssara_federated_query-cj.py'] + ssaraopt + ['--print', '--download']
print('Download data using:\n' + ' '.join(ssara_call))
message_rsmas.log(slc_dir, ' '.join(ssara_call))
ssara_process = subprocess.Popen(' '.join(ssara_call), shell=True)
logger.log(loglevel.INFO, "STARTED PROCESS")
completion_status = ssara_process.poll() # the completion status of the process
hang_status = False # whether or not the download has hung
wait_time = 2 # 10 wait time in 'minutes' to determine hang status
prev_size = -1 # initial download directory size
i = 0 # index for waiting periods (for calculation of total time only)
logger.log(loglevel.INFO, "INITIAL COMPLETION STATUS: %s", str(completion_status))
# while the process has not completed
while completion_status is None:
i = i + 1
# Computer the current download directory size
curr_size = int(subprocess.check_output(['du', '-s', os.getcwd()]).split()[0].decode('utf-8'))
# Compare the current and previous directory sizes to determine determine hang status
if prev_size == curr_size:
hang_status = True
logger.log(loglevel.WARNING, "SSARA Hung")
ssara_process.terminate() # teminate the process beacause download hung
break # break the completion loop
prev_size = curr_size # store current size for comparison after waiting
time.sleep(60 * wait_time) # wait 'wait_time' minutes before continuing (checking for completion)
completion_status = ssara_process.poll()
logger.log(loglevel.INFO,
"{} minutes: {:.1f}GB, completion_status {}".format(i * wait_time, curr_size / 1024 / 1024,
completion_status))
exit_code = completion_status # get the exit code of the command
ssara_process.terminate()
logger.log(loglevel.INFO, "EXIT CODE: %s", str(exit_code))
bad_codes = [137, -9]
# If the exit code is one that signifies an error, rerun the entire command
if exit_code in bad_codes or hang_status:
if exit_code in bad_codes:
logger.log(loglevel.WARNING, "Exited with bad exit code, running again")
if hang_status:
logger.log(loglevel.WARNING, "Hanging, running again")
run_ssara(slc_dir, template, delta_lat, logger, run_number=run_number + 1)
return 0