def update_template_file(TEMP_FILE, custom_templateObj):
"""
updates final template file in project directory based on custom template file
:param TEMP_FILE: file to be updated
:param custom_templateObj: custom template having extra or new options
:return: updated file text
"""
tempObj = Template(TEMP_FILE)
update_status = False
for key, value in custom_templateObj.options.items():
if key not in tempObj.options or tempObj.options[key] != value:
tempObj.options[key] = value
update_status = True
if update_status:
print('Updating template file')
fileText = '#####################################\n'
for key, value in tempObj.options.items():
fileText = fileText + "{:<38}".format(key) + "{:<15}".format(
"= {}".format(value.strip("'"))) + '\n'
with open(TEMP_FILE, 'w') as file:
file.writelines(fileText)
else:
print('template file exists: {}, no updates'.format(TEMP_FILE))
return
def create_default_template(temp_inps):
"""
:param temp_inps: input parsed arguments
:return Updated template file added to temp_inps.
"""
inps = temp_inps
inps.customTemplateFile = os.path.abspath(inps.customTemplateFile)
inps.template_file = os.path.join(inps.work_dir, os.path.basename(inps.customTemplateFile))
# read custom template from file
custom_tempObj = Template(os.path.abspath(inps.customTemplateFile))
# check for required options
required_template_keys = pathObj.required_template_options
for template_key in required_template_keys:
if not template_key in custom_tempObj.options:
raise Exception('ERROR: {0} is required'.format(template_key))
# find default values from template_defaults.cfg to assign to default_tempObj
default_tempObj = Template(pathObj.auto_template)
config_template = get_config_defaults(config_file='template_defaults.cfg')
for each_section in config_template.sections():
for (each_key, each_val) in config_template.items(each_section):
default_tempObj.options.update({each_key: os.path.expandvars(each_val.strip("'"))})
inps.template = default_tempObj.options
pathObj.set_isce_defaults(inps)
# update default_temObj with custom_tempObj
for key, value in custom_tempObj.options.items():
if not value in [None, 'auto']:
inps.template.update({key: os.path.expandvars(value.strip("'"))})
if os.path.exists(inps.template_file):
if not os.path.samefile(inps.customTemplateFile, inps.template_file):
print('generate template file: {}'.format(inps.template_file))
shutil.copyfile(inps.customTemplateFile, inps.template_file)
else:
print('template file exists: {}'.format(inps.template_file))
else:
print('generate template file: {}'.format(inps.template_file))
shutil.copyfile(inps.customTemplateFile, inps.template_file)
# updates tempDefault dictionary with the given templateObj adding new keys
new_file = update_template_file(inps.template_file, custom_tempObj)
with open(inps.template_file, 'w') as file:
file.write(new_file)
inps.cropbox = pathObj.grab_cropbox(inps)
# build ssaraopt string from ssara options
custom_tempObj.options.update(pathObj.correct_for_ssara_date_format(custom_tempObj.options))
inps.ssaraopt = custom_tempObj.generate_ssaraopt_string()
return inps
def generate_files_csv(slc_dir, custom_template_file):
""" Generates a csv file of the files to download serially.
Uses the `awk` command to generate a csv file containing the data files to be download
serially. The output csv file is then sent through the `sed` command to remove the first five
empty values to eliminate errors in download_ASF_serial.py.
"""
dataset_template = Template(custom_template_file)
dataset_template.options.update(
PathFind.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(),
ssaraopt.copy(),
delta_lat=0.0)
filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [
'--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>',
os.path.join(slc_dir, 'files.csv')
]
csv_command = ' '.join(filecsv_options)
message_rsmas.log(slc_dir, csv_command)
subprocess.Popen(csv_command, shell=True).wait()
# FA 8/2019: replaced new_files.csv by files.csv as infile argument
sed_command = "sed 's/^.\{5\}//' " + os.path.join(slc_dir, 'files.csv') + \
">" + os.path.join(slc_dir, 'new_files.csv')
message_rsmas.log(slc_dir, sed_command)
subprocess.Popen(sed_command, shell=True).wait()
def run_ssara(download_dir, template, delta_lat, logger, run_number=1):
""" Runs ssara_federated_query-cj.py and checks for download issues.
Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without
comleting or exited with an error code. If either of the above occur, the function is run again, for a
maxiumum of 10 times.
Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting)
Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success)
"""
# Compute SSARA options to use
dataset_template = Template(template)
dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat)
# get kml file and create listing
get_ssara_kml(download_dir, ssaraopt=ssaraopt)
# Runs ssara_federated_query.bash with proper options
ssara_call = ['ssara_federated_query.bash'] + ssaraopt + ['--print', '--download']
#FA 9/20: I could not figure out how to get the string into a bash shell variable, that is why writing a file
#print( ' '.join(ssara_call) )
with open('../ssara_command.txt', 'w') as f:
f.write(' '.join(ssara_call) + '\n')
return
def create_default_template(temp_inps):
"""
:param temp_inps: input parsed arguments
:return Updated template file added to temp_inps.
"""
inps = temp_inps
inps.custom_template_file = os.path.abspath(inps.custom_template_file)
inps.template_file = os.path.join(inps.work_dir, os.path.basename(inps.custom_template_file))
# read custom template from file
custom_tempObj = Template(inps.custom_template_file)
if not 'acquisition_mode' in custom_tempObj.options:
print('WARNING: "acquisition_mode" is not given --> default: tops (available options: tops, stripmap)')
inps.prefix = 'tops'
else:
inps.prefix = custom_tempObj.options['acquisition_mode']
# check for required options
required_template_keys = pathObj.required_template_options(inps.prefix)
for template_key in required_template_keys:
if template_key not in custom_tempObj.options:
raise Exception('ERROR: {0} is required'.format(template_key))
# find default values from minsar_template_defaults.cfg to assign to default_tempObj
default_tempObj = Template(pathObj.auto_template)
config_template = get_config_defaults(config_file='minsar_template_defaults.cfg')
for each_section in config_template.sections():
for (each_key, each_val) in config_template.items(each_section):
status = (inps.prefix == 'tops' and each_key.startswith('stripmap')) or \
(inps.prefix == 'stripmap' and each_key.startswith('tops'))
if status:
default_tempObj.options.pop(each_key)
else:
default_tempObj.options.update({each_key: os.path.expandvars(each_val.strip("'"))})
inps.template = default_tempObj.options
pathObj.set_isce_defaults(inps)
# update default_temObj with custom_tempObj
for key, value in custom_tempObj.options.items():
if value not in [None, 'auto']:
inps.template.update({key: os.path.expandvars(value.strip("'"))})
# update template file if necessary
if not os.path.exists(inps.template_file):
shutil.copyfile(inps.custom_template_file, inps.template_file)
else:
update_template_file(inps.template_file, custom_tempObj)
inps.cropbox = pathObj.grab_cropbox(inps)
# build ssaraopt string from ssara options
custom_tempObj.options.update(pathObj.correct_for_ssara_date_format(custom_tempObj.options))
inps.ssaraopt = custom_tempObj.generate_ssaraopt_string()
return inps
def get_newest_data_date(template_file):
"""
Obtains the most recent image date for a dataset
:param template_file: the template file corresponding to the dataset being obtained
:return: the newest image date in "YYYY-MM-DD T H:M:S.00000" format
"""
delta_lat = 0.0 # 8/2019: this should use the same default as download_ssara_rsmas.py which I believe is set in
# utils/process_utilities.py: flag_parser.add_argument('--delta_lat', dest='delta_lat', default='0.0', type=float,
dataset_template = Template(template_file)
dataset_template.options.update(
pathObj.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(),
ssaraopt.copy(), delta_lat)
ssaraopt_cmd = ['ssara_federated_query.py'] + ssaraopt + ['--print']
ssaraopt_cmd = ' '.join(ssaraopt_cmd[:])
print(ssaraopt_cmd)
# Yield list of images in following format:
# ASF,Sentinel-1A,15775,2017-03-20T11:49:56.000000,2017-03-20T11:50:25.000000,128,3592,3592,IW,NA,DESCENDING,R,VV+VH,https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20170320T114956_20170320T115025_015775_019FA4_097A.zip
ssara_output = subprocess.check_output(ssaraopt_cmd, shell=True)
newest_data = ssara_output.decode('utf-8').split("\n")[-2]
return datetime.strptime(newest_data.split(",")[3], DATE_FORMAT)
def run_ssara(work_dir, template, delta_lat):
""" Runs ssara_federated_query.py and checks for differences
"""
# Compute SSARA options to use
dataset_template = Template(template)
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string
ssaraopt_polygon = add_polygon_to_ssaraopt(dataset_template, ssaraopt.copy(), delta_lat)
# get kml file and create listing
compare_ssara_listings(work_dir, ssaraopt, ssaraopt_polygon)
return 0
def auto_template_not_existing_options(args):
job_options = ['QUEUENAME', 'CPUS_PER_NODE', 'THREADS_PER_CORE', 'MAX_JOBS_PER_WORKFLOW', 'MAX_JOBS_PER_QUEUE',
'WALLTIME_FACTOR', 'MEM_PER_NODE', 'job_submission_scheme']
if hasattr(args, 'custom_template_file'):
from minsar.objects.dataset_template import Template
template = Template(args.custom_template_file).options
for option in job_options:
if not option in template:
template[option] = 'auto'
else:
template = {}
for option in job_options:
template[option] = 'auto'
return template
def update_template_file(TEMP_FILE, custom_templateObj):
"""
updates final template file in project directory based on custom template file
:param TEMP_FILE: file to be updated
:param custom_templateObj: custom template having extra or new options
:return: updated file text
"""
# fileText = TEMP_FILE
with open(TEMP_FILE, 'r') as file:
fileText = file.read()
tempObj = Template(TEMP_FILE)
for key, value in custom_templateObj.options.items():
if not key in tempObj.options:
fileText = fileText + "{:<38}".format(key) + "{:<15}".format("= {}".format(value.strip("'"))) + '\n'
return fileText
def run_ssara(slc_dir, template, delta_lat, logger, run_number=1):
""" Runs ssara_federated_query-cj.py and checks for download issues.
Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without
comleting or exited with an error code. If either of the above occur, the function is run again, for a
maxiumum of 10 times.
Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting)
Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success)
"""
logger.log(loglevel.INFO, "RUN NUMBER: %s", str(run_number))
if run_number > 10:
return 0
logger.log(loglevel.INFO, "PASSED RUN NUMBER > 10")
# Compute SSARA options to use
dataset_template = Template(template)
dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
logger.log(loglevel.INFO, "GENERATED SSARAOPT STRING")
# add intersectWith to ssaraopt string
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat)
# get kml file and create listing
get_ssara_kml_and_listing(slc_dir, ssaraopt=ssaraopt)
# Runs ssara_federated_query-cj.py with proper options
ssara_call = ['ssara_federated_query-cj.py'] + ssaraopt + ['--print', '--download']
print('Download data using:\n' + ' '.join(ssara_call))
message_rsmas.log(slc_dir, ' '.join(ssara_call))
ssara_process = subprocess.Popen(' '.join(ssara_call), shell=True)
logger.log(loglevel.INFO, "STARTED PROCESS")
completion_status = ssara_process.poll() # the completion status of the process
hang_status = False # whether or not the download has hung
wait_time = 2 # 10 wait time in 'minutes' to determine hang status
prev_size = -1 # initial download directory size
i = 0 # index for waiting periods (for calculation of total time only)
logger.log(loglevel.INFO, "INITIAL COMPLETION STATUS: %s", str(completion_status))
# while the process has not completed
while completion_status is None:
i = i + 1
# Computer the current download directory size
curr_size = int(subprocess.check_output(['du', '-s', os.getcwd()]).split()[0].decode('utf-8'))
# Compare the current and previous directory sizes to determine determine hang status
if prev_size == curr_size:
hang_status = True
logger.log(loglevel.WARNING, "SSARA Hung")
ssara_process.terminate() # teminate the process beacause download hung
break # break the completion loop
prev_size = curr_size # store current size for comparison after waiting
time.sleep(60 * wait_time) # wait 'wait_time' minutes before continuing (checking for completion)
completion_status = ssara_process.poll()
logger.log(loglevel.INFO,
"{} minutes: {:.1f}GB, completion_status {}".format(i * wait_time, curr_size / 1024 / 1024,
completion_status))
exit_code = completion_status # get the exit code of the command
ssara_process.terminate()
logger.log(loglevel.INFO, "EXIT CODE: %s", str(exit_code))
bad_codes = [137, -9]
# If the exit code is one that signifies an error, rerun the entire command
if exit_code in bad_codes or hang_status:
if exit_code in bad_codes:
logger.log(loglevel.WARNING, "Exited with bad exit code, running again")
if hang_status:
logger.log(loglevel.WARNING, "Hanging, running again")
run_ssara(slc_dir, template, delta_lat, logger, run_number=run_number + 1)
return 0
def main(iargs=None):
inps = putils.cmd_line_parse(iargs, script='download_rsmas')
if not iargs is None:
input_arguments = iargs
else:
input_arguments = sys.argv[1::]
message_rsmas.log(
inps.work_dir,
os.path.basename(__file__) + ' ' + ' '.join(input_arguments))
logfile_name = inps.work_dir + '/asfserial_rsmas.log'
global logger
logger = RsmasLogger(file_name=logfile_name)
#########################################
# Submit job
#########################################
if inps.submit_flag:
job_file_name = 'download_asfserial_rsmas'
job_name = inps.custom_template_file.split(os.sep)[-1].split('.')[0]
job_obj = JOB_SUBMIT(inps)
if '--submit' in input_arguments:
input_arguments.remove('--submit')
command = [os.path.abspath(__file__)] + input_arguments
job_obj.submit_script(job_name, job_file_name, command)
sys.exit(0)
os.chdir(inps.work_dir)
if not inps.template[inps.prefix + 'Stack.slcDir'] is None:
inps.slc_dir = inps.template[inps.prefix + 'Stack.slcDir']
else:
inps.slc_dir = os.path.join(inps.work_dir, 'SLC')
global project_slc_dir
project_slc_dir = os.path.join(inps.work_dir, 'SLC')
if not os.path.exists(inps.slc_dir):
os.mkdir(inps.slc_dir)
os.chdir(inps.slc_dir)
try:
os.remove(os.path.expanduser('~') + '/.bulk_download_cookiejar.txt')
except OSError:
pass
dataset_template = Template(inps.custom_template_file)
dataset_template.options.update(
PathFind.correct_for_ssara_date_format(dataset_template.options))
subprocess.Popen("rm " + project_slc_dir + "/new_files*.csv",
shell=True).wait()
seasonal_start_date = None
seasonal_end_date = None
try:
if dataset_template.options[
'seasonalStartDate'] is not None and dataset_template.options[
'seasonalEndDate'] is not None:
seasonal_start_date = dataset_template.options['seasonalStartDate']
seasonal_end_date = dataset_template.options['seasonalEndDate']
except:
pass
if inps.seasonalStartDate is not None and inps.seasonalEndDate is not None:
seasonal_start_date = inps.seasonalStartDate
seasonal_end_date = inps.seasonalEndDate
if seasonal_start_date is not None and seasonal_end_date is not None:
generate_seasonal_files_csv(dataset_template, seasonal_start_date,
seasonal_end_date)
else:
generate_files_csv(project_slc_dir, dataset_template)
parallel = False
try:
if dataset_template.options['parallel'] == 'yes':
parallel = True
except:
pass
"""if inps.parallel == 'yes':
parallel = True"""
threads = os.cpu_count()
try:
if dataset_template.options['threads'] is not None:
threads = int(dataset_template.options['threads'])
except:
pass
"""if inps.processes is not None:
processes = inps.processes"""
if parallel:
run_parallel_download_asf_serial(project_slc_dir, threads)
else:
succesful = run_download_asf_serial(project_slc_dir, logger)
logger.log(loglevel.INFO, "SUCCESS: %s", str(succesful))
change_file_permissions()
logger.log(loglevel.INFO, "------------------------------------")
subprocess.Popen("rm " + project_slc_dir + "/new_files*.csv",
shell=True).wait()
return None
def run_generate_chunk_template_files(inps):
""" create e*chunk*.template files """
project_name = putils.get_project_name(inps.custom_template_file)
location_name, sat_direction, sat_track = putils.split_project_name(
project_name)
location_name = location_name.split('Big')[0]
chunk_templates_dir = inps.work_dir + '/chunk_templates'
chunk_templates_dir_string = '$SCRATCHDIR/' + project_name + '/chunk_templates'
os.makedirs(chunk_templates_dir, exist_ok=True)
command_list = []
sleep_time = 0
command_options = ''
if inps.start_step is None and inps.do_step is None:
inps.start_step = 'jobfiles'
if inps.do_step is not None:
command_options = command_options + ' --dostep ' + inps.do_step
else:
if inps.start_step is not None:
command_options = command_options + ' --start ' + inps.start_step
if inps.end_step is not None:
command_options = command_options + ' --end ' + inps.end_step
prefix = 'tops'
bbox_list = inps.template[prefix + 'Stack.boundingBox'].split(' ')
bbox_list[0] = bbox_list[0].replace(
"\'", ''
) # this does ["'-8.75", '-7.8', '115.0', "115.7'"] (needed for run_operations.py, run_operations
bbox_list[1] = bbox_list[1].replace(
"\'", ''
) # --> ['-8.75', '-7.8', '115.0', '115.7'] (should be modified so that this is not needed)
bbox_list[2] = bbox_list[2].replace("\'", '')
bbox_list[3] = bbox_list[3].replace("\'", '')
tmp_min_lat = float(bbox_list[0])
tmp_max_lat = float(bbox_list[1])
min_lat = math.ceil(tmp_min_lat)
max_lat = math.floor(tmp_max_lat)
lat = min_lat
chunk_number = 0
chunk1_option = ''
while lat < max_lat:
tmp_min_lat = lat
tmp_max_lat = lat + inps.lat_step
chunk_name = [
location_name + 'Chunk' + str(int(lat)) + sat_direction + sat_track
]
chunk_template_file = chunk_templates_dir + '/' + chunk_name[
0] + '.template'
chunk_template_file_base = chunk_name[0] + '.template'
shutil.copy(inps.custom_template_file, chunk_template_file)
chunk_bbox_list = bbox_list
chunk_bbox_list[0] = str(float(tmp_min_lat - inps.lat_margin))
chunk_bbox_list[1] = str(float(tmp_max_lat + inps.lat_margin))
print(chunk_name, tmp_min_lat, tmp_max_lat, chunk_bbox_list)
custom_tempObj = Template(inps.custom_template_file)
custom_tempObj.options['topsStack.boundingBox'] = ' '.join(
chunk_bbox_list)
slcDir = '$SCRATCHDIR/' + project_name + '/SLC'
demDir = '$SCRATCHDIR/' + project_name + '/DEM'
custom_tempObj.options['topsStack.slcDir'] = slcDir
custom_tempObj.options['topsStack.demDir'] = demDir
#if inps.download_flag in [ True , 'True']:
# del(custom_tempObj.options['topsStack.slcDir'])
if 'download' in command_options:
del (custom_tempObj.options['topsStack.slcDir'])
putils.write_template_file(chunk_template_file, custom_tempObj)
putils.beautify_template_file(chunk_template_file)
chunk_number = chunk_number + 1
if chunk_number > 1 and inps.bash_script == 'minsarApp.bash':
chunk1_option = ' --no_download_ECMWF '
command = inps.bash_script + ' ' + chunk_templates_dir_string + '/' + chunk_template_file_base + command_options + chunk1_option + ' --sleep ' + str(
sleep_time) + ' &'
command_list.append(command)
lat = lat + inps.lat_step
sleep_time = sleep_time + inps.wait_time
chunk1_option = ''
commands_file = inps.work_dir + '/minsar_commands.txt'
f = open(commands_file, "w")
print()
for item in command_list:
print(item)
f.write(item + '\n')
print()
f.write('\n')
return
def run_generate_chunk_template_files(inps):
""" create e*chunk*.template files """
project_name = putils.get_project_name(inps.custom_template_file)
location_name, sat_direction, sat_track = putils.split_project_name(
project_name)
chunk_templates_dir = inps.work_dir + '/chunk_templates'
os.makedirs(chunk_templates_dir, exist_ok=True)
commands_file = inps.work_dir + '/minsar_commands.txt'
f = open(commands_file, "w")
if inps.download_flag == True:
minsarApp_option = '--start download'
else:
minsarApp_option = '--start dem'
prefix = 'tops'
bbox_list = inps.template[prefix + 'Stack.boundingBox'].split(' ')
bbox_list[0] = bbox_list[0].replace(
"\'", ''
) # this does ["'-8.75", '-7.8', '115.0', "115.7'"] (needed for run_operations.py, run_operations
bbox_list[1] = bbox_list[1].replace(
"\'", ''
) # --> ['-8.75', '-7.8', '115.0', '115.7'] (should be modified so that this is not needed)
bbox_list[2] = bbox_list[2].replace("\'", '')
bbox_list[3] = bbox_list[3].replace("\'", '')
tmp_min_lat = float(bbox_list[0])
tmp_max_lat = float(bbox_list[1])
min_lat = math.ceil(tmp_min_lat)
max_lat = math.floor(tmp_max_lat)
lat = min_lat
while lat < max_lat:
tmp_min_lat = lat
tmp_max_lat = lat + inps.lat_step
chunk_name = [
location_name + 'Chunk' + str(int(lat)) + sat_direction + sat_track
]
chunk_template_file = chunk_templates_dir + '/' + chunk_name[
0] + '.template'
shutil.copy(inps.custom_template_file, chunk_template_file)
chunk_bbox_list = bbox_list
chunk_bbox_list[0] = str(float(tmp_min_lat - inps.lat_margin))
chunk_bbox_list[1] = str(float(tmp_max_lat + inps.lat_margin))
print(chunk_name, tmp_min_lat, tmp_max_lat, chunk_bbox_list)
custom_tempObj = Template(inps.custom_template_file)
custom_tempObj.options['topsStack.boundingBox'] = ' '.join(
chunk_bbox_list)
if inps.download_flag in [True, 'True']:
del (custom_tempObj.options['topsStack.slcDir'])
putils.write_template_file(chunk_template_file, custom_tempObj)
putils.beautify_template_file(chunk_template_file)
minsar_command = 'minsarApp.bash ' + chunk_template_file + ' ' + minsarApp_option
f.write(minsar_command + '\n')
lat = lat + inps.lat_step
return
def main(iargs=None):
inps = putils.cmd_line_parse(iargs, script='download_rsmas')
config = putils.get_config_defaults(config_file='job_defaults.cfg')
if not iargs is None:
message_rsmas.log(
inps.work_dir,
os.path.basename(__file__) + ' ' + ' '.join(iargs[:]))
else:
message_rsmas.log(
inps.work_dir,
os.path.basename(__file__) + ' ' + ' '.join(sys.argv[1::]))
logfile_name = inps.work_dir + '/asfserial_rsmas.log'
logger = RsmasLogger(file_name=logfile_name)
#########################################
# Submit job
#########################################
if inps.submit_flag:
job_file_name = 'download_asfserial_rsmas'
job_name = inps.custom_template_file.split(os.sep)[-1].split('.')[0]
work_dir = inps.work_dir
if inps.wall_time == 'None':
inps.wall_time = config['download_rsmas']['walltime']
js.submit_script(job_name, job_file_name, sys.argv[:], work_dir,
inps.wall_time)
os.chdir(inps.work_dir)
if not inps.template['topsStack.slcDir'] is None:
inps.slc_dir = inps.template['topsStack.slcDir']
else:
inps.slc_dir = os.path.join(inps.work_dir, 'SLC')
project_slc_dir = os.path.join(inps.work_dir, 'SLC')
os.chdir(inps.slc_dir)
try:
os.remove(os.path.expanduser('~') + '/.bulk_download_cookiejar.txt')
except OSError:
pass
dataset_template = Template(inps.custom_template_file)
dataset_template.options.update(
PathFind.correct_for_ssara_date_format(dataset_template.options))
subprocess.Popen("rm new_files.csv", shell=True).wait()
standardTuple = (inps, dataset_template)
if inps.seasonalStartDate is not None and inps.seasonalEndDate is not None:
ogStartYearInt = int(
dataset_template.options['ssaraopt.startDate'][:4])
if int(inps.seasonalStartDate) > int(inps.seasonalEndDate):
y = 1
else:
y = 0
YearRange = int(dataset_template.options['ssaraopt.endDate']
[:4]) - ogStartYearInt + 1
if YearRange > 1 and y == 1:
YearRange = YearRange - 1
seasonalStartDateAddOn = '-' + inps.seasonalStartDate[:
2] + '-' + inps.seasonalStartDate[
2:]
seasonalEndDateAddOn = '-' + inps.seasonalEndDate[:
2] + '-' + inps.seasonalEndDate[
2:]
ogEndDate = dataset_template.options['ssaraopt.endDate']
for x in range(YearRange):
seasonalTuple = standardTuple + (x, ogStartYearInt, y, YearRange,
seasonalStartDateAddOn,
seasonalEndDateAddOn, ogEndDate)
generate_files_csv(project_slc_dir, inps.custom_template_file,
seasonalTuple)
y += 1
else:
generate_files_csv(project_slc_dir, inps.custom_template_file,
standardTuple)
succesful = run_download_asf_serial(project_slc_dir, logger)
change_file_permissions()
logger.log(loglevel.INFO, "SUCCESS: %s", str(succesful))
logger.log(loglevel.INFO, "------------------------------------")
return None