def generate_files_csv(slc_dir, custom_template_file):
""" Generates a csv file of the files to download serially.
Uses the `awk` command to generate a csv file containing the data files to be download
serially. The output csv file is then sent through the `sed` command to remove the first five
empty values to eliminate errors in download_ASF_serial.py.
"""
dataset_template = Template(custom_template_file)
dataset_template.options.update(
PathFind.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(),
ssaraopt.copy(),
delta_lat=0.0)
filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [
'--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>',
os.path.join(slc_dir, 'files.csv')
]
csv_command = ' '.join(filecsv_options)
message_rsmas.log(slc_dir, csv_command)
subprocess.Popen(csv_command, shell=True).wait()
# FA 8/2019: replaced new_files.csv by files.csv as infile argument
sed_command = "sed 's/^.\{5\}//' " + os.path.join(slc_dir, 'files.csv') + \
">" + os.path.join(slc_dir, 'new_files.csv')
message_rsmas.log(slc_dir, sed_command)
subprocess.Popen(sed_command, shell=True).wait()
def get_newest_data_date(template_file):
"""
Obtains the most recent image date for a dataset
:param template_file: the template file corresponding to the dataset being obtained
:return: the newest image date in "YYYY-MM-DD T H:M:S.00000" format
"""
delta_lat = 0.0 # 8/2019: this should use the same default as download_ssara_rsmas.py which I believe is set in
# utils/process_utilities.py: flag_parser.add_argument('--delta_lat', dest='delta_lat', default='0.0', type=float,
dataset_template = Template(template_file)
dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat)
ssaraopt_cmd = ['ssara_federated_query.py'] + ssaraopt + ['--print']
ssaraopt_cmd = ' '.join(ssaraopt_cmd[:])
# Yield list of images in following format:
# ASF,Sentinel-1A,15775,2017-03-20T11:49:56.000000,2017-03-20T11:50:25.000000,128,3592,3592,IW,NA,DESCENDING,R,VV+VH,https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20170320T114956_20170320T115025_015775_019FA4_097A.zip
ssara_output = subprocess.check_output(ssaraopt_cmd, shell=True)
newest_data = ssara_output.decode('utf-8').split("\n")[-2]
return datetime.strptime(newest_data.split(",")[3], DATE_FORMAT)
def call_ssara_dem(inps, cwd):
print('DEM generation using SSARA')
sys.stdout.flush()
out_file = 'ssara_dem.log'
# need to refactor so that Josh's dataset_template will be used throughout
ssaraopt_string = inps.ssaraopt
ssaraopt_list = ssaraopt_string.split(' ')
ssaraopt_list = add_polygon_to_ssaraopt(dataset_template=inps.template,
ssaraopt=ssaraopt_list.copy(),
delta_lat=0)
ssaraopt_string = ' '.join(ssaraopt_list)
out_file = 'out_ssara_dem'
command = 'ssara_federated_query.py {ssaraopt} --dem '.format(
ssaraopt=ssaraopt_string)
message_rsmas.log(os.getcwd(), command)
command = '(' + command + ' | tee ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee ' + out_file + '.e'
print('command currently executing: ' + command)
sys.stdout.flush()
if os.getenv('DOWNLOADHOST') == 'local':
print('Command: ' + command)
sys.stdout.flush()
try:
proc = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
shell=True,
universal_newlines=True)
error, output = proc.communicate(
) # FA 8/19 error, output works better here than output, error. Could be that stderr and stdout are switched .
if proc.returncode is not 0:
raise Exception(
'ERROR starting dem.py subprocess'
) # FA 8/19: I don't think this happens, errors are is output
except subprocess.CalledProcessError as exc:
print("Command failed. Exit code, StdErr:", exc.returncode,
exc.output)
sys.exit('Error produced by ssara_federated_query.py')
else:
if not 'Downloading DEM' in output:
os.chdir('..')
shutil.rmtree('DEM')
sys.exit('Error in dem.py: Tiles are missing. Ocean???')
else:
dem_dir = os.getcwd()
ssh_command_list = ['s.bgood', 'cd {0}'.format(cwd), command]
host = os.getenv('DOWNLOADHOST')
status = ssh_with_commands(host, ssh_command_list)
print('status from ssh_with_commands:' + str(status))
sys.stdout.flush()
print('Done downloading dem.grd')
sys.stdout.flush()
grd_to_envi_and_vrt()
grd_to_xml(cwd)
def main(iargs=None):
"""Downloads data with ssara and asfserial scripts."""
inps = putils.cmd_line_parse(iargs, script='download_rsmas')
if not iargs is None:
input_arguments = iargs
else:
input_arguments = sys.argv[1::]
message_rsmas.log(inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments))
time.sleep(putils.pause_seconds(inps.wait_time))
#########################################
# Submit job
#########################################
if inps.submit_flag:
job_obj = JOB_SUBMIT(inps)
job_name = 'download_rsmas'
job_file_name = job_name
if '--submit' in input_arguments:
input_arguments.remove('--submit')
command = [os.path.abspath(__file__)] + input_arguments
job_obj.submit_script(job_name, job_file_name, command)
sys.exit(0)
if inps.prefix == 'tops':
if not inps.template[inps.prefix + 'Stack.slcDir'] is None:
download_dir = inps.template[inps.prefix + 'Stack.slcDir']
else:
download_dir = os.path.join(inps.work_dir, 'SLC')
else:
if not inps.template['raw_image_dir'] in [None, 'None']:
download_dir = inps.template['raw_image_dir']
else:
download_dir = os.path.join(inps.work_dir, 'RAW_data')
os.makedirs(inps.work_dir, exist_ok=True)
os.makedirs(download_dir, exist_ok=True)
if 'SenDT' not in inps.project_name and 'SenAT' not in inps.project_name or os.getenv('SSARA_ASF') == 'False':
try:
inps.template['ssaraopt.intersectsWithPoint']
inps.ssaraopt = ' '.join(add_point_to_ssaraopt(inps.template, inps.ssaraopt.split(' ')))
except:
inps.ssaraopt = ' '.join(add_polygon_to_ssaraopt(inps.template, inps.ssaraopt.split(' '), delta_lat=inps.delta_lat))
command = 'ssara_federated_query.py ' + inps.ssaraopt + ' --print' + ' --download'
os.chdir(download_dir)
message_rsmas.log(download_dir, command)
status = subprocess.Popen(command, shell=True).wait()
if status is not 0:
raise Exception('ERROR in ssara_federated_query.py')
os.chdir(inps.work_dir)
return
if os.getenv('SSARA_ASF') == 'False':
return
download('ssara', inps.custom_template_file, download_dir, outnum=1)
#download('asfserial', inps.custom_template_file, download_dir, outnum = 1)
for i_download in [2, 3]:
download_success = run_check_download(download_dir = download_dir)
if not download_success:
print('check_download.py: There were bad files, download again')
message_rsmas.log(inps.work_dir,'check_download.py: there were bad files, download again')
download('ssara', inps.custom_template_file, download_dir, outnum = i_download)
def generate_files_csv(slc_dir, custom_template_file, tupleParam):
""" Generates a csv file of the files to download serially.
Uses the `awk` command to generate a csv file containing the data files to be download
serially. The output csv file is then sent through the `sed` command to remove the first five
empty values to eliminate errors in download_ASF_serial.py.
"""
inps = tupleParam[0]
dataset_template = tupleParam[1]
if inps.seasonalStartDate is not None and inps.seasonalEndDate is not None:
x = tupleParam[2]
ogStartYearInt = tupleParam[3]
y = tupleParam[4]
YearRange = tupleParam[5]
seasonalStartDateAddOn = tupleParam[6]
seasonalEndDateAddOn = tupleParam[7]
ogEndDate = tupleParam[8]
if x == 0:
if YearRange == 1:
if y == 0:
if int(inps.seasonalEndDate) < int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')) or int(
inps.seasonalStartDate) > int(
ogEndDate[4:].replace('-', '')):
return
else:
if int(inps.seasonalStartDate) > int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')):
dataset_template.options[
'ssaraopt.startDate'] = str(
ogStartYearInt) + seasonalStartDateAddOn
if int(inps.seasonalEndDate) < int(
ogEndDate[4:].replace('-', '')):
dataset_template.options['ssaraopt.endDate'] = str(
ogStartYearInt) + seasonalEndDateAddOn
elif int(dataset_template.options['ssaraopt.endDate']
[:4]) - ogStartYearInt + 1 == 1:
if int(inps.seasonalStartDate) > int(ogEndDate[4:].replace(
'-', '')):
return
elif int(inps.seasonalStartDate) > int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')):
dataset_template.options['ssaraopt.startDate'] = str(
ogStartYearInt) + seasonalStartDateAddOn
else:
if int(inps.seasonalStartDate) > int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')):
dataset_template.options['ssaraopt.startDate'] = str(
ogStartYearInt) + seasonalStartDateAddOn
if int(inps.seasonalEndDate) < int(ogEndDate[4:].replace(
'-', '')):
dataset_template.options['ssaraopt.endDate'] = str(
ogStartYearInt + y) + seasonalEndDateAddOn
else:
if y == 0:
if int(inps.seasonalEndDate) < int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')):
return
else:
if int(inps.seasonalStartDate) > int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')):
dataset_template.options[
'ssaraopt.startDate'] = str(
ogStartYearInt) + seasonalStartDateAddOn
dataset_template.options['ssaraopt.endDate'] = str(
ogStartYearInt) + seasonalEndDateAddOn
else:
if int(inps.seasonalStartDate) > int(
dataset_template.options['ssaraopt.startDate']
[4:].replace('-', '')):
dataset_template.options['ssaraopt.startDate'] = str(
ogStartYearInt) + seasonalStartDateAddOn
dataset_template.options['ssaraopt.endDate'] = str(
ogStartYearInt + y) + seasonalEndDateAddOn
elif x < YearRange - 1:
dataset_template.options['ssaraopt.startDate'] = str(
ogStartYearInt + x) + seasonalStartDateAddOn
dataset_template.options['ssaraopt.endDate'] = str(
ogStartYearInt + y) + seasonalEndDateAddOn
elif x == YearRange - 1:
if int(inps.seasonalEndDate) < int(ogEndDate[4:].replace('-', '')):
dataset_template.options['ssaraopt.endDate'] = str(
ogStartYearInt + y) + seasonalEndDateAddOn
else:
dataset_template.options['ssaraopt.endDate'] = ogEndDate
dataset_template.options['ssaraopt.startDate'] = str(
ogStartYearInt + x) + seasonalStartDateAddOn
ssaraopt = dataset_template.generate_ssaraopt_string()
ssaraopt = ssaraopt.split(' ')
# add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse
ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(),
ssaraopt.copy(),
delta_lat=0.0)
filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [
'--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>',
os.path.join(slc_dir, 'files.csv')
]
csv_command = ' '.join(filecsv_options)
message_rsmas.log(slc_dir, csv_command)
subprocess.Popen(csv_command, shell=True).wait()
# FA 8/2019: replaced new_files.csv by files.csv as infile argument
sed_command = "sed 's/^.\{5\}//;s/,\{1,4\}$//' " + os.path.join(slc_dir, 'files.csv') + \
">>" + os.path.join(slc_dir, 'new_files.csv')
message_rsmas.log(slc_dir, sed_command)
subprocess.Popen(sed_command, shell=True).wait()