def generate_files_csv(slc_dir, custom_template_file): """ Generates a csv file of the files to download serially. Uses the `awk` command to generate a csv file containing the data files to be download serially. The output csv file is then sent through the `sed` command to remove the first five empty values to eliminate errors in download_ASF_serial.py. """ dataset_template = Template(custom_template_file) dataset_template.options.update( PathFind.correct_for_ssara_date_format(dataset_template.options)) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat=0.0) filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [ '--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>', os.path.join(slc_dir, 'files.csv') ] csv_command = ' '.join(filecsv_options) message_rsmas.log(slc_dir, csv_command) subprocess.Popen(csv_command, shell=True).wait() # FA 8/2019: replaced new_files.csv by files.csv as infile argument sed_command = "sed 's/^.\{5\}//' " + os.path.join(slc_dir, 'files.csv') + \ ">" + os.path.join(slc_dir, 'new_files.csv') message_rsmas.log(slc_dir, sed_command) subprocess.Popen(sed_command, shell=True).wait()
def get_newest_data_date(template_file): """ Obtains the most recent image date for a dataset :param template_file: the template file corresponding to the dataset being obtained :return: the newest image date in "YYYY-MM-DD T H:M:S.00000" format """ delta_lat = 0.0 # 8/2019: this should use the same default as download_ssara_rsmas.py which I believe is set in # utils/process_utilities.py: flag_parser.add_argument('--delta_lat', dest='delta_lat', default='0.0', type=float, dataset_template = Template(template_file) dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options)) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat) ssaraopt_cmd = ['ssara_federated_query.py'] + ssaraopt + ['--print'] ssaraopt_cmd = ' '.join(ssaraopt_cmd[:]) # Yield list of images in following format: # ASF,Sentinel-1A,15775,2017-03-20T11:49:56.000000,2017-03-20T11:50:25.000000,128,3592,3592,IW,NA,DESCENDING,R,VV+VH,https://datapool.asf.alaska.edu/SLC/SA/S1A_IW_SLC__1SDV_20170320T114956_20170320T115025_015775_019FA4_097A.zip ssara_output = subprocess.check_output(ssaraopt_cmd, shell=True) newest_data = ssara_output.decode('utf-8').split("\n")[-2] return datetime.strptime(newest_data.split(",")[3], DATE_FORMAT)
def call_ssara_dem(inps, cwd): print('DEM generation using SSARA') sys.stdout.flush() out_file = 'ssara_dem.log' # need to refactor so that Josh's dataset_template will be used throughout ssaraopt_string = inps.ssaraopt ssaraopt_list = ssaraopt_string.split(' ') ssaraopt_list = add_polygon_to_ssaraopt(dataset_template=inps.template, ssaraopt=ssaraopt_list.copy(), delta_lat=0) ssaraopt_string = ' '.join(ssaraopt_list) out_file = 'out_ssara_dem' command = 'ssara_federated_query.py {ssaraopt} --dem '.format( ssaraopt=ssaraopt_string) message_rsmas.log(os.getcwd(), command) command = '(' + command + ' | tee ' + out_file + '.o) 3>&1 1>&2 2>&3 | tee ' + out_file + '.e' print('command currently executing: ' + command) sys.stdout.flush() if os.getenv('DOWNLOADHOST') == 'local': print('Command: ' + command) sys.stdout.flush() try: proc = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, universal_newlines=True) error, output = proc.communicate( ) # FA 8/19 error, output works better here than output, error. Could be that stderr and stdout are switched . if proc.returncode is not 0: raise Exception( 'ERROR starting dem.py subprocess' ) # FA 8/19: I don't think this happens, errors are is output except subprocess.CalledProcessError as exc: print("Command failed. Exit code, StdErr:", exc.returncode, exc.output) sys.exit('Error produced by ssara_federated_query.py') else: if not 'Downloading DEM' in output: os.chdir('..') shutil.rmtree('DEM') sys.exit('Error in dem.py: Tiles are missing. Ocean???') else: dem_dir = os.getcwd() ssh_command_list = ['s.bgood', 'cd {0}'.format(cwd), command] host = os.getenv('DOWNLOADHOST') status = ssh_with_commands(host, ssh_command_list) print('status from ssh_with_commands:' + str(status)) sys.stdout.flush() print('Done downloading dem.grd') sys.stdout.flush() grd_to_envi_and_vrt() grd_to_xml(cwd)
def main(iargs=None): """Downloads data with ssara and asfserial scripts.""" inps = putils.cmd_line_parse(iargs, script='download_rsmas') if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log(inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) time.sleep(putils.pause_seconds(inps.wait_time)) ######################################### # Submit job ######################################### if inps.submit_flag: job_obj = JOB_SUBMIT(inps) job_name = 'download_rsmas' job_file_name = job_name if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) sys.exit(0) if inps.prefix == 'tops': if not inps.template[inps.prefix + 'Stack.slcDir'] is None: download_dir = inps.template[inps.prefix + 'Stack.slcDir'] else: download_dir = os.path.join(inps.work_dir, 'SLC') else: if not inps.template['raw_image_dir'] in [None, 'None']: download_dir = inps.template['raw_image_dir'] else: download_dir = os.path.join(inps.work_dir, 'RAW_data') os.makedirs(inps.work_dir, exist_ok=True) os.makedirs(download_dir, exist_ok=True) if 'SenDT' not in inps.project_name and 'SenAT' not in inps.project_name or os.getenv('SSARA_ASF') == 'False': try: inps.template['ssaraopt.intersectsWithPoint'] inps.ssaraopt = ' '.join(add_point_to_ssaraopt(inps.template, inps.ssaraopt.split(' '))) except: inps.ssaraopt = ' '.join(add_polygon_to_ssaraopt(inps.template, inps.ssaraopt.split(' '), delta_lat=inps.delta_lat)) command = 'ssara_federated_query.py ' + inps.ssaraopt + ' --print' + ' --download' os.chdir(download_dir) message_rsmas.log(download_dir, command) status = subprocess.Popen(command, shell=True).wait() if status is not 0: raise Exception('ERROR in ssara_federated_query.py') os.chdir(inps.work_dir) return if os.getenv('SSARA_ASF') == 'False': return download('ssara', inps.custom_template_file, download_dir, outnum=1) #download('asfserial', inps.custom_template_file, download_dir, outnum = 1) for i_download in [2, 3]: download_success = run_check_download(download_dir = download_dir) if not download_success: print('check_download.py: There were bad files, download again') message_rsmas.log(inps.work_dir,'check_download.py: there were bad files, download again') download('ssara', inps.custom_template_file, download_dir, outnum = i_download)
def generate_files_csv(slc_dir, custom_template_file, tupleParam): """ Generates a csv file of the files to download serially. Uses the `awk` command to generate a csv file containing the data files to be download serially. The output csv file is then sent through the `sed` command to remove the first five empty values to eliminate errors in download_ASF_serial.py. """ inps = tupleParam[0] dataset_template = tupleParam[1] if inps.seasonalStartDate is not None and inps.seasonalEndDate is not None: x = tupleParam[2] ogStartYearInt = tupleParam[3] y = tupleParam[4] YearRange = tupleParam[5] seasonalStartDateAddOn = tupleParam[6] seasonalEndDateAddOn = tupleParam[7] ogEndDate = tupleParam[8] if x == 0: if YearRange == 1: if y == 0: if int(inps.seasonalEndDate) < int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')) or int( inps.seasonalStartDate) > int( ogEndDate[4:].replace('-', '')): return else: if int(inps.seasonalStartDate) > int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')): dataset_template.options[ 'ssaraopt.startDate'] = str( ogStartYearInt) + seasonalStartDateAddOn if int(inps.seasonalEndDate) < int( ogEndDate[4:].replace('-', '')): dataset_template.options['ssaraopt.endDate'] = str( ogStartYearInt) + seasonalEndDateAddOn elif int(dataset_template.options['ssaraopt.endDate'] [:4]) - ogStartYearInt + 1 == 1: if int(inps.seasonalStartDate) > int(ogEndDate[4:].replace( '-', '')): return elif int(inps.seasonalStartDate) > int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')): dataset_template.options['ssaraopt.startDate'] = str( ogStartYearInt) + seasonalStartDateAddOn else: if int(inps.seasonalStartDate) > int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')): dataset_template.options['ssaraopt.startDate'] = str( ogStartYearInt) + seasonalStartDateAddOn if int(inps.seasonalEndDate) < int(ogEndDate[4:].replace( '-', '')): dataset_template.options['ssaraopt.endDate'] = str( ogStartYearInt + y) + seasonalEndDateAddOn else: if y == 0: if int(inps.seasonalEndDate) < int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')): return else: if int(inps.seasonalStartDate) > int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')): dataset_template.options[ 'ssaraopt.startDate'] = str( ogStartYearInt) + seasonalStartDateAddOn dataset_template.options['ssaraopt.endDate'] = str( ogStartYearInt) + seasonalEndDateAddOn else: if int(inps.seasonalStartDate) > int( dataset_template.options['ssaraopt.startDate'] [4:].replace('-', '')): dataset_template.options['ssaraopt.startDate'] = str( ogStartYearInt) + seasonalStartDateAddOn dataset_template.options['ssaraopt.endDate'] = str( ogStartYearInt + y) + seasonalEndDateAddOn elif x < YearRange - 1: dataset_template.options['ssaraopt.startDate'] = str( ogStartYearInt + x) + seasonalStartDateAddOn dataset_template.options['ssaraopt.endDate'] = str( ogStartYearInt + y) + seasonalEndDateAddOn elif x == YearRange - 1: if int(inps.seasonalEndDate) < int(ogEndDate[4:].replace('-', '')): dataset_template.options['ssaraopt.endDate'] = str( ogStartYearInt + y) + seasonalEndDateAddOn else: dataset_template.options['ssaraopt.endDate'] = ogEndDate dataset_template.options['ssaraopt.startDate'] = str( ogStartYearInt + x) + seasonalStartDateAddOn ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat=0.0) filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [ '--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>', os.path.join(slc_dir, 'files.csv') ] csv_command = ' '.join(filecsv_options) message_rsmas.log(slc_dir, csv_command) subprocess.Popen(csv_command, shell=True).wait() # FA 8/2019: replaced new_files.csv by files.csv as infile argument sed_command = "sed 's/^.\{5\}//;s/,\{1,4\}$//' " + os.path.join(slc_dir, 'files.csv') + \ ">>" + os.path.join(slc_dir, 'new_files.csv') message_rsmas.log(slc_dir, sed_command) subprocess.Popen(sed_command, shell=True).wait()