def generate_files_csv(slc_dir, custom_template_file): """ Generates a csv file of the files to download serially. Uses the `awk` command to generate a csv file containing the data files to be download serially. The output csv file is then sent through the `sed` command to remove the first five empty values to eliminate errors in download_ASF_serial.py. """ dataset_template = Template(custom_template_file) dataset_template.options.update( PathFind.correct_for_ssara_date_format(dataset_template.options)) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat=0.0) filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [ '--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>', os.path.join(slc_dir, 'files.csv') ] csv_command = ' '.join(filecsv_options) message_rsmas.log(slc_dir, csv_command) subprocess.Popen(csv_command, shell=True).wait() # FA 8/2019: replaced new_files.csv by files.csv as infile argument sed_command = "sed 's/^.\{5\}//' " + os.path.join(slc_dir, 'files.csv') + \ ">" + os.path.join(slc_dir, 'new_files.csv') message_rsmas.log(slc_dir, sed_command) subprocess.Popen(sed_command, shell=True).wait()
""" Author: Falk Amelung, Sara Mirzaee """ ############################################################################### import os import sys import glob import subprocess import h5py from mintpy.utils import readfile import minsar.utils.process_utilities as putils from minsar.objects.auto_defaults import PathFind from minsar.objects import message_rsmas pathObj = PathFind() ########################################################################################### def main(iargs=None): """ email mintpy or insarmaps results """ inps = putils.cmd_line_parse(iargs, script='email_results') email_address = os.getenv('NOTIFICATIONEMAIL') if not iargs is None: message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(iargs[:]))
# Backwards compatibility for Python 2 from __future__ import print_function import os import sys import shutil import time import argparse import minsar import minsar.workflow from minsar.objects import message_rsmas import minsar.utils.process_utilities as putils import minsar.job_submission as js from minsar.objects.auto_defaults import PathFind pathObj = PathFind() step_list, step_help = pathObj.process_rsmas_help() ############################################################################### EXAMPLE = """example: process_rsmas.py <custom_template_file> # run with default and custom templates process_rsmas.py <custom_template_file> --submit # submit as job process_rsmas.py -h / --help # help process_rsmas.py -H # print default template options # Run with --start/stop/step options process_rsmas.py GalapagosSenDT128.template --step download # run the step 'download' only process_rsmas.py GalapagosSenDT128.template --start download # start from the step 'download' process_rsmas.py GalapagosSenDT128.template --stop ifgrams # end after step 'interferogram' """
def main(iargs=None): inps = putils.cmd_line_parse(iargs, script='download_rsmas') if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) logfile_name = inps.work_dir + '/asfserial_rsmas.log' global logger logger = RsmasLogger(file_name=logfile_name) ######################################### # Submit job ######################################### if inps.submit_flag: job_file_name = 'download_asfserial_rsmas' job_name = inps.custom_template_file.split(os.sep)[-1].split('.')[0] job_obj = JOB_SUBMIT(inps) if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) sys.exit(0) os.chdir(inps.work_dir) if not inps.template[inps.prefix + 'Stack.slcDir'] is None: inps.slc_dir = inps.template[inps.prefix + 'Stack.slcDir'] else: inps.slc_dir = os.path.join(inps.work_dir, 'SLC') global project_slc_dir project_slc_dir = os.path.join(inps.work_dir, 'SLC') if not os.path.exists(inps.slc_dir): os.mkdir(inps.slc_dir) os.chdir(inps.slc_dir) try: os.remove(os.path.expanduser('~') + '/.bulk_download_cookiejar.txt') except OSError: pass dataset_template = Template(inps.custom_template_file) dataset_template.options.update( PathFind.correct_for_ssara_date_format(dataset_template.options)) subprocess.Popen("rm " + project_slc_dir + "/new_files*.csv", shell=True).wait() seasonal_start_date = None seasonal_end_date = None try: if dataset_template.options[ 'seasonalStartDate'] is not None and dataset_template.options[ 'seasonalEndDate'] is not None: seasonal_start_date = dataset_template.options['seasonalStartDate'] seasonal_end_date = dataset_template.options['seasonalEndDate'] except: pass if inps.seasonalStartDate is not None and inps.seasonalEndDate is not None: seasonal_start_date = inps.seasonalStartDate seasonal_end_date = inps.seasonalEndDate if seasonal_start_date is not None and seasonal_end_date is not None: generate_seasonal_files_csv(dataset_template, seasonal_start_date, seasonal_end_date) else: generate_files_csv(project_slc_dir, dataset_template) parallel = False try: if dataset_template.options['parallel'] == 'yes': parallel = True except: pass """if inps.parallel == 'yes': parallel = True""" threads = os.cpu_count() try: if dataset_template.options['threads'] is not None: threads = int(dataset_template.options['threads']) except: pass """if inps.processes is not None: processes = inps.processes""" if parallel: run_parallel_download_asf_serial(project_slc_dir, threads) else: succesful = run_download_asf_serial(project_slc_dir, logger) logger.log(loglevel.INFO, "SUCCESS: %s", str(succesful)) change_file_permissions() logger.log(loglevel.INFO, "------------------------------------") subprocess.Popen("rm " + project_slc_dir + "/new_files*.csv", shell=True).wait() return None
def main(iargs=None): inps = putils.cmd_line_parse(iargs, script='download_rsmas') config = putils.get_config_defaults(config_file='job_defaults.cfg') if not iargs is None: message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(iargs[:])) else: message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(sys.argv[1::])) logfile_name = inps.work_dir + '/asfserial_rsmas.log' logger = RsmasLogger(file_name=logfile_name) ######################################### # Submit job ######################################### if inps.submit_flag: job_file_name = 'download_asfserial_rsmas' job_name = inps.custom_template_file.split(os.sep)[-1].split('.')[0] work_dir = inps.work_dir if inps.wall_time == 'None': inps.wall_time = config['download_rsmas']['walltime'] js.submit_script(job_name, job_file_name, sys.argv[:], work_dir, inps.wall_time) os.chdir(inps.work_dir) if not inps.template['topsStack.slcDir'] is None: inps.slc_dir = inps.template['topsStack.slcDir'] else: inps.slc_dir = os.path.join(inps.work_dir, 'SLC') project_slc_dir = os.path.join(inps.work_dir, 'SLC') os.chdir(inps.slc_dir) try: os.remove(os.path.expanduser('~') + '/.bulk_download_cookiejar.txt') except OSError: pass dataset_template = Template(inps.custom_template_file) dataset_template.options.update( PathFind.correct_for_ssara_date_format(dataset_template.options)) subprocess.Popen("rm new_files.csv", shell=True).wait() standardTuple = (inps, dataset_template) if inps.seasonalStartDate is not None and inps.seasonalEndDate is not None: ogStartYearInt = int( dataset_template.options['ssaraopt.startDate'][:4]) if int(inps.seasonalStartDate) > int(inps.seasonalEndDate): y = 1 else: y = 0 YearRange = int(dataset_template.options['ssaraopt.endDate'] [:4]) - ogStartYearInt + 1 if YearRange > 1 and y == 1: YearRange = YearRange - 1 seasonalStartDateAddOn = '-' + inps.seasonalStartDate[: 2] + '-' + inps.seasonalStartDate[ 2:] seasonalEndDateAddOn = '-' + inps.seasonalEndDate[: 2] + '-' + inps.seasonalEndDate[ 2:] ogEndDate = dataset_template.options['ssaraopt.endDate'] for x in range(YearRange): seasonalTuple = standardTuple + (x, ogStartYearInt, y, YearRange, seasonalStartDateAddOn, seasonalEndDateAddOn, ogEndDate) generate_files_csv(project_slc_dir, inps.custom_template_file, seasonalTuple) y += 1 else: generate_files_csv(project_slc_dir, inps.custom_template_file, standardTuple) succesful = run_download_asf_serial(project_slc_dir, logger) change_file_permissions() logger.log(loglevel.INFO, "SUCCESS: %s", str(succesful)) logger.log(loglevel.INFO, "------------------------------------") return None