def generate_files_csv(slc_dir, custom_template_file): """ Generates a csv file of the files to download serially. Uses the `awk` command to generate a csv file containing the data files to be download serially. The output csv file is then sent through the `sed` command to remove the first five empty values to eliminate errors in download_ASF_serial.py. """ dataset_template = Template(custom_template_file) dataset_template.options.update( PathFind.correct_for_ssara_date_format(dataset_template.options)) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string #FA 8/19: the delta_lat default value should come from a command_linr parse ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat=0.0) filecsv_options = ['ssara_federated_query.py'] + ssaraopt + [ '--print', '|', 'awk', "'BEGIN{FS=\",\"; ORS=\",\"}{ print $14}'", '>', os.path.join(slc_dir, 'files.csv') ] csv_command = ' '.join(filecsv_options) message_rsmas.log(slc_dir, csv_command) subprocess.Popen(csv_command, shell=True).wait() # FA 8/2019: replaced new_files.csv by files.csv as infile argument sed_command = "sed 's/^.\{5\}//' " + os.path.join(slc_dir, 'files.csv') + \ ">" + os.path.join(slc_dir, 'new_files.csv') message_rsmas.log(slc_dir, sed_command) subprocess.Popen(sed_command, shell=True).wait()
def create_default_template(temp_inps): """ :param temp_inps: input parsed arguments :return Updated template file added to temp_inps. """ inps = temp_inps inps.customTemplateFile = os.path.abspath(inps.customTemplateFile) inps.template_file = os.path.join(inps.work_dir, os.path.basename(inps.customTemplateFile)) # read custom template from file custom_tempObj = Template(os.path.abspath(inps.customTemplateFile)) # check for required options required_template_keys = pathObj.required_template_options for template_key in required_template_keys: if not template_key in custom_tempObj.options: raise Exception('ERROR: {0} is required'.format(template_key)) # find default values from template_defaults.cfg to assign to default_tempObj default_tempObj = Template(pathObj.auto_template) config_template = get_config_defaults(config_file='template_defaults.cfg') for each_section in config_template.sections(): for (each_key, each_val) in config_template.items(each_section): default_tempObj.options.update({each_key: os.path.expandvars(each_val.strip("'"))}) inps.template = default_tempObj.options pathObj.set_isce_defaults(inps) # update default_temObj with custom_tempObj for key, value in custom_tempObj.options.items(): if not value in [None, 'auto']: inps.template.update({key: os.path.expandvars(value.strip("'"))}) if os.path.exists(inps.template_file): if not os.path.samefile(inps.customTemplateFile, inps.template_file): print('generate template file: {}'.format(inps.template_file)) shutil.copyfile(inps.customTemplateFile, inps.template_file) else: print('template file exists: {}'.format(inps.template_file)) else: print('generate template file: {}'.format(inps.template_file)) shutil.copyfile(inps.customTemplateFile, inps.template_file) # updates tempDefault dictionary with the given templateObj adding new keys new_file = update_template_file(inps.template_file, custom_tempObj) with open(inps.template_file, 'w') as file: file.write(new_file) inps.cropbox = pathObj.grab_cropbox(inps) # build ssaraopt string from ssara options custom_tempObj.options.update(pathObj.correct_for_ssara_date_format(custom_tempObj.options)) inps.ssaraopt = custom_tempObj.generate_ssaraopt_string() return inps
def run_ssara(download_dir, template, delta_lat, logger, run_number=1): """ Runs ssara_federated_query-cj.py and checks for download issues. Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without comleting or exited with an error code. If either of the above occur, the function is run again, for a maxiumum of 10 times. Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting) Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success) """ # Compute SSARA options to use dataset_template = Template(template) dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options)) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat) # get kml file and create listing get_ssara_kml(download_dir, ssaraopt=ssaraopt) # Runs ssara_federated_query.bash with proper options ssara_call = ['ssara_federated_query.bash'] + ssaraopt + ['--print', '--download'] #FA 9/20: I could not figure out how to get the string into a bash shell variable, that is why writing a file #print( ' '.join(ssara_call) ) with open('../ssara_command.txt', 'w') as f: f.write(' '.join(ssara_call) + '\n') return
def create_default_template(temp_inps): """ :param temp_inps: input parsed arguments :return Updated template file added to temp_inps. """ inps = temp_inps inps.custom_template_file = os.path.abspath(inps.custom_template_file) inps.template_file = os.path.join(inps.work_dir, os.path.basename(inps.custom_template_file)) # read custom template from file custom_tempObj = Template(inps.custom_template_file) if not 'acquisition_mode' in custom_tempObj.options: print('WARNING: "acquisition_mode" is not given --> default: tops (available options: tops, stripmap)') inps.prefix = 'tops' else: inps.prefix = custom_tempObj.options['acquisition_mode'] # check for required options required_template_keys = pathObj.required_template_options(inps.prefix) for template_key in required_template_keys: if template_key not in custom_tempObj.options: raise Exception('ERROR: {0} is required'.format(template_key)) # find default values from minsar_template_defaults.cfg to assign to default_tempObj default_tempObj = Template(pathObj.auto_template) config_template = get_config_defaults(config_file='minsar_template_defaults.cfg') for each_section in config_template.sections(): for (each_key, each_val) in config_template.items(each_section): status = (inps.prefix == 'tops' and each_key.startswith('stripmap')) or \ (inps.prefix == 'stripmap' and each_key.startswith('tops')) if status: default_tempObj.options.pop(each_key) else: default_tempObj.options.update({each_key: os.path.expandvars(each_val.strip("'"))}) inps.template = default_tempObj.options pathObj.set_isce_defaults(inps) # update default_temObj with custom_tempObj for key, value in custom_tempObj.options.items(): if value not in [None, 'auto']: inps.template.update({key: os.path.expandvars(value.strip("'"))}) # update template file if necessary if not os.path.exists(inps.template_file): shutil.copyfile(inps.custom_template_file, inps.template_file) else: update_template_file(inps.template_file, custom_tempObj) inps.cropbox = pathObj.grab_cropbox(inps) # build ssaraopt string from ssara options custom_tempObj.options.update(pathObj.correct_for_ssara_date_format(custom_tempObj.options)) inps.ssaraopt = custom_tempObj.generate_ssaraopt_string() return inps
def run_ssara(work_dir, template, delta_lat): """ Runs ssara_federated_query.py and checks for differences """ # Compute SSARA options to use dataset_template = Template(template) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') # add intersectWith to ssaraopt string ssaraopt_polygon = add_polygon_to_ssaraopt(dataset_template, ssaraopt.copy(), delta_lat) # get kml file and create listing compare_ssara_listings(work_dir, ssaraopt, ssaraopt_polygon) return 0
def run_ssara(slc_dir, template, delta_lat, logger, run_number=1): """ Runs ssara_federated_query-cj.py and checks for download issues. Runs ssara_federated_query-cj.py and checks continuously for whether the data download has hung without comleting or exited with an error code. If either of the above occur, the function is run again, for a maxiumum of 10 times. Parameters: run_number: int, the current iteration the wrapper is on (maxiumum 10 before quitting) Returns: status_cod: int, the status of the donwload (0 for failed, 1 for success) """ logger.log(loglevel.INFO, "RUN NUMBER: %s", str(run_number)) if run_number > 10: return 0 logger.log(loglevel.INFO, "PASSED RUN NUMBER > 10") # Compute SSARA options to use dataset_template = Template(template) dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options)) ssaraopt = dataset_template.generate_ssaraopt_string() ssaraopt = ssaraopt.split(' ') logger.log(loglevel.INFO, "GENERATED SSARAOPT STRING") # add intersectWith to ssaraopt string ssaraopt = add_polygon_to_ssaraopt(dataset_template.get_options(), ssaraopt.copy(), delta_lat) # get kml file and create listing get_ssara_kml_and_listing(slc_dir, ssaraopt=ssaraopt) # Runs ssara_federated_query-cj.py with proper options ssara_call = ['ssara_federated_query-cj.py'] + ssaraopt + ['--print', '--download'] print('Download data using:\n' + ' '.join(ssara_call)) message_rsmas.log(slc_dir, ' '.join(ssara_call)) ssara_process = subprocess.Popen(' '.join(ssara_call), shell=True) logger.log(loglevel.INFO, "STARTED PROCESS") completion_status = ssara_process.poll() # the completion status of the process hang_status = False # whether or not the download has hung wait_time = 2 # 10 wait time in 'minutes' to determine hang status prev_size = -1 # initial download directory size i = 0 # index for waiting periods (for calculation of total time only) logger.log(loglevel.INFO, "INITIAL COMPLETION STATUS: %s", str(completion_status)) # while the process has not completed while completion_status is None: i = i + 1 # Computer the current download directory size curr_size = int(subprocess.check_output(['du', '-s', os.getcwd()]).split()[0].decode('utf-8')) # Compare the current and previous directory sizes to determine determine hang status if prev_size == curr_size: hang_status = True logger.log(loglevel.WARNING, "SSARA Hung") ssara_process.terminate() # teminate the process beacause download hung break # break the completion loop prev_size = curr_size # store current size for comparison after waiting time.sleep(60 * wait_time) # wait 'wait_time' minutes before continuing (checking for completion) completion_status = ssara_process.poll() logger.log(loglevel.INFO, "{} minutes: {:.1f}GB, completion_status {}".format(i * wait_time, curr_size / 1024 / 1024, completion_status)) exit_code = completion_status # get the exit code of the command ssara_process.terminate() logger.log(loglevel.INFO, "EXIT CODE: %s", str(exit_code)) bad_codes = [137, -9] # If the exit code is one that signifies an error, rerun the entire command if exit_code in bad_codes or hang_status: if exit_code in bad_codes: logger.log(loglevel.WARNING, "Exited with bad exit code, running again") if hang_status: logger.log(loglevel.WARNING, "Hanging, running again") run_ssara(slc_dir, template, delta_lat, logger, run_number=run_number + 1) return 0