def rerun_job_if_exit_code_140(run_file, inps_dict): """Find files that exited because walltime exceeed and run again with twice the walltime""" from minsar.job_submission import JOB_SUBMIT inps = inps_dict search_string = 'Exited with exit code 140.' files, job_files = find_completed_jobs_matching_search_string( run_file, search_string) if len(files) == 0: return rerun_file = create_rerun_run_file(job_files) wall_time = extract_walltime_from_job_file(job_files[0]) memory = extract_memory_from_job_file(job_files[0]) new_wall_time = multiply_walltime(wall_time, factor=2) print(new_wall_time) print(memory) for file in files: os.remove(file.replace('.o*', '.e*')) move_out_job_files_to_stdout(run_file) # renaming of stdout dir as otherwise it will get deleted in later move_out_job_files_to_stdout step stdout_dir = os.path.dirname(run_file) + '/stdout_' + os.path.basename( run_file) os.rename(stdout_dir, stdout_dir + '_pre_rerun') remove_last_job_running_products(run_file) #queuename = os.getenv('QUEUENAME') inps.wall_time = new_wall_time inps.work_dir = os.path.dirname(os.path.dirname(rerun_file)) inps.out_dir = os.path.dirname(rerun_file) #inps.queue = queuename job_obj = JOB_SUBMIT(inps) job_obj.write_batch_jobs(batch_file=rerun_file) jobs = job_obj.submit_batch_jobs(batch_file=rerun_file) remove_zero_size_or_length_error_files(run_file=rerun_file) raise_exception_if_job_exited(run_file=rerun_file) concatenate_error_files(run_file=rerun_file, work_dir=os.path.dirname( os.path.dirname(run_file))) move_out_job_files_to_stdout(rerun_file)
def main(iargs=None): inps = putils.cmd_line_parse(iargs, script='create_runfiles') if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) os.chdir(inps.work_dir) #time.sleep(putils.pause_seconds(inps.wait_time)) inps.out_dir = inps.work_dir inps.num_data = 1 job_obj = JOB_SUBMIT(inps) ######################################### # Submit job ######################################### if inps.submit_flag: job_name = 'create_runfiles' job_file_name = job_name if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) sys.exit(0) try: dem_dir = inps.template[inps.prefix + 'Stack.demDir'] except: dem_dir = 'DEM' try: dem_file = glob.glob(dem_dir + '/*.wgs84')[0] inps.template[inps.prefix + 'Stack.demDir'] = dem_file except: raise SystemExit('DEM does not exist') slc_dir = inps.template[inps.prefix + 'Stack.slcDir'] os.makedirs(slc_dir, exist_ok=True) if int( get_size(slc_dir) / 1024**2 ) < 500: # calculate slc_dir size in MB and see if there are SLCs according to size # Unpack Raw data: if not inps.template['raw_image_dir'] in [None, 'None']: raw_image_dir = inps.template['raw_image_dir'] else: raw_image_dir = os.path.join(inps.work_dir, 'RAW_data') if os.path.exists(raw_image_dir): unpackObj = Sensors( raw_image_dir, slc_dir, remove_file='False', multiple_raw_frame=inps.template['multiple_raw_frame']) unpack_run_file = unpackObj.start() unpackObj.close() job_obj.write_batch_jobs(batch_file=unpack_run_file) job_status = job_obj.submit_batch_jobs(batch_file=unpack_run_file) if not job_status: raise Exception('ERROR: Unpacking was failed') else: raise Exception('ERROR: No data (SLC or Raw) available') # make run file: run_files_dirname = "run_files" config_dirnane = "configs" if inps.copy_to_tmp: run_files_dirname += "_tmp" config_dirnane += "_tmp" run_dir = os.path.join(inps.work_dir, run_files_dirname) config_dir = os.path.join(inps.work_dir, config_dirnane) for directory in [run_dir, config_dir]: if os.path.exists(directory): shutil.rmtree(directory) inps.Stack_template = pathObj.correct_for_isce_naming_convention(inps) if inps.ignore_stack and os.path.exists(inps.work_dir + '/coreg_secondarys'): shutil.rmtree(inps.work_dir + '/tmp_coreg_secondarys', ignore_errors=True) shutil.move(inps.work_dir + '/coreg_secondarys', inps.work_dir + '/tmp_coreg_secondarys') runObj = CreateRun(inps) runObj.run_stack_workflow() if inps.ignore_stack and os.path.exists(inps.work_dir + '/tmp_coreg_secondarys'): shutil.move(inps.work_dir + '/tmp_coreg_secondarys', inps.work_dir + '/coreg_secondarys') if os.path.isfile(run_dir + '/run_06_extract_stack_valid_region'): with open(run_dir + '/run_06_extract_stack_valid_region', 'r') as f: line = f.readlines() with open(run_dir + '/run_06_extract_stack_valid_region', 'w') as f: f.writelines(['rm -rf ./stack; '] + line) run_file_list = putils.make_run_list(inps.work_dir) with open(inps.work_dir + '/run_files_list', 'w') as run_file: for item in run_file_list: run_file.writelines(item + '\n') if inps.copy_to_tmp: run_file_list = [ item.replace("/run_files/", "/run_files_tmp/") for item in run_file_list ] with open(inps.work_dir + '/run_files_tmp_list', 'w') as run_file: for item in run_file_list: run_file.writelines(item + '\n') shutil.copytree(pathObj.rundir, run_dir) if inps.prefix == 'tops': # check for orbits orbit_dir = os.getenv('SENTINEL_ORBITS') local_orbit = os.path.join(inps.work_dir, 'orbits') precise_orbits_in_local = glob.glob(local_orbit + '/*/*POEORB*') if len(precise_orbits_in_local) > 0: for orbit_file in precise_orbits_in_local: os.system('cp {} {}'.format(orbit_file, orbit_dir)) # Writing job files if inps.write_jobs: for item in run_file_list: job_obj.write_batch_jobs(batch_file=item) if inps.template['processingMethod'] == 'smallbaseline': job_name = 'smallbaseline_wrapper' job_file_name = job_name command = [ 'smallbaselineApp.py', inps.custom_template_file, '--dir', 'mintpy' ] # pre_command = ["""[[ $(ls mintpy/time* | wc -l) -eq 1 ]] && rm mintpy/time*"""] pre_command = ["clean_mintpy_dir.bash;"] command = pre_command + command job_obj.submit_script(job_name, job_file_name, command, writeOnly='True') else: job_name = 'minopy_wrapper' job_file_name = job_name command = [ 'minopyApp.py', inps.custom_template_file, '--dir', 'minopy' ] job_obj.submit_script(job_name, job_file_name, command, writeOnly='True') job_name = 'insarmaps' job_file_name = job_name command = ['ingest_insarmaps.py', inps.custom_template_file] job_obj.submit_script(job_name, job_file_name, command, writeOnly='True') print("copy_to_tmp: {}".format(inps.copy_to_tmp)) if inps.copy_to_tmp: #run_dir_tmp = os.path.join(inps.work_dir, 'run_files_tmp') config_dir_tmp = os.path.join(inps.work_dir, 'configs_tmp') shutil.copytree(os.path.join(inps.work_dir, "configs"), config_dir_tmp) cmd = "update_configs_for_tmp.bash {}".format(inps.work_dir) subprocess.Popen(cmd, shell=True) return None
def main(iargs=None): """ create orth and geo rectifying run jobs and submit them. """ inps = putils.cmd_line_parse(iargs) import s1a_isce_utils as ut import mergeBursts as mb if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) inps.geom_masterDir = os.path.join(inps.work_dir, pathObj.geomlatlondir) inps.master = os.path.join(inps.work_dir, pathObj.masterdir) try: inps.dem = glob.glob('{}/DEM/*.wgs84'.format(inps.work_dir))[0] except: print('DEM not exists!') sys.exit(1) if not os.path.exists(inps.geom_masterDir): os.mkdir(inps.geom_masterDir) time.sleep(putils.pause_seconds(inps.wait_time)) inps.out_dir = os.path.join(inps.work_dir, 'run_files') job_obj = JOB_SUBMIT(inps) ######################################### # Submit job ######################################### if inps.submit_flag: job_name = 'export_ortho_geo' job_file_name = job_name if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) sys.exit(0) pic_dir = os.path.join(inps.work_dir, pathObj.tiffdir) if not os.path.exists(pic_dir): os.mkdir(pic_dir) demZero = create_demZero(inps.dem, inps.geom_masterDir) swathList = ut.getSwathList(inps.master) create_georectified_lat_lon(swathList, inps.master, inps.geom_masterDir, demZero) merge_burst_lat_lon(inps) multilook_images(inps) run_file_list = make_run_list(inps) for item in run_file_list: putils.remove_last_job_running_products(run_file=item) job_status = job_obj.submit_batch_jobs(batch_file=item) if job_status: putils.remove_zero_size_or_length_error_files(run_file=item) putils.rerun_job_if_exit_code_140(run_file=item, inps_dict=inps) putils.raise_exception_if_job_exited(run_file=item) putils.concatenate_error_files(run_file=item, work_dir=inps.work_dir) putils.move_out_job_files_to_stdout(run_file=item) #upload_to_s3(pic_dir) minsar.upload_data_products.main( [inps.custom_template_file, '--imageProducts']) return
def main(iargs=None): inps = putils.cmd_line_parse(iargs, script='execute_runfiles') if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) os.chdir(inps.work_dir) time.sleep(putils.pause_seconds(inps.wait_time)) if inps.prefix == 'stripmap': inps.num_bursts = 1 inps.out_dir = os.path.join(inps.work_dir, 'run_files') job_obj = JOB_SUBMIT(inps) ######################################### # Submit job ######################################### if inps.submit_flag: job_name = 'execute_runfiles' job_file_name = job_name if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) sys.exit(0) run_file_list = putils.read_run_list(inps.work_dir) if inps.end_run == 0: inps.end_run = len(run_file_list) if not inps.start_run == 0: inps.start_run = inps.start_run - 1 if inps.step: inps.start_run = inps.step - 1 inps.end_run = inps.step run_file_list = run_file_list[inps.start_run:inps.end_run] for item in run_file_list: putils.remove_last_job_running_products(run_file=item) job_status = job_obj.submit_batch_jobs(batch_file=item) if job_status: putils.remove_zero_size_or_length_error_files(run_file=item) putils.rerun_job_if_exit_code_140(run_file=item, inps_dict=inps) putils.raise_exception_if_job_exited(run_file=item) putils.concatenate_error_files(run_file=item, work_dir=inps.work_dir) putils.move_out_job_files_to_stdout(run_file=item) date_str = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d:%H%M%S') print(date_str + ' * Job {} completed'.format(item)) date_str = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d:%H%M%S') print(date_str + ' * all jobs from {} to {} have been completed'.format( os.path.basename(run_file_list[0]), os.path.basename( run_file_list[-1]))) return None
def main(iargs=None): inps = putils.cmd_line_parse(iargs, script='create_runfiles') if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) os.chdir(inps.work_dir) time.sleep(putils.pause_seconds(inps.wait_time)) inps.out_dir = inps.work_dir job_obj = JOB_SUBMIT(inps) ######################################### # Submit job ######################################### if inps.submit_flag: job_name = 'create_runfiles' job_file_name = job_name if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) sys.exit(0) try: dem_file = glob.glob('DEM/*.wgs84')[0] inps.template[inps.prefix + 'Stack.demDir'] = dem_file except: raise SystemExit('DEM does not exist') slc_dir = inps.template[inps.prefix + 'Stack.slcDir'] os.makedirs(slc_dir, exist_ok=True) if int( get_size(slc_dir) / 1024**2 ) < 500: # calculate slc_dir size in MB and see if there are SLCs according to size # Unpack Raw data: if not inps.template['raw_image_dir'] in [None, 'None']: raw_image_dir = inps.template['raw_image_dir'] else: raw_image_dir = os.path.join(inps.work_dir, 'RAW_data') if os.path.exists(raw_image_dir): unpackObj = Sensors( raw_image_dir, slc_dir, remove_file='False', multiple_raw_frame=inps.template['multiple_raw_frame']) unpack_run_file = unpackObj.start() unpackObj.close() job_obj.write_batch_jobs(batch_file=unpack_run_file) job_status = job_obj.submit_batch_jobs(batch_file=unpack_run_file) if not job_status: raise Exception('ERROR: Unpacking was failed') else: raise Exception('ERROR: No data (SLC or Raw) available') # make run file: run_dir = os.path.join(inps.work_dir, 'run_files') config_dir = os.path.join(inps.work_dir, 'configs') for directory in [run_dir, config_dir]: if os.path.exists(directory): shutil.rmtree(directory) inps.Stack_template = pathObj.correct_for_isce_naming_convention(inps) runObj = CreateRun(inps) runObj.run_stack_workflow() run_file_list = putils.make_run_list(inps.work_dir) with open(inps.work_dir + '/run_files_list', 'w') as run_file: for item in run_file_list: run_file.writelines(item + '\n') if inps.prefix == 'tops': # check for orbits orbit_dir = os.getenv('SENTINEL_ORBITS') local_orbit = os.path.join(inps.work_dir, 'orbits') precise_orbits_in_local = glob.glob(local_orbit + '/*/*POEORB*') if len(precise_orbits_in_local) > 0: for orbit_file in precise_orbits_in_local: os.system('cp {} {}'.format(orbit_file, orbit_dir)) # Writing job files if inps.write_jobs: for item in run_file_list: job_obj.write_batch_jobs(batch_file=item) if inps.template['processingMethod'] == 'smallbaseline': job_name = 'smallbaseline_wrapper' job_file_name = job_name command = [ 'smallbaselineApp.py', inps.custom_template_file, '--dir', 'mintpy' ] job_obj.submit_script(job_name, job_file_name, command, writeOnly='True') else: job_name = 'minopy_wrapper' job_file_name = job_name command = [ 'minopyApp.py', inps.custom_template_file, '--dir', 'minopy' ] job_obj.submit_script(job_name, job_file_name, command, writeOnly='True') job_name = 'insarmaps' job_file_name = job_name command = ['ingest_insarmaps.py', inps.custom_template_file] job_obj.submit_script(job_name, job_file_name, command, writeOnly='True') return None
def main(iargs=None): """ create orth and geo rectifying run jobs and submit them. """ inps = putils.cmd_line_parse(iargs) if 'stripmap' in inps.prefix: sys.path.append(os.path.join(os.getenv('ISCE_STACK'), 'stripmapStack')) else: sys.path.append(os.path.join(os.getenv('ISCE_STACK'), 'topsStack')) from s1a_isce_utils import loadProduct, getSwathList import mergeBursts if not iargs is None: input_arguments = iargs else: input_arguments = sys.argv[1::] message_rsmas.log( inps.work_dir, os.path.basename(__file__) + ' ' + ' '.join(input_arguments)) inps.geom_referenceDir = os.path.join(inps.work_dir, pathObj.geomlatlondir) inps.reference = os.path.join(inps.work_dir, pathObj.referencedir) try: inps.dem = glob.glob('{}/DEM/*.wgs84'.format(inps.work_dir))[0] except: print('DEM not exists!') sys.exit(1) if not os.path.exists(inps.geom_referenceDir): os.mkdir(inps.geom_referenceDir) time.sleep(putils.pause_seconds(inps.wait_time)) inps.out_dir = os.path.join(inps.work_dir, 'run_files') job_obj = JOB_SUBMIT(inps) ######################################### # Submit job ######################################### if inps.submit_flag: job_name = 'export_ortho_geo' job_file_name = job_name if '--submit' in input_arguments: input_arguments.remove('--submit') command = [os.path.abspath(__file__)] + input_arguments job_obj.submit_script(job_name, job_file_name, command) pic_dir = os.path.join(inps.work_dir, pathObj.tiffdir) if not os.path.exists(pic_dir): os.mkdir(pic_dir) demZero = create_demZero(inps.dem, inps.geom_referenceDir) swathList = getSwathList(inps.reference) create_georectified_lat_lon(swathList, inps.reference, inps.geom_referenceDir, demZero, loadProduct) merge_burst_lat_lon(inps, mergeBursts) multilook_images(inps, mergeBursts) run_file_list = make_run_list(inps) for item in run_file_list: putils.remove_last_job_running_products(run_file=item) job_obj.write_batch_jobs(batch_file=item) job_status = job_obj.submit_batch_jobs(batch_file=item) if job_status: putils.remove_zero_size_or_length_error_files(run_file=item) putils.rerun_job_if_exit_code_140(run_file=item, inps_dict=inps) putils.raise_exception_if_job_exited(run_file=item) putils.concatenate_error_files(run_file=item, work_dir=inps.work_dir) putils.move_out_job_files_to_stdout(run_file=item) return