def test_args(self): positional_arg_keys = ['positional'] arg_keys_to_remove = ['toremove', 'to_remove'] arg_str = taskhandler.convert_optional_args_to_string(self.args, positional_arg_keys, arg_keys_to_remove) self.assertIn('--tuple item1 item2', arg_str) self.assertIn('--list item1 item2', arg_str) self.assertIn('--boolean', arg_str) self.assertIn('--multi-word-key multi-word-key', arg_str) self.assertNotIn('positional', arg_str) self.assertNotIn('toremove', arg_str) self.assertNotIn('to-remove', arg_str) self.assertNotIn('to_remove', arg_str)
def main(): parser = argparse.ArgumentParser() #### Set Up Options parser.add_argument("src", help="source directory or image") parser.add_argument("--dstdir", help="dstination directory") parser.add_argument("-o", "--overwrite", action="store_true", default=False, help="overwrite existing files if present") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "qsub script to use in scheduler submission (PBS default is qsub_resample.sh, SLURM default is slurm_resample.sh)" ) parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") pos_arg_keys = ['src'] #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) #### Validate Required Arguments if not os.path.isdir(src) and not os.path.isfile(src): parser.error('src must be avalid directory or file') ## Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_resample.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_resample.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" % qsubpath) ## Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error( "HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive" ) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) task_queue = [] i = 0 logger.info("Searching for SETSM rasters") if os.path.isfile(src): if src.endswith(('dem.tif', 'matchtag.tif', 'ortho.tif')): srcfp = src regfp = srcfp.replace('dem.tif', 'reg.txt').replace( 'matchtag.tif', 'reg.txt').replace('ortho.tif', 'reg.txt') if not os.path.isfile(regfp): logger.info("No regfile found for {}".format(src)) else: if args.dstdir: dstfp = "{}_reg.tif".format( os.path.join( args.dstdir, os.path.basename(os.path.splitext(srcfp)[0]))) else: dstfp = "{}_reg.tif".format(os.path.splitext(srcfp)[0]) if not os.path.isfile(dstfp): i += 1 task = taskhandler.Task( os.path.basename(srcfp), 'Reg{:04g}'.format(i), 'python', '{} {} {}'.format(scriptpath, arg_str_base, srcfp), apply_reg, [srcfp, args]) task_queue.append(task) else: for root, dirs, files in os.walk(src): for f in files: if f.endswith(('dem.tif', 'matchtag.tif', 'ortho.tif')): srcfp = os.path.join(root, f) regfp = srcfp.replace('dem.tif', 'reg.txt').replace( 'matchtag.tif', 'reg.txt').replace('ortho.tif', 'reg.txt') if not os.path.isfile(regfp): logger.info("No regfile found for {}".format(srcfp)) else: if args.dstdir: dstfp = "{}_reg.tif".format( os.path.join( args.dstdir, os.path.basename( os.path.splitext(srcfp)[0]))) else: dstfp = "{}_reg.tif".format( os.path.splitext(srcfp)[0]) if not os.path.isfile(dstfp): i += 1 task = taskhandler.Task( f, 'Reg{:04g}'.format(i), 'python', '{} {} {}'.format(scriptpath, arg_str_base, srcfp), apply_reg, [srcfp, args]) task_queue.append(task) logger.info('Number of incomplete tasks: {}'.format(i)) if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: try: task_handler = taskhandler.PBSTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: {0}".format( task_handler.num_processes)) if not args.dryrun: task_handler.run_tasks(task_queue) else: for task in task_queue: src, task_arg_obj = task.method_arg_list #### Set up processing log handler if args.dstdir: logfile = "{}.log".format( os.path.join( args.dstdir, os.path.basename(os.path.splitext(src)[0]))) else: logfile = os.path.splitext(src)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: task.method(src, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) else: logger.info("No tasks found to process")
def main(): parser = argparse.ArgumentParser() #### Set Up Options parser.add_argument("srcdir", help="source directory or image") parser.add_argument("dstdir", help="destination directory") parser.add_argument("epsg", type=int, help="target epsg code") parser.add_argument( "-r", "--resolution", default=default_res, type=int, help="output resolution (default={})".format(default_res)) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "qsub script to use in PBS submission (default is qsub_resample.sh in script root folder)" ) parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") pos_arg_keys = ['srcdir', 'dstdir', 'epsg'] #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) srcpath = os.path.abspath(args.srcdir) #### Validate Required Arguments if not os.path.isdir(srcpath) and not os.path.isfile(srcpath): parser.error('src must be a valid directory or file') dstdir = os.path.abspath(args.dstdir) if not os.path.isdir(dstdir): os.makedirs(dstdir) ## Verify EPSG test_sr = osr.SpatialReference() status = test_sr.ImportFromEPSG(args.epsg) if status != 0: parser.error( 'EPSG test osr.SpatialReference.ImportFromEPSG returns error code %d' % status) ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_resample.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" % qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error( "Options --pbs and --parallel-processes > 1 are mutually exclusive" ) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) task_queue = [] i = 0 logger.info("Searching for SETSM rasters") if os.path.isfile(srcpath): if srcpath.endswith('dem.tif'): raster = srcpath new_raster = os.path.join(args.dstdir, os.path.basename(raster)) if not os.path.isfile(new_raster): i += 1 task = taskhandler.Task( os.path.basename(raster), 'Reproject{:04g}'.format(i), 'python', '{} {} {} {} {}'.format(scriptpath, arg_str_base, raster, dstdir, args.epsg), resample_setsm, [raster, dstdir, args]) task_queue.append(task) else: for root, dirs, files in os.walk(srcpath): for f in files: if f.endswith('dem.tif'): raster = os.path.join(root, f) new_raster = os.path.join(args.dstdir, os.path.basename(raster)) if not os.path.isfile(new_raster): i += 1 task = taskhandler.Task( f, 'Reproject{:04g}'.format(i), 'python', '{} {} {} {} {}'.format(scriptpath, arg_str_base, raster, dstdir, args.epsg), resample_setsm, [raster, dstdir, args]) task_queue.append(task) logger.info('Number of incomplete tasks: {}'.format(i)) if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: task_handler = taskhandler.PBSTaskHandler(qsubpath) if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) logger.info("Number of child processes to spawn: {0}".format( task_handler.num_processes)) if not args.dryrun: task_handler.run_tasks(task_queue) else: for task in task_queue: src, dst, task_arg_obj = task.method_arg_list if not args.dryrun: #### Set up processing log handler logfile = os.path.join( dst, os.path.splitext(os.path.basename(src))[0] + ".log") lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) task.method(src, dst, task_arg_obj) else: logger.info("No tasks found to process")
if args.dem: if not os.path.isfile(args.dem): parser.error("DEM does not exist: %s" % args.dem) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] ## Group Ikonos image_list2 = [] for srcfp in image_list1: srcdir, srcfn = os.path.split(srcfp) if "IK01" in srcfn and sum( [b in srcfn for b in ortho_functions.ikMsiBands]) > 0:
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description="divide setsm mosaics into subtiles") #### Positional Arguments parser.add_argument('src', help="source directory or dem") pos_arg_keys = ['src'] #### Optionsl Arguments parser.add_argument('--log', help="directory for log output") parser.add_argument('--num-rows', type=int, default=1, help="number of subtile rows") parser.add_argument('--num-cols', type=int, default=1, help="number of subtile columns") parser.add_argument('--res', type=int, default=2, help="resolution in meters") parser.add_argument('--tiles', help="list of tiles to process, comma delimited") parser.add_argument("--version", help="version string (ex: v1.2)") parser.add_argument( "--cutline-loc", help="directory containing cutline shps indicating areas of bad data") parser.add_argument('--build-ovr', action='store_true', default=False, help="build overviews") parser.add_argument( '--resample', default="bilinear", help= "dem_resampling strategy (default=bilinear). matchtag resampling is always nearest neighbor" ) parser.add_argument('--dryrun', action='store_true', default=False, help="print actions without executing") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "qsub script to use in PBS submission (default is qsub_divide.sh in script root folder)" ) #### Parse Arguments args = parser.parse_args() #### Verify Arguments src = os.path.abspath(args.src) if not os.path.isdir(src) and not os.path.isfile(src): parser.error("Source directory or file does not exist: %s" % src) if args.cutline_loc: if not os.path.isdir(args.cutline_loc): parser.error("Cutline directory does not exist: {}".format( args.cutline_loc)) scriptpath = os.path.abspath(sys.argv[0]) ## Verify qsubscript if args.pbs: if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_divide.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" % qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error( "Options --pbs and --parallel-processes > 1 are mutually exclusive" ) if args.version: version_str = '_{}'.format(args.version) else: version_str = '' if args.tiles: tiles = args.tiles.split(',') #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes', 'tiles') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) task_queue = [] i = 0 #### ID rasters logger.info('Identifying DEM mosaics') if os.path.isfile(src): logger.info(src) try: raster = dem.SetsmTile(src) except RuntimeError as e: logger.error(e) else: if src.endswith('reg_dem.tif'): dstfp_list = glob.glob('{}*{}m{}_reg_dem.tif'.format( src[:-15], args.res, version_str)) else: dstfp_list = glob.glob('{}*{}m{}_dem.tif'.format( src[:-12], args.res, version_str)) #### verify that cutlines can be found if requested if args.cutline_loc: tile = raster.tile_name cutline_shp = os.path.join(args.cutline_loc, tile + '_cut.shp') if not os.path.isfile(cutline_shp): logger.warning("Cutline shp not found for tile {}".format( raster.tileid)) if len(dstfp_list) == 0: i += 1 task = taskhandler.Task( raster.tileid, 'div_{}'.format(raster.tileid), 'python', '{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp), divide_tile, [raster.srcfp, args]) task_queue.append(task) else: logger.info("output tile(s) already exist: {}".format(src)) else: for root, dirs, files in os.walk(src): for f in files: if f.endswith(("2m_dem.tif", "2m_reg_dem.tif")): if args.tiles: tile = f[:5] process = True if tile in tiles else False else: process = True if process: srcfp = os.path.join(root, f) try: raster = dem.SetsmTile(srcfp) except RuntimeError as e: logger.error(e) else: if srcfp.endswith('reg_dem.tif'): dstfp_list = glob.glob( '{}*{}m{}_reg_dem.tif'.format( srcfp[:-15], args.res, version_str)) else: dstfp_list = glob.glob( '{}*{}m{}_dem.tif'.format( srcfp[:-12], args.res, version_str)) if len(dstfp_list) == 0: logger.info("computing tile: {}".format(srcfp)) #### verify that cutlines can be found if requested if args.cutline_loc: tile = raster.tilename cutline_shp = os.path.join( args.cutline_loc, tile + '_cut.shp') if not os.path.isfile(cutline_shp): logger.warning( "Cutline shp not found for tile {}" .format(raster.tileid)) i += 1 task = taskhandler.Task( raster.tileid, 'div_{}'.format(raster.tileid), 'python', '{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp), divide_tile, [raster.srcfp, args]) #print '{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp) task_queue.append(task) else: logger.info( "output tile(s) already exist: {}".format( ','.join(dstfp_list))) logger.info('Number of incomplete tasks: {}'.format(i)) if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: task_handler = taskhandler.PBSTaskHandler(qsubpath) if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) logger.info("Number of child processes to spawn: {0}".format( task_handler.num_processes)) if not args.dryrun: task_handler.run_tasks(task_queue) else: for task in task_queue: src, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(src)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: task.method(src, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) else: logger.info("No tasks found to process")
def main(): #### Set Up Arguments parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Run/submit batch image ortho and conversion tasks" ) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument("--tasks-per-job", type=int, help="Number of tasks to bundle into a single job. (requires --pbs or --slurm option) (Warning:" " a higher number of tasks per job may require modification of default wallclock limit.)") parser.add_argument('--scratch', default=ARGDEF_SCRATCH, help="Scratch space to build task bundle text files. (default={})".format(ARGDEF_SCRATCH)) parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM " "default is slurm_ortho.py, in script root folder)") parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") parser.add_argument("--dryrun", action='store_true', default=False, help='print actions without executing') #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) scratch = os.path.abspath(args.scratch) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error("Error arg1 is not a recognized file path or file type: {}".format(src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) ## Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ortho.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ortho.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ## Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error("HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive") if args.tasks_per_job: if not (args.pbs or args.slurm): parser.error("--jobs-per-task option requires the (--pbs or --slurm) option") if not os.path.isdir(args.scratch): print("Creating --scratch directory: {}".format(args.scratch)) os.makedirs(args.scratch) #### Verify EPSG try: spatial_ref = utils.SpatialRef(args.epsg) except RuntimeError as e: parser.error(e) #### Verify that dem and ortho_height are not both specified if args.dem is not None and args.ortho_height is not None: parser.error("--dem and --ortho_height options are mutually exclusive. Please choose only one.") #### Test if DEM exists if args.dem: if not os.path.isfile(args.dem): parser.error("DEM does not exist: {}".format(args.dem)) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes', 'tasks_per_job') arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] ## Group Ikonos image_list2 = [] for srcfp in image_list1: srcdir, srcfn = os.path.split(srcfp) if "IK01" in srcfn and sum([b in srcfn for b in ortho_functions.ikMsiBands]) > 0: for b in ortho_functions.ikMsiBands: if b in srcfn: newname = os.path.join(srcdir, srcfn.replace(b, "msi")) break image_list2.append(newname) else: image_list2.append(srcfp) image_list = list(set(image_list2)) logger.info('Number of src images: %i', len(image_list)) ## Build task queue i = 0 images_to_process = [] for srcfp in image_list: srcdir, srcfn = os.path.split(srcfp) dstfp = os.path.join(dstdir, "{}_{}{}{}{}".format( os.path.splitext(srcfn)[0], utils.get_bit_depth(args.outtype), args.stretch, spatial_ref.epsg, ortho_functions.formats[args.format] )) done = os.path.isfile(dstfp) if done is False: i += 1 images_to_process.append(srcfp) logger.info('Number of incomplete tasks: %i', i) if len(images_to_process) == 0: logger.info("No images found to process") sys.exit(0) tm = datetime.now() job_count = 0 image_count = 0 scenes_in_job_count = 0 task_queue = [] for srcfp in images_to_process: image_count += 1 srcdir, srcfn = os.path.split(srcfp) dstfp = os.path.join(dstdir, "{}_{}{}{}{}".format( os.path.splitext(srcfn)[0], utils.get_bit_depth(args.outtype), args.stretch, spatial_ref.epsg, ortho_functions.formats[args.format] )) if args.tasks_per_job: # bundle tasks into text files in the dst dir and pass the text file in as src scenes_in_job_count += 1 src_txt = os.path.join(scratch, 'Or_src_{}_{}.txt'.format(tm.strftime("%Y%m%d%H%M%S"), job_count + 1)) if scenes_in_job_count == 1: # remove text file if dst already exists try: os.remove(src_txt) except OSError: pass if scenes_in_job_count <= args.tasks_per_job: # add to txt file fh = open(src_txt, 'a') fh.write("{}\n".format(srcfp)) fh.close() if scenes_in_job_count == args.tasks_per_job or image_count == len(images_to_process): scenes_in_job_count = 0 job_count += 1 task = taskhandler.Task( srcfn, 'Or{:04g}'.format(job_count), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, src_txt, dstdir), ortho_functions.process_image, [srcfp, dstfp, args] ) task_queue.append(task) else: job_count += 1 task = taskhandler.Task( srcfn, 'Or{:04g}'.format(job_count), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, dstdir), ortho_functions.process_image, [srcfp, dstfp, args] ) task_queue.append(task) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} for task in task_queue: src, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(src, dstfp, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) #### Print Images with Errors for k, v in results.items(): if v != 0: logger.warning("Failed Image: %s", k) logger.info("Done") else: logger.info("No images found to process")
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description="Sumbit/run batch mosaic tasks") parser.add_argument( "src", help="textfile or directory of input rasters (tif only)") parser.add_argument("mosaicname", help="output mosaic name excluding extension") pos_arg_keys = ["src", "mosaicname"] parser.add_argument( "-r", "--resolution", nargs=2, type=float, help= "output pixel resolution -- xres yres (default is same as first input file)" ) parser.add_argument( "-e", "--extent", nargs=4, type=float, help= "extent of output mosaic -- xmin xmax ymin ymax (default is union of all inputs)" ) parser.add_argument( "-t", "--tilesize", nargs=2, type=float, help= "tile size in coordinate system units -- xsize ysize (default is 40,000 times output " "resolution)") parser.add_argument( "--force-pan-to-multi", action="store_true", default=False, help="if output is multiband, force script to also use 1 band images") parser.add_argument( "-b", "--bands", type=int, help= "number of output bands( default is number of bands in the first image)" ) parser.add_argument( "--tday", help= "month and day of the year to use as target for image suitability ranking -- 04-05" ) parser.add_argument( "--tyear", help= "year (or year range) to use as target for image suitability ranking -- 2017 or 2015-2017" ) parser.add_argument( "--nosort", action="store_true", default=False, help= "do not sort images by metadata. script uses the order of the input textfile or directory " "(first image is first drawn). Not recommended if input is a directory; order will be " "random") parser.add_argument( "--use-exposure", action="store_true", default=False, help="use exposure settings in metadata to inform score") parser.add_argument( "--exclude", help= "file of file name patterns (text only, no wildcards or regexs) to exclude" ) parser.add_argument( "--max-cc", type=float, default=0.2, help="maximum fractional cloud cover (0.0-1.0, default 0.5)") parser.add_argument( "--include-all-ms", action="store_true", default=False, help= "include all multispectral imagery, even if the imagery has differing numbers of bands" ) parser.add_argument( "--min-contribution-area", type=int, default=20000000, help= "minimum area contribution threshold in target projection units (default=20000000). " "Higher values remove more image slivers from the resulting mosaic") parser.add_argument( "--median-remove", action="store_true", default=False, help= "subtract the median from each input image before forming the mosaic in order to correct " "for contrast") parser.add_argument( "--allow-invalid-geom", action="store_true", default=False, help= "normally, if 1 or more images has a invalid geometry, a tile will not be created. this " "option will attempt to create a mosaic with the remaining valid geometries, if any." ) parser.add_argument( "--mode", choices=mosaic.MODES, default="ALL", help= " mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, " "TEST- create log only") parser.add_argument("--wd", help="scratch space (default is mosaic directory)") parser.add_argument("--component-shp", action="store_true", default=False, help="create shp of all componenet images") parser.add_argument( "--cutline-step", type=int, default=2, help="cutline calculator pixel skip interval (default=2)") parser.add_argument( "--calc-stats", action="store_true", default=False, help="calculate image stats and record them in the index") parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw", help="GTiff compression type. Default=lzw ({})".format( ",".join(mosaic.GTIFF_COMPRESSIONS))) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "submission script to use in PBS/SLURM submission (PBS default is qsub_mosaic.sh, SLURM " "default is slurm_mosaic.py, in script root folder)") parser.add_argument( "-l", help= "PBS resources requested (mimicks qsub syntax). Use only on HPC systems." ) parser.add_argument( "--log", help="file to log progress (default is <output dir>\{}".format( default_logfile)) parser.add_argument("--version", action='version', version="imagery_utils v{}".format( utils.package_version)) #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) inpath = os.path.abspath(args.src) mosaicname = os.path.abspath(args.mosaicname) mosaicname = os.path.splitext(mosaicname)[0] mosaic_dir = os.path.dirname(mosaicname) tile_builder_script = os.path.join(os.path.dirname(scriptpath), 'pgc_mosaic_build_tile.py') ## Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_mosaic.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_mosaic.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ## Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") #### Validate Arguments if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False else: parser.error( "Arg1 is not a valid file path or directory: {}".format(inpath)) #### Validate target day option if args.tday is not None: try: m = int(args.tday.split("-")[0]) d = int(args.tday.split("-")[1]) td = date(2000, m, d) except ValueError: parser.error("Target day must be in mm-dd format (i.e 04-05)") sys.exit(1) else: m = 0 d = 0 #### Validate target year/year range option if args.tyear is not None: if len(str(args.tyear)) == 4: ## ensure single year is valid try: tyear_test = datetime(year=args.tyear, month=1, day=1) except ValueError: parser.error("Supplied year {0} is not valid".format( args.tyear)) sys.exit(1) elif len(str(args.tyear)) == 9: if '-' in args.tyear: ## decouple range and build year yrs = args.tyear.split('-') yrs_range = range(int(yrs[0]), int(yrs[1]) + 1) for yy in yrs_range: try: tyear_test = datetime(year=yy, month=1, day=1) except ValueError: parser.error( "Supplied year {0} in range {1} is not valid". format(yy, args.tyear)) sys.exit(1) else: parser.error( "Supplied year range {0} is not valid; should be like: 2015 OR 2015-2017" .format(args.tyear)) sys.exit(1) else: parser.error( "Supplied year {0} is not valid, or its format is incorrect; should be 4 digits for single " "year (e.g., 2017), eight digits and dash for range (e.g., 2015-2017)" .format(args.tyear)) sys.exit(1) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): parser.error("Value for option --exclude-list is not a valid file") ## Build tasks task_queue = [] #### Create task for mosaic arg_keys_to_remove = ('l', 'qsubscript', 'pbs', 'slurm') mos_arg_str = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) cmd = r'{} {} {} {}'.format(scriptpath, mos_arg_str, inpath, mosaicname) task = taskhandler.Task('Mosaic {}'.format(os.path.basename(mosaicname)), 'Mos{:04g}'.format(1), 'python', cmd) task_queue.append(task) #logger.info(task_queue) if len(task_queue) > 0: if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: task_handler.run_tasks(task_queue) else: try: run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_arg_keys) except RuntimeError as e: logger.error(e) else: logger.info("No tasks to process")
def main(): #### Set Up Arguments parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Run/Submit batch pansharpening in parallel") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "submission script to use in PBS/SLURM submission (PBS default is qsub_pansharpen.sh, " "SLURM default is slurm_pansharpen.py, in script root folder)") parser.add_argument( "-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext( src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext( src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error( "Error arg1 is not a recognized file path or file type: {}".format( src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) # Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_pansharpen.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_pansharpen.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ### Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error( "HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive" ) #### Verify EPSG try: spatial_ref = utils.SpatialRef(args.epsg) except RuntimeError as e: parser.error(e) ## Check GDAL version (2.1.0 minimum) gdal_version = gdal.VersionInfo() try: if int(gdal_version) < 2010000: parser.error( "gdal_pansharpen requires GDAL version 2.1.0 or higher") except ValueError: parser.error("Cannot parse GDAL version: {}".format(gdal_version)) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] pair_list = [] for srcfp in image_list1: #print(srcfp) try: image_pair = ImagePair(srcfp, spatial_ref) except RuntimeError as e: logger.error(e) else: logger.info("Image: %s, Sensor: %s", image_pair.mul_srcfn, image_pair.sensor) pair_list.append(image_pair) logger.info('Number of src image pairs: %i', len(pair_list)) ## Build task queue i = 0 task_queue = [] for image_pair in pair_list: bittype = utils.get_bit_depth(args.outtype) pansh_dstfp = os.path.join( dstdir, "{}_{}{}{}_pansh.tif".format( os.path.splitext(image_pair.mul_srcfn)[0], bittype, args.stretch, args.epsg)) if not os.path.isfile(pansh_dstfp): i += 1 task = taskhandler.Task( image_pair.mul_srcfn, 'Psh{:04g}'.format(i), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, image_pair.mul_srcfp, dstdir), exec_pansharpen, [image_pair, pansh_dstfp, args]) task_queue.append(task) logger.info('Number of incomplete tasks: %i', i) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} lfh = None for task in task_queue: src, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(src, dstfp, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) #### remove existing file handler logger.removeHandler(lfh) #### Print Images with Errors for k, v in results.items(): if v != 0: logger.warning("Failed Image: %s", k) logger.info("Done") else: logger.info("No images found to process")
def main(): #### Set Up Arguments parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Run/Submit batch pansharpening in parallel" ) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="submission script to use in PBS/SLURM submission (PBS default is qsub_pansharpen.sh, " "SLURM default is slurm_pansharpen.py, in script root folder)") parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error("Error arg1 is not a recognized file path or file type: {}".format(src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) # Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_pansharpen.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_pansharpen.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ### Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error("HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive") #### Verify EPSG try: spatial_ref = utils.SpatialRef(args.epsg) except RuntimeError as e: parser.error(e) ## Check GDAL version (2.1.0 minimum) gdal_version = gdal.VersionInfo() try: if int(gdal_version) < 2010000: parser.error("gdal_pansharpen requires GDAL version 2.1.0 or higher") except ValueError: parser.error("Cannot parse GDAL version: {}".format(gdal_version)) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] pair_list = [] for srcfp in image_list1: #print(srcfp) try: image_pair = ImagePair(srcfp, spatial_ref) except RuntimeError as e: logger.error(e) else: logger.info("Image: %s, Sensor: %s", image_pair.mul_srcfn, image_pair.sensor) pair_list.append(image_pair) logger.info('Number of src image pairs: %i', len(pair_list)) ## Build task queue i = 0 task_queue = [] for image_pair in pair_list: bittype = utils.get_bit_depth(args.outtype) pansh_dstfp = os.path.join(dstdir, "{}_{}{}{}_pansh.tif".format(os.path.splitext(image_pair.mul_srcfn)[0], bittype, args.stretch, args.epsg)) if not os.path.isfile(pansh_dstfp): i += 1 task = taskhandler.Task( image_pair.mul_srcfn, 'Psh{:04g}'.format(i), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, image_pair.mul_srcfp, dstdir), exec_pansharpen, [image_pair, pansh_dstfp, args] ) task_queue.append(task) logger.info('Number of incomplete tasks: %i', i) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} lfh = None for task in task_queue: src, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(src, dstfp, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) #### remove existing file handler logger.removeHandler(lfh) #### Print Images with Errors for k, v in results.items(): if v != 0: logger.warning("Failed Image: %s", k) logger.info("Done") else: logger.info("No images found to process")
def main(): parser = argparse.ArgumentParser() #### Set Up Options parser.add_argument("srcdir", help="source directory or image") parser.add_argument("-c", "--component", choices=suffixes, default='dem', help="SETSM DEM component to resample") parser.add_argument("-r", "--resolution", default=default_res, type=int, help="output resolution (default={})".format(default_res)) parser.add_argument("-o", "--overwrite", action="store_true", default=False, help="overwrite existing files if present") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="qsub script to use in PBS submission (default is qsub_resample.sh in script root folder)") parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") pos_arg_keys = ['srcdir'] #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) path = os.path.abspath(args.srcdir) #### Validate Required Arguments if not os.path.isdir(path) and not os.path.isfile(path): parser.error('src must be a valid directory or file') ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_resample.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) task_queue = [] i=0 logger.info("Searching for SETSM rasters") if os.path.isfile(path): if path.endswith('{}.tif'.format(args.component)): dem = path low_res_dem = "{}.tif".format(os.path.splitext(dem)[0]) low_res_dem = os.path.join(os.path.dirname(low_res_dem),os.path.basename(low_res_dem.replace("_2m","_{}m".format(args.resolution)))) if dem != low_res_dem and not os.path.isfile(low_res_dem): i+=1 task = taskhandler.Task( os.path.basename(dem), 'Resample{:04g}'.format(i), 'python', '{} {} {}'.format(scriptpath, arg_str_base, dem), resample_setsm, [dem, args] ) task_queue.append(task) else: for root,dirs,files in os.walk(path): for f in files: if f.endswith('{}.tif'.format(args.component)): dem = os.path.join(root,f) low_res_dem = "{}.tif".format(os.path.splitext(dem)[0]) low_res_dem = os.path.join(os.path.dirname(low_res_dem),os.path.basename(low_res_dem.replace("_2m","_{}m".format(args.resolution)))) if dem != low_res_dem and not os.path.isfile(low_res_dem): i+=1 task = taskhandler.Task( f, 'Resample{:04g}'.format(i), 'python', '{} {} {}'.format(scriptpath, arg_str_base, dem), resample_setsm, [dem, args] ) task_queue.append(task) logger.info('Number of incomplete tasks: {}'.format(i)) if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: task_handler = taskhandler.PBSTaskHandler(qsubpath) if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes)) if not args.dryrun: task_handler.run_tasks(task_queue) else: for task in task_queue: src, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(src)[0]+".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: task.method(src, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) else: logger.info("No tasks found to process")
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description="Run/Submit batch ndvi calculation in parallel" ) parser.add_argument("src", help="source image, text file, or directory") parser.add_argument("dst", help="destination directory") pos_arg_keys = ["src", "dst"] parser.add_argument("-t", "--outtype", choices=outtypes, default='Float32', help="output data type (for Int16, output values are scaled from -1000 to 1000)") parser.add_argument("-s", "--save-temps", action="store_true", default=False, help="save temp files") parser.add_argument("--wd", help="local working directory for cluster jobs (default is dst dir)") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="submission script to use in PBS/SLURM submission (PBS default is qsub_ndvi.sh, SLURM " "default is slurm_ndvi.py, in script root folder)") parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") parser.add_argument("--version", action='version', version="imagery_utils v{}".format(utils.package_version)) #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error("Error arg1 is not a recognized file path or file type: {}".format(src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) ## Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ndvi.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ndvi.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ## Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error("HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive") #### Set concole logging handler lso = logging.StreamHandler() lso.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'pbs', 'slurm', 'parallel_processes', 'dryrun') arg_str = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list = utils.find_images(src, True, ortho_functions.exts) else: image_list = [src] logger.info('Number of src images: %i', len(image_list)) ## Build task queue i = 0 task_queue = [] for srcfp in image_list: srcdir, srcfn = os.path.split(srcfp) bn, ext = os.path.splitext(srcfn) dstfp = os.path.join(dstdir, bn + '_ndvi.tif') if not os.path.isfile(dstfp): i += 1 task = taskhandler.Task( srcfn, 'NDVI{:04g}'.format(i), 'python', '{} {} {} {}'.format(scriptpath, arg_str, srcfp, dstdir), calc_ndvi, [srcfp, dstfp, args] ) task_queue.append(task) logger.info('Number of incomplete tasks: %i', i) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} for task in task_queue: srcfp, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(srcfp, dstfp, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) #### Print Images with Errors for k, v in results.items(): if v != 0: logger.warning("Failed Image: %s", k) logger.info("Done") else: logger.info("No images found to process")
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description="calculate density for setsm tile or subtile DEMS and save to a density.txt file" ) #### Positional Arguments parser.add_argument('src', help="source directory or dem") parser.add_argument('scratch', help="scratch space") #### Optionsl Arguments parser.add_argument('-v', action='store_true', default=False, help="verbose output") parser.add_argument("--tasks-per-job", type=int, help="number of tasks to bundle into a single job (requires pbs option)") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="qsub script to use in PBS submission (default is qsub_package.sh in script root folder)") parser.add_argument('--dryrun', action='store_true', default=False, help="print actions without executing") pos_arg_keys = ['src','scratch'] #### Parse Arguments scriptpath = os.path.abspath(sys.argv[0]) args = parser.parse_args() src = os.path.abspath(args.src) scratch = os.path.abspath(args.scratch) #### Verify Arguments if not os.path.isdir(args.src) and not os.path.isfile(args.src): parser.error("Source directory or file does not exist: %s" %args.src) if not os.path.isdir(args.scratch) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs parser.error("Scratch directory does not exist: %s" %args.scratch) ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_package.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) if args.tasks_per_job and not args.pbs: parser.error("jobs-per-task argument requires the pbs option") ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") if args.v: log_level = logging.DEBUG else: log_level = logging.INFO lsh = logging.StreamHandler() lsh.setLevel(log_level) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes', 'tasks_per_job') arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) j=0 scenes = [] #### ID rasters logger.info('Identifying DEMs') if os.path.isfile(src) and src.endswith('.tif'): logger.debug(src) try: raster = dem.SetsmTile(src) except RuntimeError, e: logger.error( e ) else: j+=1 if not os.path.isfile(raster.density_file): scenes.append(src)
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description="package setsm dems (build mdf and readme files and create archive) in place in the filesystem" ) #### Positional Arguments parser.add_argument('src', help="source directory or dem") parser.add_argument('scratch', help="scratch space to build index shps") #### Optionsl Arguments parser.add_argument('--log', help="directory for log output") parser.add_argument('--epsg', type=int, default=3413, help="egsg code for output index projection (default epsg:3413)") parser.add_argument('-v', action='store_true', default=False, help="verbose output") parser.add_argument('--overwrite', action='store_true', default=False, help="overwrite existing index") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="qsub script to use in PBS submission (default is qsub_package.sh in script root folder)") parser.add_argument('--dryrun', action='store_true', default=False, help="print actions without executing") #### Parse Arguments args = parser.parse_args() #### Verify Arguments if not os.path.isdir(args.src) and not os.path.isfile(args.src): parser.error("Source directory or file does not exist: %s" %args.src) if not os.path.isdir(args.scratch) and not os.path.isfile(args.scratch): parser.error("Source directory or file does not exist: %s" %args.scratch) scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) scratch = os.path.abspath(args.scratch) ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_package.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") if args.v: log_level = logging.DEBUG else: log_level = logging.INFO lsh = logging.StreamHandler() lsh.setLevel(log_level) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get args ready to pass to task handler pos_arg_keys = ['src','scratch'] arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) if args.log: if os.path.isdir(args.log): tm = datetime.now() logfile = os.path.join(args.log,"package_setsm_tiles_{}.log".format(tm.strftime("%Y%m%d%H%M%S"))) else: parser.error('log folder does not exist: {}'.format(args.log)) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) rasters = [] j=0 #### ID rasters logger.info('Identifying DEMs') if os.path.isfile(src): j+=1 logger.debug(src) try: raster = dem.SetsmTile(os.path.join(src)) except RuntimeError, e: logger.error( e ) else: if not os.path.isfile(raster.archive) or args.overwrite: rasters.append(raster)
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description= "package setsm dems (build mdf and readme files and create archive) in place in the filesystem" ) #### Positional Arguments parser.add_argument('src', help="source directory or dem") parser.add_argument('scratch', help="scratch space to build index shps") #### Optionsl Arguments parser.add_argument('--mdf-only', action='store_true', default=False, help="build mdf and readme files only, do not archive") parser.add_argument( '--lsf', action='store_true', default=False, help="package LSF DEM instead of original DEM. Includes metadata flag." ) parser.add_argument( '--filter-dems', action='store_true', default=False, help="filter dems with area < 5.6 sqkm and density < 0.1") parser.add_argument( '--force-filter-dems', action='store_true', default=False, help= "filter dems where tar has already been built with area < 5.6 sqkm and density < 0.1" ) parser.add_argument('-v', action='store_true', default=False, help="verbose output") parser.add_argument('--overwrite', action='store_true', default=False, help="overwrite existing index") parser.add_argument( "--tasks-per-job", type=int, help="number of tasks to bundle into a single job (requires pbs option)" ) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "qsub script to use in PBS submission (default is qsub_package.sh in script root folder)" ) parser.add_argument('--dryrun', action='store_true', default=False, help="print actions without executing") pos_arg_keys = ['src', 'scratch'] #### Parse Arguments scriptpath = os.path.abspath(sys.argv[0]) args = parser.parse_args() src = os.path.abspath(args.src) scratch = os.path.abspath(args.scratch) #### Verify Arguments if not os.path.isdir(args.src) and not os.path.isfile(args.src): parser.error("Source directory or file does not exist: %s" % args.src) if not os.path.isdir( args.scratch ) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs parser.error("Scratch directory does not exist: %s" % args.scratch) ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" % qsubpath) if args.tasks_per_job and not args.pbs: parser.error("jobs-per-task argument requires the pbs option") ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error( "Options --pbs and --parallel-processes > 1 are mutually exclusive" ) if args.v: log_level = logging.DEBUG else: log_level = logging.INFO lsh = logging.StreamHandler() lsh.setLevel(log_level) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes', 'tasks_per_job') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) if args.lsf: logger.info('Packaging LSF DEMs') else: logger.info('Packaging non-LSF DEMs') j = 0 scenes = [] #### ID rasters logger.info('Identifying DEMs') if os.path.isfile(src) and src.endswith('.tif'): logger.debug(src) try: raster = dem.SetsmDem(src) except RuntimeError as e: logger.error(e) else: j += 1 if args.overwrite or args.force_filter_dems: scenes.append(src) elif args.mdf_only: if (not os.path.isfile(raster.mdf) or not os.path.isfile(raster.readme)): scenes.append(src) elif not os.path.isfile(raster.archive) or not os.path.isfile( raster.mdf) or not os.path.isfile(raster.readme): scenes.append(src) elif os.path.isfile(src) and src.endswith('.txt'): fh = open(src, 'r') for line in fh.readlines(): sceneid = line.strip() try: raster = dem.SetsmDem(sceneid) except RuntimeError as e: logger.error(e) else: j += 1 if args.overwrite or args.force_filter_dems: scenes.append(sceneid) elif args.mdf_only: if (not os.path.isfile(raster.mdf) or not os.path.isfile(raster.readme)): scenes.append(sceneid) elif not os.path.isfile(raster.archive) or not os.path.isfile( raster.mdf) or not os.path.isfile(raster.readme): scenes.append(sceneid) elif os.path.isdir(src): for root, dirs, files in os.walk(src): for f in files: if f.endswith("_dem.tif") and "m_" in f: srcfp = os.path.join(root, f) logger.debug(srcfp) try: raster = dem.SetsmDem(srcfp) except RuntimeError as e: logger.error(e) else: j += 1 if args.overwrite or args.force_filter_dems: scenes.append(srcfp) elif args.mdf_only: if (not os.path.isfile(raster.mdf) or not os.path.isfile(raster.readme)): scenes.append(srcfp) elif not os.path.isfile( raster.archive) or not os.path.isfile( raster.mdf) or not os.path.isfile( raster.readme): scenes.append(srcfp) else: logger.error("src must be a directory, a strip dem, or a text file") scenes = list(set(scenes)) logger.info('Number of src rasters: {}'.format(j)) logger.info('Number of incomplete tasks: {}'.format(len(scenes))) tm = datetime.now() job_count = 0 scene_count = 0 scenes_in_job_count = 0 task_queue = [] for srcfp in scenes: scene_count += 1 srcdir, srcfn = os.path.split(srcfp) if args.tasks_per_job: # bundle tasks into text files in the dst dir and pass the text file in as src scenes_in_job_count += 1 src_txt = os.path.join( scratch, 'src_dems_{}_{}.txt'.format(tm.strftime("%Y%m%d%H%M%S"), job_count)) if scenes_in_job_count == 1: # remove text file if dst already exists try: os.remove(src_txt) except OSError: pass if scenes_in_job_count <= args.tasks_per_job: # add to txt file fh = open(src_txt, 'a') fh.write("{}\n".format(srcfp)) fh.close() if scenes_in_job_count == args.tasks_per_job or scene_count == len( scenes): scenes_in_job_count = 0 job_count += 1 task = taskhandler.Task( 'Pkg{:04g}'.format(job_count), 'Pkg{:04g}'.format(job_count), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, src_txt, scratch), build_archive, [srcfp, scratch, args]) task_queue.append(task) else: job_count += 1 task = taskhandler.Task( srcfn, 'Pkg{:04g}'.format(job_count), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, scratch), build_archive, [srcfp, scratch, args]) task_queue.append(task) if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: task_handler = taskhandler.PBSTaskHandler(qsubpath) if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) logger.info("Number of child processes to spawn: {0}".format( task_handler.num_processes)) if not args.dryrun: task_handler.run_tasks(task_queue) else: for task in task_queue: src, scratch, task_arg_obj = task.method_arg_list if not args.dryrun: task.method(src, scratch, task_arg_obj) else: logger.info("No tasks found to process")
def main(): #### Set Up Arguments parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Run/submit batch image ortho and conversion tasks") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM " "default is slurm_ortho.py, in script root folder)") parser.add_argument( "-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") parser.add_argument("--dryrun", action='store_true', default=False, help='print actions without executing') #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext( src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext( src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error( "Error arg1 is not a recognized file path or file type: {}".format( src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) ## Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ortho.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ortho.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ## Verify processing options do not conflict if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error( "HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive" ) #### Verify EPSG try: spatial_ref = utils.SpatialRef(args.epsg) except RuntimeError as e: parser.error(e) #### Verify that dem and ortho_height are not both specified if args.dem is not None and args.ortho_height is not None: parser.error( "--dem and --ortho_height options are mutually exclusive. Please choose only one." ) #### Test if DEM exists if args.dem: if not os.path.isfile(args.dem): parser.error("DEM does not exist: {}".format(args.dem)) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] ## Group Ikonos image_list2 = [] for srcfp in image_list1: srcdir, srcfn = os.path.split(srcfp) if "IK01" in srcfn and sum( [b in srcfn for b in ortho_functions.ikMsiBands]) > 0: for b in ortho_functions.ikMsiBands: if b in srcfn: newname = os.path.join(srcdir, srcfn.replace(b, "msi")) break image_list2.append(newname) else: image_list2.append(srcfp) image_list = list(set(image_list2)) logger.info('Number of src images: %i', len(image_list)) ## Build task queue i = 0 task_queue = [] for srcfp in image_list: srcdir, srcfn = os.path.split(srcfp) dstfp = os.path.join( dstdir, "{}_{}{}{}{}".format( os.path.splitext(srcfn)[0], utils.get_bit_depth(args.outtype), args.stretch, spatial_ref.epsg, ortho_functions.formats[args.format])) done = os.path.isfile(dstfp) if done is False: i += 1 task = taskhandler.Task( srcfn, 'Or{:04g}'.format(i), 'python', '{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, dstdir), ortho_functions.process_image, [srcfp, dstfp, args]) task_queue.append(task) logger.info('Number of incomplete tasks: %i', i) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} for task in task_queue: src, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(src, dstfp, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) #### Print Images with Errors for k, v in results.items(): if v != 0: logger.warning("Failed Image: %s", k) logger.info("Done") else: logger.info("No images found to process")
def main(): #### Set Up Arguments parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Run/submit batch image ortho and conversion tasks") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--slurm", action='store_true', default=False, help="submit tasks to SLURM") parser.add_argument( "--tasks-per-job", type=int, help= "Number of tasks to bundle into a single job. (requires --pbs or --slurm option) (Warning:" " a higher number of tasks per job may require modification of default wallclock limit.)" ) parser.add_argument( '--scratch', default=ARGDEF_SCRATCH, help="Scratch space to build task bundle text files. (default={})". format(ARGDEF_SCRATCH)) parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM " "default is slurm_ortho.py, in script root folder)") parser.add_argument( "-l", help="PBS resources requested (mimicks qsub syntax, PBS only)") parser.add_argument("--dryrun", action='store_true', default=False, help='print actions without executing') #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) scratch = os.path.abspath(args.scratch) bittype = utils.get_bit_depth(args.outtype) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext( src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext( src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error( "Error arg1 is not a recognized file path or file type: {}".format( src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: {}".format(dstdir)) ## Verify qsubscript if args.pbs or args.slurm: if args.qsubscript is None: if args.pbs: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ortho.sh') if args.slurm: qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ortho.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: {}".format(qsubpath)) ## Verify processing options do not conflict requested_threads = ortho_functions.ARGDEF_CPUS_AVAIL if args.threads == "ALL_CPUS" else args.threads if args.pbs and args.slurm: parser.error("Options --pbs and --slurm are mutually exclusive") if (args.pbs or args.slurm) and args.parallel_processes > 1: parser.error( "HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive" ) if (args.pbs or args.slurm) and requested_threads > 1: parser.error( "HPC Options (--pbs or --slurm) and --threads > 1 are mutually exclusive" ) if requested_threads < 1: parser.error( "--threads count must be positive, nonzero integer or ALL_CPUS") if args.parallel_processes > 1: total_proc_count = requested_threads * args.parallel_processes if total_proc_count > ortho_functions.ARGDEF_CPUS_AVAIL: parser.error( "the (threads * number of processes requested) ({0}) exceeds number of available threads " "({1}); reduce --threads and/or --parallel-processes count". format(total_proc_count, ortho_functions.ARGDEF_CPUS_AVAIL)) if args.tasks_per_job: if not (args.pbs or args.slurm): parser.error( "--tasks-per-job option requires the (--pbs or --slurm) option" ) if not os.path.isdir(args.scratch): print("Creating --scratch directory: {}".format(args.scratch)) os.makedirs(args.scratch) #### Verify EPSG try: spatial_ref = utils.SpatialRef(args.epsg) except RuntimeError as e: parser.error(e) #### Verify that dem and ortho_height are not both specified if args.dem is not None and args.ortho_height is not None: parser.error( "--dem and --ortho_height options are mutually exclusive. Please choose only one." ) #### Test if DEM exists if args.dem: if not os.path.isfile(args.dem): parser.error("DEM does not exist: {}".format(args.dem)) if args.l is None: if args.dem.endswith('.vrt'): total_dem_filesz_gb = 0.0 tree = ET.parse(args.dem) root = tree.getroot() for sourceFilename in root.iter('SourceFilename'): dem_filename = sourceFilename.text if not os.path.isfile(dem_filename): parser.error( "VRT DEM component raster does not exist: {}". format(dem_filename)) dem_filesz_gb = os.path.getsize( dem_filename) / 1024.0 / 1024 / 1024 total_dem_filesz_gb += dem_filesz_gb dem_filesz_gb = total_dem_filesz_gb else: dem_filesz_gb = os.path.getsize( args.dem) / 1024.0 / 1024 / 1024 pbs_req_mem_gb = int(max(math.ceil(dem_filesz_gb) + 2, 4)) args.l = 'mem={}gb'.format(pbs_req_mem_gb) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Handle thread count that exceeds system limits if requested_threads > ortho_functions.ARGDEF_CPUS_AVAIL: logger.info( "threads requested ({0}) exceeds number available on system ({1}), setting thread count to " "'ALL_CPUS'".format(requested_threads, ortho_functions.ARGDEF_CPUS_AVAIL)) args.threads = 'ALL_CPUS' #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm', 'parallel_processes', 'tasks_per_job') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] ## Group Ikonos image_list2 = [] for srcfp in image_list1: srcdir, srcfn = os.path.split(srcfp) if "IK01" in srcfn and sum( [b in srcfn for b in ortho_functions.ikMsiBands]) > 0: for b in ortho_functions.ikMsiBands: if b in srcfn: newname = os.path.join(srcdir, srcfn.replace(b, "msi")) break image_list2.append(newname) else: image_list2.append(srcfp) image_list = list(set(image_list2)) logger.info('Number of src images: %i', len(image_list)) ## Build task queue i = 0 images_to_process = [] for srcfp in image_list: srcdir, srcfn = os.path.split(srcfp) dstfp = os.path.join( dstdir, "{}_{}{}{}{}".format( os.path.splitext(srcfn)[0], bittype, args.stretch, spatial_ref.epsg, ortho_functions.formats[args.format])) done = os.path.isfile(dstfp) if done is False: i += 1 images_to_process.append(srcfp) logger.info('Number of incomplete tasks: %i', i) if len(images_to_process) == 0: logger.info("No images found to process") sys.exit(0) task_queue = [] if args.tasks_per_job and args.tasks_per_job > 1: task_srcfp_list = utils.write_task_bundles(images_to_process, args.tasks_per_job, scratch, 'Or_src') else: task_srcfp_list = images_to_process for job_count, srcfp in enumerate(task_srcfp_list, 1): srcdir, srcfn = os.path.split(srcfp) if task_srcfp_list is images_to_process: dstfp = os.path.join( dstdir, "{}_{}{}{}{}".format( os.path.splitext(srcfn)[0], bittype, args.stretch, spatial_ref.epsg, ortho_functions.formats[args.format])) else: dstfp = None task = taskhandler.Task( srcfn, 'Or{:04g}'.format(job_count), 'python', '{} {} {} {}'.format(argval2str(scriptpath), arg_str_base, argval2str(srcfp), argval2str(dstdir)), ortho_functions.process_image, [srcfp, dstfp, args]) task_queue.append(task) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: l = "-l {}".format(args.l) if args.l else "" try: task_handler = taskhandler.PBSTaskHandler(qsubpath, l) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue, dryrun=args.dryrun) elif args.slurm: try: task_handler = taskhandler.SLURMTaskHandler(qsubpath) except RuntimeError as e: logger.error(e) else: if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} for task in task_queue: src, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0] + ".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter( '%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(src, dstfp, task_arg_obj) #### remove existing file handler logger.removeHandler(lfh) #### Print Images with Errors for k, v in results.items(): if v != 0: logger.warning("Failed Image: %s", k) logger.info("Done") else: logger.info("No images found to process")
def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_arg_keys): if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False if not os.path.isdir(mosaic_dir): os.makedirs(mosaic_dir) ## TODO: verify logger woks for both interactive and hpc jobs #### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(mosaic_dir, default_logfile) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): logger.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() #### Get Images #logger.info("Reading input images") xs = [] ys = [] image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list) if len(image_list) == 0: raise RuntimeError( "No images found in input file or directory: {}".format(inpath)) # remove duplicate images image_list_unique = list(set(image_list)) dupes = [x for n, x in enumerate(image_list) if x in image_list[:n]] if len(dupes) > 0: logger.info("Removed %i duplicate image paths", len(dupes)) logger.debug("Dupes: %s", dupes) image_list = image_list_unique logger.info("%i existing images found", len(image_list)) #### gather image info list logger.info("Getting image info") imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list] #### Get mosaic parameters logger.info("Setting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0], args) logger.info("Mosaic parameters: band count=%i, datatype=%s", params.bands, params.datatype) logger.info("Mosaic parameters: projection=%s", params.proj) #### Remove images that do not match ref logger.info("Applying attribute filter") imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, params) if len(imginfo_list2) == 0: raise RuntimeError( "No valid images found. Check input filter parameters.") logger.info("%i of %i images match filter", len(imginfo_list2), len(image_list)) #### if extent is specified, build tile params and compare extent to input image geom if args.extent: imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params) #### else set extent after image geoms computed else: #### Get geom for each image imginfo_list3 = [] for iinfo in imginfo_list2: if iinfo.geom is not None: xs = xs + iinfo.xs ys = ys + iinfo.ys imginfo_list3.append(iinfo) else: # remove from list if no geom logger.debug("Null geometry for image: %s", iinfo.srcfn) params.xmin = min(xs) params.xmax = max(xs) params.ymin = min(ys) params.ymax = max(ys) poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format( params.xmin, params.ymin, params.xmin, params.ymax, params.xmax, params.ymax, params.xmax, params.ymin, params.xmin, params.ymin) params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) if len(imginfo_list3) == 0: raise RuntimeError("No images found that intersect mosaic extent") logger.info( "Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f", params.xres, params.yres, params.xtilesize, params.ytilesize, params.xmin, params.xmax, params.ymin, params.ymax) logger.info("%d of %d input images intersect mosaic extent", len(imginfo_list3), len(imginfo_list2)) ## Sort images by score logger.info("Reading image metadata and determining sort order") for iinfo in imginfo_list3: iinfo.getScore(params) if not args.nosort: imginfo_list3.sort(key=lambda x: x.score) logger.info("Getting Exact Image geometry") imginfo_list4 = [] all_valid = True for iinfo in imginfo_list3: if iinfo.score > 0 or args.nosort: simplify_tolerance = 2.0 * ( (params.xres + params.yres) / 2.0 ) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m geom, xs1, ys1 = mosaic.GetExactTrimmedGeom( iinfo.srcfp, step=args.cutline_step, tolerance=simplify_tolerance) if geom is None: logger.warning( "%s: geometry could not be determined, verify image is valid", iinfo.srcfn) all_valid = False elif geom.IsEmpty(): logger.warning("%s: geometry is empty", iinfo.srcfn) all_valid = False else: iinfo.geom = geom tm = datetime.today() imginfo_list4.append(iinfo) centroid = geom.Centroid() logger.info("%s: geometry acquired - centroid: %f, %f", iinfo.srcfn, centroid.GetX(), centroid.GetY()) #print(geom) else: logger.debug("Image has an invalid score: %s --> %i", iinfo.srcfp, iinfo.score) if not all_valid: if not args.allow_invalid_geom: raise RuntimeError( "Some source images do not have valid geometries. Cannot proceeed" ) else: logger.info( "--allow-invalid-geom used; mosaic will be created using %i valid images (%i invalid \ images not used.)".format( len(imginfo_list4), len(imginfo_list3) - len(imginfo_list4))) # Get stats if needed logger.info("Getting image metadata") for iinfo in imginfo_list4: logger.info(iinfo.srcfn) if args.calc_stats or args.median_remove: iinfo.get_raster_stats(args.calc_stats, args.median_remove) # Build componenet index if args.component_shp: if args.mode == "ALL" or args.mode == "SHP": contribs = [(iinfo, iinfo.geom) for iinfo in imginfo_list4] logger.info("Number of contributors: %d", len(contribs)) logger.info("Building component index") comp_shp = mosaicname + "_components.shp" if len(contribs) > 0: if os.path.isfile(comp_shp): logger.info("Components shapefile already exists: %s", comp_shp) else: build_shp(contribs, comp_shp, args, params) else: logger.error("No contributing images") # Build cutlines index #### Overlay geoms and remove non-contributors logger.info("Overlaying images to determine contribution geom") contribs = mosaic.determine_contributors(imginfo_list4, params.extent_geom, args.min_contribution_area) logger.info("Number of contributors: %d", len(contribs)) if args.mode == "ALL" or args.mode == "SHP": logger.info("Building cutlines index") shp = mosaicname + "_cutlines.shp" if len(contribs) > 0: if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s", shp) else: build_shp(contribs, shp, args, params) else: logger.error("No contributing images") ## Create tile objects tiles = [] xtiledim = math.ceil((params.xmax - params.xmin) / params.xtilesize) ytiledim = math.ceil((params.ymax - params.ymin) / params.ytilesize) logger.info("Tiles: %d rows, %d columns", ytiledim, xtiledim) xtdb = len(str(int(xtiledim))) ytdb = len(str(int(ytiledim))) i = 1 for x in mosaic.drange(params.xmin, params.xmax, params.xtilesize): # Columns if x + params.xtilesize > params.xmax: x2 = params.xmax else: x2 = x + params.xtilesize j = 1 for y in mosaic.drange(params.ymin, params.ymax, params.ytilesize): # Rows if y + params.ytilesize > params.ymax: y2 = params.ymax else: y2 = y + params.ytilesize tilename = "{}_{}_{}.tif".format(mosaicname, mosaic.buffernum(j, ytdb), mosaic.buffernum(i, xtdb)) tile = mosaic.TileParams(x, x2, y, y2, j, i, tilename) tiles.append(tile) j += 1 i += 1 #### Write shapefile of tiles if len(tiles) == 0: raise RuntimeError("No tile objects created") if args.mode == "ALL" or args.mode == "SHP": build_tiles_shp(mosaicname, tiles, params) ## Build tile tasks task_queue = [] #### Create task for each tile arg_keys_to_remove = ('l', 'qsubscript', 'parallel_processes', 'log', 'mode', 'extent', 'resolution', 'bands', 'max_cc', 'exclude', 'nosort', 'component_shp', 'cutline_step', 'min_contribution_area', 'calc_stats', 'pbs', 'slurm', 'tday', 'tyear', 'allow_invalid_geom') tile_arg_str = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) logger.debug("Identifying components of %i subtiles", len(tiles)) i = 0 for t in tiles: logger.debug("Identifying components of tile %i of %i: %s", i, len(tiles), os.path.basename(t.name)) #### determine which images in each tile - create geom and query image geoms logger.debug("Running intersect with imagery") intersects = [] for iinfo, contrib_geom in contribs: if contrib_geom.Intersects(t.geom): if args.median_remove: ## parse median dct into text median_string = ";".join([ "{}:{}".format(k, v) for k, v in iinfo.median.items() ]) intersects.append("{},{}".format(iinfo.srcfp, median_string)) else: intersects.append(iinfo.srcfp) #### If any images are in the tile, mosaic them if len(intersects) > 0: tile_basename = os.path.basename(os.path.splitext(t.name)[0]) logger.info("Number of contributors to subtile %s: %i", tile_basename, len(intersects)) itpath = os.path.join(mosaic_dir, tile_basename + "_intersects.txt") it = open(itpath, "w") it.write("\n".join(intersects)) it.close() #### Submit QSUB job logger.debug("Building mosaicking job for tile: %s", os.path.basename(t.name)) if not os.path.isfile(t.name): cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format( tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin, t.ymax, params.xres, params.yres, params.bands, t.name, itpath) task = taskhandler.Task( 'Tile {0}'.format(os.path.basename(t.name)), 'Mos{:04g}'.format(i), 'python', cmd) if args.mode == "ALL" or args.mode == "MOSAIC": logger.debug(cmd) task_queue.append(task) else: logger.info("Tile already exists: %s", os.path.basename(t.name)) i += 1 if args.mode == "ALL" or args.mode == "MOSAIC": logger.info("Submitting Tasks") #logger.info(task_queue) if len(task_queue) > 0: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: if task_handler.num_processes > 1: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) task_handler.run_tasks(task_queue) logger.info("Done") else: logger.info("No tasks to process")
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description= "package setsm dems (build mdf and readme files and create archive) in place in the filesystem" ) #### Positional Arguments parser.add_argument('src', help="source directory or dem") parser.add_argument('scratch', help="scratch space to build index shps") #### Optionsl Arguments parser.add_argument('--mdf-only', action='store_true', default=False, help="build mdf and readme files only, do not archive") parser.add_argument( '--lsf', action='store_true', default=False, help="package LSF DEM instead of original DEM. Includes metadata flag." ) parser.add_argument( '--filter-dems', action='store_true', default=False, help="filter dems with area < 5.6 sqkm and density < 0.1") parser.add_argument( '--force-filter-dems', action='store_true', default=False, help= "filter dems where tar has already been built with area < 5.6 sqkm and density < 0.1" ) parser.add_argument('-v', action='store_true', default=False, help="verbose output") parser.add_argument('--overwrite', action='store_true', default=False, help="overwrite existing index") parser.add_argument( "--tasks-per-job", type=int, help="number of tasks to bundle into a single job (requires pbs option)" ) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument( "--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument( "--qsubscript", help= "qsub script to use in PBS submission (default is qsub_package.sh in script root folder)" ) parser.add_argument('--dryrun', action='store_true', default=False, help="print actions without executing") pos_arg_keys = ['src', 'scratch'] #### Parse Arguments scriptpath = os.path.abspath(sys.argv[0]) args = parser.parse_args() src = os.path.abspath(args.src) scratch = os.path.abspath(args.scratch) #### Verify Arguments if not os.path.isdir(args.src) and not os.path.isfile(args.src): parser.error("Source directory or file does not exist: %s" % args.src) if not os.path.isdir( args.scratch ) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs parser.error("Scratch directory does not exist: %s" % args.scratch) ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" % qsubpath) if args.tasks_per_job and not args.pbs: parser.error("jobs-per-task argument requires the pbs option") ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error( "Options --pbs and --parallel-processes > 1 are mutually exclusive" ) if args.v: log_level = logging.DEBUG else: log_level = logging.INFO lsh = logging.StreamHandler() lsh.setLevel(log_level) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get args ready to pass to task handler arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes', 'tasks_per_job') arg_str_base = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) if args.lsf: logger.info('Packaging LSF DEMs') else: logger.info('Packaging non-LSF DEMs') j = 0 scenes = [] #### ID rasters logger.info('Identifying DEMs') if os.path.isfile(src) and src.endswith('.tif'): logger.debug(src) try: raster = dem.SetsmDem(src) except RuntimeError, e: logger.error(e) else: j += 1 if args.overwrite or args.force_filter_dems: scenes.append(src) elif args.mdf_only: if (not os.path.isfile(raster.mdf) or not os.path.isfile(raster.readme)): scenes.append(src) elif not os.path.isfile(raster.archive) or not os.path.isfile( raster.mdf) or not os.path.isfile(raster.readme): scenes.append(src)