def test_args(self): positional_arg_keys = ['positional'] arg_keys_to_remove = ['toremove', 'to_remove'] arg_str = utils.convert_optional_args_to_string(self.args, positional_arg_keys, arg_keys_to_remove) self.assertIn('--tuple item1 item2', arg_str) self.assertIn('--list item1 item2', arg_str) self.assertIn('--boolean', arg_str) self.assertIn('--multi-word-key multi-word-key', arg_str) self.assertNotIn('positional', arg_str) self.assertNotIn('toremove', arg_str) self.assertNotIn('to-remove', arg_str) self.assertNotIn('to_remove', arg_str)
def main(): #### Set Up Arguments parent_parser = mosaic.buildMosaicParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Sumbit/run batch mosaic tasks" ) parser.add_argument("src", help="textfile or directory of input rasters (tif only)") parser.add_argument("mosaicname", help="output mosaic name excluding extension") pos_arg_keys = ["src","mosaicname"] parser.add_argument("--mode", choices=mosaic.MODES , default="ALL", help=" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, TEST- create log only") parser.add_argument("--wd", help="scratch space (default is mosaic directory)") parser.add_argument("--component-shp", action="store_true", default=False, help="create shp of all componenet images") parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw", help="GTiff compression type. Default=lzw (%s)"%(",".join(mosaic.GTIFF_COMPRESSIONS))) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="qsub script to use in cluster job submission (default is <script_dir>/%s)" %default_qsub_script) parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax). Use only on HPC systems.") parser.add_argument("--log", help="file to log progress (default is <output dir>\%s" %default_logfile) #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) inpath = os.path.abspath(args.src) mosaicname = os.path.abspath(args.mosaicname) mosaicname = os.path.splitext(mosaicname)[0] mosaic_dir = os.path.dirname(mosaicname) cutline_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_cutlines.py') tile_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_tile.py') ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),default_qsub_script) else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") #### Validate Arguments if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False else: parser.error("Arg1 is not a valid file path or directory: %s" %inpath) if not os.path.isdir(mosaic_dir): os.makedirs(mosaic_dir) if not os.path.isfile(qsubpath): parser.error("Arg3 is not a valid file path: %s" %qsubpath) #### Validate target day option if args.tday is not None: try: m = int(args.tday.split("-")[0]) d = int(args.tday.split("-")[1]) td = date(2000,m,d) except ValueError: logger.error("Target day must be in mm-dd format (i.e 04-05)") sys.exit(1) else: m = 0 d = 0 #### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(mosaic_dir,default_logfile) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): parser.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() #### Get Images #logger.info("Reading input images") xs = [] ys = [] image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list) if len(image_list) == 0: logger.error("No images found in input file or directory: %s" %inpath) sys.exit() else: logger.info("%i existing images found" %len(image_list)) #### gather image info list logger.info("Getting image info") imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in image_list] #### Get mosaic parameters logger.info("Setting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0],args) #### Remove images that do not match ref logger.info("Applying attribute filter") imginfo_list2 = mosaic.filterMatchingImages(imginfo_list,params) if len(imginfo_list2) == 0: logger.error("No valid images found. Check input filter parameters.") sys.exit() else: logger.info("%i images match filter" %len(imginfo_list2)) #### if extent is specified, build tile params and compare extent to input image geom if args.extent: imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params) #### else set extent after image geoms computed else: #### Get geom for each image imginfo_list3 = [] for iinfo in imginfo_list2: if iinfo.geom is not None: xs = xs + iinfo.xs ys = ys + iinfo.ys imginfo_list3.append(iinfo) else: # remove from list if no geom logger.debug("Null geometry for image: %s" %iinfo.srcfn) params.xmin = min(xs) params.xmax = max(xs) params.ymin = min(ys) params.ymax = max(ys) poly_wkt = 'POLYGON (( %f %f, %f %f, %f %f, %f %f, %f %f ))' %(params.xmin,params.ymin,params.xmin,params.ymax,params.xmax,params.ymax,params.xmax,params.ymin,params.xmin,params.ymin) params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) #### Check number of remaining images num_images = len(imginfo_list3) if num_images > 0: logger.info("%d of %d input images intersect mosaic extent" %(num_images,len(image_list))) logger.info("Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f" %(params.xres,params.yres,params.xtilesize,params.ytilesize,params.xmin,params.xmax,params.ymin,params.ymax)) else: logger.error("No valid images found") sys.exit(0) ## Sort images by score logger.info("Reading image metadata and determining sort order") for iinfo in imginfo_list3: iinfo.getScore(params) #### Sort by score if not args.nosort: imginfo_list3.sort(key=lambda x: x.score) #### Write all intersects file intersects_all = [] for iinfo in imginfo_list3: if params.extent_geom.Intersect(iinfo.geom) is True: if iinfo.score > 0: intersects_all.append(iinfo) elif args.nosort: intersects_all.append(iinfo) else: logger.debug("Image has an invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score)) else: logger.debug("Image does not intersect mosaic extent: %s" %iinfo.srcfp) ### this line should never be needed. non-intersecting images should be removed earlier if extent is provided, otherwise all images are in the extent. aitpath = mosaicname+"_intersects.txt" ait = open(aitpath,"w") intersects_fps = [intersect.srcfp for intersect in intersects_all] ait.write(string.join(intersects_fps,"\n")) ait.close() ## Create tiles logger.info("Creating tiles") tiles = [] xtiledim = math.ceil((params.xmax-params.xmin)/params.xtilesize) ytiledim = math.ceil((params.ymax-params.ymin)/params.ytilesize) logger.info("Tiles: %d rows, %d columns" %(ytiledim,xtiledim)) xtdb = len(str(int(xtiledim))) ytdb = len(str(int(ytiledim))) i = 1 for x in mosaic.drange(params.xmin,params.xmax,params.xtilesize): # Columns if x+params.xtilesize > params.xmax: x2 = params.xmax else: x2 = x+params.xtilesize j = 1 for y in mosaic.drange(params.ymin,params.ymax,params.ytilesize): # Rows if y+params.ytilesize > params.ymax: y2 = params.ymax else: y2 = y+params.ytilesize tilename = "%s_%s_%s.tif" %(mosaicname,mosaic.buffernum(j,ytdb),mosaic.buffernum(i,xtdb)) tile = mosaic.TileParams(x,x2,y,y2,j,i,tilename) tiles.append(tile) j += 1 i += 1 num_tiles = len(tiles) #### Write shapefile of tiles if args.mode == "ALL" or args.mode == "SHP": shp = mosaicname + "_tiles.shp" if os.path.isfile(shp): logger.info("Tiles shapefile already exists: %s" %os.path.basename(shp)) else: logger.info("Creating shapefile of tiles: %s" %os.path.basename(shp)) fields = [('ROW', ogr.OFTInteger, 4), ('COL', ogr.OFTInteger, 4), ("TILENAME", ogr.OFTString, 100), ('TILEPATH', ogr.OFTString, 254), ('XMIN', ogr.OFTReal, 0), ('XMAX', ogr.OFTReal, 0), ('YMIN', ogr.OFTReal, 0), ('YMAX', ogr.OFTReal, 0)] OGR_DRIVER = "ESRI Shapefile" ogrDriver = ogr.GetDriverByName(OGR_DRIVER) if ogrDriver is None: logger.error("OGR: Driver %s is not available" % OGR_DRIVER) sys.exit(-1) if os.path.isfile(shp): ogrDriver.DeleteDataSource(shp) vds = ogrDriver.CreateDataSource(shp) if vds is None: logger.error("Could not create shp") sys.exit(-1) shpd, shpn = os.path.split(shp) shpbn, shpe = os.path.splitext(shpn) rp = osr.SpatialReference() rp.ImportFromWkt(params.proj) lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon) if lyr is None: logger.error("ERROR: Failed to create layer: %s" % shpbn) sys.exit(-1) for fld, fdef, flen in fields: field_defn = ogr.FieldDefn(fld, fdef) if fdef == ogr.OFTString: field_defn.SetWidth(flen) if lyr.CreateField(field_defn) != 0: logger.error("ERROR: Failed to create field: %s" % fld) for t in tiles: feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField("TILENAME",os.path.basename(t.name)) feat.SetField("TILEPATH",t.name) feat.SetField("ROW",t.j) feat.SetField("COL",t.i) feat.SetField("XMIN",t.xmin) feat.SetField("XMAX",t.xmax) feat.SetField("YMIN",t.ymin) feat.SetField("YMAX",t.ymax) feat.SetGeometry(t.geom) if lyr.CreateFeature(feat) != 0: logger.error("ERROR: Could not create feature for tile %s" % tile) feat.Destroy() ## Build tasks task_queue = [] #### Create task for shapefile of mosaic components if args.component_shp is True: arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'gtiff_compression', 'mode', 'extent', 'resolution', 'pbs', 'wd' ) shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) comp_shp = mosaicname + "_components.shp" if os.path.isfile(comp_shp): logger.info("Components shapefile already exists: %s" %os.path.basename(comp_shp)) else: logger.info("Processing components: %s" %os.path.basename(comp_shp)) ## Make task and add to queue cmd = '{} --cutline-step 512 {} -e {} {} {} {} {} {}'.format( cutline_builder_script, shp_arg_str, params.xmin, params.xmax, params.ymin, params.ymax, comp_shp, aitpath ) task = utils.Task( 'Components', 'Components', 'python', cmd ) if args.mode == "ALL" or args.mode == "SHP": logger.debug(cmd) task_queue.append(task) #### Create task for shapefile of image cutlines shp = mosaicname + "_cutlines.shp" arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'gtiff_compression', 'mode', 'extent', 'resolution', 'component_shp', 'pbs', 'wd' ) shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s" %os.path.basename(shp)) else: logger.info("Processing cutlines: %s" %os.path.basename(shp)) ## Make task and add to queue cmd = '{} {} -e {} {} {} {} {} {}'.format( cutline_builder_script, shp_arg_str, params.xmin, params.xmax, params.ymin, params.ymax, shp, aitpath ) task = utils.Task( 'Cutlines', 'Cutlines', 'python', cmd ) if args.mode == "ALL" or args.mode == "SHP": logger.debug(cmd) task_queue.append(task) #### Create task for each tile arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'mode', 'extent', 'resolution', 'bands', 'component_shp', 'pbs' ) tile_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) logger.debug("Identifying components of {0} subtiles".format(num_tiles)) i = 0 for t in tiles: logger.debug("Identifying components of tile %d of %d: %s" %(i,num_tiles,os.path.basename(t.name))) #### determine which images in each tile - create geom and query image geoms logger.debug("Running intersect with imagery") intersects = [] for iinfo in intersects_all: if t.geom.Intersect(iinfo.geom) is True: if iinfo.score > 0: logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score)) intersects.append(iinfo.srcfp) elif args.nosort: logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score)) intersects.append(iinfo.srcfp) else: logger.warning("Invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score)) #### If any images are in the tile, mosaic them if len(intersects) > 0: tile_basename = os.path.basename(os.path.splitext(t.name)[0]) itpath = os.path.join(mosaic_dir,tile_basename+"_intersects.txt") it = open(itpath,"w") it.write(string.join(intersects,"\n")) it.close() #### Submit QSUB job logger.debug("Building mosaicking job for tile: %s" %os.path.basename(t.name)) if not os.path.isfile(t.name): cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format( tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin, t.ymax, params.xres, params.yres, params.bands, t.name, itpath ) task = utils.Task( 'Tile {0}'.format(os.path.basename(t.name)), 'Mosaic{:04g}'.format(i), 'python', cmd ) if args.mode == "ALL" or args.mode == "MOSAIC": logger.debug(cmd) task_queue.append(task) else: logger.info("Tile already exists: %s" %os.path.basename(t.name)) i += 1 logger.info("Submitting Tasks") #logger.info(task_queue) if len(task_queue) > 0: if args.pbs: if args.l: task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l)) else: task_handler = utils.PBSTaskHandler(qsubpath) task_handler.run_tasks(task_queue) else: task_handler = utils.ParallelTaskHandler(args.parallel_processes) if task_handler.num_processes > 1: logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes)) task_handler.run_tasks(task_queue) logger.info("Done") else: logger.info("No tasks to process")
#### Test if DEM exists if args.dem: if not os.path.isfile(args.dem): parser.error("DEM does not exist: %s" %args.dem) #### Set up console logging handler lso = logging.StreamHandler() lso.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'parallel_processes') arg_str_base = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list1 = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list1 = utils.find_images(src, True, ortho_functions.exts) else: image_list1 = [src] ## Group Ikonos image_list2 = [] for srcfp in image_list1: srcdir,srcfn = os.path.split(srcfp) if "IK01" in srcfn and sum([b in srcfn for b in ortho_functions.ikMsiBands]) > 0: for b in ortho_functions.ikMsiBands:
def main(): #### Set Up Arguments parser = argparse.ArgumentParser( description="Run/Submit batch ndvi calculation in parallel" ) parser.add_argument("src", help="source image, text file, or directory") parser.add_argument("dst", help="destination directory") pos_arg_keys = ["src","dst"] parser.add_argument("-t", "--outtype", choices=outtypes, default='Float32', help="output data type (for Int16, output values are scaled from -1000 to 1000)") parser.add_argument("-s", "--save-temps", action="store_true", default=False, help="save temp files") parser.add_argument("--wd", help="local working directory for cluster jobs (default is dst dir)") parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax)") parser.add_argument("--qsubscript", help="qsub script to use in cluster job submission (default is qsub_ndvi.sh in script root folder)") parser.add_argument("--dryrun", action="store_true", default=False, help="print actions without executing") #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) src = os.path.abspath(args.src) dstdir = os.path.abspath(args.dst) #### Validate Required Arguments if os.path.isdir(src): srctype = 'dir' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt': srctype = 'textfile' elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' elif os.path.isfile(src.replace('msi','blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts: srctype = 'image' else: parser.error("Error arg1 is not a recognized file path or file type: %s" %(src)) if not os.path.isdir(dstdir): parser.error("Error arg2 is not a valid file path: %s" %(dstdir)) ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_ndvi.sh') else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") #### Set concole logging handler lso = logging.StreamHandler() lso.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lso.setFormatter(formatter) logger.addHandler(lso) #### Get args ready to pass to task handler arg_keys_to_remove = ('l', 'qsubscript', 'pbs', 'parallel_processes', 'dryrun') arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) ## Identify source images if srctype == 'dir': image_list = utils.find_images(src, False, ortho_functions.exts) elif srctype == 'textfile': image_list = utils.find_images(src, True, ortho_functions.exts) else: image_list = [src] logger.info('Number of src images: %i' %len(image_list)) ## Build task queue i = 0 task_queue = [] for srcfp in image_list: srcdir, srcfn = os.path.split(srcfp) bn, ext = os.path.splitext(srcfn) dstfp = os.path.join(dstdir, bn + '_ndvi.tif') if not os.path.isfile(dstfp): i+=1 task = utils.Task( srcfn, 'NDVI{:04g}'.format(i), 'python', '{} {} {} {}'.format(scriptpath, arg_str, srcfp, dstdir), calc_ndvi, [srcfp, dstfp, args] ) task_queue.append(task) logger.info('Number of incomplete tasks: {}'.format(i)) ## Run tasks if len(task_queue) > 0: logger.info("Submitting Tasks") if args.pbs: if args.l: task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l)) else: task_handler = utils.PBSTaskHandler(qsubpath) if not args.dryrun: task_handler.run_tasks(task_queue) elif args.parallel_processes > 1: task_handler = utils.ParallelTaskHandler(args.parallel_processes) logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes)) if not args.dryrun: task_handler.run_tasks(task_queue) else: results = {} for task in task_queue: srcfp, dstfp, task_arg_obj = task.method_arg_list #### Set up processing log handler logfile = os.path.splitext(dstfp)[0]+".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) if not args.dryrun: results[task.name] = task.method(srcfp, dstfp, task_arg_obj) #### Print Images with Errors for k,v in results.iteritems(): if v != 0: logger.warning("Failed Image: {}".format(k)) logger.info("Done") else: logger.info("No images found to process")