def test_filter_images(self): image_list = glob.glob(os.path.join(self.srcdir, '*.tif')) imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list] filter_list = [iinfo.srcfn for iinfo in imginfo_list] self.assertIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413.tif', filter_list) self.assertIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif', filter_list) self.assertIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif', filter_list) mosaic_args = MosaicArgs() mosaic_args.extent = [-820000.0, -800000.0, -2420000.0, -2400000.0] mosaic_args.tilesize = [20000, 20000] mosaic_args.bands = 1 mosaic_params = mosaic.getMosaicParameters(imginfo_list[0], mosaic_args) imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, mosaic_params) filter_list = [iinfo.srcfn for iinfo in imginfo_list2] self.assertEqual(len(imginfo_list2), 8) self.assertNotIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413.tif', filter_list) self.assertNotIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif', filter_list) imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, mosaic_params) filter_list = [iinfo.srcfn for iinfo in imginfo_list3] self.assertEqual(len(imginfo_list3), 7) self.assertNotIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif', filter_list)
def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_arg_keys): if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False if not os.path.isdir(mosaic_dir): os.makedirs(mosaic_dir) ## TODO: verify logger woks for both interactive and hpc jobs #### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(mosaic_dir, default_logfile) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): logger.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() #### Get Images #logger.info("Reading input images") xs = [] ys = [] image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list) if len(image_list) == 0: raise RuntimeError( "No images found in input file or directory: {}".format(inpath)) # remove duplicate images image_list_unique = list(set(image_list)) dupes = [x for n, x in enumerate(image_list) if x in image_list[:n]] if len(dupes) > 0: logger.info("Removed %i duplicate image paths", len(dupes)) logger.debug("Dupes: %s", dupes) image_list = image_list_unique logger.info("%i existing images found", len(image_list)) #### gather image info list logger.info("Getting image info") imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list] #### Get mosaic parameters logger.info("Setting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0], args) logger.info("Mosaic parameters: band count=%i, datatype=%s", params.bands, params.datatype) logger.info("Mosaic parameters: projection=%s", params.proj) #### Remove images that do not match ref logger.info("Applying attribute filter") imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, params) if len(imginfo_list2) == 0: raise RuntimeError( "No valid images found. Check input filter parameters.") logger.info("%i of %i images match filter", len(imginfo_list2), len(image_list)) #### if extent is specified, build tile params and compare extent to input image geom if args.extent: imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params) #### else set extent after image geoms computed else: #### Get geom for each image imginfo_list3 = [] for iinfo in imginfo_list2: if iinfo.geom is not None: xs = xs + iinfo.xs ys = ys + iinfo.ys imginfo_list3.append(iinfo) else: # remove from list if no geom logger.debug("Null geometry for image: %s", iinfo.srcfn) params.xmin = min(xs) params.xmax = max(xs) params.ymin = min(ys) params.ymax = max(ys) poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format( params.xmin, params.ymin, params.xmin, params.ymax, params.xmax, params.ymax, params.xmax, params.ymin, params.xmin, params.ymin) params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) if len(imginfo_list3) == 0: raise RuntimeError("No images found that intersect mosaic extent") logger.info( "Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f", params.xres, params.yres, params.xtilesize, params.ytilesize, params.xmin, params.xmax, params.ymin, params.ymax) logger.info("%d of %d input images intersect mosaic extent", len(imginfo_list3), len(imginfo_list2)) ## Sort images by score logger.info("Reading image metadata and determining sort order") for iinfo in imginfo_list3: iinfo.getScore(params) if not args.nosort: imginfo_list3.sort(key=lambda x: x.score) logger.info("Getting Exact Image geometry") imginfo_list4 = [] all_valid = True for iinfo in imginfo_list3: if iinfo.score > 0 or args.nosort: simplify_tolerance = 2.0 * ( (params.xres + params.yres) / 2.0 ) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m geom, xs1, ys1 = mosaic.GetExactTrimmedGeom( iinfo.srcfp, step=args.cutline_step, tolerance=simplify_tolerance) if geom is None: logger.warning( "%s: geometry could not be determined, verify image is valid", iinfo.srcfn) all_valid = False elif geom.IsEmpty(): logger.warning("%s: geometry is empty", iinfo.srcfn) all_valid = False else: iinfo.geom = geom tm = datetime.today() imginfo_list4.append(iinfo) centroid = geom.Centroid() logger.info("%s: geometry acquired - centroid: %f, %f", iinfo.srcfn, centroid.GetX(), centroid.GetY()) #print(geom) else: logger.debug("Image has an invalid score: %s --> %i", iinfo.srcfp, iinfo.score) if not all_valid: if not args.allow_invalid_geom: raise RuntimeError( "Some source images do not have valid geometries. Cannot proceeed" ) else: logger.info( "--allow-invalid-geom used; mosaic will be created using %i valid images (%i invalid \ images not used.)".format( len(imginfo_list4), len(imginfo_list3) - len(imginfo_list4))) # Get stats if needed logger.info("Getting image metadata") for iinfo in imginfo_list4: logger.info(iinfo.srcfn) if args.calc_stats or args.median_remove: iinfo.get_raster_stats(args.calc_stats, args.median_remove) # Build componenet index if args.component_shp: if args.mode == "ALL" or args.mode == "SHP": contribs = [(iinfo, iinfo.geom) for iinfo in imginfo_list4] logger.info("Number of contributors: %d", len(contribs)) logger.info("Building component index") comp_shp = mosaicname + "_components.shp" if len(contribs) > 0: if os.path.isfile(comp_shp): logger.info("Components shapefile already exists: %s", comp_shp) else: build_shp(contribs, comp_shp, args, params) else: logger.error("No contributing images") # Build cutlines index #### Overlay geoms and remove non-contributors logger.info("Overlaying images to determine contribution geom") contribs = mosaic.determine_contributors(imginfo_list4, params.extent_geom, args.min_contribution_area) logger.info("Number of contributors: %d", len(contribs)) if args.mode == "ALL" or args.mode == "SHP": logger.info("Building cutlines index") shp = mosaicname + "_cutlines.shp" if len(contribs) > 0: if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s", shp) else: build_shp(contribs, shp, args, params) else: logger.error("No contributing images") ## Create tile objects tiles = [] xtiledim = math.ceil((params.xmax - params.xmin) / params.xtilesize) ytiledim = math.ceil((params.ymax - params.ymin) / params.ytilesize) logger.info("Tiles: %d rows, %d columns", ytiledim, xtiledim) xtdb = len(str(int(xtiledim))) ytdb = len(str(int(ytiledim))) i = 1 for x in mosaic.drange(params.xmin, params.xmax, params.xtilesize): # Columns if x + params.xtilesize > params.xmax: x2 = params.xmax else: x2 = x + params.xtilesize j = 1 for y in mosaic.drange(params.ymin, params.ymax, params.ytilesize): # Rows if y + params.ytilesize > params.ymax: y2 = params.ymax else: y2 = y + params.ytilesize tilename = "{}_{}_{}.tif".format(mosaicname, mosaic.buffernum(j, ytdb), mosaic.buffernum(i, xtdb)) tile = mosaic.TileParams(x, x2, y, y2, j, i, tilename) tiles.append(tile) j += 1 i += 1 #### Write shapefile of tiles if len(tiles) == 0: raise RuntimeError("No tile objects created") if args.mode == "ALL" or args.mode == "SHP": build_tiles_shp(mosaicname, tiles, params) ## Build tile tasks task_queue = [] #### Create task for each tile arg_keys_to_remove = ('l', 'qsubscript', 'parallel_processes', 'log', 'mode', 'extent', 'resolution', 'bands', 'max_cc', 'exclude', 'nosort', 'component_shp', 'cutline_step', 'min_contribution_area', 'calc_stats', 'pbs', 'slurm', 'tday', 'tyear', 'allow_invalid_geom') tile_arg_str = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) logger.debug("Identifying components of %i subtiles", len(tiles)) i = 0 for t in tiles: logger.debug("Identifying components of tile %i of %i: %s", i, len(tiles), os.path.basename(t.name)) #### determine which images in each tile - create geom and query image geoms logger.debug("Running intersect with imagery") intersects = [] for iinfo, contrib_geom in contribs: if contrib_geom.Intersects(t.geom): if args.median_remove: ## parse median dct into text median_string = ";".join([ "{}:{}".format(k, v) for k, v in iinfo.median.items() ]) intersects.append("{},{}".format(iinfo.srcfp, median_string)) else: intersects.append(iinfo.srcfp) #### If any images are in the tile, mosaic them if len(intersects) > 0: tile_basename = os.path.basename(os.path.splitext(t.name)[0]) logger.info("Number of contributors to subtile %s: %i", tile_basename, len(intersects)) itpath = os.path.join(mosaic_dir, tile_basename + "_intersects.txt") it = open(itpath, "w") it.write("\n".join(intersects)) it.close() #### Submit QSUB job logger.debug("Building mosaicking job for tile: %s", os.path.basename(t.name)) if not os.path.isfile(t.name): cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format( tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin, t.ymax, params.xres, params.yres, params.bands, t.name, itpath) task = taskhandler.Task( 'Tile {0}'.format(os.path.basename(t.name)), 'Mos{:04g}'.format(i), 'python', cmd) if args.mode == "ALL" or args.mode == "MOSAIC": logger.debug(cmd) task_queue.append(task) else: logger.info("Tile already exists: %s", os.path.basename(t.name)) i += 1 if args.mode == "ALL" or args.mode == "MOSAIC": logger.info("Submitting Tasks") #logger.info(task_queue) if len(task_queue) > 0: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: if task_handler.num_processes > 1: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) task_handler.run_tasks(task_queue) logger.info("Done") else: logger.info("No tasks to process")
def main(): #### Set Up Arguments parent_parser = mosaic.buildMosaicParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Sumbit/run batch mosaic tasks" ) parser.add_argument("src", help="textfile or directory of input rasters (tif only)") parser.add_argument("mosaicname", help="output mosaic name excluding extension") pos_arg_keys = ["src","mosaicname"] parser.add_argument("--mode", choices=mosaic.MODES , default="ALL", help=" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, TEST- create log only") parser.add_argument("--wd", help="scratch space (default is mosaic directory)") parser.add_argument("--component-shp", action="store_true", default=False, help="create shp of all componenet images") parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw", help="GTiff compression type. Default=lzw (%s)"%(",".join(mosaic.GTIFF_COMPRESSIONS))) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="qsub script to use in cluster job submission (default is <script_dir>/%s)" %default_qsub_script) parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax). Use only on HPC systems.") parser.add_argument("--log", help="file to log progress (default is <output dir>\%s" %default_logfile) #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) inpath = os.path.abspath(args.src) mosaicname = os.path.abspath(args.mosaicname) mosaicname = os.path.splitext(mosaicname)[0] mosaic_dir = os.path.dirname(mosaicname) cutline_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_cutlines.py') tile_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_tile.py') ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),default_qsub_script) else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") #### Validate Arguments if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False else: parser.error("Arg1 is not a valid file path or directory: %s" %inpath) if not os.path.isdir(mosaic_dir): os.makedirs(mosaic_dir) if not os.path.isfile(qsubpath): parser.error("Arg3 is not a valid file path: %s" %qsubpath) #### Validate target day option if args.tday is not None: try: m = int(args.tday.split("-")[0]) d = int(args.tday.split("-")[1]) td = date(2000,m,d) except ValueError: logger.error("Target day must be in mm-dd format (i.e 04-05)") sys.exit(1) else: m = 0 d = 0 #### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(mosaic_dir,default_logfile) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): parser.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() #### Get Images #logger.info("Reading input images") xs = [] ys = [] image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list) if len(image_list) == 0: logger.error("No images found in input file or directory: %s" %inpath) sys.exit() else: logger.info("%i existing images found" %len(image_list)) #### gather image info list logger.info("Getting image info") imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in image_list] #### Get mosaic parameters logger.info("Setting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0],args) #### Remove images that do not match ref logger.info("Applying attribute filter") imginfo_list2 = mosaic.filterMatchingImages(imginfo_list,params) if len(imginfo_list2) == 0: logger.error("No valid images found. Check input filter parameters.") sys.exit() else: logger.info("%i images match filter" %len(imginfo_list2)) #### if extent is specified, build tile params and compare extent to input image geom if args.extent: imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params) #### else set extent after image geoms computed else: #### Get geom for each image imginfo_list3 = [] for iinfo in imginfo_list2: if iinfo.geom is not None: xs = xs + iinfo.xs ys = ys + iinfo.ys imginfo_list3.append(iinfo) else: # remove from list if no geom logger.debug("Null geometry for image: %s" %iinfo.srcfn) params.xmin = min(xs) params.xmax = max(xs) params.ymin = min(ys) params.ymax = max(ys) poly_wkt = 'POLYGON (( %f %f, %f %f, %f %f, %f %f, %f %f ))' %(params.xmin,params.ymin,params.xmin,params.ymax,params.xmax,params.ymax,params.xmax,params.ymin,params.xmin,params.ymin) params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) #### Check number of remaining images num_images = len(imginfo_list3) if num_images > 0: logger.info("%d of %d input images intersect mosaic extent" %(num_images,len(image_list))) logger.info("Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f" %(params.xres,params.yres,params.xtilesize,params.ytilesize,params.xmin,params.xmax,params.ymin,params.ymax)) else: logger.error("No valid images found") sys.exit(0) ## Sort images by score logger.info("Reading image metadata and determining sort order") for iinfo in imginfo_list3: iinfo.getScore(params) #### Sort by score if not args.nosort: imginfo_list3.sort(key=lambda x: x.score) #### Write all intersects file intersects_all = [] for iinfo in imginfo_list3: if params.extent_geom.Intersect(iinfo.geom) is True: if iinfo.score > 0: intersects_all.append(iinfo) elif args.nosort: intersects_all.append(iinfo) else: logger.debug("Image has an invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score)) else: logger.debug("Image does not intersect mosaic extent: %s" %iinfo.srcfp) ### this line should never be needed. non-intersecting images should be removed earlier if extent is provided, otherwise all images are in the extent. aitpath = mosaicname+"_intersects.txt" ait = open(aitpath,"w") intersects_fps = [intersect.srcfp for intersect in intersects_all] ait.write(string.join(intersects_fps,"\n")) ait.close() ## Create tiles logger.info("Creating tiles") tiles = [] xtiledim = math.ceil((params.xmax-params.xmin)/params.xtilesize) ytiledim = math.ceil((params.ymax-params.ymin)/params.ytilesize) logger.info("Tiles: %d rows, %d columns" %(ytiledim,xtiledim)) xtdb = len(str(int(xtiledim))) ytdb = len(str(int(ytiledim))) i = 1 for x in mosaic.drange(params.xmin,params.xmax,params.xtilesize): # Columns if x+params.xtilesize > params.xmax: x2 = params.xmax else: x2 = x+params.xtilesize j = 1 for y in mosaic.drange(params.ymin,params.ymax,params.ytilesize): # Rows if y+params.ytilesize > params.ymax: y2 = params.ymax else: y2 = y+params.ytilesize tilename = "%s_%s_%s.tif" %(mosaicname,mosaic.buffernum(j,ytdb),mosaic.buffernum(i,xtdb)) tile = mosaic.TileParams(x,x2,y,y2,j,i,tilename) tiles.append(tile) j += 1 i += 1 num_tiles = len(tiles) #### Write shapefile of tiles if args.mode == "ALL" or args.mode == "SHP": shp = mosaicname + "_tiles.shp" if os.path.isfile(shp): logger.info("Tiles shapefile already exists: %s" %os.path.basename(shp)) else: logger.info("Creating shapefile of tiles: %s" %os.path.basename(shp)) fields = [('ROW', ogr.OFTInteger, 4), ('COL', ogr.OFTInteger, 4), ("TILENAME", ogr.OFTString, 100), ('TILEPATH', ogr.OFTString, 254), ('XMIN', ogr.OFTReal, 0), ('XMAX', ogr.OFTReal, 0), ('YMIN', ogr.OFTReal, 0), ('YMAX', ogr.OFTReal, 0)] OGR_DRIVER = "ESRI Shapefile" ogrDriver = ogr.GetDriverByName(OGR_DRIVER) if ogrDriver is None: logger.error("OGR: Driver %s is not available" % OGR_DRIVER) sys.exit(-1) if os.path.isfile(shp): ogrDriver.DeleteDataSource(shp) vds = ogrDriver.CreateDataSource(shp) if vds is None: logger.error("Could not create shp") sys.exit(-1) shpd, shpn = os.path.split(shp) shpbn, shpe = os.path.splitext(shpn) rp = osr.SpatialReference() rp.ImportFromWkt(params.proj) lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon) if lyr is None: logger.error("ERROR: Failed to create layer: %s" % shpbn) sys.exit(-1) for fld, fdef, flen in fields: field_defn = ogr.FieldDefn(fld, fdef) if fdef == ogr.OFTString: field_defn.SetWidth(flen) if lyr.CreateField(field_defn) != 0: logger.error("ERROR: Failed to create field: %s" % fld) for t in tiles: feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField("TILENAME",os.path.basename(t.name)) feat.SetField("TILEPATH",t.name) feat.SetField("ROW",t.j) feat.SetField("COL",t.i) feat.SetField("XMIN",t.xmin) feat.SetField("XMAX",t.xmax) feat.SetField("YMIN",t.ymin) feat.SetField("YMAX",t.ymax) feat.SetGeometry(t.geom) if lyr.CreateFeature(feat) != 0: logger.error("ERROR: Could not create feature for tile %s" % tile) feat.Destroy() ## Build tasks task_queue = [] #### Create task for shapefile of mosaic components if args.component_shp is True: arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'gtiff_compression', 'mode', 'extent', 'resolution', 'pbs', 'wd' ) shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) comp_shp = mosaicname + "_components.shp" if os.path.isfile(comp_shp): logger.info("Components shapefile already exists: %s" %os.path.basename(comp_shp)) else: logger.info("Processing components: %s" %os.path.basename(comp_shp)) ## Make task and add to queue cmd = '{} --cutline-step 512 {} -e {} {} {} {} {} {}'.format( cutline_builder_script, shp_arg_str, params.xmin, params.xmax, params.ymin, params.ymax, comp_shp, aitpath ) task = utils.Task( 'Components', 'Components', 'python', cmd ) if args.mode == "ALL" or args.mode == "SHP": logger.debug(cmd) task_queue.append(task) #### Create task for shapefile of image cutlines shp = mosaicname + "_cutlines.shp" arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'gtiff_compression', 'mode', 'extent', 'resolution', 'component_shp', 'pbs', 'wd' ) shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s" %os.path.basename(shp)) else: logger.info("Processing cutlines: %s" %os.path.basename(shp)) ## Make task and add to queue cmd = '{} {} -e {} {} {} {} {} {}'.format( cutline_builder_script, shp_arg_str, params.xmin, params.xmax, params.ymin, params.ymax, shp, aitpath ) task = utils.Task( 'Cutlines', 'Cutlines', 'python', cmd ) if args.mode == "ALL" or args.mode == "SHP": logger.debug(cmd) task_queue.append(task) #### Create task for each tile arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'mode', 'extent', 'resolution', 'bands', 'component_shp', 'pbs' ) tile_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) logger.debug("Identifying components of {0} subtiles".format(num_tiles)) i = 0 for t in tiles: logger.debug("Identifying components of tile %d of %d: %s" %(i,num_tiles,os.path.basename(t.name))) #### determine which images in each tile - create geom and query image geoms logger.debug("Running intersect with imagery") intersects = [] for iinfo in intersects_all: if t.geom.Intersect(iinfo.geom) is True: if iinfo.score > 0: logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score)) intersects.append(iinfo.srcfp) elif args.nosort: logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score)) intersects.append(iinfo.srcfp) else: logger.warning("Invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score)) #### If any images are in the tile, mosaic them if len(intersects) > 0: tile_basename = os.path.basename(os.path.splitext(t.name)[0]) itpath = os.path.join(mosaic_dir,tile_basename+"_intersects.txt") it = open(itpath,"w") it.write(string.join(intersects,"\n")) it.close() #### Submit QSUB job logger.debug("Building mosaicking job for tile: %s" %os.path.basename(t.name)) if not os.path.isfile(t.name): cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format( tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin, t.ymax, params.xres, params.yres, params.bands, t.name, itpath ) task = utils.Task( 'Tile {0}'.format(os.path.basename(t.name)), 'Mosaic{:04g}'.format(i), 'python', cmd ) if args.mode == "ALL" or args.mode == "MOSAIC": logger.debug(cmd) task_queue.append(task) else: logger.info("Tile already exists: %s" %os.path.basename(t.name)) i += 1 logger.info("Submitting Tasks") #logger.info(task_queue) if len(task_queue) > 0: if args.pbs: if args.l: task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l)) else: task_handler = utils.PBSTaskHandler(qsubpath) task_handler.run_tasks(task_queue) else: task_handler = utils.ParallelTaskHandler(args.parallel_processes) if task_handler.num_processes > 1: logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes)) task_handler.run_tasks(task_queue) logger.info("Done") else: logger.info("No tasks to process")