def main(): ######################################################### #### Handle args ######################################################### #### Set Up Arguments parent_parser = mosaic.buildMosaicParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Create cutline or component shapefile" ) parser.add_argument("shp", help="output shapefile name") parser.add_argument("src", help="textfile or directory of input rasters (tif only)") parser.add_argument("--cutline-step", type=int, default=2, help="cutline calculator pixel skip interval (default=2)") parser.add_argument("--component-shp", action="store_true", default=False, help="create shp of all component images") #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) inpath = os.path.abspath(args.src) shp = os.path.abspath(args.shp) #print (" ".join(sys.argv)) #### Validate target day option if args.tday is not None: try: m = int(args.tday.split("-")[0]) d = int(args.tday.split("-")[1]) td = date(2000,m,d) except ValueError: logger.error("Target day must be in mm-dd format (i.e 04-05)") sys.exit(1) else: m = 0 d = 0 ##### Configure Logger logfile = os.path.splitext(shp)[0]+".log" lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) stm = datetime.today() logger.info("Start Time: %s\n" %(stm)) minx = args.extent[0] maxx = args.extent[1] miny = args.extent[2] maxy = args.extent[3] poly_wkt = 'POLYGON (( %s %s, %s %s, %s %s, %s %s, %s %s ))' %(minx,miny,minx,maxy,maxx,maxy,maxx,miny,minx,miny) extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s" %shp) else: intersects = [] t = open(inpath,'r') for line in t.readlines(): if os.path.isfile(line.rstrip('\n').rstrip('\r')): intersects.append(line.rstrip('\n').rstrip('\r')) else: logger.warning("Imagepath in intersects textfile does not exist: %s" %line.rstrip('\n').rstrip('\r')) t.close() if len(intersects) == 0: logger.error("No images found: %s" %inpath) sys.exit() else: logger.info("Number of intersecting images: %i" %len(intersects)) #### gather image info list logger.info("Gathering image info") imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in intersects] #### Get mosaic parameters logger.info("Getting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0],args) logger.info("Mosaic extent: %f %f %f %f" %(params.xmin, params.xmax, params.ymin, params.ymax)) logger.info("Mosaic tilesize: %f %f" %(params.xtilesize, params.ytilesize)) logger.info("Mosaic resolution: %.10f %.10f" %(params.xres, params.yres)) logger.info("Mosaic projection: %s" %(params.proj)) logger.info("Getting Exact Image geometry") imginfo_list2 =[] for iinfo in imginfo_list: simplify_tolerance = 2.0 * ((params.xres + params.yres) / 2.0) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m geom,xs1,ys1 = mosaic.GetExactTrimmedGeom(iinfo.srcfp,step=args.cutline_step,tolerance=simplify_tolerance) if geom is None: logger.info("%s: geometry could not be determined" %iinfo.srcfn) elif geom.IsEmpty(): logger.info("%s: geometry is empty" %iinfo.srcfn) else: iinfo.geom = geom tm = datetime.today() imginfo_list2.append(iinfo) centroid = geom.Centroid() logger.info("%s: geometry acquired - centroid: %f, %f" %(iinfo.srcfn, centroid.GetX(), centroid.GetY())) #print geom logger.info("Getting image metadata and calculating image scores") for iinfo in imginfo_list2: iinfo.getScore(params) if (params.median_remove is True): iinfo.get_raster_median() iinfo.get_raster_stats() logger.info("%s: %s" %(iinfo.srcfn,iinfo.score)) #### Overlay geoms and remove non-contributors if args.component_shp: contribs = [(iinfo,iinfo.geom) for iinfo in imginfo_list2] else: logger.info("Overlaying images to determine contributors") contribs = [] for i in xrange(0,len(imginfo_list2)): iinfo = imginfo_list2[i] basegeom = iinfo.geom for j in range(i+1,len(imginfo_list2)): iinfo2 = imginfo_list2[j] geom2 = iinfo2.geom if basegeom.Intersects(geom2): basegeom = basegeom.Difference(geom2) if basegeom is None or basegeom.IsEmpty(): break if basegeom is None: logger.info("Function Error: %s" %iinfo.srcfn) elif basegeom.IsEmpty(): logger.info("Removing non-contributing image: %s" %iinfo.srcfn) else: basegeom = basegeom.Intersection(extent_geom) if basegeom is None: logger.info("Function Error: %s" %iinfo.srcfn) elif basegeom.IsEmpty(): logger.info("Removing non-contributing image: %s" %iinfo.srcfn) else: contribs.append((iinfo,basegeom)) tm = datetime.today() logger.info("Image: %s" %(os.path.basename(image))) logger.info("Number of contributors: %d" %len(contribs)) ####################################################### #### Create Shp logger.info("Creating shapefile of image boundaries: %s" %shp) if (params.median_remove is True): fields = ( ("IMAGENAME", ogr.OFTString, 100), ("SENSOR", ogr.OFTString, 10), ("ACQDATE", ogr.OFTString, 10), ("CAT_ID", ogr.OFTString, 30), ("RESOLUTION", ogr.OFTReal, 0), ("OFF_NADIR", ogr.OFTReal, 0), ("SUN_ELEV", ogr.OFTReal, 0), ("SUN_AZ", ogr.OFTReal, 0), ("SAT_ELEV", ogr.OFTReal, 0), ("SAT_AZ", ogr.OFTReal, 0), ("STATS_MIN", ogr.OFTString, 80), ("STATS_MAX", ogr.OFTString, 80), ("STATS_STD", ogr.OFTString, 80), ("STATS_MEAN", ogr.OFTString, 80), ("STATS_PXCT", ogr.OFTString, 80), ("MEDIAN", ogr.OFTString, 80), ("CLOUDCOVER", ogr.OFTReal, 0), ("TDI", ogr.OFTReal, 0), ("DATE_DIFF", ogr.OFTReal, 0), ("SCORE", ogr.OFTReal, 0), ) else: fields = ( ("IMAGENAME", ogr.OFTString, 100), ("SENSOR", ogr.OFTString, 10), ("ACQDATE", ogr.OFTString, 10), ("CAT_ID", ogr.OFTString, 30), ("RESOLUTION", ogr.OFTReal, 0), ("OFF_NADIR", ogr.OFTReal, 0), ("SUN_ELEV", ogr.OFTReal, 0), ("SUN_AZ", ogr.OFTReal, 0), ("SAT_ELEV", ogr.OFTReal, 0), ("SAT_AZ", ogr.OFTReal, 0), ("STATS_MIN", ogr.OFTString, 80), ("STATS_MAX", ogr.OFTString, 80), ("STATS_STD", ogr.OFTString, 80), ("STATS_MEAN", ogr.OFTString, 80), ("STATS_PXCT", ogr.OFTString, 80), ("CLOUDCOVER", ogr.OFTReal, 0), ("TDI", ogr.OFTReal, 0), ("DATE_DIFF", ogr.OFTReal, 0), ("SCORE", ogr.OFTReal, 0), ) OGR_DRIVER = "ESRI Shapefile" ogrDriver = ogr.GetDriverByName(OGR_DRIVER) if ogrDriver is None: logger.info("OGR: Driver %s is not available" % OGR_DRIVER) sys.exit(-1) if os.path.isfile(shp): ogrDriver.DeleteDataSource(shp) vds = ogrDriver.CreateDataSource(shp) if vds is None: logger.info("Could not create shp") sys.exit(-1) shpd, shpn = os.path.split(shp) shpbn, shpe = os.path.splitext(shpn) rp = osr.SpatialReference() rp.ImportFromWkt(params.proj) lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon) if lyr is None: logger.info("ERROR: Failed to create layer: %s" % shpbn) sys.exit(-1) for fld, fdef, flen in fields: field_defn = ogr.FieldDefn(fld, fdef) if fdef == ogr.OFTString: field_defn.SetWidth(flen) if lyr.CreateField(field_defn) != 0: logger.info("ERROR: Failed to create field: %s" % fld) for iinfo,geom in contribs: logger.info("Image: %s" %(iinfo.srcfn)) feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField("IMAGENAME", iinfo.srcfn) feat.SetField("SENSOR", iinfo.sensor) feat.SetField("ACQDATE", iinfo.acqdate.strftime("%Y-%m-%d")) feat.SetField("CAT_ID", iinfo.catid) feat.SetField("OFF_NADIR", iinfo.ona) feat.SetField("SUN_ELEV" ,iinfo.sunel) feat.SetField("SUN_AZ", iinfo.sunaz) feat.SetField("SAT_ELEV", iinfo.satel) feat.SetField("SAT_AZ", iinfo.sataz) feat.SetField("CLOUDCOVER", iinfo.cloudcover) feat.SetField("SCORE", iinfo.score) tdi = iinfo.tdi if iinfo.tdi else 0 feat.SetField("TDI", tdi) date_diff = iinfo.date_diff if iinfo.date_diff else -9999 feat.SetField("DATE_DIFF", date_diff) res = ((iinfo.xres+iinfo.yres)/2.0) if iinfo.xres else 0 feat.SetField("RESOLUTION", res) if len(iinfo.stat_dct) > 0: min_list = [] max_list = [] mean_list = [] stdev_list = [] px_cnt_list = [] keys = iinfo.stat_dct.keys() keys.sort() for band in keys: imin, imax, imean, istdev = iinfo.stat_dct[band] ipx_cnt = iinfo.datapixelcount_dct[band] min_list.append(str(imin)) max_list.append(str(imax)) mean_list.append(str(imean)) stdev_list.append(str(istdev)) px_cnt_list.append(str(ipx_cnt)) feat.SetField("STATS_MIN", ",".join(min_list)) feat.SetField("STATS_MAX", ",".join(max_list)) feat.SetField("STATS_MEAN", ",".join(mean_list)) feat.SetField("STATS_STD", ",".join(stdev_list)) feat.SetField("STATS_PXCT", ",".join(px_cnt_list)) if (params.median_remove is True): median_list = [] keys = iinfo.median.keys() keys.sort() for band in keys: band_median = iinfo.median[band] median_list.append(str(band_median)) feat.SetField("MEDIAN", ",".join(median_list)) logger.info("median = {}".format(",".join(median_list))) feat.SetGeometry(geom) if lyr.CreateFeature(feat) != 0: logger.info("ERROR: Could not create feature for image %s" % image) else: logger.info("Created feature for image: %s" %image) feat.Destroy() etm = datetime.today() td = (etm-stm) logger.info("Total Processing Time: %s\n" %(td))
def main(): #### Set Up Arguments parent_parser = mosaic.buildMosaicParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="Sumbit/run batch mosaic tasks" ) parser.add_argument("src", help="textfile or directory of input rasters (tif only)") parser.add_argument("mosaicname", help="output mosaic name excluding extension") pos_arg_keys = ["src","mosaicname"] parser.add_argument("--mode", choices=mosaic.MODES , default="ALL", help=" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, TEST- create log only") parser.add_argument("--wd", help="scratch space (default is mosaic directory)") parser.add_argument("--component-shp", action="store_true", default=False, help="create shp of all componenet images") parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw", help="GTiff compression type. Default=lzw (%s)"%(",".join(mosaic.GTIFF_COMPRESSIONS))) parser.add_argument("--pbs", action='store_true', default=False, help="submit tasks to PBS") parser.add_argument("--parallel-processes", type=int, default=1, help="number of parallel processes to spawn (default 1)") parser.add_argument("--qsubscript", help="qsub script to use in cluster job submission (default is <script_dir>/%s)" %default_qsub_script) parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax). Use only on HPC systems.") parser.add_argument("--log", help="file to log progress (default is <output dir>\%s" %default_logfile) #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) inpath = os.path.abspath(args.src) mosaicname = os.path.abspath(args.mosaicname) mosaicname = os.path.splitext(mosaicname)[0] mosaic_dir = os.path.dirname(mosaicname) cutline_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_cutlines.py') tile_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_tile.py') ## Verify qsubscript if args.qsubscript is None: qsubpath = os.path.join(os.path.dirname(scriptpath),default_qsub_script) else: qsubpath = os.path.abspath(args.qsubscript) if not os.path.isfile(qsubpath): parser.error("qsub script path is not valid: %s" %qsubpath) ## Verify processing options do not conflict if args.pbs and args.parallel_processes > 1: parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive") #### Validate Arguments if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False else: parser.error("Arg1 is not a valid file path or directory: %s" %inpath) if not os.path.isdir(mosaic_dir): os.makedirs(mosaic_dir) if not os.path.isfile(qsubpath): parser.error("Arg3 is not a valid file path: %s" %qsubpath) #### Validate target day option if args.tday is not None: try: m = int(args.tday.split("-")[0]) d = int(args.tday.split("-")[1]) td = date(2000,m,d) except ValueError: logger.error("Target day must be in mm-dd format (i.e 04-05)") sys.exit(1) else: m = 0 d = 0 #### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(mosaic_dir,default_logfile) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): parser.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() #### Get Images #logger.info("Reading input images") xs = [] ys = [] image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list) if len(image_list) == 0: logger.error("No images found in input file or directory: %s" %inpath) sys.exit() else: logger.info("%i existing images found" %len(image_list)) #### gather image info list logger.info("Getting image info") imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in image_list] #### Get mosaic parameters logger.info("Setting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0],args) #### Remove images that do not match ref logger.info("Applying attribute filter") imginfo_list2 = mosaic.filterMatchingImages(imginfo_list,params) if len(imginfo_list2) == 0: logger.error("No valid images found. Check input filter parameters.") sys.exit() else: logger.info("%i images match filter" %len(imginfo_list2)) #### if extent is specified, build tile params and compare extent to input image geom if args.extent: imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params) #### else set extent after image geoms computed else: #### Get geom for each image imginfo_list3 = [] for iinfo in imginfo_list2: if iinfo.geom is not None: xs = xs + iinfo.xs ys = ys + iinfo.ys imginfo_list3.append(iinfo) else: # remove from list if no geom logger.debug("Null geometry for image: %s" %iinfo.srcfn) params.xmin = min(xs) params.xmax = max(xs) params.ymin = min(ys) params.ymax = max(ys) poly_wkt = 'POLYGON (( %f %f, %f %f, %f %f, %f %f, %f %f ))' %(params.xmin,params.ymin,params.xmin,params.ymax,params.xmax,params.ymax,params.xmax,params.ymin,params.xmin,params.ymin) params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) #### Check number of remaining images num_images = len(imginfo_list3) if num_images > 0: logger.info("%d of %d input images intersect mosaic extent" %(num_images,len(image_list))) logger.info("Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f" %(params.xres,params.yres,params.xtilesize,params.ytilesize,params.xmin,params.xmax,params.ymin,params.ymax)) else: logger.error("No valid images found") sys.exit(0) ## Sort images by score logger.info("Reading image metadata and determining sort order") for iinfo in imginfo_list3: iinfo.getScore(params) #### Sort by score if not args.nosort: imginfo_list3.sort(key=lambda x: x.score) #### Write all intersects file intersects_all = [] for iinfo in imginfo_list3: if params.extent_geom.Intersect(iinfo.geom) is True: if iinfo.score > 0: intersects_all.append(iinfo) elif args.nosort: intersects_all.append(iinfo) else: logger.debug("Image has an invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score)) else: logger.debug("Image does not intersect mosaic extent: %s" %iinfo.srcfp) ### this line should never be needed. non-intersecting images should be removed earlier if extent is provided, otherwise all images are in the extent. aitpath = mosaicname+"_intersects.txt" ait = open(aitpath,"w") intersects_fps = [intersect.srcfp for intersect in intersects_all] ait.write(string.join(intersects_fps,"\n")) ait.close() ## Create tiles logger.info("Creating tiles") tiles = [] xtiledim = math.ceil((params.xmax-params.xmin)/params.xtilesize) ytiledim = math.ceil((params.ymax-params.ymin)/params.ytilesize) logger.info("Tiles: %d rows, %d columns" %(ytiledim,xtiledim)) xtdb = len(str(int(xtiledim))) ytdb = len(str(int(ytiledim))) i = 1 for x in mosaic.drange(params.xmin,params.xmax,params.xtilesize): # Columns if x+params.xtilesize > params.xmax: x2 = params.xmax else: x2 = x+params.xtilesize j = 1 for y in mosaic.drange(params.ymin,params.ymax,params.ytilesize): # Rows if y+params.ytilesize > params.ymax: y2 = params.ymax else: y2 = y+params.ytilesize tilename = "%s_%s_%s.tif" %(mosaicname,mosaic.buffernum(j,ytdb),mosaic.buffernum(i,xtdb)) tile = mosaic.TileParams(x,x2,y,y2,j,i,tilename) tiles.append(tile) j += 1 i += 1 num_tiles = len(tiles) #### Write shapefile of tiles if args.mode == "ALL" or args.mode == "SHP": shp = mosaicname + "_tiles.shp" if os.path.isfile(shp): logger.info("Tiles shapefile already exists: %s" %os.path.basename(shp)) else: logger.info("Creating shapefile of tiles: %s" %os.path.basename(shp)) fields = [('ROW', ogr.OFTInteger, 4), ('COL', ogr.OFTInteger, 4), ("TILENAME", ogr.OFTString, 100), ('TILEPATH', ogr.OFTString, 254), ('XMIN', ogr.OFTReal, 0), ('XMAX', ogr.OFTReal, 0), ('YMIN', ogr.OFTReal, 0), ('YMAX', ogr.OFTReal, 0)] OGR_DRIVER = "ESRI Shapefile" ogrDriver = ogr.GetDriverByName(OGR_DRIVER) if ogrDriver is None: logger.error("OGR: Driver %s is not available" % OGR_DRIVER) sys.exit(-1) if os.path.isfile(shp): ogrDriver.DeleteDataSource(shp) vds = ogrDriver.CreateDataSource(shp) if vds is None: logger.error("Could not create shp") sys.exit(-1) shpd, shpn = os.path.split(shp) shpbn, shpe = os.path.splitext(shpn) rp = osr.SpatialReference() rp.ImportFromWkt(params.proj) lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon) if lyr is None: logger.error("ERROR: Failed to create layer: %s" % shpbn) sys.exit(-1) for fld, fdef, flen in fields: field_defn = ogr.FieldDefn(fld, fdef) if fdef == ogr.OFTString: field_defn.SetWidth(flen) if lyr.CreateField(field_defn) != 0: logger.error("ERROR: Failed to create field: %s" % fld) for t in tiles: feat = ogr.Feature(lyr.GetLayerDefn()) feat.SetField("TILENAME",os.path.basename(t.name)) feat.SetField("TILEPATH",t.name) feat.SetField("ROW",t.j) feat.SetField("COL",t.i) feat.SetField("XMIN",t.xmin) feat.SetField("XMAX",t.xmax) feat.SetField("YMIN",t.ymin) feat.SetField("YMAX",t.ymax) feat.SetGeometry(t.geom) if lyr.CreateFeature(feat) != 0: logger.error("ERROR: Could not create feature for tile %s" % tile) feat.Destroy() ## Build tasks task_queue = [] #### Create task for shapefile of mosaic components if args.component_shp is True: arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'gtiff_compression', 'mode', 'extent', 'resolution', 'pbs', 'wd' ) shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) comp_shp = mosaicname + "_components.shp" if os.path.isfile(comp_shp): logger.info("Components shapefile already exists: %s" %os.path.basename(comp_shp)) else: logger.info("Processing components: %s" %os.path.basename(comp_shp)) ## Make task and add to queue cmd = '{} --cutline-step 512 {} -e {} {} {} {} {} {}'.format( cutline_builder_script, shp_arg_str, params.xmin, params.xmax, params.ymin, params.ymax, comp_shp, aitpath ) task = utils.Task( 'Components', 'Components', 'python', cmd ) if args.mode == "ALL" or args.mode == "SHP": logger.debug(cmd) task_queue.append(task) #### Create task for shapefile of image cutlines shp = mosaicname + "_cutlines.shp" arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'gtiff_compression', 'mode', 'extent', 'resolution', 'component_shp', 'pbs', 'wd' ) shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s" %os.path.basename(shp)) else: logger.info("Processing cutlines: %s" %os.path.basename(shp)) ## Make task and add to queue cmd = '{} {} -e {} {} {} {} {} {}'.format( cutline_builder_script, shp_arg_str, params.xmin, params.xmax, params.ymin, params.ymax, shp, aitpath ) task = utils.Task( 'Cutlines', 'Cutlines', 'python', cmd ) if args.mode == "ALL" or args.mode == "SHP": logger.debug(cmd) task_queue.append(task) #### Create task for each tile arg_keys_to_remove = ( 'l', 'qsubscript', 'parallel_processes', 'log', 'mode', 'extent', 'resolution', 'bands', 'component_shp', 'pbs' ) tile_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove) logger.debug("Identifying components of {0} subtiles".format(num_tiles)) i = 0 for t in tiles: logger.debug("Identifying components of tile %d of %d: %s" %(i,num_tiles,os.path.basename(t.name))) #### determine which images in each tile - create geom and query image geoms logger.debug("Running intersect with imagery") intersects = [] for iinfo in intersects_all: if t.geom.Intersect(iinfo.geom) is True: if iinfo.score > 0: logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score)) intersects.append(iinfo.srcfp) elif args.nosort: logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score)) intersects.append(iinfo.srcfp) else: logger.warning("Invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score)) #### If any images are in the tile, mosaic them if len(intersects) > 0: tile_basename = os.path.basename(os.path.splitext(t.name)[0]) itpath = os.path.join(mosaic_dir,tile_basename+"_intersects.txt") it = open(itpath,"w") it.write(string.join(intersects,"\n")) it.close() #### Submit QSUB job logger.debug("Building mosaicking job for tile: %s" %os.path.basename(t.name)) if not os.path.isfile(t.name): cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format( tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin, t.ymax, params.xres, params.yres, params.bands, t.name, itpath ) task = utils.Task( 'Tile {0}'.format(os.path.basename(t.name)), 'Mosaic{:04g}'.format(i), 'python', cmd ) if args.mode == "ALL" or args.mode == "MOSAIC": logger.debug(cmd) task_queue.append(task) else: logger.info("Tile already exists: %s" %os.path.basename(t.name)) i += 1 logger.info("Submitting Tasks") #logger.info(task_queue) if len(task_queue) > 0: if args.pbs: if args.l: task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l)) else: task_handler = utils.PBSTaskHandler(qsubpath) task_handler.run_tasks(task_queue) else: task_handler = utils.ParallelTaskHandler(args.parallel_processes) if task_handler.num_processes > 1: logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes)) task_handler.run_tasks(task_queue) logger.info("Done") else: logger.info("No tasks to process")
def main(): #### Set Up Arguments parent_parser = mosaic.buildMosaicParentArgumentParser() parser = argparse.ArgumentParser( parents=[parent_parser], description="query PGC index for images contributing to a mosaic" ) parser.add_argument("index", help="PGC index shapefile") parser.add_argument("tile_csv", help="tile schema csv") parser.add_argument("dstdir", help="textfile output directory") #pos_arg_keys = ["index","tile_csv","dstdir"] parser.add_argument("--log", help="output log file (default is queryFP.log in the output folder)") parser.add_argument("--ttile", help="target tile (default is to compute all valid tiles. multiple tiles should be delimited by a comma [ex: 23_24,23_25])") parser.add_argument("--overwrite", action="store_true", default=False, help="overwrite any existing files") parser.add_argument("--stretch", choices=ortho_functions.stretches, default="rf", help="stretch abbreviation used in image processing (default=rf)") parser.add_argument("--build-shp", action='store_true', default=False, help="build shapefile of intersecting images (only invoked if --no_sort is not used)") parser.add_argument("--online-only", action='store_true', default=False, help="limit search to those records where status = online and image is found on the file system") parser.add_argument("--require-pan", action='store_true', default=False, help="limit search to imagery with both a multispectral and a panchromatic component") #### Parse Arguments args = parser.parse_args() scriptpath = os.path.abspath(sys.argv[0]) shp = os.path.abspath(args.index) csvpath = os.path.abspath(args.tile_csv) dstdir = os.path.abspath(args.dstdir) #### Validate Required Arguments if not os.path.isfile(shp): parser.error("Arg1 is not a valid file path: %s" %shp) if not os.path.isfile(csvpath): parser.error("Arg2 is not a valid file path: %s" %csvpath) #### Validate target day option if args.tday is not None: try: m = int(args.tday.split("-")[0]) d = int(args.tday.split("-")[1]) td = date(2000,m,d) except ValueError: logger.error("Target day must be in mm-dd format (i.e 04-05)") sys.exit(1) else: m = 0 d = 0 ##### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(dstdir,"queryFP_%s.log" %datetime.today().strftime("%Y%m%d%H%M%S")) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude_list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): parser.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() logger.debug("Exclude list: %s" %str(exclude_list)) #### Parse csv, validate tile ID and get tilegeom tiles = {} csv = open(csvpath,'r') for line in csv: tile = line.rstrip().split(",") if len(tile) != 9: logger.warning("funny csv line: %s" %line.strip('\n')) else: name = tile[2] if name != "name": ### Tile csv schema: row, column, name, status, xmin, xmax, ymin, ymax, epsg code t = mosaic.TileParams(float(tile[4]),float(tile[5]),float(tile[6]),float(tile[7]),int(tile[0]),int(tile[1]),tile[2]) t.status = tile[3] t.epsg = int(tile[8]) tiles[name] = t csv.close() if args.ttile is not None: if "," in args.ttile: ttiles = args.ttile.split(",") else: ttiles = [args.ttile] for ttile in ttiles: if ttile not in tiles: logger.info("Target tile is not in the tile csv: %s" %ttile) else: t = tiles[ttile] if t.status == "0": logger.error("Tile status indicates it should not be created: %s, %s" %(ttile,t.status)) else: HandleTile(t,shp,dstdir,csvpath,args,exclude_list) else: keys = tiles.keys() keys.sort() for tile in keys: t = tiles[tile] if t.status == "1": HandleTile(t,shp,dstdir,csvpath,args,exclude_list)