def test_get_exact_trimmed_geom(self): xs_expected = [ 2502000.0, 2867000.0, 2867000.0, -2868000.0, -2868000.0, -2503000.0 ] ys_expected = [ 2455500.0, 455500.0, -1544500.0, -1544500.0, 455500.0, 2455500.0 ] geom, xs, ys = mosaic.GetExactTrimmedGeom(self.dem, step=400) self.assertEqual(xs, xs_expected) self.assertEqual(ys, ys_expected)
def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args, pos_arg_keys): if os.path.isfile(inpath): bTextfile = True elif os.path.isdir(inpath): bTextfile = False if not os.path.isdir(mosaic_dir): os.makedirs(mosaic_dir) ## TODO: verify logger woks for both interactive and hpc jobs #### Configure Logger if args.log is not None: logfile = os.path.abspath(args.log) else: logfile = os.path.join(mosaic_dir, default_logfile) lfh = logging.FileHandler(logfile) lfh.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lfh.setFormatter(formatter) logger.addHandler(lfh) lsh = logging.StreamHandler() lsh.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S') lsh.setFormatter(formatter) logger.addHandler(lsh) #### Get exclude list if specified if args.exclude is not None: if not os.path.isfile(args.exclude): logger.error("Value for option --exclude-list is not a valid file") f = open(args.exclude, 'r') exclude_list = set([line.rstrip() for line in f.readlines()]) else: exclude_list = set() #### Get Images #logger.info("Reading input images") xs = [] ys = [] image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list) if len(image_list) == 0: raise RuntimeError( "No images found in input file or directory: {}".format(inpath)) # remove duplicate images image_list_unique = list(set(image_list)) dupes = [x for n, x in enumerate(image_list) if x in image_list[:n]] if len(dupes) > 0: logger.info("Removed %i duplicate image paths", len(dupes)) logger.debug("Dupes: %s", dupes) image_list = image_list_unique logger.info("%i existing images found", len(image_list)) #### gather image info list logger.info("Getting image info") imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list] #### Get mosaic parameters logger.info("Setting mosaic parameters") params = mosaic.getMosaicParameters(imginfo_list[0], args) logger.info("Mosaic parameters: band count=%i, datatype=%s", params.bands, params.datatype) logger.info("Mosaic parameters: projection=%s", params.proj) #### Remove images that do not match ref logger.info("Applying attribute filter") imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, params) if len(imginfo_list2) == 0: raise RuntimeError( "No valid images found. Check input filter parameters.") logger.info("%i of %i images match filter", len(imginfo_list2), len(image_list)) #### if extent is specified, build tile params and compare extent to input image geom if args.extent: imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params) #### else set extent after image geoms computed else: #### Get geom for each image imginfo_list3 = [] for iinfo in imginfo_list2: if iinfo.geom is not None: xs = xs + iinfo.xs ys = ys + iinfo.ys imginfo_list3.append(iinfo) else: # remove from list if no geom logger.debug("Null geometry for image: %s", iinfo.srcfn) params.xmin = min(xs) params.xmax = max(xs) params.ymin = min(ys) params.ymax = max(ys) poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format( params.xmin, params.ymin, params.xmin, params.ymax, params.xmax, params.ymax, params.xmax, params.ymin, params.xmin, params.ymin) params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt) if len(imginfo_list3) == 0: raise RuntimeError("No images found that intersect mosaic extent") logger.info( "Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f", params.xres, params.yres, params.xtilesize, params.ytilesize, params.xmin, params.xmax, params.ymin, params.ymax) logger.info("%d of %d input images intersect mosaic extent", len(imginfo_list3), len(imginfo_list2)) ## Sort images by score logger.info("Reading image metadata and determining sort order") for iinfo in imginfo_list3: iinfo.getScore(params) if not args.nosort: imginfo_list3.sort(key=lambda x: x.score) logger.info("Getting Exact Image geometry") imginfo_list4 = [] all_valid = True for iinfo in imginfo_list3: if iinfo.score > 0 or args.nosort: simplify_tolerance = 2.0 * ( (params.xres + params.yres) / 2.0 ) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m geom, xs1, ys1 = mosaic.GetExactTrimmedGeom( iinfo.srcfp, step=args.cutline_step, tolerance=simplify_tolerance) if geom is None: logger.warning( "%s: geometry could not be determined, verify image is valid", iinfo.srcfn) all_valid = False elif geom.IsEmpty(): logger.warning("%s: geometry is empty", iinfo.srcfn) all_valid = False else: iinfo.geom = geom tm = datetime.today() imginfo_list4.append(iinfo) centroid = geom.Centroid() logger.info("%s: geometry acquired - centroid: %f, %f", iinfo.srcfn, centroid.GetX(), centroid.GetY()) #print(geom) else: logger.debug("Image has an invalid score: %s --> %i", iinfo.srcfp, iinfo.score) if not all_valid: if not args.allow_invalid_geom: raise RuntimeError( "Some source images do not have valid geometries. Cannot proceeed" ) else: logger.info( "--allow-invalid-geom used; mosaic will be created using %i valid images (%i invalid \ images not used.)".format( len(imginfo_list4), len(imginfo_list3) - len(imginfo_list4))) # Get stats if needed logger.info("Getting image metadata") for iinfo in imginfo_list4: logger.info(iinfo.srcfn) if args.calc_stats or args.median_remove: iinfo.get_raster_stats(args.calc_stats, args.median_remove) # Build componenet index if args.component_shp: if args.mode == "ALL" or args.mode == "SHP": contribs = [(iinfo, iinfo.geom) for iinfo in imginfo_list4] logger.info("Number of contributors: %d", len(contribs)) logger.info("Building component index") comp_shp = mosaicname + "_components.shp" if len(contribs) > 0: if os.path.isfile(comp_shp): logger.info("Components shapefile already exists: %s", comp_shp) else: build_shp(contribs, comp_shp, args, params) else: logger.error("No contributing images") # Build cutlines index #### Overlay geoms and remove non-contributors logger.info("Overlaying images to determine contribution geom") contribs = mosaic.determine_contributors(imginfo_list4, params.extent_geom, args.min_contribution_area) logger.info("Number of contributors: %d", len(contribs)) if args.mode == "ALL" or args.mode == "SHP": logger.info("Building cutlines index") shp = mosaicname + "_cutlines.shp" if len(contribs) > 0: if os.path.isfile(shp): logger.info("Cutlines shapefile already exists: %s", shp) else: build_shp(contribs, shp, args, params) else: logger.error("No contributing images") ## Create tile objects tiles = [] xtiledim = math.ceil((params.xmax - params.xmin) / params.xtilesize) ytiledim = math.ceil((params.ymax - params.ymin) / params.ytilesize) logger.info("Tiles: %d rows, %d columns", ytiledim, xtiledim) xtdb = len(str(int(xtiledim))) ytdb = len(str(int(ytiledim))) i = 1 for x in mosaic.drange(params.xmin, params.xmax, params.xtilesize): # Columns if x + params.xtilesize > params.xmax: x2 = params.xmax else: x2 = x + params.xtilesize j = 1 for y in mosaic.drange(params.ymin, params.ymax, params.ytilesize): # Rows if y + params.ytilesize > params.ymax: y2 = params.ymax else: y2 = y + params.ytilesize tilename = "{}_{}_{}.tif".format(mosaicname, mosaic.buffernum(j, ytdb), mosaic.buffernum(i, xtdb)) tile = mosaic.TileParams(x, x2, y, y2, j, i, tilename) tiles.append(tile) j += 1 i += 1 #### Write shapefile of tiles if len(tiles) == 0: raise RuntimeError("No tile objects created") if args.mode == "ALL" or args.mode == "SHP": build_tiles_shp(mosaicname, tiles, params) ## Build tile tasks task_queue = [] #### Create task for each tile arg_keys_to_remove = ('l', 'qsubscript', 'parallel_processes', 'log', 'mode', 'extent', 'resolution', 'bands', 'max_cc', 'exclude', 'nosort', 'component_shp', 'cutline_step', 'min_contribution_area', 'calc_stats', 'pbs', 'slurm', 'tday', 'tyear', 'allow_invalid_geom') tile_arg_str = taskhandler.convert_optional_args_to_string( args, pos_arg_keys, arg_keys_to_remove) logger.debug("Identifying components of %i subtiles", len(tiles)) i = 0 for t in tiles: logger.debug("Identifying components of tile %i of %i: %s", i, len(tiles), os.path.basename(t.name)) #### determine which images in each tile - create geom and query image geoms logger.debug("Running intersect with imagery") intersects = [] for iinfo, contrib_geom in contribs: if contrib_geom.Intersects(t.geom): if args.median_remove: ## parse median dct into text median_string = ";".join([ "{}:{}".format(k, v) for k, v in iinfo.median.items() ]) intersects.append("{},{}".format(iinfo.srcfp, median_string)) else: intersects.append(iinfo.srcfp) #### If any images are in the tile, mosaic them if len(intersects) > 0: tile_basename = os.path.basename(os.path.splitext(t.name)[0]) logger.info("Number of contributors to subtile %s: %i", tile_basename, len(intersects)) itpath = os.path.join(mosaic_dir, tile_basename + "_intersects.txt") it = open(itpath, "w") it.write("\n".join(intersects)) it.close() #### Submit QSUB job logger.debug("Building mosaicking job for tile: %s", os.path.basename(t.name)) if not os.path.isfile(t.name): cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format( tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin, t.ymax, params.xres, params.yres, params.bands, t.name, itpath) task = taskhandler.Task( 'Tile {0}'.format(os.path.basename(t.name)), 'Mos{:04g}'.format(i), 'python', cmd) if args.mode == "ALL" or args.mode == "MOSAIC": logger.debug(cmd) task_queue.append(task) else: logger.info("Tile already exists: %s", os.path.basename(t.name)) i += 1 if args.mode == "ALL" or args.mode == "MOSAIC": logger.info("Submitting Tasks") #logger.info(task_queue) if len(task_queue) > 0: try: task_handler = taskhandler.ParallelTaskHandler( args.parallel_processes) except RuntimeError as e: logger.error(e) else: if task_handler.num_processes > 1: logger.info("Number of child processes to spawn: %i", task_handler.num_processes) task_handler.run_tasks(task_queue) logger.info("Done") else: logger.info("No tasks to process")