Example #1
0
    def test_filter_images(self):
        image_list = glob.glob(os.path.join(self.srcdir, '*.tif'))
        imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list]
        filter_list = [iinfo.srcfn for iinfo in imginfo_list]
        self.assertIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413.tif',
                      filter_list)
        self.assertIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif',
                      filter_list)
        self.assertIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif',
                      filter_list)

        mosaic_args = MosaicArgs()
        mosaic_args.extent = [-820000.0, -800000.0, -2420000.0, -2400000.0]
        mosaic_args.tilesize = [20000, 20000]
        mosaic_args.bands = 1
        mosaic_params = mosaic.getMosaicParameters(imginfo_list[0], mosaic_args)
        imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, mosaic_params)
        filter_list = [iinfo.srcfn for iinfo in imginfo_list2]
        
        self.assertEqual(len(imginfo_list2), 8)
        self.assertNotIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413.tif',
                         filter_list)
        self.assertNotIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif',
                         filter_list)
        
        imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, mosaic_params)
        filter_list = [iinfo.srcfn for iinfo in imginfo_list3]
        self.assertEqual(len(imginfo_list3), 7)
        self.assertNotIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif',
                         filter_list)
Example #2
0
def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args,
               pos_arg_keys):

    if os.path.isfile(inpath):
        bTextfile = True
    elif os.path.isdir(inpath):
        bTextfile = False
    if not os.path.isdir(mosaic_dir):
        os.makedirs(mosaic_dir)

    ## TODO: verify logger woks for both interactive and hpc jobs
    #### Configure Logger
    if args.log is not None:
        logfile = os.path.abspath(args.log)
    else:
        logfile = os.path.join(mosaic_dir, default_logfile)

    lfh = logging.FileHandler(logfile)
    lfh.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
                                  '%m-%d-%Y %H:%M:%S')
    lfh.setFormatter(formatter)
    logger.addHandler(lfh)

    lsh = logging.StreamHandler()
    lsh.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
                                  '%m-%d-%Y %H:%M:%S')
    lsh.setFormatter(formatter)
    logger.addHandler(lsh)

    #### Get exclude list if specified
    if args.exclude is not None:
        if not os.path.isfile(args.exclude):
            logger.error("Value for option --exclude-list is not a valid file")

        f = open(args.exclude, 'r')
        exclude_list = set([line.rstrip() for line in f.readlines()])
    else:
        exclude_list = set()

    #### Get Images
    #logger.info("Reading input images")
    xs = []
    ys = []

    image_list = utils.find_images_with_exclude_list(inpath, bTextfile,
                                                     mosaic.EXTS, exclude_list)

    if len(image_list) == 0:
        raise RuntimeError(
            "No images found in input file or directory: {}".format(inpath))

    # remove duplicate images
    image_list_unique = list(set(image_list))
    dupes = [x for n, x in enumerate(image_list) if x in image_list[:n]]
    if len(dupes) > 0:
        logger.info("Removed %i duplicate image paths", len(dupes))
        logger.debug("Dupes: %s", dupes)
    image_list = image_list_unique

    logger.info("%i existing images found", len(image_list))

    #### gather image info list
    logger.info("Getting image info")
    imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list]

    #### Get mosaic parameters
    logger.info("Setting mosaic parameters")
    params = mosaic.getMosaicParameters(imginfo_list[0], args)
    logger.info("Mosaic parameters: band count=%i, datatype=%s", params.bands,
                params.datatype)
    logger.info("Mosaic parameters: projection=%s", params.proj)

    #### Remove images that do not match ref
    logger.info("Applying attribute filter")
    imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, params)

    if len(imginfo_list2) == 0:
        raise RuntimeError(
            "No valid images found.  Check input filter parameters.")

    logger.info("%i of %i images match filter", len(imginfo_list2),
                len(image_list))

    #### if extent is specified, build tile params and compare extent to input image geom
    if args.extent:
        imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params)

    #### else set extent after image geoms computed
    else:

        #### Get geom for each image
        imginfo_list3 = []
        for iinfo in imginfo_list2:
            if iinfo.geom is not None:
                xs = xs + iinfo.xs
                ys = ys + iinfo.ys
                imginfo_list3.append(iinfo)
            else:  # remove from list if no geom
                logger.debug("Null geometry for image: %s", iinfo.srcfn)

        params.xmin = min(xs)
        params.xmax = max(xs)
        params.ymin = min(ys)
        params.ymax = max(ys)

        poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(
            params.xmin, params.ymin, params.xmin, params.ymax, params.xmax,
            params.ymax, params.xmax, params.ymin, params.xmin, params.ymin)
        params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)

    if len(imginfo_list3) == 0:
        raise RuntimeError("No images found that intersect mosaic extent")

    logger.info(
        "Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f",
        params.xres, params.yres, params.xtilesize, params.ytilesize,
        params.xmin, params.xmax, params.ymin, params.ymax)
    logger.info("%d of %d input images intersect mosaic extent",
                len(imginfo_list3), len(imginfo_list2))

    ## Sort images by score
    logger.info("Reading image metadata and determining sort order")
    for iinfo in imginfo_list3:
        iinfo.getScore(params)

    if not args.nosort:
        imginfo_list3.sort(key=lambda x: x.score)

    logger.info("Getting Exact Image geometry")
    imginfo_list4 = []
    all_valid = True

    for iinfo in imginfo_list3:
        if iinfo.score > 0 or args.nosort:
            simplify_tolerance = 2.0 * (
                (params.xres + params.yres) / 2.0
            )  ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m
            geom, xs1, ys1 = mosaic.GetExactTrimmedGeom(
                iinfo.srcfp,
                step=args.cutline_step,
                tolerance=simplify_tolerance)

            if geom is None:
                logger.warning(
                    "%s: geometry could not be determined, verify image is valid",
                    iinfo.srcfn)
                all_valid = False
            elif geom.IsEmpty():
                logger.warning("%s: geometry is empty", iinfo.srcfn)
                all_valid = False
            else:
                iinfo.geom = geom
                tm = datetime.today()
                imginfo_list4.append(iinfo)
                centroid = geom.Centroid()
                logger.info("%s: geometry acquired - centroid: %f, %f",
                            iinfo.srcfn, centroid.GetX(), centroid.GetY())
                #print(geom)
        else:
            logger.debug("Image has an invalid score: %s --> %i", iinfo.srcfp,
                         iinfo.score)

    if not all_valid:
        if not args.allow_invalid_geom:
            raise RuntimeError(
                "Some source images do not have valid geometries.  Cannot proceeed"
            )
        else:
            logger.info(
                "--allow-invalid-geom used; mosaic will be created using %i valid images (%i invalid \
                        images not used.)".format(
                    len(imginfo_list4),
                    len(imginfo_list3) - len(imginfo_list4)))

    # Get stats if needed
    logger.info("Getting image metadata")
    for iinfo in imginfo_list4:
        logger.info(iinfo.srcfn)
        if args.calc_stats or args.median_remove:
            iinfo.get_raster_stats(args.calc_stats, args.median_remove)

    # Build componenet index
    if args.component_shp:

        if args.mode == "ALL" or args.mode == "SHP":
            contribs = [(iinfo, iinfo.geom) for iinfo in imginfo_list4]
            logger.info("Number of contributors: %d", len(contribs))

            logger.info("Building component index")
            comp_shp = mosaicname + "_components.shp"
            if len(contribs) > 0:
                if os.path.isfile(comp_shp):
                    logger.info("Components shapefile already exists: %s",
                                comp_shp)
                else:
                    build_shp(contribs, comp_shp, args, params)

            else:
                logger.error("No contributing images")

    # Build cutlines index
    ####  Overlay geoms and remove non-contributors
    logger.info("Overlaying images to determine contribution geom")
    contribs = mosaic.determine_contributors(imginfo_list4, params.extent_geom,
                                             args.min_contribution_area)
    logger.info("Number of contributors: %d", len(contribs))

    if args.mode == "ALL" or args.mode == "SHP":
        logger.info("Building cutlines index")
        shp = mosaicname + "_cutlines.shp"
        if len(contribs) > 0:
            if os.path.isfile(shp):
                logger.info("Cutlines shapefile already exists: %s", shp)
            else:
                build_shp(contribs, shp, args, params)

        else:
            logger.error("No contributing images")

    ## Create tile objects
    tiles = []

    xtiledim = math.ceil((params.xmax - params.xmin) / params.xtilesize)
    ytiledim = math.ceil((params.ymax - params.ymin) / params.ytilesize)
    logger.info("Tiles: %d rows, %d columns", ytiledim, xtiledim)

    xtdb = len(str(int(xtiledim)))
    ytdb = len(str(int(ytiledim)))

    i = 1
    for x in mosaic.drange(params.xmin, params.xmax,
                           params.xtilesize):  # Columns
        if x + params.xtilesize > params.xmax:
            x2 = params.xmax
        else:
            x2 = x + params.xtilesize

        j = 1
        for y in mosaic.drange(params.ymin, params.ymax,
                               params.ytilesize):  # Rows
            if y + params.ytilesize > params.ymax:
                y2 = params.ymax
            else:
                y2 = y + params.ytilesize

            tilename = "{}_{}_{}.tif".format(mosaicname,
                                             mosaic.buffernum(j, ytdb),
                                             mosaic.buffernum(i, xtdb))
            tile = mosaic.TileParams(x, x2, y, y2, j, i, tilename)
            tiles.append(tile)
            j += 1
        i += 1

    ####  Write shapefile of tiles
    if len(tiles) == 0:
        raise RuntimeError("No tile objects created")

    if args.mode == "ALL" or args.mode == "SHP":
        build_tiles_shp(mosaicname, tiles, params)

    ## Build tile tasks
    task_queue = []

    ####  Create task for each tile
    arg_keys_to_remove = ('l', 'qsubscript', 'parallel_processes', 'log',
                          'mode', 'extent', 'resolution', 'bands', 'max_cc',
                          'exclude', 'nosort', 'component_shp', 'cutline_step',
                          'min_contribution_area', 'calc_stats', 'pbs',
                          'slurm', 'tday', 'tyear', 'allow_invalid_geom')
    tile_arg_str = taskhandler.convert_optional_args_to_string(
        args, pos_arg_keys, arg_keys_to_remove)

    logger.debug("Identifying components of %i subtiles", len(tiles))
    i = 0
    for t in tiles:
        logger.debug("Identifying components of tile %i of %i: %s", i,
                     len(tiles), os.path.basename(t.name))

        ####    determine which images in each tile - create geom and query image geoms
        logger.debug("Running intersect with imagery")

        intersects = []
        for iinfo, contrib_geom in contribs:
            if contrib_geom.Intersects(t.geom):
                if args.median_remove:
                    ## parse median dct into text
                    median_string = ";".join([
                        "{}:{}".format(k, v) for k, v in iinfo.median.items()
                    ])
                    intersects.append("{},{}".format(iinfo.srcfp,
                                                     median_string))
                else:
                    intersects.append(iinfo.srcfp)

        ####  If any images are in the tile, mosaic them
        if len(intersects) > 0:

            tile_basename = os.path.basename(os.path.splitext(t.name)[0])
            logger.info("Number of contributors to subtile %s: %i",
                        tile_basename, len(intersects))
            itpath = os.path.join(mosaic_dir,
                                  tile_basename + "_intersects.txt")
            it = open(itpath, "w")
            it.write("\n".join(intersects))
            it.close()

            #### Submit QSUB job
            logger.debug("Building mosaicking job for tile: %s",
                         os.path.basename(t.name))
            if not os.path.isfile(t.name):

                cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format(
                    tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin,
                    t.ymax, params.xres, params.yres, params.bands, t.name,
                    itpath)

                task = taskhandler.Task(
                    'Tile {0}'.format(os.path.basename(t.name)),
                    'Mos{:04g}'.format(i), 'python', cmd)

                if args.mode == "ALL" or args.mode == "MOSAIC":
                    logger.debug(cmd)
                    task_queue.append(task)

            else:
                logger.info("Tile already exists: %s",
                            os.path.basename(t.name))
        i += 1

    if args.mode == "ALL" or args.mode == "MOSAIC":
        logger.info("Submitting Tasks")
        #logger.info(task_queue)
        if len(task_queue) > 0:

            try:
                task_handler = taskhandler.ParallelTaskHandler(
                    args.parallel_processes)
            except RuntimeError as e:
                logger.error(e)
            else:
                if task_handler.num_processes > 1:
                    logger.info("Number of child processes to spawn: %i",
                                task_handler.num_processes)
                task_handler.run_tasks(task_queue)

            logger.info("Done")

        else:
            logger.info("No tasks to process")
Example #3
0
def HandleTile(t, src, dstdir, csvpath, args, exclude_list):

    otxtpath = os.path.join(
        dstdir,
        "{}_{}_orig.txt".format(os.path.basename(csvpath)[:-4], t.name))
    mtxtpath = os.path.join(
        dstdir,
        "{}_{}_ortho.txt".format(os.path.basename(csvpath)[:-4], t.name))

    if os.path.isfile(otxtpath) and os.path.isfile(
            mtxtpath) and args.overwrite is False:
        logger.info("Tile %s processing files already exist", t.name)
    else:
        logger.info("Tile %s", t.name)

        t_srs = osr.SpatialReference()
        t_srs.ImportFromEPSG(t.epsg)

        #### Open mfp
        dsp, lyrn = utils.get_source_names(src)

        ds = ogr.Open(dsp)
        if ds is None:
            logger.error("Open failed")

        else:
            lyr = ds.GetLayerByName(lyrn)

            if not lyr:
                raise RuntimeError(
                    "Layer {} does not exist in dataset {}".format(lyrn, dsp))
            else:

                s_srs = lyr.GetSpatialRef()
                #logger.debug(str(s_srs))
                #logger.debug(str(t.geom))

                tile_geom_in_s_srs = t.geom.Clone()

                if not t_srs.IsSame(s_srs):
                    ict = osr.CoordinateTransformation(t_srs, s_srs)
                    ct = osr.CoordinateTransformation(s_srs, t_srs)
                    tile_geom_in_s_srs.Transform(ict)

                # if the geometry crosses meridian, split it into multipolygon (else this breaks SetSpatialFilter)
                if utils.doesCross180(tile_geom_in_s_srs):
                    logger.debug(
                        "tile_geom_in_s_srs crosses 180 meridian; splitting to multiple polygons..."
                    )
                    tile_geom_in_s_srs = utils.getWrappedGeometry(
                        tile_geom_in_s_srs)

                lyr.ResetReading()
                lyr.SetSpatialFilter(tile_geom_in_s_srs)
                feat = lyr.GetNextFeature()

                imginfo_list1 = []

                while feat:

                    iinfo = mosaic.ImageInfo(feat, "RECORD", srs=s_srs)

                    if iinfo.geom is not None and iinfo.geom.GetGeometryType(
                    ) in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
                        if not t_srs.IsSame(s_srs):
                            iinfo.geom.Transform(ct)
                            ## fix self-intersection errors caused by reprojecting over 180
                            temp = iinfo.geom.Buffer(
                                0.1
                            )  # assumes a projected coordinate system with meters or feet as units
                            iinfo.geom = temp

                        if iinfo.geom.Intersects(t.geom):

                            if iinfo.scene_id in exclude_list:
                                logger.debug(
                                    "Scene in exclude list, excluding: %s",
                                    iinfo.srcfp)

                            elif not os.path.isfile(iinfo.srcfp):
                                logger.warning(
                                    "Scene path is invalid, excluding %s (path = %s)",
                                    iinfo.scene_id, iinfo.srcfp)
                            elif args.require_pan:
                                srcfp = iinfo.srcfp
                                srcdir, mul_name = os.path.split(srcfp)
                                if iinfo.sensor in ["WV02", "WV03", "QB02"]:
                                    pan_name = mul_name.replace("-M", "-P")
                                elif iinfo.sensor == "GE01":
                                    if "_5V" in mul_name:
                                        pan_name_base = srcfp[:-24].replace(
                                            "M0", "P0")
                                        candidates = glob.glob(pan_name_base +
                                                               "*")
                                        candidates2 = [
                                            f for f in candidates
                                            if f.endswith(('.ntf', '.NTF',
                                                           '.tif', '.TIF'))
                                        ]
                                        if len(candidates2) == 0:
                                            pan_name = ''
                                        elif len(candidates2) == 1:
                                            pan_name = os.path.basename(
                                                candidates2[0])
                                        else:
                                            pan_name = ''
                                            logger.error(
                                                '%i panchromatic images match the multispectral image name '
                                                '%s', len(candidates2),
                                                mul_name)
                                    else:
                                        pan_name = mul_name.replace("-M", "-P")
                                elif iinfo.sensor == "IK01":
                                    pan_name = mul_name.replace("blu", "pan")
                                    pan_name = mul_name.replace("msi", "pan")
                                    pan_name = mul_name.replace("bgrn", "pan")
                                pan_srcfp = os.path.join(srcdir, pan_name)
                                if not os.path.isfile(pan_srcfp):
                                    logger.debug(
                                        "Image does not have a panchromatic component, excluding: %s",
                                        iinfo.srcfp)
                                else:
                                    logger.debug("Intersect %s, %s: %s",
                                                 iinfo.scene_id, iinfo.srcfp,
                                                 str(iinfo.geom))
                                    imginfo_list1.append(iinfo)

                            else:
                                logger.debug("Intersect %s, %s: %s",
                                             iinfo.scene_id, iinfo.srcfp,
                                             str(iinfo.geom))
                                imginfo_list1.append(iinfo)

                    feat = lyr.GetNextFeature()

            ds = None

            logger.info("Number of intersects in tile %s: %i", t.name,
                        len(imginfo_list1))

            if len(imginfo_list1) > 0:

                #### Get mosaic parameters
                logger.debug("Getting mosaic parameters")
                params = mosaic.getMosaicParameters(imginfo_list1[0], args)

                #### Remove images that do not match ref
                logger.debug("Setting image pattern filter")
                imginfo_list2 = mosaic.filterMatchingImages(
                    imginfo_list1, params)
                logger.info("Number of images matching filter: %i",
                            len(imginfo_list2))

                if args.nosort is False:
                    #### Sort by quality
                    logger.debug("Sorting images by quality")
                    imginfo_list3 = []
                    for iinfo in imginfo_list2:

                        iinfo.getScore(params)
                        if iinfo.score > 0:
                            imginfo_list3.append(iinfo)

                    # sort so highest score is last
                    imginfo_list3.sort(key=lambda x: x.score)

                else:
                    imginfo_list3 = list(imginfo_list2)

                ####  Overlay geoms and remove non-contributors
                logger.debug("Overlaying images to determine contributors")
                contribs = mosaic.determine_contributors(
                    imginfo_list3, t.geom, args.min_contribution_area)

                logger.info("Number of contributing images: %i", len(contribs))

                if len(contribs) > 0:

                    if args.build_shp:

                        #######################################################
                        #### Create Shp

                        shp = os.path.join(
                            dstdir,
                            "{}_{}_imagery.shp".format(args.mosaic, t.name))

                        logger.debug("Creating shapefile of geoms: %s", shp)

                        fields = [("IMAGENAME", ogr.OFTString, 100),
                                  ("SCORE", ogr.OFTReal, 0)]

                        OGR_DRIVER = "ESRI Shapefile"

                        ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
                        if ogrDriver is None:
                            logger.debug("OGR: Driver %s is not available",
                                         OGR_DRIVER)
                            sys.exit(-1)

                        if os.path.isfile(shp):
                            ogrDriver.DeleteDataSource(shp)
                        vds = ogrDriver.CreateDataSource(shp)
                        if vds is None:
                            logger.debug("Could not create shp")
                            sys.exit(-1)

                        shpd, shpn = os.path.split(shp)
                        shpbn, shpe = os.path.splitext(shpn)

                        lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
                        if lyr is None:
                            logger.debug("ERROR: Failed to create layer: %s",
                                         shpbn)
                            sys.exit(-1)

                        for fld, fdef, flen in fields:
                            field_defn = ogr.FieldDefn(fld, fdef)
                            if fdef == ogr.OFTString:
                                field_defn.SetWidth(flen)
                            if lyr.CreateField(field_defn) != 0:
                                logger.debug(
                                    "ERROR: Failed to create field: %s", fld)

                        for iinfo, geom in contribs:

                            logger.debug("Image: %s", iinfo.srcfn)

                            feat = ogr.Feature(lyr.GetLayerDefn())

                            feat.SetField("IMAGENAME", iinfo.srcfn)
                            feat.SetField("SCORE", iinfo.score)

                            feat.SetGeometry(geom)
                            if lyr.CreateFeature(feat) != 0:
                                logger.debug(
                                    "ERROR: Could not create feature for image %s",
                                    iinfo.srcfn)
                            else:
                                logger.debug("Created feature for image: %s",
                                             iinfo.srcfn)

                            feat.Destroy()

                    #### Write textfiles
                    if not os.path.isdir(dstdir):
                        os.makedirs(dstdir)

                    otxtpath = os.path.join(
                        dstdir, "{}_{}_orig.txt".format(args.mosaic, t.name))
                    mtxtpath = os.path.join(
                        dstdir, "{}_{}_ortho.txt".format(args.mosaic, t.name))
                    otxt = open(otxtpath, 'w')
                    mtxt = open(mtxtpath, 'w')

                    for iinfo, geom in contribs:

                        if not os.path.isfile(iinfo.srcfp):
                            logger.warning("Image does not exist: %s",
                                           iinfo.srcfp)

                        otxt.write("{}\n".format(iinfo.srcfp))
                        m_fn = "{0}_u08{1}{2}.tif".format(
                            os.path.splitext(iinfo.srcfn)[0], args.stretch,
                            t.epsg)

                        mtxt.write(
                            os.path.join(dstdir, 'ortho', t.name, m_fn) + "\n")

                    otxt.close()
Example #4
0
def HandleTile(t, src, dstdir, csvpath, args, exclude_list):
    
    
    otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(os.path.basename(csvpath)[:-4], t.name))
    mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(os.path.basename(csvpath)[:-4], t.name))
    
    if os.path.isfile(otxtpath) and os.path.isfile(mtxtpath) and args.overwrite is False:
        logger.info("Tile %s processing files already exist", t.name)
    else:
        logger.info("Tile %s", t.name)
    
        t_srs = utils.osr_srs_preserve_axis_order(osr.SpatialReference())
        t_srs.ImportFromEPSG(t.epsg)
        
        #### Open mfp
        dsp, lyrn = utils.get_source_names(src)
        
        ds = ogr.Open(dsp)
        if ds is None:
            logger.error("Open failed")
            
        else:
            lyr = ds.GetLayerByName(lyrn)
            
            if not lyr:
                raise RuntimeError("Layer {} does not exist in dataset {}".format(lyrn, dsp))
            else:

                s_srs = lyr.GetSpatialRef()
                #logger.debug(str(s_srs))
                #logger.debug(str(t.geom))
                
                tile_geom_in_s_srs = t.geom.Clone()

                if not t_srs.IsSame(s_srs):
                    ict = osr.CoordinateTransformation(t_srs, s_srs)
                    ct = osr.CoordinateTransformation(s_srs, t_srs)
                    tile_geom_in_s_srs.Transform(ict)

                # if the geometry crosses meridian, split it into multipolygon (else this breaks SetSpatialFilter)
                if utils.doesCross180(tile_geom_in_s_srs):
                    logger.debug("tile_geom_in_s_srs crosses 180 meridian; splitting to multiple polygons...")
                    tile_geom_in_s_srs = utils.getWrappedGeometry(tile_geom_in_s_srs)

                lyr.ResetReading()
                lyr.SetSpatialFilter(tile_geom_in_s_srs)
                feat = lyr.GetNextFeature()
                
                imginfo_list1 = []
                
                while feat:
                    
                    iinfo = mosaic.ImageInfo(feat, "RECORD", srs=s_srs)
                    
                    if iinfo.geom is not None and iinfo.geom.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
                        if not t_srs.IsSame(s_srs):
                            iinfo.geom.Transform(ct)
                            ## fix self-intersection errors caused by reprojecting over 180
                            temp = iinfo.geom.Buffer(0.1) # assumes a projected coordinate system with meters or feet as units
                            iinfo.geom = temp
                        
                        if iinfo.geom.Intersects(t.geom):
                            
                            if iinfo.scene_id in exclude_list:
                                logger.debug("Scene in exclude list, excluding: %s", iinfo.srcfp)
                                
                            elif not os.path.isfile(iinfo.srcfp) and iinfo.status != "tape":
                                #logger.info("iinfo.status != tape: {0}".format(iinfo.status != "tape"))
                                logger.warning("Scene path is invalid, excluding %s (path = %s) (status = %s)",
                                               iinfo.scene_id, iinfo.srcfp, iinfo.status)
                            elif args.require_pan:
                                srcfp = iinfo.srcfp
                                srcdir, mul_name = os.path.split(srcfp)
                                if iinfo.sensor in ["WV02", "WV03", "QB02"]:
                                    pan_name = mul_name.replace("-M", "-P")
                                elif iinfo.sensor == "GE01":
                                    if "_5V" in mul_name:
                                        pan_name_base = srcfp[:-24].replace("M0", "P0")
                                        candidates = glob.glob(pan_name_base + "*")
                                        candidates2 = [f for f in candidates if f.endswith(('.ntf', '.NTF', '.tif',
                                                                                            '.TIF'))]
                                        if len(candidates2) == 0:
                                            pan_name = ''
                                        elif len(candidates2) == 1:
                                            pan_name = os.path.basename(candidates2[0])
                                        else:
                                            pan_name = ''
                                            logger.error('%i panchromatic images match the multispectral image name '
                                                         '%s', len(candidates2), mul_name)
                                    else:
                                        pan_name = mul_name.replace("-M", "-P")
                                elif iinfo.sensor == "IK01":
                                    pan_name = mul_name.replace("blu", "pan")
                                    pan_name = mul_name.replace("msi", "pan")
                                    pan_name = mul_name.replace("bgrn", "pan")
                                pan_srcfp = os.path.join(srcdir, pan_name)
                                if not os.path.isfile(pan_srcfp):
                                    logger.debug("Image does not have a panchromatic component, excluding: %s",
                                                 iinfo.srcfp)
                                else:
                                    logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
                                    imginfo_list1.append(iinfo)
                                
                            else:
                                logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
                                imginfo_list1.append(iinfo)                                
                                
                    feat = lyr.GetNextFeature()
            
            ds = None
        
            logger.info("Number of intersects in tile %s: %i", t.name, len(imginfo_list1))
            
            if len(imginfo_list1) > 0:

                #### Get mosaic parameters
                logger.debug("Getting mosaic parameters")
                params = mosaic.getMosaicParameters(imginfo_list1[0], args)
                
                #### Remove images that do not match ref
                logger.debug("Setting image pattern filter")
                imginfo_list2 = mosaic.filterMatchingImages(imginfo_list1, params)
                logger.info("Number of images matching filter: %i", len(imginfo_list2))
                    
                if args.nosort is False:    
                    #### Sort by quality
                    logger.debug("Sorting images by quality")
                    imginfo_list3 = []
                    for iinfo in imginfo_list2:
                        
                        iinfo.getScore(params)
                        if iinfo.score > 0:
                            imginfo_list3.append(iinfo)
                    
                    # sort so highest score is last
                    imginfo_list3.sort(key=lambda x: x.score)
                    
                else:
                    imginfo_list3 = list(imginfo_list2)
                    
                ####  Overlay geoms and remove non-contributors
                logger.debug("Overlaying images to determine contributors")
                contribs = mosaic.determine_contributors(imginfo_list3, t.geom, args.min_contribution_area)
                                            
                logger.info("Number of contributing images: %i", len(contribs))
            
                if len(contribs) > 0:
                    
                    if args.build_shp:
                        
                        #######################################################
                        #### Create Shp
                        
                        shp = os.path.join(dstdir, "{}_{}_imagery.shp".format(args.mosaic, t.name))
                   
                        logger.debug("Creating shapefile of geoms: %s", shp)
                    
                        fields = [("IMAGENAME", ogr.OFTString, 100), ("SCORE", ogr.OFTReal, 0)]
                        
                        OGR_DRIVER = "ESRI Shapefile"
                        
                        ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
                        if ogrDriver is None:
                            logger.debug("OGR: Driver %s is not available", OGR_DRIVER)
                            sys.exit(-1)
                        
                        if os.path.isfile(shp):
                            ogrDriver.DeleteDataSource(shp)
                        vds = ogrDriver.CreateDataSource(shp)
                        if vds is None:
                            logger.debug("Could not create shp")
                            sys.exit(-1)
                        
                        shpd, shpn = os.path.split(shp)
                        shpbn, shpe = os.path.splitext(shpn)
                        
                        lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
                        if lyr is None:
                            logger.debug("ERROR: Failed to create layer: %s", shpbn)
                            sys.exit(-1)
                        
                        for fld, fdef, flen in fields:
                            field_defn = ogr.FieldDefn(fld, fdef)
                            if fdef == ogr.OFTString:
                                field_defn.SetWidth(flen)
                            if lyr.CreateField(field_defn) != 0:
                                logger.debug("ERROR: Failed to create field: %s", fld)
                        
                        for iinfo, geom in contribs:
                        
                            logger.debug("Image: %s", iinfo.srcfn)
                            
                            feat = ogr.Feature(lyr.GetLayerDefn())
                            
                            feat.SetField("IMAGENAME", iinfo.srcfn)
                            feat.SetField("SCORE", iinfo.score)
    
                            feat.SetGeometry(geom)
                            if lyr.CreateFeature(feat) != 0:
                                logger.debug("ERROR: Could not create feature for image %s", iinfo.srcfn)
                            else:
                                logger.debug("Created feature for image: %s", iinfo.srcfn)
                                
                            feat.Destroy()
                    
                    #### Write textfiles
                    if not os.path.isdir(dstdir):
                        os.makedirs(dstdir)
                    
                    otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(args.mosaic, t.name))
                    otxtpath_ontape = os.path.join(dstdir, "{}_{}_orig_ontape.csv".format(args.mosaic, t.name))
                    mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(args.mosaic, t.name))

                    rn_fromtape_basedir = os.path.join(dstdir, "renamed_fromtape")
                    rn_fromtape_path = os.path.join(rn_fromtape_basedir, t.name)

                    otxt = open(otxtpath, 'w')
                    ttxt = open(otxtpath_ontape, 'w')
                    mtxt = open(mtxtpath, 'w')

                    # write header
                    ttxt.write("{0},{1},{2}\n".format("SCENE_ID", "S_FILEPATH", "STATUS"))

                    tape_ct = 0
                    
                    for iinfo, geom in contribs:
                        
                        if not os.path.isfile(iinfo.srcfp) and iinfo.status != "tape":
                            logger.warning("Image does not exist: %s", iinfo.srcfp)
                            
                        if iinfo.status == "tape":
                            tape_ct += 1
                            ttxt.write("{0},{1},{2}\n".format(iinfo.scene_id, iinfo.srcfp, iinfo.status))
                            # get srcfp with file extension
                            srcfp_file = os.path.basename(iinfo.srcfp)
                            otxt.write("{}\n".format(os.path.join(rn_fromtape_path, srcfp_file)))

                        else:
                            otxt.write("{}\n".format(iinfo.srcfp))

                        m_fn = "{0}_u08{1}{2}.tif".format(
                            os.path.splitext(iinfo.srcfn)[0],
                            args.stretch,
                            t.epsg
                        )
                        
                        mtxt.write(os.path.join(dstdir, 'ortho', t.name, m_fn) + "\n")
 
                    otxt.close()

                    if tape_ct == 0:
                        logger.debug("No files need to be pulled from tape.")
                        os.remove(otxtpath_ontape)

                    else:
                        # make output dirs from tape
                        if not os.path.isdir(rn_fromtape_basedir):
                            os.mkdir(rn_fromtape_basedir)
                        if not os.path.isdir(rn_fromtape_path):
                            os.mkdir(rn_fromtape_path)

                        tape_tmp = os.path.join(dstdir, "{0}_{1}_tmp".format(args.mosaic, t.name))
                        if not os.path.isdir(tape_tmp):
                            os.mkdir(tape_tmp)
                        logger.warning("{0} scenes are not accessible, as they are on tape. Please use ir.py to pull "
                                       "scenes using file '{1}'. They must be put in directory '{2}', as file '{3}' "
                                       "contains hard-coded paths to said files (necessary to perform "
                                       "orthorectification). Please set a --tmp path (use '{4}').\n"
                                       "Note that if some (or all) scenes have already been pulled from tape, ir.py "
                                       "will not pull them again.\n".
                                       format(tape_ct, otxtpath_ontape, rn_fromtape_path, otxtpath, tape_tmp))

                        tape_log = "{0}_{1}_ir_log_{2}.log".format(args.mosaic, t.name,
                                                                   datetime.today().strftime("%Y%m%d%H%M%S"))
                        root_pgclib_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
                                                        "pgclib", "")
                        logger.info("Suggested ir.py command:\n\n"
                                    ""
                                    "python {}ir.py -i {} -o {} --tmp {} -tm link 2>&1 | tee {}"
                        .format(root_pgclib_path, otxtpath_ontape, rn_fromtape_path, tape_tmp,
                                os.path.join(dstdir, tape_log)))
def HandleTile(t, src, dstdir, csvpath, args, exclude_list):
    
    
    otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(os.path.basename(csvpath)[:-4], t.name))
    mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(os.path.basename(csvpath)[:-4], t.name))
    
    if os.path.isfile(otxtpath) and os.path.isfile(mtxtpath) and args.overwrite is False:
        logger.info("Tile %s processing files already exist", t.name)
    else:
        logger.info("Tile %s", t.name)
    
        t_srs = osr.SpatialReference()
        t_srs.ImportFromEPSG(t.epsg)
        
        #### Open mfp
        dsp, lyrn = utils.get_source_names(src)
        
        ds = ogr.Open(dsp)
        if ds is None:
            logger.error("Open failed")
            
        else:
            lyr = ds.GetLayerByName(lyrn)
            
            if not lyr:
                raise RuntimeError("Layer {} does not exist in dataset {}".format(lyrn, dsp))
            else:

                s_srs = lyr.GetSpatialRef()
                #logger.debug(str(s_srs))
                #logger.debug(str(t.geom))
                
                tile_geom_in_s_srs = t.geom.Clone()

                if not t_srs.IsSame(s_srs):
                    ict = osr.CoordinateTransformation(t_srs, s_srs)
                    ct = osr.CoordinateTransformation(s_srs, t_srs)
                    tile_geom_in_s_srs.Transform(ict)

                # if the geometry crosses meridian, split it into multipolygon (else this breaks SetSpatialFilter)
                if utils.doesCross180(tile_geom_in_s_srs):
                    logger.debug("tile_geom_in_s_srs crosses 180 meridian; splitting to multiple polygons...")
                    tile_geom_in_s_srs = utils.getWrappedGeometry(tile_geom_in_s_srs)

                lyr.ResetReading()
                lyr.SetSpatialFilter(tile_geom_in_s_srs)
                feat = lyr.GetNextFeature()
                
                imginfo_list1 = []
                
                while feat:
                    
                    iinfo = mosaic.ImageInfo(feat, "RECORD", srs=s_srs)
                    
                    if iinfo.geom is not None and iinfo.geom.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
                        if not t_srs.IsSame(s_srs):
                            iinfo.geom.Transform(ct)
                            ## fix self-intersection errors caused by reprojecting over 180
                            temp = iinfo.geom.Buffer(0.1) # assumes a projected coordinate system with meters or feet as units
                            iinfo.geom = temp
                        
                        if iinfo.geom.Intersects(t.geom):
                            
                            if iinfo.scene_id in exclude_list:
                                logger.debug("Scene in exclude list, excluding: %s", iinfo.srcfp)
                                
                            elif not os.path.isfile(iinfo.srcfp):
                                logger.warning("Scene path is invalid, excluding %s (path = %s)", iinfo.scene_id,
                                               iinfo.srcfp)
                            elif args.require_pan:
                                srcfp = iinfo.srcfp
                                srcdir, mul_name = os.path.split(srcfp)
                                if iinfo.sensor in ["WV02", "WV03", "QB02"]:
                                    pan_name = mul_name.replace("-M", "-P")
                                elif iinfo.sensor == "GE01":
                                    if "_5V" in mul_name:
                                        pan_name_base = srcfp[:-24].replace("M0", "P0")
                                        candidates = glob.glob(pan_name_base + "*")
                                        candidates2 = [f for f in candidates if f.endswith(('.ntf', '.NTF', '.tif',
                                                                                            '.TIF'))]
                                        if len(candidates2) == 0:
                                            pan_name = ''
                                        elif len(candidates2) == 1:
                                            pan_name = os.path.basename(candidates2[0])
                                        else:
                                            pan_name = ''
                                            logger.error('%i panchromatic images match the multispectral image name '
                                                         '%s', len(candidates2), mul_name)
                                    else:
                                        pan_name = mul_name.replace("-M", "-P")
                                elif iinfo.sensor == "IK01":
                                    pan_name = mul_name.replace("blu", "pan")
                                    pan_name = mul_name.replace("msi", "pan")
                                    pan_name = mul_name.replace("bgrn", "pan")
                                pan_srcfp = os.path.join(srcdir, pan_name)
                                if not os.path.isfile(pan_srcfp):
                                    logger.debug("Image does not have a panchromatic component, excluding: %s",
                                                 iinfo.srcfp)
                                else:
                                    logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
                                    imginfo_list1.append(iinfo)
                                
                            else:
                                logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
                                imginfo_list1.append(iinfo)                                
                                
                    feat = lyr.GetNextFeature()
            
            ds = None
        
            logger.info("Number of intersects in tile %s: %i", t.name, len(imginfo_list1))
            
            if len(imginfo_list1) > 0:

                #### Get mosaic parameters
                logger.debug("Getting mosaic parameters")
                params = mosaic.getMosaicParameters(imginfo_list1[0], args)
                
                #### Remove images that do not match ref
                logger.debug("Setting image pattern filter")
                imginfo_list2 = mosaic.filterMatchingImages(imginfo_list1, params)
                logger.info("Number of images matching filter: %i", len(imginfo_list2))
                    
                if args.nosort is False:    
                    #### Sort by quality
                    logger.debug("Sorting images by quality")
                    imginfo_list3 = []
                    for iinfo in imginfo_list2:
                        
                        iinfo.getScore(params)
                        if iinfo.score > 0:
                            imginfo_list3.append(iinfo)
                    
                    # sort so highest score is last
                    imginfo_list3.sort(key=lambda x: x.score)
                    
                else:
                    imginfo_list3 = list(imginfo_list2)
                    
                ####  Overlay geoms and remove non-contributors
                logger.debug("Overlaying images to determine contributors")
                contribs = mosaic.determine_contributors(imginfo_list3, t.geom, args.min_contribution_area)
                                            
                logger.info("Number of contributing images: %i", len(contribs))
            
                if len(contribs) > 0:
                    
                    if args.build_shp:
                        
                        #######################################################
                        #### Create Shp
                        
                        shp = os.path.join(dstdir, "{}_{}_imagery.shp".format(args.mosaic, t.name))
                   
                        logger.debug("Creating shapefile of geoms: %s", shp)
                    
                        fields = [("IMAGENAME", ogr.OFTString, 100), ("SCORE", ogr.OFTReal, 0)]
                        
                        OGR_DRIVER = "ESRI Shapefile"
                        
                        ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
                        if ogrDriver is None:
                            logger.debug("OGR: Driver %s is not available", OGR_DRIVER)
                            sys.exit(-1)
                        
                        if os.path.isfile(shp):
                            ogrDriver.DeleteDataSource(shp)
                        vds = ogrDriver.CreateDataSource(shp)
                        if vds is None:
                            logger.debug("Could not create shp")
                            sys.exit(-1)
                        
                        shpd, shpn = os.path.split(shp)
                        shpbn, shpe = os.path.splitext(shpn)
                        
                        lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
                        if lyr is None:
                            logger.debug("ERROR: Failed to create layer: %s", shpbn)
                            sys.exit(-1)
                        
                        for fld, fdef, flen in fields:
                            field_defn = ogr.FieldDefn(fld, fdef)
                            if fdef == ogr.OFTString:
                                field_defn.SetWidth(flen)
                            if lyr.CreateField(field_defn) != 0:
                                logger.debug("ERROR: Failed to create field: %s", fld)
                        
                        for iinfo, geom in contribs:
                        
                            logger.debug("Image: %s", iinfo.srcfn)
                            
                            feat = ogr.Feature(lyr.GetLayerDefn())
                            
                            feat.SetField("IMAGENAME", iinfo.srcfn)
                            feat.SetField("SCORE", iinfo.score)
    
                            feat.SetGeometry(geom)
                            if lyr.CreateFeature(feat) != 0:
                                logger.debug("ERROR: Could not create feature for image %s", iinfo.srcfn)
                            else:
                                logger.debug("Created feature for image: %s", iinfo.srcfn)
                                
                            feat.Destroy()
                    
                    #### Write textfiles
                    if not os.path.isdir(dstdir):
                        os.makedirs(dstdir)
                    
                    otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(args.mosaic, t.name))
                    mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(args.mosaic, t.name))
                    otxt = open(otxtpath, 'w')
                    mtxt = open(mtxtpath, 'w')
                    
                    for iinfo, geom in contribs:
                        
                        if not os.path.isfile(iinfo.srcfp):
                            logger.warning("Image does not exist: %s", iinfo.srcfp)
                            
                        otxt.write("{}\n".format(iinfo.srcfp))
                        m_fn = "{0}_u08{1}{2}.tif".format(
                            os.path.splitext(iinfo.srcfn)[0],
                            args.stretch,
                            t.epsg
                        )
                        
                        mtxt.write(os.path.join(dstdir, 'ortho', t.name, m_fn) + "\n")
 
                    otxt.close()
Example #6
0
def main():

    #### Set Up Arguments 
    parent_parser = mosaic.buildMosaicParentArgumentParser()
    parser = argparse.ArgumentParser(
        parents=[parent_parser],
        description="Sumbit/run batch mosaic tasks"
    )
    
    parser.add_argument("src", help="textfile or directory of input rasters (tif only)")
    parser.add_argument("mosaicname", help="output mosaic name excluding extension")
    pos_arg_keys = ["src","mosaicname"]

    parser.add_argument("--mode", choices=mosaic.MODES , default="ALL",
                        help=" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, TEST- create log only")
    parser.add_argument("--wd",
                        help="scratch space (default is mosaic directory)")
    parser.add_argument("--component-shp", action="store_true", default=False,
                        help="create shp of all componenet images")
    parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw",
                        help="GTiff compression type. Default=lzw (%s)"%(",".join(mosaic.GTIFF_COMPRESSIONS)))
    parser.add_argument("--pbs", action='store_true', default=False,
                        help="submit tasks to PBS")
    parser.add_argument("--parallel-processes", type=int, default=1,
                        help="number of parallel processes to spawn (default 1)")
    parser.add_argument("--qsubscript",
                        help="qsub script to use in cluster job submission (default is <script_dir>/%s)" %default_qsub_script)
    parser.add_argument("-l",
                        help="PBS resources requested (mimicks qsub syntax). Use only on HPC systems.")
    parser.add_argument("--log",
                        help="file to log progress (default is <output dir>\%s" %default_logfile)
    
    #### Parse Arguments
    args = parser.parse_args()
    scriptpath = os.path.abspath(sys.argv[0])
    inpath = os.path.abspath(args.src)
    mosaicname = os.path.abspath(args.mosaicname)
    mosaicname = os.path.splitext(mosaicname)[0]
    mosaic_dir = os.path.dirname(mosaicname)
    cutline_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_cutlines.py')
    tile_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_tile.py')
    
    ## Verify qsubscript
    if args.qsubscript is None: 
        qsubpath = os.path.join(os.path.dirname(scriptpath),default_qsub_script)
    else:
        qsubpath = os.path.abspath(args.qsubscript)
    if not os.path.isfile(qsubpath):
        parser.error("qsub script path is not valid: %s" %qsubpath)
        
    ## Verify processing options do not conflict
    if args.pbs and args.parallel_processes > 1:
        parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive")

    #### Validate Arguments
    if os.path.isfile(inpath):
        bTextfile = True
    elif os.path.isdir(inpath):
        bTextfile = False
    else:
        parser.error("Arg1 is not a valid file path or directory: %s" %inpath)    
    if not os.path.isdir(mosaic_dir):
        os.makedirs(mosaic_dir)
    if not os.path.isfile(qsubpath):
        parser.error("Arg3 is not a valid file path: %s" %qsubpath)
        
    #### Validate target day option
    if args.tday is not None:
        try:
            m = int(args.tday.split("-")[0])
            d = int(args.tday.split("-")[1])
            td = date(2000,m,d)
        except ValueError:
            logger.error("Target day must be in mm-dd format (i.e 04-05)")
            sys.exit(1)
    else:
        m = 0
        d = 0
    
    #### Configure Logger
    if args.log is not None:
        logfile = os.path.abspath(args.log)
    else:
        logfile = os.path.join(mosaic_dir,default_logfile)
    
    lfh = logging.FileHandler(logfile)
    lfh.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
    lfh.setFormatter(formatter)
    logger.addHandler(lfh)
    
    lsh = logging.StreamHandler()
    lsh.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
    lsh.setFormatter(formatter)
    logger.addHandler(lsh)
  
    #### Get exclude list if specified
    if args.exclude is not None:
        if not os.path.isfile(args.exclude):
            parser.error("Value for option --exclude-list is not a valid file")
        
        f = open(args.exclude, 'r')
        exclude_list = set([line.rstrip() for line in f.readlines()])
    else:
        exclude_list = set()

    #### Get Images
    #logger.info("Reading input images")
    xs = []
    ys = []
    
    image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list)
        
    if len(image_list) == 0:
        logger.error("No images found in input file or directory: %s" %inpath)
        sys.exit()
    else:
        logger.info("%i existing images found" %len(image_list))
    
    #### gather image info list
    logger.info("Getting image info")
    imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in image_list]
    
    #### Get mosaic parameters
    logger.info("Setting mosaic parameters")
    params = mosaic.getMosaicParameters(imginfo_list[0],args)
    
    #### Remove images that do not match ref
    logger.info("Applying attribute filter")
    imginfo_list2 = mosaic.filterMatchingImages(imginfo_list,params)
    
    if len(imginfo_list2) == 0:
        logger.error("No valid images found.  Check input filter parameters.")
        sys.exit()
    else:
        logger.info("%i images match filter" %len(imginfo_list2))

    #### if extent is specified, build tile params and compare extent to input image geom
    if args.extent:
        imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params)
    
    #### else set extent after image geoms computed
    else:
        
        #### Get geom for each image
        imginfo_list3 = []
        for iinfo in imginfo_list2:
            if iinfo.geom is not None:
                xs = xs + iinfo.xs
                ys = ys + iinfo.ys
                imginfo_list3.append(iinfo)
            else: # remove from list if no geom
                logger.debug("Null geometry for image: %s" %iinfo.srcfn)
        
        params.xmin = min(xs)
        params.xmax = max(xs)
        params.ymin = min(ys)
        params.ymax = max(ys)
    
        poly_wkt = 'POLYGON (( %f %f, %f %f, %f %f, %f %f, %f %f ))' %(params.xmin,params.ymin,params.xmin,params.ymax,params.xmax,params.ymax,params.xmax,params.ymin,params.xmin,params.ymin)
        params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)
     
    #### Check number of remaining images
    num_images = len(imginfo_list3)

    if num_images > 0:            
        logger.info("%d of %d input images intersect mosaic extent" %(num_images,len(image_list)))
        logger.info("Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f" %(params.xres,params.yres,params.xtilesize,params.ytilesize,params.xmin,params.xmax,params.ymin,params.ymax))
        
    else:
        logger.error("No valid images found")
        sys.exit(0)

    ## Sort images by score
    logger.info("Reading image metadata and determining sort order")
    for iinfo in imginfo_list3:
        iinfo.getScore(params)
            
    ####  Sort by score
    if not args.nosort:
        imginfo_list3.sort(key=lambda x: x.score)
    
    #### Write all intersects file
    intersects_all = []
    
    for iinfo in imginfo_list3:
        if params.extent_geom.Intersect(iinfo.geom) is True:
            if iinfo.score > 0:
                intersects_all.append(iinfo)
            elif args.nosort:
                intersects_all.append(iinfo)
            else:
                logger.debug("Image has an invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score))
        else:
            logger.debug("Image does not intersect mosaic extent: %s" %iinfo.srcfp)  ### this line should never be needed.  non-intersecting images should be removed earlier if extent is provided, otherwise all images are in the extent.
    
    aitpath = mosaicname+"_intersects.txt"
    ait = open(aitpath,"w")
    intersects_fps = [intersect.srcfp for intersect in intersects_all]
    ait.write(string.join(intersects_fps,"\n"))
    ait.close()

    ## Create tiles
    logger.info("Creating tiles")
    tiles = []
    
    xtiledim = math.ceil((params.xmax-params.xmin)/params.xtilesize)
    ytiledim = math.ceil((params.ymax-params.ymin)/params.ytilesize)
    logger.info("Tiles: %d rows, %d columns" %(ytiledim,xtiledim))
    
    xtdb = len(str(int(xtiledim)))
    ytdb = len(str(int(ytiledim)))
    
    i = 1   
    for x in mosaic.drange(params.xmin,params.xmax,params.xtilesize):  # Columns
        if x+params.xtilesize > params.xmax:
            x2 = params.xmax
        else:
            x2 = x+params.xtilesize
      
        j = 1
        for y in mosaic.drange(params.ymin,params.ymax,params.ytilesize):  # Rows
            if y+params.ytilesize > params.ymax:
                y2 = params.ymax
            else:
                y2 = y+params.ytilesize
                        
            tilename = "%s_%s_%s.tif" %(mosaicname,mosaic.buffernum(j,ytdb),mosaic.buffernum(i,xtdb))
            tile = mosaic.TileParams(x,x2,y,y2,j,i,tilename)
            tiles.append(tile)
            j += 1
        i += 1
    num_tiles = len(tiles)
       
    ####  Write shapefile of tiles
    if args.mode == "ALL" or args.mode == "SHP":
        
        shp = mosaicname + "_tiles.shp"
        if os.path.isfile(shp):
            logger.info("Tiles shapefile already exists: %s" %os.path.basename(shp))
        else:
            logger.info("Creating shapefile of tiles: %s" %os.path.basename(shp))
            fields = [('ROW', ogr.OFTInteger, 4),
                    ('COL', ogr.OFTInteger, 4),
                    ("TILENAME", ogr.OFTString, 100),
                    ('TILEPATH', ogr.OFTString, 254),
                    ('XMIN', ogr.OFTReal, 0),
                    ('XMAX', ogr.OFTReal, 0),
                    ('YMIN', ogr.OFTReal, 0),
                    ('YMAX', ogr.OFTReal, 0)]
                      
            OGR_DRIVER = "ESRI Shapefile"
            ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
            if ogrDriver is None:
                logger.error("OGR: Driver %s is not available" % OGR_DRIVER)
                sys.exit(-1)
        
            if os.path.isfile(shp):
                ogrDriver.DeleteDataSource(shp)
            vds = ogrDriver.CreateDataSource(shp)
            if vds is None:
                logger.error("Could not create shp")
                sys.exit(-1)
            
            shpd, shpn = os.path.split(shp)
            shpbn, shpe = os.path.splitext(shpn)
            
            rp = osr.SpatialReference()
            rp.ImportFromWkt(params.proj)
            
            lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon)
            if lyr is None:
                logger.error("ERROR: Failed to create layer: %s" % shpbn)
                sys.exit(-1)
            
            for fld, fdef, flen in fields:
                field_defn = ogr.FieldDefn(fld, fdef)
                if fdef == ogr.OFTString:
                    field_defn.SetWidth(flen)
                if lyr.CreateField(field_defn) != 0:
                    logger.error("ERROR: Failed to create field: %s" % fld)
            
            for t in tiles:
                feat = ogr.Feature(lyr.GetLayerDefn())
                feat.SetField("TILENAME",os.path.basename(t.name))
                feat.SetField("TILEPATH",t.name)
                feat.SetField("ROW",t.j)
                feat.SetField("COL",t.i)
                feat.SetField("XMIN",t.xmin)
                feat.SetField("XMAX",t.xmax)
                feat.SetField("YMIN",t.ymin)
                feat.SetField("YMAX",t.ymax)
                feat.SetGeometry(t.geom)
                
                if lyr.CreateFeature(feat) != 0:
                    logger.error("ERROR: Could not create feature for tile %s" % tile)
                feat.Destroy()
     
    ## Build tasks
    task_queue = []
    
    ####  Create task for shapefile of mosaic components
    if args.component_shp is True:
        
        arg_keys_to_remove = (
            'l',
            'qsubscript',
            'parallel_processes',
            'log',
            'gtiff_compression',
            'mode',
            'extent',
            'resolution',
            'pbs',
            'wd'
        )
        shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
        
        comp_shp = mosaicname + "_components.shp"
        if os.path.isfile(comp_shp):
            logger.info("Components shapefile already exists: %s" %os.path.basename(comp_shp))
        else:
            logger.info("Processing components: %s" %os.path.basename(comp_shp))
            
            ## Make task and add to queue
            cmd = '{} --cutline-step 512 {} -e {} {} {} {} {} {}'.format(
                cutline_builder_script,
                shp_arg_str,
                params.xmin,
                params.xmax,
                params.ymin,
                params.ymax,
                comp_shp,
                aitpath
            )
            
            task = utils.Task(
                'Components',
                'Components',
                'python',
                cmd
            )
            
            if args.mode == "ALL" or args.mode == "SHP":
                logger.debug(cmd)
                task_queue.append(task)
            
    ####  Create task for shapefile of image cutlines
    shp = mosaicname + "_cutlines.shp"
    
    arg_keys_to_remove = (
        'l',
        'qsubscript',
        'parallel_processes',
        'log',
        'gtiff_compression',
        'mode',
        'extent',
        'resolution',
        'component_shp',
        'pbs',
        'wd'
    )
    shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
    
    if os.path.isfile(shp):
        logger.info("Cutlines shapefile already exists: %s" %os.path.basename(shp))
    else:
        logger.info("Processing cutlines: %s" %os.path.basename(shp))
        
        ## Make task and add to queue
        cmd = '{} {} -e {} {} {} {} {} {}'.format(
            cutline_builder_script,
            shp_arg_str,
            params.xmin,
            params.xmax,
            params.ymin,
            params.ymax,
            shp,
            aitpath
        )
        
        task = utils.Task(
            'Cutlines',
            'Cutlines',
            'python',
            cmd
        )
        
        if args.mode == "ALL" or args.mode == "SHP":
            logger.debug(cmd)
            task_queue.append(task)
   
    ####  Create task for each tile
    arg_keys_to_remove = (
        'l',
        'qsubscript',
        'parallel_processes',
        'log',
        'mode',
        'extent',
        'resolution',
        'bands',
        'component_shp',
        'pbs'
    )
    tile_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
    
    logger.debug("Identifying components of {0} subtiles".format(num_tiles))
    i = 0
    for t in tiles:
        logger.debug("Identifying components of tile %d of %d: %s" %(i,num_tiles,os.path.basename(t.name)))
        
        ####    determine which images in each tile - create geom and query image geoms
        logger.debug("Running intersect with imagery")       
        
        intersects = []
        for iinfo in intersects_all:
            if t.geom.Intersect(iinfo.geom) is True:
                if iinfo.score > 0:
                    logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score))
                    intersects.append(iinfo.srcfp)
                elif args.nosort:
                    logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score))
                    intersects.append(iinfo.srcfp)
                else:
                    logger.warning("Invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score))
        
        ####  If any images are in the tile, mosaic them        
        if len(intersects) > 0:
            
            tile_basename = os.path.basename(os.path.splitext(t.name)[0])                                    
            itpath = os.path.join(mosaic_dir,tile_basename+"_intersects.txt")
            it = open(itpath,"w")
            it.write(string.join(intersects,"\n"))
            it.close()
            
            #### Submit QSUB job
            logger.debug("Building mosaicking job for tile: %s" %os.path.basename(t.name))
            if not os.path.isfile(t.name):
                
                cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format(
                    tile_builder_script,
                    tile_arg_str,
                    t.xmin,
                    t.xmax,
                    t.ymin,
                    t.ymax,
                    params.xres,
                    params.yres,
                    params.bands,
                    t.name,
                    itpath
                )
                
                task = utils.Task(
                    'Tile {0}'.format(os.path.basename(t.name)),
                    'Mosaic{:04g}'.format(i),
                    'python',
                    cmd
                )
                    
                if args.mode == "ALL" or args.mode == "MOSAIC":
                    logger.debug(cmd)
                    task_queue.append(task)
                
            else:
                logger.info("Tile already exists: %s" %os.path.basename(t.name))
        i += 1
    
    logger.info("Submitting Tasks")
    #logger.info(task_queue)
    if len(task_queue) > 0:
        if args.pbs:
            if args.l:
                task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l))
            else:
                task_handler = utils.PBSTaskHandler(qsubpath)
            task_handler.run_tasks(task_queue)
            
        else:
            task_handler = utils.ParallelTaskHandler(args.parallel_processes)
            if task_handler.num_processes > 1:
                logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes))
            task_handler.run_tasks(task_queue)
            
        logger.info("Done")
        
    else:
        logger.info("No tasks to process")
def HandleTile(t,shp,dstdir,csvpath,args,exclude_list):
    
    
    otxtpath = os.path.join(dstdir,"%s_%s_orig.txt" %(os.path.basename(csvpath)[:-4],t.name))
    mtxtpath = os.path.join(dstdir,"%s_%s_ortho.txt" %(os.path.basename(csvpath)[:-4],t.name))
    
    if os.path.isfile(otxtpath) and os.path.isfile(mtxtpath) and args.overwrite is False:
        logger.info("Tile %s processing files already exist" %t.name)
    else:
        logger.debug("Tile %s" %(t.name))
    
        t_srs = osr.SpatialReference()
        t_srs.ImportFromEPSG(t.epsg)
        
        #### Open Shp
        shpd, shpn = os.path.split(shp)
        shpbn, shpe = os.path.splitext(shpn)
        
        ds = ogr.Open(shp)
        if ds is None:
            logger.warning("Open failed")
            
        else:
            lyr = ds.GetLayerByName( shpbn )
            
            #### attribute filter for online images
            if args.online_only:
                lyr.SetAttributeFilter('STATUS = "online"')
            
            s_srs = lyr.GetSpatialRef()
            #logger.debug(str(s_srs))
            logger.debug(str(t.geom))
        
            if not t_srs.IsSame(s_srs):
                ict = osr.CoordinateTransformation(t_srs, s_srs)
                ct = osr.CoordinateTransformation(s_srs, t_srs)
            
            lyr.ResetReading()
            feat = lyr.GetNextFeature()
            
            imginfo_list1 = []
            
            while feat:
                
                iinfo = mosaic.ImageInfo(feat,"RECORD",srs=s_srs)
                
                if iinfo.geom is not None and iinfo.geom.GetGeometryType() == ogr.wkbPolygon:
                    if not t_srs.IsSame(s_srs):
                        iinfo.geom.Transform(ct)
                    
                    if iinfo.geom.Intersect(t.geom):
                        
                        if iinfo.scene_id in exclude_list:
                            logger.debug("Scene in exclude list, excluding: %s" %iinfo.srcfp)
                            
                        elif args.online_only and not os.path.isfile(iinfo.srcfp):
                            logger.warning("Scene does not exist, excluding: {0}".format(iinfo.srcfp))
                        elif args.require_pan:
                            srcfp = iinfo.srcfp
                            srcdir, mul_name = os.path.split(srcfp)
                            if iinfo.sensor in ["WV02","WV03","QB02"]:
                                 pan_name = mul_name.replace("-M","-P")
                            elif iinfo.sensor == "GE01":
                                 if "_5V" in mul_name:
                                      pan_name_base = srcfp[:-24].replace("M0","P0")
                                      candidates = glob.glob(pan_name_base + "*")
                                      candidates2 = [f for f in candidates if f.endswith(('.ntf','.NTF','.tif','.TIF'))]
                                      if len(candidates2) == 0:
                                          pan_name = ''
                                      elif len(candidates2) == 1:
                                          pan_name = os.path.basename(candidates2[0])
                                      else:
                                          pan_name = ''
                                          logger.error('{} panchromatic images match the multispectral image name {}'.format(len(candidates2),mul_name))
                                 else:
                                      pan_name = mul_name.replace("-M","-P")
                            elif sensor == "IK01":
                                 pan_name = mul_name.replace("blu","pan")
                                 pan_name = mul_name.replace("msi","pan")
                                 pan_name = mul_name.replace("bgrn","pan")    
                            pan_srcfp = os.path.join(srcdir,pan_name)
                            if not os.path.isfile(pan_srcfp):
                                 logger.debug("Image does not have a panchromatic component, excluding: %s" %iinfo.srcfp)
                            else:
                                 logger.debug( "Intersect %s, %s: %s" %(iinfo.scene_id, iinfo.srcfp, str(iinfo.geom)))
                                 imginfo_list1.append(iinfo)
                            
                        else:
                            logger.debug( "Intersect %s, %s: %s" %(iinfo.scene_id, iinfo.srcfp, str(iinfo.geom)))
                            imginfo_list1.append(iinfo)                                
                            
                feat = lyr.GetNextFeature()
            
            ds = None
        
            logger.info("Number of intersects in tile %s: %i" %(t.name,len(imginfo_list1)))
            
            if len(imginfo_list1) > 0:
                if args.nosort is False:
                
                    #### Get mosaic parameters
                    logger.debug("Getting mosaic parameters")
                    params = mosaic.getMosaicParameters(imginfo_list1[0],args)
                    
                    #### Remove images that do not match ref
                    logger.debug("Setting image pattern filter")
                    imginfo_list2 = mosaic.filterMatchingImages(imginfo_list1,params)
                    logger.info("Number of images matching filter: %i" %(len(imginfo_list2)))
                    
                    #### Sort by quality
                    logger.debug("Sorting images by quality")
                    imginfo_list3 = []
                    for iinfo in imginfo_list2:
                        
                        iinfo.getScore(params)
                        if iinfo.score > 0:
                            imginfo_list3.append(iinfo)
                    
                    imginfo_list3.sort(key=lambda x: x.score)
                    
                    if args.build_shp:
                        
                        #######################################################
                        #### Create Shp
                        
                        shp = os.path.join(dstdir,"%s_%s_imagery.shp" %(os.path.basename(csvpath)[:-4],t.name))
                   
                        logger.debug("Creating shapefile of geoms: %s" %shp)
                    
                        fields = [("IMAGENAME", ogr.OFTString, 100),
                            ("SCORE", ogr.OFTReal, 0)]
                        
                        OGR_DRIVER = "ESRI Shapefile"
                        
                        ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
                        if ogrDriver is None:
                            logger.debug("OGR: Driver %s is not available" % OGR_DRIVER)
                            sys.exit(-1)
                        
                        if os.path.isfile(shp):
                            ogrDriver.DeleteDataSource(shp)
                        vds = ogrDriver.CreateDataSource(shp)
                        if vds is None:
                            logger.debug("Could not create shp")
                            sys.exit(-1)
                        
                        shpd, shpn = os.path.split(shp)
                        shpbn, shpe = os.path.splitext(shpn)
                        
                        
                        lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
                        if lyr is None:
                            logger.debug("ERROR: Failed to create layer: %s" % shpbn)
                            sys.exit(-1)
                        
                        for fld, fdef, flen in fields:
                            field_defn = ogr.FieldDefn(fld, fdef)
                            if fdef == ogr.OFTString:
                                field_defn.SetWidth(flen)
                            if lyr.CreateField(field_defn) != 0:
                                logger.debug("ERROR: Failed to create field: %s" % fld)
                        
                        for iinfo in imginfo_list3:
                        
                            logger.debug("Image: %s" %(iinfo.srcfn))
                            
                            feat = ogr.Feature(lyr.GetLayerDefn())
                            
                            feat.SetField("IMAGENAME",iinfo.srcfn)
                            feat.SetField("SCORE",iinfo.score)
    
                            feat.SetGeometry(iinfo.geom)
                            if lyr.CreateFeature(feat) != 0:
                                logger.debug("ERROR: Could not create feature for image %s" % iinfo.srcfn)
                            else:
                                logger.debug("Created feature for image: %s" %iinfo.srcfn)
                                
                            feat.Destroy()
                    
                    
                    ####  Overlay geoms and remove non-contributors
                    logger.debug("Overlaying images to determine contributors")
                    contribs = []
                    
                    for i in xrange(0,len(imginfo_list3)):
                        iinfo = imginfo_list3[i]
                        basegeom = iinfo.geom
    
                        for j in range(i+1,len(imginfo_list3)):
                            iinfo2 = imginfo_list3[j]
                            geom2 = iinfo2.geom
                            
                            if basegeom.Intersects(geom2):
                                basegeom = basegeom.Difference(geom2)
                                if basegeom is None or basegeom.IsEmpty():
                                    #logger.debug("Broke after %i comparisons" %j)
                                    break
                                    
                        if basegeom is None:
                            logger.debug("Function Error: %s" %iinfo.srcfp)
                        elif basegeom.IsEmpty():
                            logger.debug("Removing non-contributing image: %s" %iinfo.srcfp)
                        else:
                            basegeom = basegeom.Intersection(t.geom)
                            if basegeom is None:
                                logger.debug("Function Error: %s" %iinfo.srcfp)
                            elif basegeom.IsEmpty():
                                logger.debug("Removing non-contributing image: %s" %iinfo.srcfp)
                            else:
                                contribs.append(iinfo.srcfp)
                                                
                elif args.nosort is True:
                    contribs = image_list
            
                logger.info("Number of contributing images: %i" %(len(contribs)))      
            
                if len(contribs) > 0:
                    
                    #### Write textfiles
                    if not os.path.isdir(dstdir):
                        os.makedirs(dstdir)
                    
                    otxtpath = os.path.join(dstdir,"%s_%s_orig.txt" %(os.path.basename(csvpath)[:-4],t.name))
                    mtxtpath = os.path.join(dstdir,"%s_%s_ortho.txt" %(os.path.basename(csvpath)[:-4],t.name))
                    otxt = open(otxtpath,'w')
                    mtxt = open(mtxtpath,'w')
                    
                    for contrib in contribs:
                        
                        if not os.path.isfile(contrib):
                            logger.warning("Image does not exist: %s" %(contrib))
                            
                        otxt.write("%s\n" %contrib)
                        m_fn = "{0}_u08{1}{2}.tif".format(
                            os.path.splitext(os.path.basename(contrib))[0],
                            args.stretch,
                            t.epsg
                        )
                        
                        mtxt.write(os.path.join(dstdir,'orthos',t.name,m_fn)+"\n")
 
                    otxt.close()