def test_filter_images(self):
image_list = glob.glob(os.path.join(self.srcdir, '*.tif'))
imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list]
filter_list = [iinfo.srcfn for iinfo in imginfo_list]
self.assertIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413.tif',
filter_list)
self.assertIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif',
filter_list)
self.assertIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif',
filter_list)
mosaic_args = MosaicArgs()
mosaic_args.extent = [-820000.0, -800000.0, -2420000.0, -2400000.0]
mosaic_args.tilesize = [20000, 20000]
mosaic_args.bands = 1
mosaic_params = mosaic.getMosaicParameters(imginfo_list[0], mosaic_args)
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, mosaic_params)
filter_list = [iinfo.srcfn for iinfo in imginfo_list2]
self.assertEqual(len(imginfo_list2), 8)
self.assertNotIn('WV02_20110901210434_103001000B41DC00_11SEP01210434-M1BS-052730735130_01_P007_u08rf3413.tif',
filter_list)
self.assertNotIn('GE01_20130728161916_1050410002608900_13JUL28161916-P1BS-054448357040_01_P002_u16rf3413.tif',
filter_list)
imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, mosaic_params)
filter_list = [iinfo.srcfn for iinfo in imginfo_list3]
self.assertEqual(len(imginfo_list3), 7)
self.assertNotIn('WV02_20131123162834_10300100293C3400_13NOV23162834-P1BS-500408660030_01_P005_u08mr3413.tif',
filter_list)
def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args,
pos_arg_keys):
if os.path.isfile(inpath):
bTextfile = True
elif os.path.isdir(inpath):
bTextfile = False
if not os.path.isdir(mosaic_dir):
os.makedirs(mosaic_dir)
## TODO: verify logger woks for both interactive and hpc jobs
#### Configure Logger
if args.log is not None:
logfile = os.path.abspath(args.log)
else:
logfile = os.path.join(mosaic_dir, default_logfile)
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get exclude list if specified
if args.exclude is not None:
if not os.path.isfile(args.exclude):
logger.error("Value for option --exclude-list is not a valid file")
f = open(args.exclude, 'r')
exclude_list = set([line.rstrip() for line in f.readlines()])
else:
exclude_list = set()
#### Get Images
#logger.info("Reading input images")
xs = []
ys = []
image_list = utils.find_images_with_exclude_list(inpath, bTextfile,
mosaic.EXTS, exclude_list)
if len(image_list) == 0:
raise RuntimeError(
"No images found in input file or directory: {}".format(inpath))
# remove duplicate images
image_list_unique = list(set(image_list))
dupes = [x for n, x in enumerate(image_list) if x in image_list[:n]]
if len(dupes) > 0:
logger.info("Removed %i duplicate image paths", len(dupes))
logger.debug("Dupes: %s", dupes)
image_list = image_list_unique
logger.info("%i existing images found", len(image_list))
#### gather image info list
logger.info("Getting image info")
imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list]
#### Get mosaic parameters
logger.info("Setting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list[0], args)
logger.info("Mosaic parameters: band count=%i, datatype=%s", params.bands,
params.datatype)
logger.info("Mosaic parameters: projection=%s", params.proj)
#### Remove images that do not match ref
logger.info("Applying attribute filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, params)
if len(imginfo_list2) == 0:
raise RuntimeError(
"No valid images found. Check input filter parameters.")
logger.info("%i of %i images match filter", len(imginfo_list2),
len(image_list))
#### if extent is specified, build tile params and compare extent to input image geom
if args.extent:
imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params)
#### else set extent after image geoms computed
else:
#### Get geom for each image
imginfo_list3 = []
for iinfo in imginfo_list2:
if iinfo.geom is not None:
xs = xs + iinfo.xs
ys = ys + iinfo.ys
imginfo_list3.append(iinfo)
else: # remove from list if no geom
logger.debug("Null geometry for image: %s", iinfo.srcfn)
params.xmin = min(xs)
params.xmax = max(xs)
params.ymin = min(ys)
params.ymax = max(ys)
poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(
params.xmin, params.ymin, params.xmin, params.ymax, params.xmax,
params.ymax, params.xmax, params.ymin, params.xmin, params.ymin)
params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)
if len(imginfo_list3) == 0:
raise RuntimeError("No images found that intersect mosaic extent")
logger.info(
"Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f",
params.xres, params.yres, params.xtilesize, params.ytilesize,
params.xmin, params.xmax, params.ymin, params.ymax)
logger.info("%d of %d input images intersect mosaic extent",
len(imginfo_list3), len(imginfo_list2))
## Sort images by score
logger.info("Reading image metadata and determining sort order")
for iinfo in imginfo_list3:
iinfo.getScore(params)
if not args.nosort:
imginfo_list3.sort(key=lambda x: x.score)
logger.info("Getting Exact Image geometry")
imginfo_list4 = []
all_valid = True
for iinfo in imginfo_list3:
if iinfo.score > 0 or args.nosort:
simplify_tolerance = 2.0 * (
(params.xres + params.yres) / 2.0
) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m
geom, xs1, ys1 = mosaic.GetExactTrimmedGeom(
iinfo.srcfp,
step=args.cutline_step,
tolerance=simplify_tolerance)
if geom is None:
logger.warning(
"%s: geometry could not be determined, verify image is valid",
iinfo.srcfn)
all_valid = False
elif geom.IsEmpty():
logger.warning("%s: geometry is empty", iinfo.srcfn)
all_valid = False
else:
iinfo.geom = geom
tm = datetime.today()
imginfo_list4.append(iinfo)
centroid = geom.Centroid()
logger.info("%s: geometry acquired - centroid: %f, %f",
iinfo.srcfn, centroid.GetX(), centroid.GetY())
#print(geom)
else:
logger.debug("Image has an invalid score: %s --> %i", iinfo.srcfp,
iinfo.score)
if not all_valid:
if not args.allow_invalid_geom:
raise RuntimeError(
"Some source images do not have valid geometries. Cannot proceeed"
)
else:
logger.info(
"--allow-invalid-geom used; mosaic will be created using %i valid images (%i invalid \
images not used.)".format(
len(imginfo_list4),
len(imginfo_list3) - len(imginfo_list4)))
# Get stats if needed
logger.info("Getting image metadata")
for iinfo in imginfo_list4:
logger.info(iinfo.srcfn)
if args.calc_stats or args.median_remove:
iinfo.get_raster_stats(args.calc_stats, args.median_remove)
# Build componenet index
if args.component_shp:
if args.mode == "ALL" or args.mode == "SHP":
contribs = [(iinfo, iinfo.geom) for iinfo in imginfo_list4]
logger.info("Number of contributors: %d", len(contribs))
logger.info("Building component index")
comp_shp = mosaicname + "_components.shp"
if len(contribs) > 0:
if os.path.isfile(comp_shp):
logger.info("Components shapefile already exists: %s",
comp_shp)
else:
build_shp(contribs, comp_shp, args, params)
else:
logger.error("No contributing images")
# Build cutlines index
#### Overlay geoms and remove non-contributors
logger.info("Overlaying images to determine contribution geom")
contribs = mosaic.determine_contributors(imginfo_list4, params.extent_geom,
args.min_contribution_area)
logger.info("Number of contributors: %d", len(contribs))
if args.mode == "ALL" or args.mode == "SHP":
logger.info("Building cutlines index")
shp = mosaicname + "_cutlines.shp"
if len(contribs) > 0:
if os.path.isfile(shp):
logger.info("Cutlines shapefile already exists: %s", shp)
else:
build_shp(contribs, shp, args, params)
else:
logger.error("No contributing images")
## Create tile objects
tiles = []
xtiledim = math.ceil((params.xmax - params.xmin) / params.xtilesize)
ytiledim = math.ceil((params.ymax - params.ymin) / params.ytilesize)
logger.info("Tiles: %d rows, %d columns", ytiledim, xtiledim)
xtdb = len(str(int(xtiledim)))
ytdb = len(str(int(ytiledim)))
i = 1
for x in mosaic.drange(params.xmin, params.xmax,
params.xtilesize): # Columns
if x + params.xtilesize > params.xmax:
x2 = params.xmax
else:
x2 = x + params.xtilesize
j = 1
for y in mosaic.drange(params.ymin, params.ymax,
params.ytilesize): # Rows
if y + params.ytilesize > params.ymax:
y2 = params.ymax
else:
y2 = y + params.ytilesize
tilename = "{}_{}_{}.tif".format(mosaicname,
mosaic.buffernum(j, ytdb),
mosaic.buffernum(i, xtdb))
tile = mosaic.TileParams(x, x2, y, y2, j, i, tilename)
tiles.append(tile)
j += 1
i += 1
#### Write shapefile of tiles
if len(tiles) == 0:
raise RuntimeError("No tile objects created")
if args.mode == "ALL" or args.mode == "SHP":
build_tiles_shp(mosaicname, tiles, params)
## Build tile tasks
task_queue = []
#### Create task for each tile
arg_keys_to_remove = ('l', 'qsubscript', 'parallel_processes', 'log',
'mode', 'extent', 'resolution', 'bands', 'max_cc',
'exclude', 'nosort', 'component_shp', 'cutline_step',
'min_contribution_area', 'calc_stats', 'pbs',
'slurm', 'tday', 'tyear', 'allow_invalid_geom')
tile_arg_str = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
logger.debug("Identifying components of %i subtiles", len(tiles))
i = 0
for t in tiles:
logger.debug("Identifying components of tile %i of %i: %s", i,
len(tiles), os.path.basename(t.name))
#### determine which images in each tile - create geom and query image geoms
logger.debug("Running intersect with imagery")
intersects = []
for iinfo, contrib_geom in contribs:
if contrib_geom.Intersects(t.geom):
if args.median_remove:
## parse median dct into text
median_string = ";".join([
"{}:{}".format(k, v) for k, v in iinfo.median.items()
])
intersects.append("{},{}".format(iinfo.srcfp,
median_string))
else:
intersects.append(iinfo.srcfp)
#### If any images are in the tile, mosaic them
if len(intersects) > 0:
tile_basename = os.path.basename(os.path.splitext(t.name)[0])
logger.info("Number of contributors to subtile %s: %i",
tile_basename, len(intersects))
itpath = os.path.join(mosaic_dir,
tile_basename + "_intersects.txt")
it = open(itpath, "w")
it.write("\n".join(intersects))
it.close()
#### Submit QSUB job
logger.debug("Building mosaicking job for tile: %s",
os.path.basename(t.name))
if not os.path.isfile(t.name):
cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format(
tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin,
t.ymax, params.xres, params.yres, params.bands, t.name,
itpath)
task = taskhandler.Task(
'Tile {0}'.format(os.path.basename(t.name)),
'Mos{:04g}'.format(i), 'python', cmd)
if args.mode == "ALL" or args.mode == "MOSAIC":
logger.debug(cmd)
task_queue.append(task)
else:
logger.info("Tile already exists: %s",
os.path.basename(t.name))
i += 1
if args.mode == "ALL" or args.mode == "MOSAIC":
logger.info("Submitting Tasks")
#logger.info(task_queue)
if len(task_queue) > 0:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
if task_handler.num_processes > 1:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
task_handler.run_tasks(task_queue)
logger.info("Done")
else:
logger.info("No tasks to process")
def HandleTile(t, src, dstdir, csvpath, args, exclude_list):
otxtpath = os.path.join(
dstdir,
"{}_{}_orig.txt".format(os.path.basename(csvpath)[:-4], t.name))
mtxtpath = os.path.join(
dstdir,
"{}_{}_ortho.txt".format(os.path.basename(csvpath)[:-4], t.name))
if os.path.isfile(otxtpath) and os.path.isfile(
mtxtpath) and args.overwrite is False:
logger.info("Tile %s processing files already exist", t.name)
else:
logger.info("Tile %s", t.name)
t_srs = osr.SpatialReference()
t_srs.ImportFromEPSG(t.epsg)
#### Open mfp
dsp, lyrn = utils.get_source_names(src)
ds = ogr.Open(dsp)
if ds is None:
logger.error("Open failed")
else:
lyr = ds.GetLayerByName(lyrn)
if not lyr:
raise RuntimeError(
"Layer {} does not exist in dataset {}".format(lyrn, dsp))
else:
s_srs = lyr.GetSpatialRef()
#logger.debug(str(s_srs))
#logger.debug(str(t.geom))
tile_geom_in_s_srs = t.geom.Clone()
if not t_srs.IsSame(s_srs):
ict = osr.CoordinateTransformation(t_srs, s_srs)
ct = osr.CoordinateTransformation(s_srs, t_srs)
tile_geom_in_s_srs.Transform(ict)
# if the geometry crosses meridian, split it into multipolygon (else this breaks SetSpatialFilter)
if utils.doesCross180(tile_geom_in_s_srs):
logger.debug(
"tile_geom_in_s_srs crosses 180 meridian; splitting to multiple polygons..."
)
tile_geom_in_s_srs = utils.getWrappedGeometry(
tile_geom_in_s_srs)
lyr.ResetReading()
lyr.SetSpatialFilter(tile_geom_in_s_srs)
feat = lyr.GetNextFeature()
imginfo_list1 = []
while feat:
iinfo = mosaic.ImageInfo(feat, "RECORD", srs=s_srs)
if iinfo.geom is not None and iinfo.geom.GetGeometryType(
) in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
if not t_srs.IsSame(s_srs):
iinfo.geom.Transform(ct)
## fix self-intersection errors caused by reprojecting over 180
temp = iinfo.geom.Buffer(
0.1
) # assumes a projected coordinate system with meters or feet as units
iinfo.geom = temp
if iinfo.geom.Intersects(t.geom):
if iinfo.scene_id in exclude_list:
logger.debug(
"Scene in exclude list, excluding: %s",
iinfo.srcfp)
elif not os.path.isfile(iinfo.srcfp):
logger.warning(
"Scene path is invalid, excluding %s (path = %s)",
iinfo.scene_id, iinfo.srcfp)
elif args.require_pan:
srcfp = iinfo.srcfp
srcdir, mul_name = os.path.split(srcfp)
if iinfo.sensor in ["WV02", "WV03", "QB02"]:
pan_name = mul_name.replace("-M", "-P")
elif iinfo.sensor == "GE01":
if "_5V" in mul_name:
pan_name_base = srcfp[:-24].replace(
"M0", "P0")
candidates = glob.glob(pan_name_base +
"*")
candidates2 = [
f for f in candidates
if f.endswith(('.ntf', '.NTF',
'.tif', '.TIF'))
]
if len(candidates2) == 0:
pan_name = ''
elif len(candidates2) == 1:
pan_name = os.path.basename(
candidates2[0])
else:
pan_name = ''
logger.error(
'%i panchromatic images match the multispectral image name '
'%s', len(candidates2),
mul_name)
else:
pan_name = mul_name.replace("-M", "-P")
elif iinfo.sensor == "IK01":
pan_name = mul_name.replace("blu", "pan")
pan_name = mul_name.replace("msi", "pan")
pan_name = mul_name.replace("bgrn", "pan")
pan_srcfp = os.path.join(srcdir, pan_name)
if not os.path.isfile(pan_srcfp):
logger.debug(
"Image does not have a panchromatic component, excluding: %s",
iinfo.srcfp)
else:
logger.debug("Intersect %s, %s: %s",
iinfo.scene_id, iinfo.srcfp,
str(iinfo.geom))
imginfo_list1.append(iinfo)
else:
logger.debug("Intersect %s, %s: %s",
iinfo.scene_id, iinfo.srcfp,
str(iinfo.geom))
imginfo_list1.append(iinfo)
feat = lyr.GetNextFeature()
ds = None
logger.info("Number of intersects in tile %s: %i", t.name,
len(imginfo_list1))
if len(imginfo_list1) > 0:
#### Get mosaic parameters
logger.debug("Getting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list1[0], args)
#### Remove images that do not match ref
logger.debug("Setting image pattern filter")
imginfo_list2 = mosaic.filterMatchingImages(
imginfo_list1, params)
logger.info("Number of images matching filter: %i",
len(imginfo_list2))
if args.nosort is False:
#### Sort by quality
logger.debug("Sorting images by quality")
imginfo_list3 = []
for iinfo in imginfo_list2:
iinfo.getScore(params)
if iinfo.score > 0:
imginfo_list3.append(iinfo)
# sort so highest score is last
imginfo_list3.sort(key=lambda x: x.score)
else:
imginfo_list3 = list(imginfo_list2)
#### Overlay geoms and remove non-contributors
logger.debug("Overlaying images to determine contributors")
contribs = mosaic.determine_contributors(
imginfo_list3, t.geom, args.min_contribution_area)
logger.info("Number of contributing images: %i", len(contribs))
if len(contribs) > 0:
if args.build_shp:
#######################################################
#### Create Shp
shp = os.path.join(
dstdir,
"{}_{}_imagery.shp".format(args.mosaic, t.name))
logger.debug("Creating shapefile of geoms: %s", shp)
fields = [("IMAGENAME", ogr.OFTString, 100),
("SCORE", ogr.OFTReal, 0)]
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.debug("OGR: Driver %s is not available",
OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.debug("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
if lyr is None:
logger.debug("ERROR: Failed to create layer: %s",
shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.debug(
"ERROR: Failed to create field: %s", fld)
for iinfo, geom in contribs:
logger.debug("Image: %s", iinfo.srcfn)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("IMAGENAME", iinfo.srcfn)
feat.SetField("SCORE", iinfo.score)
feat.SetGeometry(geom)
if lyr.CreateFeature(feat) != 0:
logger.debug(
"ERROR: Could not create feature for image %s",
iinfo.srcfn)
else:
logger.debug("Created feature for image: %s",
iinfo.srcfn)
feat.Destroy()
#### Write textfiles
if not os.path.isdir(dstdir):
os.makedirs(dstdir)
otxtpath = os.path.join(
dstdir, "{}_{}_orig.txt".format(args.mosaic, t.name))
mtxtpath = os.path.join(
dstdir, "{}_{}_ortho.txt".format(args.mosaic, t.name))
otxt = open(otxtpath, 'w')
mtxt = open(mtxtpath, 'w')
for iinfo, geom in contribs:
if not os.path.isfile(iinfo.srcfp):
logger.warning("Image does not exist: %s",
iinfo.srcfp)
otxt.write("{}\n".format(iinfo.srcfp))
m_fn = "{0}_u08{1}{2}.tif".format(
os.path.splitext(iinfo.srcfn)[0], args.stretch,
t.epsg)
mtxt.write(
os.path.join(dstdir, 'ortho', t.name, m_fn) + "\n")
otxt.close()
def HandleTile(t, src, dstdir, csvpath, args, exclude_list):
otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(os.path.basename(csvpath)[:-4], t.name))
mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(os.path.basename(csvpath)[:-4], t.name))
if os.path.isfile(otxtpath) and os.path.isfile(mtxtpath) and args.overwrite is False:
logger.info("Tile %s processing files already exist", t.name)
else:
logger.info("Tile %s", t.name)
t_srs = utils.osr_srs_preserve_axis_order(osr.SpatialReference())
t_srs.ImportFromEPSG(t.epsg)
#### Open mfp
dsp, lyrn = utils.get_source_names(src)
ds = ogr.Open(dsp)
if ds is None:
logger.error("Open failed")
else:
lyr = ds.GetLayerByName(lyrn)
if not lyr:
raise RuntimeError("Layer {} does not exist in dataset {}".format(lyrn, dsp))
else:
s_srs = lyr.GetSpatialRef()
#logger.debug(str(s_srs))
#logger.debug(str(t.geom))
tile_geom_in_s_srs = t.geom.Clone()
if not t_srs.IsSame(s_srs):
ict = osr.CoordinateTransformation(t_srs, s_srs)
ct = osr.CoordinateTransformation(s_srs, t_srs)
tile_geom_in_s_srs.Transform(ict)
# if the geometry crosses meridian, split it into multipolygon (else this breaks SetSpatialFilter)
if utils.doesCross180(tile_geom_in_s_srs):
logger.debug("tile_geom_in_s_srs crosses 180 meridian; splitting to multiple polygons...")
tile_geom_in_s_srs = utils.getWrappedGeometry(tile_geom_in_s_srs)
lyr.ResetReading()
lyr.SetSpatialFilter(tile_geom_in_s_srs)
feat = lyr.GetNextFeature()
imginfo_list1 = []
while feat:
iinfo = mosaic.ImageInfo(feat, "RECORD", srs=s_srs)
if iinfo.geom is not None and iinfo.geom.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
if not t_srs.IsSame(s_srs):
iinfo.geom.Transform(ct)
## fix self-intersection errors caused by reprojecting over 180
temp = iinfo.geom.Buffer(0.1) # assumes a projected coordinate system with meters or feet as units
iinfo.geom = temp
if iinfo.geom.Intersects(t.geom):
if iinfo.scene_id in exclude_list:
logger.debug("Scene in exclude list, excluding: %s", iinfo.srcfp)
elif not os.path.isfile(iinfo.srcfp) and iinfo.status != "tape":
#logger.info("iinfo.status != tape: {0}".format(iinfo.status != "tape"))
logger.warning("Scene path is invalid, excluding %s (path = %s) (status = %s)",
iinfo.scene_id, iinfo.srcfp, iinfo.status)
elif args.require_pan:
srcfp = iinfo.srcfp
srcdir, mul_name = os.path.split(srcfp)
if iinfo.sensor in ["WV02", "WV03", "QB02"]:
pan_name = mul_name.replace("-M", "-P")
elif iinfo.sensor == "GE01":
if "_5V" in mul_name:
pan_name_base = srcfp[:-24].replace("M0", "P0")
candidates = glob.glob(pan_name_base + "*")
candidates2 = [f for f in candidates if f.endswith(('.ntf', '.NTF', '.tif',
'.TIF'))]
if len(candidates2) == 0:
pan_name = ''
elif len(candidates2) == 1:
pan_name = os.path.basename(candidates2[0])
else:
pan_name = ''
logger.error('%i panchromatic images match the multispectral image name '
'%s', len(candidates2), mul_name)
else:
pan_name = mul_name.replace("-M", "-P")
elif iinfo.sensor == "IK01":
pan_name = mul_name.replace("blu", "pan")
pan_name = mul_name.replace("msi", "pan")
pan_name = mul_name.replace("bgrn", "pan")
pan_srcfp = os.path.join(srcdir, pan_name)
if not os.path.isfile(pan_srcfp):
logger.debug("Image does not have a panchromatic component, excluding: %s",
iinfo.srcfp)
else:
logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
imginfo_list1.append(iinfo)
else:
logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
imginfo_list1.append(iinfo)
feat = lyr.GetNextFeature()
ds = None
logger.info("Number of intersects in tile %s: %i", t.name, len(imginfo_list1))
if len(imginfo_list1) > 0:
#### Get mosaic parameters
logger.debug("Getting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list1[0], args)
#### Remove images that do not match ref
logger.debug("Setting image pattern filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list1, params)
logger.info("Number of images matching filter: %i", len(imginfo_list2))
if args.nosort is False:
#### Sort by quality
logger.debug("Sorting images by quality")
imginfo_list3 = []
for iinfo in imginfo_list2:
iinfo.getScore(params)
if iinfo.score > 0:
imginfo_list3.append(iinfo)
# sort so highest score is last
imginfo_list3.sort(key=lambda x: x.score)
else:
imginfo_list3 = list(imginfo_list2)
#### Overlay geoms and remove non-contributors
logger.debug("Overlaying images to determine contributors")
contribs = mosaic.determine_contributors(imginfo_list3, t.geom, args.min_contribution_area)
logger.info("Number of contributing images: %i", len(contribs))
if len(contribs) > 0:
if args.build_shp:
#######################################################
#### Create Shp
shp = os.path.join(dstdir, "{}_{}_imagery.shp".format(args.mosaic, t.name))
logger.debug("Creating shapefile of geoms: %s", shp)
fields = [("IMAGENAME", ogr.OFTString, 100), ("SCORE", ogr.OFTReal, 0)]
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.debug("OGR: Driver %s is not available", OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.debug("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
if lyr is None:
logger.debug("ERROR: Failed to create layer: %s", shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.debug("ERROR: Failed to create field: %s", fld)
for iinfo, geom in contribs:
logger.debug("Image: %s", iinfo.srcfn)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("IMAGENAME", iinfo.srcfn)
feat.SetField("SCORE", iinfo.score)
feat.SetGeometry(geom)
if lyr.CreateFeature(feat) != 0:
logger.debug("ERROR: Could not create feature for image %s", iinfo.srcfn)
else:
logger.debug("Created feature for image: %s", iinfo.srcfn)
feat.Destroy()
#### Write textfiles
if not os.path.isdir(dstdir):
os.makedirs(dstdir)
otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(args.mosaic, t.name))
otxtpath_ontape = os.path.join(dstdir, "{}_{}_orig_ontape.csv".format(args.mosaic, t.name))
mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(args.mosaic, t.name))
rn_fromtape_basedir = os.path.join(dstdir, "renamed_fromtape")
rn_fromtape_path = os.path.join(rn_fromtape_basedir, t.name)
otxt = open(otxtpath, 'w')
ttxt = open(otxtpath_ontape, 'w')
mtxt = open(mtxtpath, 'w')
# write header
ttxt.write("{0},{1},{2}\n".format("SCENE_ID", "S_FILEPATH", "STATUS"))
tape_ct = 0
for iinfo, geom in contribs:
if not os.path.isfile(iinfo.srcfp) and iinfo.status != "tape":
logger.warning("Image does not exist: %s", iinfo.srcfp)
if iinfo.status == "tape":
tape_ct += 1
ttxt.write("{0},{1},{2}\n".format(iinfo.scene_id, iinfo.srcfp, iinfo.status))
# get srcfp with file extension
srcfp_file = os.path.basename(iinfo.srcfp)
otxt.write("{}\n".format(os.path.join(rn_fromtape_path, srcfp_file)))
else:
otxt.write("{}\n".format(iinfo.srcfp))
m_fn = "{0}_u08{1}{2}.tif".format(
os.path.splitext(iinfo.srcfn)[0],
args.stretch,
t.epsg
)
mtxt.write(os.path.join(dstdir, 'ortho', t.name, m_fn) + "\n")
otxt.close()
if tape_ct == 0:
logger.debug("No files need to be pulled from tape.")
os.remove(otxtpath_ontape)
else:
# make output dirs from tape
if not os.path.isdir(rn_fromtape_basedir):
os.mkdir(rn_fromtape_basedir)
if not os.path.isdir(rn_fromtape_path):
os.mkdir(rn_fromtape_path)
tape_tmp = os.path.join(dstdir, "{0}_{1}_tmp".format(args.mosaic, t.name))
if not os.path.isdir(tape_tmp):
os.mkdir(tape_tmp)
logger.warning("{0} scenes are not accessible, as they are on tape. Please use ir.py to pull "
"scenes using file '{1}'. They must be put in directory '{2}', as file '{3}' "
"contains hard-coded paths to said files (necessary to perform "
"orthorectification). Please set a --tmp path (use '{4}').\n"
"Note that if some (or all) scenes have already been pulled from tape, ir.py "
"will not pull them again.\n".
format(tape_ct, otxtpath_ontape, rn_fromtape_path, otxtpath, tape_tmp))
tape_log = "{0}_{1}_ir_log_{2}.log".format(args.mosaic, t.name,
datetime.today().strftime("%Y%m%d%H%M%S"))
root_pgclib_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
"pgclib", "")
logger.info("Suggested ir.py command:\n\n"
""
"python {}ir.py -i {} -o {} --tmp {} -tm link 2>&1 | tee {}"
.format(root_pgclib_path, otxtpath_ontape, rn_fromtape_path, tape_tmp,
os.path.join(dstdir, tape_log)))
def HandleTile(t, src, dstdir, csvpath, args, exclude_list):
otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(os.path.basename(csvpath)[:-4], t.name))
mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(os.path.basename(csvpath)[:-4], t.name))
if os.path.isfile(otxtpath) and os.path.isfile(mtxtpath) and args.overwrite is False:
logger.info("Tile %s processing files already exist", t.name)
else:
logger.info("Tile %s", t.name)
t_srs = osr.SpatialReference()
t_srs.ImportFromEPSG(t.epsg)
#### Open mfp
dsp, lyrn = utils.get_source_names(src)
ds = ogr.Open(dsp)
if ds is None:
logger.error("Open failed")
else:
lyr = ds.GetLayerByName(lyrn)
if not lyr:
raise RuntimeError("Layer {} does not exist in dataset {}".format(lyrn, dsp))
else:
s_srs = lyr.GetSpatialRef()
#logger.debug(str(s_srs))
#logger.debug(str(t.geom))
tile_geom_in_s_srs = t.geom.Clone()
if not t_srs.IsSame(s_srs):
ict = osr.CoordinateTransformation(t_srs, s_srs)
ct = osr.CoordinateTransformation(s_srs, t_srs)
tile_geom_in_s_srs.Transform(ict)
# if the geometry crosses meridian, split it into multipolygon (else this breaks SetSpatialFilter)
if utils.doesCross180(tile_geom_in_s_srs):
logger.debug("tile_geom_in_s_srs crosses 180 meridian; splitting to multiple polygons...")
tile_geom_in_s_srs = utils.getWrappedGeometry(tile_geom_in_s_srs)
lyr.ResetReading()
lyr.SetSpatialFilter(tile_geom_in_s_srs)
feat = lyr.GetNextFeature()
imginfo_list1 = []
while feat:
iinfo = mosaic.ImageInfo(feat, "RECORD", srs=s_srs)
if iinfo.geom is not None and iinfo.geom.GetGeometryType() in (ogr.wkbPolygon, ogr.wkbMultiPolygon):
if not t_srs.IsSame(s_srs):
iinfo.geom.Transform(ct)
## fix self-intersection errors caused by reprojecting over 180
temp = iinfo.geom.Buffer(0.1) # assumes a projected coordinate system with meters or feet as units
iinfo.geom = temp
if iinfo.geom.Intersects(t.geom):
if iinfo.scene_id in exclude_list:
logger.debug("Scene in exclude list, excluding: %s", iinfo.srcfp)
elif not os.path.isfile(iinfo.srcfp):
logger.warning("Scene path is invalid, excluding %s (path = %s)", iinfo.scene_id,
iinfo.srcfp)
elif args.require_pan:
srcfp = iinfo.srcfp
srcdir, mul_name = os.path.split(srcfp)
if iinfo.sensor in ["WV02", "WV03", "QB02"]:
pan_name = mul_name.replace("-M", "-P")
elif iinfo.sensor == "GE01":
if "_5V" in mul_name:
pan_name_base = srcfp[:-24].replace("M0", "P0")
candidates = glob.glob(pan_name_base + "*")
candidates2 = [f for f in candidates if f.endswith(('.ntf', '.NTF', '.tif',
'.TIF'))]
if len(candidates2) == 0:
pan_name = ''
elif len(candidates2) == 1:
pan_name = os.path.basename(candidates2[0])
else:
pan_name = ''
logger.error('%i panchromatic images match the multispectral image name '
'%s', len(candidates2), mul_name)
else:
pan_name = mul_name.replace("-M", "-P")
elif iinfo.sensor == "IK01":
pan_name = mul_name.replace("blu", "pan")
pan_name = mul_name.replace("msi", "pan")
pan_name = mul_name.replace("bgrn", "pan")
pan_srcfp = os.path.join(srcdir, pan_name)
if not os.path.isfile(pan_srcfp):
logger.debug("Image does not have a panchromatic component, excluding: %s",
iinfo.srcfp)
else:
logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
imginfo_list1.append(iinfo)
else:
logger.debug("Intersect %s, %s: %s", iinfo.scene_id, iinfo.srcfp, str(iinfo.geom))
imginfo_list1.append(iinfo)
feat = lyr.GetNextFeature()
ds = None
logger.info("Number of intersects in tile %s: %i", t.name, len(imginfo_list1))
if len(imginfo_list1) > 0:
#### Get mosaic parameters
logger.debug("Getting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list1[0], args)
#### Remove images that do not match ref
logger.debug("Setting image pattern filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list1, params)
logger.info("Number of images matching filter: %i", len(imginfo_list2))
if args.nosort is False:
#### Sort by quality
logger.debug("Sorting images by quality")
imginfo_list3 = []
for iinfo in imginfo_list2:
iinfo.getScore(params)
if iinfo.score > 0:
imginfo_list3.append(iinfo)
# sort so highest score is last
imginfo_list3.sort(key=lambda x: x.score)
else:
imginfo_list3 = list(imginfo_list2)
#### Overlay geoms and remove non-contributors
logger.debug("Overlaying images to determine contributors")
contribs = mosaic.determine_contributors(imginfo_list3, t.geom, args.min_contribution_area)
logger.info("Number of contributing images: %i", len(contribs))
if len(contribs) > 0:
if args.build_shp:
#######################################################
#### Create Shp
shp = os.path.join(dstdir, "{}_{}_imagery.shp".format(args.mosaic, t.name))
logger.debug("Creating shapefile of geoms: %s", shp)
fields = [("IMAGENAME", ogr.OFTString, 100), ("SCORE", ogr.OFTReal, 0)]
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.debug("OGR: Driver %s is not available", OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.debug("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
if lyr is None:
logger.debug("ERROR: Failed to create layer: %s", shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.debug("ERROR: Failed to create field: %s", fld)
for iinfo, geom in contribs:
logger.debug("Image: %s", iinfo.srcfn)
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("IMAGENAME", iinfo.srcfn)
feat.SetField("SCORE", iinfo.score)
feat.SetGeometry(geom)
if lyr.CreateFeature(feat) != 0:
logger.debug("ERROR: Could not create feature for image %s", iinfo.srcfn)
else:
logger.debug("Created feature for image: %s", iinfo.srcfn)
feat.Destroy()
#### Write textfiles
if not os.path.isdir(dstdir):
os.makedirs(dstdir)
otxtpath = os.path.join(dstdir, "{}_{}_orig.txt".format(args.mosaic, t.name))
mtxtpath = os.path.join(dstdir, "{}_{}_ortho.txt".format(args.mosaic, t.name))
otxt = open(otxtpath, 'w')
mtxt = open(mtxtpath, 'w')
for iinfo, geom in contribs:
if not os.path.isfile(iinfo.srcfp):
logger.warning("Image does not exist: %s", iinfo.srcfp)
otxt.write("{}\n".format(iinfo.srcfp))
m_fn = "{0}_u08{1}{2}.tif".format(
os.path.splitext(iinfo.srcfn)[0],
args.stretch,
t.epsg
)
mtxt.write(os.path.join(dstdir, 'ortho', t.name, m_fn) + "\n")
otxt.close()
def main():
#### Set Up Arguments
parent_parser = mosaic.buildMosaicParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Sumbit/run batch mosaic tasks"
)
parser.add_argument("src", help="textfile or directory of input rasters (tif only)")
parser.add_argument("mosaicname", help="output mosaic name excluding extension")
pos_arg_keys = ["src","mosaicname"]
parser.add_argument("--mode", choices=mosaic.MODES , default="ALL",
help=" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, TEST- create log only")
parser.add_argument("--wd",
help="scratch space (default is mosaic directory)")
parser.add_argument("--component-shp", action="store_true", default=False,
help="create shp of all componenet images")
parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw",
help="GTiff compression type. Default=lzw (%s)"%(",".join(mosaic.GTIFF_COMPRESSIONS)))
parser.add_argument("--pbs", action='store_true', default=False,
help="submit tasks to PBS")
parser.add_argument("--parallel-processes", type=int, default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument("--qsubscript",
help="qsub script to use in cluster job submission (default is <script_dir>/%s)" %default_qsub_script)
parser.add_argument("-l",
help="PBS resources requested (mimicks qsub syntax). Use only on HPC systems.")
parser.add_argument("--log",
help="file to log progress (default is <output dir>\%s" %default_logfile)
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
inpath = os.path.abspath(args.src)
mosaicname = os.path.abspath(args.mosaicname)
mosaicname = os.path.splitext(mosaicname)[0]
mosaic_dir = os.path.dirname(mosaicname)
cutline_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_cutlines.py')
tile_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_tile.py')
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),default_qsub_script)
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" %qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive")
#### Validate Arguments
if os.path.isfile(inpath):
bTextfile = True
elif os.path.isdir(inpath):
bTextfile = False
else:
parser.error("Arg1 is not a valid file path or directory: %s" %inpath)
if not os.path.isdir(mosaic_dir):
os.makedirs(mosaic_dir)
if not os.path.isfile(qsubpath):
parser.error("Arg3 is not a valid file path: %s" %qsubpath)
#### Validate target day option
if args.tday is not None:
try:
m = int(args.tday.split("-")[0])
d = int(args.tday.split("-")[1])
td = date(2000,m,d)
except ValueError:
logger.error("Target day must be in mm-dd format (i.e 04-05)")
sys.exit(1)
else:
m = 0
d = 0
#### Configure Logger
if args.log is not None:
logfile = os.path.abspath(args.log)
else:
logfile = os.path.join(mosaic_dir,default_logfile)
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get exclude list if specified
if args.exclude is not None:
if not os.path.isfile(args.exclude):
parser.error("Value for option --exclude-list is not a valid file")
f = open(args.exclude, 'r')
exclude_list = set([line.rstrip() for line in f.readlines()])
else:
exclude_list = set()
#### Get Images
#logger.info("Reading input images")
xs = []
ys = []
image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list)
if len(image_list) == 0:
logger.error("No images found in input file or directory: %s" %inpath)
sys.exit()
else:
logger.info("%i existing images found" %len(image_list))
#### gather image info list
logger.info("Getting image info")
imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in image_list]
#### Get mosaic parameters
logger.info("Setting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list[0],args)
#### Remove images that do not match ref
logger.info("Applying attribute filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list,params)
if len(imginfo_list2) == 0:
logger.error("No valid images found. Check input filter parameters.")
sys.exit()
else:
logger.info("%i images match filter" %len(imginfo_list2))
#### if extent is specified, build tile params and compare extent to input image geom
if args.extent:
imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params)
#### else set extent after image geoms computed
else:
#### Get geom for each image
imginfo_list3 = []
for iinfo in imginfo_list2:
if iinfo.geom is not None:
xs = xs + iinfo.xs
ys = ys + iinfo.ys
imginfo_list3.append(iinfo)
else: # remove from list if no geom
logger.debug("Null geometry for image: %s" %iinfo.srcfn)
params.xmin = min(xs)
params.xmax = max(xs)
params.ymin = min(ys)
params.ymax = max(ys)
poly_wkt = 'POLYGON (( %f %f, %f %f, %f %f, %f %f, %f %f ))' %(params.xmin,params.ymin,params.xmin,params.ymax,params.xmax,params.ymax,params.xmax,params.ymin,params.xmin,params.ymin)
params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)
#### Check number of remaining images
num_images = len(imginfo_list3)
if num_images > 0:
logger.info("%d of %d input images intersect mosaic extent" %(num_images,len(image_list)))
logger.info("Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f" %(params.xres,params.yres,params.xtilesize,params.ytilesize,params.xmin,params.xmax,params.ymin,params.ymax))
else:
logger.error("No valid images found")
sys.exit(0)
## Sort images by score
logger.info("Reading image metadata and determining sort order")
for iinfo in imginfo_list3:
iinfo.getScore(params)
#### Sort by score
if not args.nosort:
imginfo_list3.sort(key=lambda x: x.score)
#### Write all intersects file
intersects_all = []
for iinfo in imginfo_list3:
if params.extent_geom.Intersect(iinfo.geom) is True:
if iinfo.score > 0:
intersects_all.append(iinfo)
elif args.nosort:
intersects_all.append(iinfo)
else:
logger.debug("Image has an invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score))
else:
logger.debug("Image does not intersect mosaic extent: %s" %iinfo.srcfp) ### this line should never be needed. non-intersecting images should be removed earlier if extent is provided, otherwise all images are in the extent.
aitpath = mosaicname+"_intersects.txt"
ait = open(aitpath,"w")
intersects_fps = [intersect.srcfp for intersect in intersects_all]
ait.write(string.join(intersects_fps,"\n"))
ait.close()
## Create tiles
logger.info("Creating tiles")
tiles = []
xtiledim = math.ceil((params.xmax-params.xmin)/params.xtilesize)
ytiledim = math.ceil((params.ymax-params.ymin)/params.ytilesize)
logger.info("Tiles: %d rows, %d columns" %(ytiledim,xtiledim))
xtdb = len(str(int(xtiledim)))
ytdb = len(str(int(ytiledim)))
i = 1
for x in mosaic.drange(params.xmin,params.xmax,params.xtilesize): # Columns
if x+params.xtilesize > params.xmax:
x2 = params.xmax
else:
x2 = x+params.xtilesize
j = 1
for y in mosaic.drange(params.ymin,params.ymax,params.ytilesize): # Rows
if y+params.ytilesize > params.ymax:
y2 = params.ymax
else:
y2 = y+params.ytilesize
tilename = "%s_%s_%s.tif" %(mosaicname,mosaic.buffernum(j,ytdb),mosaic.buffernum(i,xtdb))
tile = mosaic.TileParams(x,x2,y,y2,j,i,tilename)
tiles.append(tile)
j += 1
i += 1
num_tiles = len(tiles)
#### Write shapefile of tiles
if args.mode == "ALL" or args.mode == "SHP":
shp = mosaicname + "_tiles.shp"
if os.path.isfile(shp):
logger.info("Tiles shapefile already exists: %s" %os.path.basename(shp))
else:
logger.info("Creating shapefile of tiles: %s" %os.path.basename(shp))
fields = [('ROW', ogr.OFTInteger, 4),
('COL', ogr.OFTInteger, 4),
("TILENAME", ogr.OFTString, 100),
('TILEPATH', ogr.OFTString, 254),
('XMIN', ogr.OFTReal, 0),
('XMAX', ogr.OFTReal, 0),
('YMIN', ogr.OFTReal, 0),
('YMAX', ogr.OFTReal, 0)]
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.error("OGR: Driver %s is not available" % OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.error("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
rp = osr.SpatialReference()
rp.ImportFromWkt(params.proj)
lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon)
if lyr is None:
logger.error("ERROR: Failed to create layer: %s" % shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.error("ERROR: Failed to create field: %s" % fld)
for t in tiles:
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("TILENAME",os.path.basename(t.name))
feat.SetField("TILEPATH",t.name)
feat.SetField("ROW",t.j)
feat.SetField("COL",t.i)
feat.SetField("XMIN",t.xmin)
feat.SetField("XMAX",t.xmax)
feat.SetField("YMIN",t.ymin)
feat.SetField("YMAX",t.ymax)
feat.SetGeometry(t.geom)
if lyr.CreateFeature(feat) != 0:
logger.error("ERROR: Could not create feature for tile %s" % tile)
feat.Destroy()
## Build tasks
task_queue = []
#### Create task for shapefile of mosaic components
if args.component_shp is True:
arg_keys_to_remove = (
'l',
'qsubscript',
'parallel_processes',
'log',
'gtiff_compression',
'mode',
'extent',
'resolution',
'pbs',
'wd'
)
shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
comp_shp = mosaicname + "_components.shp"
if os.path.isfile(comp_shp):
logger.info("Components shapefile already exists: %s" %os.path.basename(comp_shp))
else:
logger.info("Processing components: %s" %os.path.basename(comp_shp))
## Make task and add to queue
cmd = '{} --cutline-step 512 {} -e {} {} {} {} {} {}'.format(
cutline_builder_script,
shp_arg_str,
params.xmin,
params.xmax,
params.ymin,
params.ymax,
comp_shp,
aitpath
)
task = utils.Task(
'Components',
'Components',
'python',
cmd
)
if args.mode == "ALL" or args.mode == "SHP":
logger.debug(cmd)
task_queue.append(task)
#### Create task for shapefile of image cutlines
shp = mosaicname + "_cutlines.shp"
arg_keys_to_remove = (
'l',
'qsubscript',
'parallel_processes',
'log',
'gtiff_compression',
'mode',
'extent',
'resolution',
'component_shp',
'pbs',
'wd'
)
shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
if os.path.isfile(shp):
logger.info("Cutlines shapefile already exists: %s" %os.path.basename(shp))
else:
logger.info("Processing cutlines: %s" %os.path.basename(shp))
## Make task and add to queue
cmd = '{} {} -e {} {} {} {} {} {}'.format(
cutline_builder_script,
shp_arg_str,
params.xmin,
params.xmax,
params.ymin,
params.ymax,
shp,
aitpath
)
task = utils.Task(
'Cutlines',
'Cutlines',
'python',
cmd
)
if args.mode == "ALL" or args.mode == "SHP":
logger.debug(cmd)
task_queue.append(task)
#### Create task for each tile
arg_keys_to_remove = (
'l',
'qsubscript',
'parallel_processes',
'log',
'mode',
'extent',
'resolution',
'bands',
'component_shp',
'pbs'
)
tile_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
logger.debug("Identifying components of {0} subtiles".format(num_tiles))
i = 0
for t in tiles:
logger.debug("Identifying components of tile %d of %d: %s" %(i,num_tiles,os.path.basename(t.name)))
#### determine which images in each tile - create geom and query image geoms
logger.debug("Running intersect with imagery")
intersects = []
for iinfo in intersects_all:
if t.geom.Intersect(iinfo.geom) is True:
if iinfo.score > 0:
logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score))
intersects.append(iinfo.srcfp)
elif args.nosort:
logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score))
intersects.append(iinfo.srcfp)
else:
logger.warning("Invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score))
#### If any images are in the tile, mosaic them
if len(intersects) > 0:
tile_basename = os.path.basename(os.path.splitext(t.name)[0])
itpath = os.path.join(mosaic_dir,tile_basename+"_intersects.txt")
it = open(itpath,"w")
it.write(string.join(intersects,"\n"))
it.close()
#### Submit QSUB job
logger.debug("Building mosaicking job for tile: %s" %os.path.basename(t.name))
if not os.path.isfile(t.name):
cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format(
tile_builder_script,
tile_arg_str,
t.xmin,
t.xmax,
t.ymin,
t.ymax,
params.xres,
params.yres,
params.bands,
t.name,
itpath
)
task = utils.Task(
'Tile {0}'.format(os.path.basename(t.name)),
'Mosaic{:04g}'.format(i),
'python',
cmd
)
if args.mode == "ALL" or args.mode == "MOSAIC":
logger.debug(cmd)
task_queue.append(task)
else:
logger.info("Tile already exists: %s" %os.path.basename(t.name))
i += 1
logger.info("Submitting Tasks")
#logger.info(task_queue)
if len(task_queue) > 0:
if args.pbs:
if args.l:
task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l))
else:
task_handler = utils.PBSTaskHandler(qsubpath)
task_handler.run_tasks(task_queue)
else:
task_handler = utils.ParallelTaskHandler(args.parallel_processes)
if task_handler.num_processes > 1:
logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes))
task_handler.run_tasks(task_queue)
logger.info("Done")
else:
logger.info("No tasks to process")
def HandleTile(t,shp,dstdir,csvpath,args,exclude_list):
otxtpath = os.path.join(dstdir,"%s_%s_orig.txt" %(os.path.basename(csvpath)[:-4],t.name))
mtxtpath = os.path.join(dstdir,"%s_%s_ortho.txt" %(os.path.basename(csvpath)[:-4],t.name))
if os.path.isfile(otxtpath) and os.path.isfile(mtxtpath) and args.overwrite is False:
logger.info("Tile %s processing files already exist" %t.name)
else:
logger.debug("Tile %s" %(t.name))
t_srs = osr.SpatialReference()
t_srs.ImportFromEPSG(t.epsg)
#### Open Shp
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
ds = ogr.Open(shp)
if ds is None:
logger.warning("Open failed")
else:
lyr = ds.GetLayerByName( shpbn )
#### attribute filter for online images
if args.online_only:
lyr.SetAttributeFilter('STATUS = "online"')
s_srs = lyr.GetSpatialRef()
#logger.debug(str(s_srs))
logger.debug(str(t.geom))
if not t_srs.IsSame(s_srs):
ict = osr.CoordinateTransformation(t_srs, s_srs)
ct = osr.CoordinateTransformation(s_srs, t_srs)
lyr.ResetReading()
feat = lyr.GetNextFeature()
imginfo_list1 = []
while feat:
iinfo = mosaic.ImageInfo(feat,"RECORD",srs=s_srs)
if iinfo.geom is not None and iinfo.geom.GetGeometryType() == ogr.wkbPolygon:
if not t_srs.IsSame(s_srs):
iinfo.geom.Transform(ct)
if iinfo.geom.Intersect(t.geom):
if iinfo.scene_id in exclude_list:
logger.debug("Scene in exclude list, excluding: %s" %iinfo.srcfp)
elif args.online_only and not os.path.isfile(iinfo.srcfp):
logger.warning("Scene does not exist, excluding: {0}".format(iinfo.srcfp))
elif args.require_pan:
srcfp = iinfo.srcfp
srcdir, mul_name = os.path.split(srcfp)
if iinfo.sensor in ["WV02","WV03","QB02"]:
pan_name = mul_name.replace("-M","-P")
elif iinfo.sensor == "GE01":
if "_5V" in mul_name:
pan_name_base = srcfp[:-24].replace("M0","P0")
candidates = glob.glob(pan_name_base + "*")
candidates2 = [f for f in candidates if f.endswith(('.ntf','.NTF','.tif','.TIF'))]
if len(candidates2) == 0:
pan_name = ''
elif len(candidates2) == 1:
pan_name = os.path.basename(candidates2[0])
else:
pan_name = ''
logger.error('{} panchromatic images match the multispectral image name {}'.format(len(candidates2),mul_name))
else:
pan_name = mul_name.replace("-M","-P")
elif sensor == "IK01":
pan_name = mul_name.replace("blu","pan")
pan_name = mul_name.replace("msi","pan")
pan_name = mul_name.replace("bgrn","pan")
pan_srcfp = os.path.join(srcdir,pan_name)
if not os.path.isfile(pan_srcfp):
logger.debug("Image does not have a panchromatic component, excluding: %s" %iinfo.srcfp)
else:
logger.debug( "Intersect %s, %s: %s" %(iinfo.scene_id, iinfo.srcfp, str(iinfo.geom)))
imginfo_list1.append(iinfo)
else:
logger.debug( "Intersect %s, %s: %s" %(iinfo.scene_id, iinfo.srcfp, str(iinfo.geom)))
imginfo_list1.append(iinfo)
feat = lyr.GetNextFeature()
ds = None
logger.info("Number of intersects in tile %s: %i" %(t.name,len(imginfo_list1)))
if len(imginfo_list1) > 0:
if args.nosort is False:
#### Get mosaic parameters
logger.debug("Getting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list1[0],args)
#### Remove images that do not match ref
logger.debug("Setting image pattern filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list1,params)
logger.info("Number of images matching filter: %i" %(len(imginfo_list2)))
#### Sort by quality
logger.debug("Sorting images by quality")
imginfo_list3 = []
for iinfo in imginfo_list2:
iinfo.getScore(params)
if iinfo.score > 0:
imginfo_list3.append(iinfo)
imginfo_list3.sort(key=lambda x: x.score)
if args.build_shp:
#######################################################
#### Create Shp
shp = os.path.join(dstdir,"%s_%s_imagery.shp" %(os.path.basename(csvpath)[:-4],t.name))
logger.debug("Creating shapefile of geoms: %s" %shp)
fields = [("IMAGENAME", ogr.OFTString, 100),
("SCORE", ogr.OFTReal, 0)]
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.debug("OGR: Driver %s is not available" % OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.debug("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
lyr = vds.CreateLayer(shpbn, t_srs, ogr.wkbPolygon)
if lyr is None:
logger.debug("ERROR: Failed to create layer: %s" % shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.debug("ERROR: Failed to create field: %s" % fld)
for iinfo in imginfo_list3:
logger.debug("Image: %s" %(iinfo.srcfn))
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("IMAGENAME",iinfo.srcfn)
feat.SetField("SCORE",iinfo.score)
feat.SetGeometry(iinfo.geom)
if lyr.CreateFeature(feat) != 0:
logger.debug("ERROR: Could not create feature for image %s" % iinfo.srcfn)
else:
logger.debug("Created feature for image: %s" %iinfo.srcfn)
feat.Destroy()
#### Overlay geoms and remove non-contributors
logger.debug("Overlaying images to determine contributors")
contribs = []
for i in xrange(0,len(imginfo_list3)):
iinfo = imginfo_list3[i]
basegeom = iinfo.geom
for j in range(i+1,len(imginfo_list3)):
iinfo2 = imginfo_list3[j]
geom2 = iinfo2.geom
if basegeom.Intersects(geom2):
basegeom = basegeom.Difference(geom2)
if basegeom is None or basegeom.IsEmpty():
#logger.debug("Broke after %i comparisons" %j)
break
if basegeom is None:
logger.debug("Function Error: %s" %iinfo.srcfp)
elif basegeom.IsEmpty():
logger.debug("Removing non-contributing image: %s" %iinfo.srcfp)
else:
basegeom = basegeom.Intersection(t.geom)
if basegeom is None:
logger.debug("Function Error: %s" %iinfo.srcfp)
elif basegeom.IsEmpty():
logger.debug("Removing non-contributing image: %s" %iinfo.srcfp)
else:
contribs.append(iinfo.srcfp)
elif args.nosort is True:
contribs = image_list
logger.info("Number of contributing images: %i" %(len(contribs)))
if len(contribs) > 0:
#### Write textfiles
if not os.path.isdir(dstdir):
os.makedirs(dstdir)
otxtpath = os.path.join(dstdir,"%s_%s_orig.txt" %(os.path.basename(csvpath)[:-4],t.name))
mtxtpath = os.path.join(dstdir,"%s_%s_ortho.txt" %(os.path.basename(csvpath)[:-4],t.name))
otxt = open(otxtpath,'w')
mtxt = open(mtxtpath,'w')
for contrib in contribs:
if not os.path.isfile(contrib):
logger.warning("Image does not exist: %s" %(contrib))
otxt.write("%s\n" %contrib)
m_fn = "{0}_u08{1}{2}.tif".format(
os.path.splitext(os.path.basename(contrib))[0],
args.stretch,
t.epsg
)
mtxt.write(os.path.join(dstdir,'orthos',t.name,m_fn)+"\n")
otxt.close()
