def main():
#########################################################
#### Handle args
#########################################################
#### Set Up Arguments
parent_parser = mosaic.buildMosaicParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Create cutline or component shapefile"
)
parser.add_argument("shp", help="output shapefile name")
parser.add_argument("src", help="textfile or directory of input rasters (tif only)")
parser.add_argument("--cutline-step", type=int, default=2,
help="cutline calculator pixel skip interval (default=2)")
parser.add_argument("--component-shp", action="store_true", default=False,
help="create shp of all component images")
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
inpath = os.path.abspath(args.src)
shp = os.path.abspath(args.shp)
#print (" ".join(sys.argv))
#### Validate target day option
if args.tday is not None:
try:
m = int(args.tday.split("-")[0])
d = int(args.tday.split("-")[1])
td = date(2000,m,d)
except ValueError:
logger.error("Target day must be in mm-dd format (i.e 04-05)")
sys.exit(1)
else:
m = 0
d = 0
##### Configure Logger
logfile = os.path.splitext(shp)[0]+".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
stm = datetime.today()
logger.info("Start Time: %s\n" %(stm))
minx = args.extent[0]
maxx = args.extent[1]
miny = args.extent[2]
maxy = args.extent[3]
poly_wkt = 'POLYGON (( %s %s, %s %s, %s %s, %s %s, %s %s ))' %(minx,miny,minx,maxy,maxx,maxy,maxx,miny,minx,miny)
extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)
if os.path.isfile(shp):
logger.info("Cutlines shapefile already exists: %s" %shp)
else:
intersects = []
t = open(inpath,'r')
for line in t.readlines():
if os.path.isfile(line.rstrip('\n').rstrip('\r')):
intersects.append(line.rstrip('\n').rstrip('\r'))
else:
logger.warning("Imagepath in intersects textfile does not exist: %s" %line.rstrip('\n').rstrip('\r'))
t.close()
if len(intersects) == 0:
logger.error("No images found: %s" %inpath)
sys.exit()
else:
logger.info("Number of intersecting images: %i" %len(intersects))
#### gather image info list
logger.info("Gathering image info")
imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in intersects]
#### Get mosaic parameters
logger.info("Getting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list[0],args)
logger.info("Mosaic extent: %f %f %f %f" %(params.xmin, params.xmax, params.ymin, params.ymax))
logger.info("Mosaic tilesize: %f %f" %(params.xtilesize, params.ytilesize))
logger.info("Mosaic resolution: %.10f %.10f" %(params.xres, params.yres))
logger.info("Mosaic projection: %s" %(params.proj))
logger.info("Getting Exact Image geometry")
imginfo_list2 =[]
for iinfo in imginfo_list:
simplify_tolerance = 2.0 * ((params.xres + params.yres) / 2.0) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m
geom,xs1,ys1 = mosaic.GetExactTrimmedGeom(iinfo.srcfp,step=args.cutline_step,tolerance=simplify_tolerance)
if geom is None:
logger.info("%s: geometry could not be determined" %iinfo.srcfn)
elif geom.IsEmpty():
logger.info("%s: geometry is empty" %iinfo.srcfn)
else:
iinfo.geom = geom
tm = datetime.today()
imginfo_list2.append(iinfo)
centroid = geom.Centroid()
logger.info("%s: geometry acquired - centroid: %f, %f" %(iinfo.srcfn, centroid.GetX(), centroid.GetY()))
#print geom
logger.info("Getting image metadata and calculating image scores")
for iinfo in imginfo_list2:
iinfo.getScore(params)
if (params.median_remove is True):
iinfo.get_raster_median()
iinfo.get_raster_stats()
logger.info("%s: %s" %(iinfo.srcfn,iinfo.score))
#### Overlay geoms and remove non-contributors
if args.component_shp:
contribs = [(iinfo,iinfo.geom) for iinfo in imginfo_list2]
else:
logger.info("Overlaying images to determine contributors")
contribs = []
for i in xrange(0,len(imginfo_list2)):
iinfo = imginfo_list2[i]
basegeom = iinfo.geom
for j in range(i+1,len(imginfo_list2)):
iinfo2 = imginfo_list2[j]
geom2 = iinfo2.geom
if basegeom.Intersects(geom2):
basegeom = basegeom.Difference(geom2)
if basegeom is None or basegeom.IsEmpty():
break
if basegeom is None:
logger.info("Function Error: %s" %iinfo.srcfn)
elif basegeom.IsEmpty():
logger.info("Removing non-contributing image: %s" %iinfo.srcfn)
else:
basegeom = basegeom.Intersection(extent_geom)
if basegeom is None:
logger.info("Function Error: %s" %iinfo.srcfn)
elif basegeom.IsEmpty():
logger.info("Removing non-contributing image: %s" %iinfo.srcfn)
else:
contribs.append((iinfo,basegeom))
tm = datetime.today()
logger.info("Image: %s" %(os.path.basename(image)))
logger.info("Number of contributors: %d" %len(contribs))
#######################################################
#### Create Shp
logger.info("Creating shapefile of image boundaries: %s" %shp)
if (params.median_remove is True):
fields = (
("IMAGENAME", ogr.OFTString, 100),
("SENSOR", ogr.OFTString, 10),
("ACQDATE", ogr.OFTString, 10),
("CAT_ID", ogr.OFTString, 30),
("RESOLUTION", ogr.OFTReal, 0),
("OFF_NADIR", ogr.OFTReal, 0),
("SUN_ELEV", ogr.OFTReal, 0),
("SUN_AZ", ogr.OFTReal, 0),
("SAT_ELEV", ogr.OFTReal, 0),
("SAT_AZ", ogr.OFTReal, 0),
("STATS_MIN", ogr.OFTString, 80),
("STATS_MAX", ogr.OFTString, 80),
("STATS_STD", ogr.OFTString, 80),
("STATS_MEAN", ogr.OFTString, 80),
("STATS_PXCT", ogr.OFTString, 80),
("MEDIAN", ogr.OFTString, 80),
("CLOUDCOVER", ogr.OFTReal, 0),
("TDI", ogr.OFTReal, 0),
("DATE_DIFF", ogr.OFTReal, 0),
("SCORE", ogr.OFTReal, 0),
)
else:
fields = (
("IMAGENAME", ogr.OFTString, 100),
("SENSOR", ogr.OFTString, 10),
("ACQDATE", ogr.OFTString, 10),
("CAT_ID", ogr.OFTString, 30),
("RESOLUTION", ogr.OFTReal, 0),
("OFF_NADIR", ogr.OFTReal, 0),
("SUN_ELEV", ogr.OFTReal, 0),
("SUN_AZ", ogr.OFTReal, 0),
("SAT_ELEV", ogr.OFTReal, 0),
("SAT_AZ", ogr.OFTReal, 0),
("STATS_MIN", ogr.OFTString, 80),
("STATS_MAX", ogr.OFTString, 80),
("STATS_STD", ogr.OFTString, 80),
("STATS_MEAN", ogr.OFTString, 80),
("STATS_PXCT", ogr.OFTString, 80),
("CLOUDCOVER", ogr.OFTReal, 0),
("TDI", ogr.OFTReal, 0),
("DATE_DIFF", ogr.OFTReal, 0),
("SCORE", ogr.OFTReal, 0),
)
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.info("OGR: Driver %s is not available" % OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.info("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
rp = osr.SpatialReference()
rp.ImportFromWkt(params.proj)
lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon)
if lyr is None:
logger.info("ERROR: Failed to create layer: %s" % shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.info("ERROR: Failed to create field: %s" % fld)
for iinfo,geom in contribs:
logger.info("Image: %s" %(iinfo.srcfn))
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("IMAGENAME", iinfo.srcfn)
feat.SetField("SENSOR", iinfo.sensor)
feat.SetField("ACQDATE", iinfo.acqdate.strftime("%Y-%m-%d"))
feat.SetField("CAT_ID", iinfo.catid)
feat.SetField("OFF_NADIR", iinfo.ona)
feat.SetField("SUN_ELEV" ,iinfo.sunel)
feat.SetField("SUN_AZ", iinfo.sunaz)
feat.SetField("SAT_ELEV", iinfo.satel)
feat.SetField("SAT_AZ", iinfo.sataz)
feat.SetField("CLOUDCOVER", iinfo.cloudcover)
feat.SetField("SCORE", iinfo.score)
tdi = iinfo.tdi if iinfo.tdi else 0
feat.SetField("TDI", tdi)
date_diff = iinfo.date_diff if iinfo.date_diff else -9999
feat.SetField("DATE_DIFF", date_diff)
res = ((iinfo.xres+iinfo.yres)/2.0) if iinfo.xres else 0
feat.SetField("RESOLUTION", res)
if len(iinfo.stat_dct) > 0:
min_list = []
max_list = []
mean_list = []
stdev_list = []
px_cnt_list = []
keys = iinfo.stat_dct.keys()
keys.sort()
for band in keys:
imin, imax, imean, istdev = iinfo.stat_dct[band]
ipx_cnt = iinfo.datapixelcount_dct[band]
min_list.append(str(imin))
max_list.append(str(imax))
mean_list.append(str(imean))
stdev_list.append(str(istdev))
px_cnt_list.append(str(ipx_cnt))
feat.SetField("STATS_MIN", ",".join(min_list))
feat.SetField("STATS_MAX", ",".join(max_list))
feat.SetField("STATS_MEAN", ",".join(mean_list))
feat.SetField("STATS_STD", ",".join(stdev_list))
feat.SetField("STATS_PXCT", ",".join(px_cnt_list))
if (params.median_remove is True):
median_list = []
keys = iinfo.median.keys()
keys.sort()
for band in keys:
band_median = iinfo.median[band]
median_list.append(str(band_median))
feat.SetField("MEDIAN", ",".join(median_list))
logger.info("median = {}".format(",".join(median_list)))
feat.SetGeometry(geom)
if lyr.CreateFeature(feat) != 0:
logger.info("ERROR: Could not create feature for image %s" % image)
else:
logger.info("Created feature for image: %s" %image)
feat.Destroy()
etm = datetime.today()
td = (etm-stm)
logger.info("Total Processing Time: %s\n" %(td))
def main():
#### Set Up Arguments
parent_parser = mosaic.buildMosaicParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Sumbit/run batch mosaic tasks"
)
parser.add_argument("src", help="textfile or directory of input rasters (tif only)")
parser.add_argument("mosaicname", help="output mosaic name excluding extension")
pos_arg_keys = ["src","mosaicname"]
parser.add_argument("--mode", choices=mosaic.MODES , default="ALL",
help=" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, TEST- create log only")
parser.add_argument("--wd",
help="scratch space (default is mosaic directory)")
parser.add_argument("--component-shp", action="store_true", default=False,
help="create shp of all componenet images")
parser.add_argument("--gtiff-compression", choices=mosaic.GTIFF_COMPRESSIONS, default="lzw",
help="GTiff compression type. Default=lzw (%s)"%(",".join(mosaic.GTIFF_COMPRESSIONS)))
parser.add_argument("--pbs", action='store_true', default=False,
help="submit tasks to PBS")
parser.add_argument("--parallel-processes", type=int, default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument("--qsubscript",
help="qsub script to use in cluster job submission (default is <script_dir>/%s)" %default_qsub_script)
parser.add_argument("-l",
help="PBS resources requested (mimicks qsub syntax). Use only on HPC systems.")
parser.add_argument("--log",
help="file to log progress (default is <output dir>\%s" %default_logfile)
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
inpath = os.path.abspath(args.src)
mosaicname = os.path.abspath(args.mosaicname)
mosaicname = os.path.splitext(mosaicname)[0]
mosaic_dir = os.path.dirname(mosaicname)
cutline_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_cutlines.py')
tile_builder_script = os.path.join(os.path.dirname(scriptpath),'pgc_mosaic_build_tile.py')
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),default_qsub_script)
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" %qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive")
#### Validate Arguments
if os.path.isfile(inpath):
bTextfile = True
elif os.path.isdir(inpath):
bTextfile = False
else:
parser.error("Arg1 is not a valid file path or directory: %s" %inpath)
if not os.path.isdir(mosaic_dir):
os.makedirs(mosaic_dir)
if not os.path.isfile(qsubpath):
parser.error("Arg3 is not a valid file path: %s" %qsubpath)
#### Validate target day option
if args.tday is not None:
try:
m = int(args.tday.split("-")[0])
d = int(args.tday.split("-")[1])
td = date(2000,m,d)
except ValueError:
logger.error("Target day must be in mm-dd format (i.e 04-05)")
sys.exit(1)
else:
m = 0
d = 0
#### Configure Logger
if args.log is not None:
logfile = os.path.abspath(args.log)
else:
logfile = os.path.join(mosaic_dir,default_logfile)
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get exclude list if specified
if args.exclude is not None:
if not os.path.isfile(args.exclude):
parser.error("Value for option --exclude-list is not a valid file")
f = open(args.exclude, 'r')
exclude_list = set([line.rstrip() for line in f.readlines()])
else:
exclude_list = set()
#### Get Images
#logger.info("Reading input images")
xs = []
ys = []
image_list = utils.find_images_with_exclude_list(inpath, bTextfile, mosaic.EXTS, exclude_list)
if len(image_list) == 0:
logger.error("No images found in input file or directory: %s" %inpath)
sys.exit()
else:
logger.info("%i existing images found" %len(image_list))
#### gather image info list
logger.info("Getting image info")
imginfo_list = [mosaic.ImageInfo(image,"IMAGE") for image in image_list]
#### Get mosaic parameters
logger.info("Setting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list[0],args)
#### Remove images that do not match ref
logger.info("Applying attribute filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list,params)
if len(imginfo_list2) == 0:
logger.error("No valid images found. Check input filter parameters.")
sys.exit()
else:
logger.info("%i images match filter" %len(imginfo_list2))
#### if extent is specified, build tile params and compare extent to input image geom
if args.extent:
imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params)
#### else set extent after image geoms computed
else:
#### Get geom for each image
imginfo_list3 = []
for iinfo in imginfo_list2:
if iinfo.geom is not None:
xs = xs + iinfo.xs
ys = ys + iinfo.ys
imginfo_list3.append(iinfo)
else: # remove from list if no geom
logger.debug("Null geometry for image: %s" %iinfo.srcfn)
params.xmin = min(xs)
params.xmax = max(xs)
params.ymin = min(ys)
params.ymax = max(ys)
poly_wkt = 'POLYGON (( %f %f, %f %f, %f %f, %f %f, %f %f ))' %(params.xmin,params.ymin,params.xmin,params.ymax,params.xmax,params.ymax,params.xmax,params.ymin,params.xmin,params.ymin)
params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)
#### Check number of remaining images
num_images = len(imginfo_list3)
if num_images > 0:
logger.info("%d of %d input images intersect mosaic extent" %(num_images,len(image_list)))
logger.info("Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f" %(params.xres,params.yres,params.xtilesize,params.ytilesize,params.xmin,params.xmax,params.ymin,params.ymax))
else:
logger.error("No valid images found")
sys.exit(0)
## Sort images by score
logger.info("Reading image metadata and determining sort order")
for iinfo in imginfo_list3:
iinfo.getScore(params)
#### Sort by score
if not args.nosort:
imginfo_list3.sort(key=lambda x: x.score)
#### Write all intersects file
intersects_all = []
for iinfo in imginfo_list3:
if params.extent_geom.Intersect(iinfo.geom) is True:
if iinfo.score > 0:
intersects_all.append(iinfo)
elif args.nosort:
intersects_all.append(iinfo)
else:
logger.debug("Image has an invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score))
else:
logger.debug("Image does not intersect mosaic extent: %s" %iinfo.srcfp) ### this line should never be needed. non-intersecting images should be removed earlier if extent is provided, otherwise all images are in the extent.
aitpath = mosaicname+"_intersects.txt"
ait = open(aitpath,"w")
intersects_fps = [intersect.srcfp for intersect in intersects_all]
ait.write(string.join(intersects_fps,"\n"))
ait.close()
## Create tiles
logger.info("Creating tiles")
tiles = []
xtiledim = math.ceil((params.xmax-params.xmin)/params.xtilesize)
ytiledim = math.ceil((params.ymax-params.ymin)/params.ytilesize)
logger.info("Tiles: %d rows, %d columns" %(ytiledim,xtiledim))
xtdb = len(str(int(xtiledim)))
ytdb = len(str(int(ytiledim)))
i = 1
for x in mosaic.drange(params.xmin,params.xmax,params.xtilesize): # Columns
if x+params.xtilesize > params.xmax:
x2 = params.xmax
else:
x2 = x+params.xtilesize
j = 1
for y in mosaic.drange(params.ymin,params.ymax,params.ytilesize): # Rows
if y+params.ytilesize > params.ymax:
y2 = params.ymax
else:
y2 = y+params.ytilesize
tilename = "%s_%s_%s.tif" %(mosaicname,mosaic.buffernum(j,ytdb),mosaic.buffernum(i,xtdb))
tile = mosaic.TileParams(x,x2,y,y2,j,i,tilename)
tiles.append(tile)
j += 1
i += 1
num_tiles = len(tiles)
#### Write shapefile of tiles
if args.mode == "ALL" or args.mode == "SHP":
shp = mosaicname + "_tiles.shp"
if os.path.isfile(shp):
logger.info("Tiles shapefile already exists: %s" %os.path.basename(shp))
else:
logger.info("Creating shapefile of tiles: %s" %os.path.basename(shp))
fields = [('ROW', ogr.OFTInteger, 4),
('COL', ogr.OFTInteger, 4),
("TILENAME", ogr.OFTString, 100),
('TILEPATH', ogr.OFTString, 254),
('XMIN', ogr.OFTReal, 0),
('XMAX', ogr.OFTReal, 0),
('YMIN', ogr.OFTReal, 0),
('YMAX', ogr.OFTReal, 0)]
OGR_DRIVER = "ESRI Shapefile"
ogrDriver = ogr.GetDriverByName(OGR_DRIVER)
if ogrDriver is None:
logger.error("OGR: Driver %s is not available" % OGR_DRIVER)
sys.exit(-1)
if os.path.isfile(shp):
ogrDriver.DeleteDataSource(shp)
vds = ogrDriver.CreateDataSource(shp)
if vds is None:
logger.error("Could not create shp")
sys.exit(-1)
shpd, shpn = os.path.split(shp)
shpbn, shpe = os.path.splitext(shpn)
rp = osr.SpatialReference()
rp.ImportFromWkt(params.proj)
lyr = vds.CreateLayer(shpbn, rp, ogr.wkbPolygon)
if lyr is None:
logger.error("ERROR: Failed to create layer: %s" % shpbn)
sys.exit(-1)
for fld, fdef, flen in fields:
field_defn = ogr.FieldDefn(fld, fdef)
if fdef == ogr.OFTString:
field_defn.SetWidth(flen)
if lyr.CreateField(field_defn) != 0:
logger.error("ERROR: Failed to create field: %s" % fld)
for t in tiles:
feat = ogr.Feature(lyr.GetLayerDefn())
feat.SetField("TILENAME",os.path.basename(t.name))
feat.SetField("TILEPATH",t.name)
feat.SetField("ROW",t.j)
feat.SetField("COL",t.i)
feat.SetField("XMIN",t.xmin)
feat.SetField("XMAX",t.xmax)
feat.SetField("YMIN",t.ymin)
feat.SetField("YMAX",t.ymax)
feat.SetGeometry(t.geom)
if lyr.CreateFeature(feat) != 0:
logger.error("ERROR: Could not create feature for tile %s" % tile)
feat.Destroy()
## Build tasks
task_queue = []
#### Create task for shapefile of mosaic components
if args.component_shp is True:
arg_keys_to_remove = (
'l',
'qsubscript',
'parallel_processes',
'log',
'gtiff_compression',
'mode',
'extent',
'resolution',
'pbs',
'wd'
)
shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
comp_shp = mosaicname + "_components.shp"
if os.path.isfile(comp_shp):
logger.info("Components shapefile already exists: %s" %os.path.basename(comp_shp))
else:
logger.info("Processing components: %s" %os.path.basename(comp_shp))
## Make task and add to queue
cmd = '{} --cutline-step 512 {} -e {} {} {} {} {} {}'.format(
cutline_builder_script,
shp_arg_str,
params.xmin,
params.xmax,
params.ymin,
params.ymax,
comp_shp,
aitpath
)
task = utils.Task(
'Components',
'Components',
'python',
cmd
)
if args.mode == "ALL" or args.mode == "SHP":
logger.debug(cmd)
task_queue.append(task)
#### Create task for shapefile of image cutlines
shp = mosaicname + "_cutlines.shp"
arg_keys_to_remove = (
'l',
'qsubscript',
'parallel_processes',
'log',
'gtiff_compression',
'mode',
'extent',
'resolution',
'component_shp',
'pbs',
'wd'
)
shp_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
if os.path.isfile(shp):
logger.info("Cutlines shapefile already exists: %s" %os.path.basename(shp))
else:
logger.info("Processing cutlines: %s" %os.path.basename(shp))
## Make task and add to queue
cmd = '{} {} -e {} {} {} {} {} {}'.format(
cutline_builder_script,
shp_arg_str,
params.xmin,
params.xmax,
params.ymin,
params.ymax,
shp,
aitpath
)
task = utils.Task(
'Cutlines',
'Cutlines',
'python',
cmd
)
if args.mode == "ALL" or args.mode == "SHP":
logger.debug(cmd)
task_queue.append(task)
#### Create task for each tile
arg_keys_to_remove = (
'l',
'qsubscript',
'parallel_processes',
'log',
'mode',
'extent',
'resolution',
'bands',
'component_shp',
'pbs'
)
tile_arg_str = utils.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
logger.debug("Identifying components of {0} subtiles".format(num_tiles))
i = 0
for t in tiles:
logger.debug("Identifying components of tile %d of %d: %s" %(i,num_tiles,os.path.basename(t.name)))
#### determine which images in each tile - create geom and query image geoms
logger.debug("Running intersect with imagery")
intersects = []
for iinfo in intersects_all:
if t.geom.Intersect(iinfo.geom) is True:
if iinfo.score > 0:
logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score))
intersects.append(iinfo.srcfp)
elif args.nosort:
logger.debug("intersects tile: %s - score %f" %(iinfo.srcfn,iinfo.score))
intersects.append(iinfo.srcfp)
else:
logger.warning("Invalid score: %s --> %i" %(iinfo.srcfp, iinfo.score))
#### If any images are in the tile, mosaic them
if len(intersects) > 0:
tile_basename = os.path.basename(os.path.splitext(t.name)[0])
itpath = os.path.join(mosaic_dir,tile_basename+"_intersects.txt")
it = open(itpath,"w")
it.write(string.join(intersects,"\n"))
it.close()
#### Submit QSUB job
logger.debug("Building mosaicking job for tile: %s" %os.path.basename(t.name))
if not os.path.isfile(t.name):
cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format(
tile_builder_script,
tile_arg_str,
t.xmin,
t.xmax,
t.ymin,
t.ymax,
params.xres,
params.yres,
params.bands,
t.name,
itpath
)
task = utils.Task(
'Tile {0}'.format(os.path.basename(t.name)),
'Mosaic{:04g}'.format(i),
'python',
cmd
)
if args.mode == "ALL" or args.mode == "MOSAIC":
logger.debug(cmd)
task_queue.append(task)
else:
logger.info("Tile already exists: %s" %os.path.basename(t.name))
i += 1
logger.info("Submitting Tasks")
#logger.info(task_queue)
if len(task_queue) > 0:
if args.pbs:
if args.l:
task_handler = utils.PBSTaskHandler(qsubpath, "-l {}".format(args.l))
else:
task_handler = utils.PBSTaskHandler(qsubpath)
task_handler.run_tasks(task_queue)
else:
task_handler = utils.ParallelTaskHandler(args.parallel_processes)
if task_handler.num_processes > 1:
logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes))
task_handler.run_tasks(task_queue)
logger.info("Done")
else:
logger.info("No tasks to process")
def main():
#### Set Up Arguments
parent_parser = mosaic.buildMosaicParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="query PGC index for images contributing to a mosaic"
)
parser.add_argument("index", help="PGC index shapefile")
parser.add_argument("tile_csv", help="tile schema csv")
parser.add_argument("dstdir", help="textfile output directory")
#pos_arg_keys = ["index","tile_csv","dstdir"]
parser.add_argument("--log",
help="output log file (default is queryFP.log in the output folder)")
parser.add_argument("--ttile",
help="target tile (default is to compute all valid tiles. multiple tiles should be delimited by a comma [ex: 23_24,23_25])")
parser.add_argument("--overwrite", action="store_true", default=False,
help="overwrite any existing files")
parser.add_argument("--stretch", choices=ortho_functions.stretches, default="rf",
help="stretch abbreviation used in image processing (default=rf)")
parser.add_argument("--build-shp", action='store_true', default=False,
help="build shapefile of intersecting images (only invoked if --no_sort is not used)")
parser.add_argument("--online-only", action='store_true', default=False,
help="limit search to those records where status = online and image is found on the file system")
parser.add_argument("--require-pan", action='store_true', default=False,
help="limit search to imagery with both a multispectral and a panchromatic component")
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
shp = os.path.abspath(args.index)
csvpath = os.path.abspath(args.tile_csv)
dstdir = os.path.abspath(args.dstdir)
#### Validate Required Arguments
if not os.path.isfile(shp):
parser.error("Arg1 is not a valid file path: %s" %shp)
if not os.path.isfile(csvpath):
parser.error("Arg2 is not a valid file path: %s" %csvpath)
#### Validate target day option
if args.tday is not None:
try:
m = int(args.tday.split("-")[0])
d = int(args.tday.split("-")[1])
td = date(2000,m,d)
except ValueError:
logger.error("Target day must be in mm-dd format (i.e 04-05)")
sys.exit(1)
else:
m = 0
d = 0
##### Configure Logger
if args.log is not None:
logfile = os.path.abspath(args.log)
else:
logfile = os.path.join(dstdir,"queryFP_%s.log" %datetime.today().strftime("%Y%m%d%H%M%S"))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get exclude_list if specified
if args.exclude is not None:
if not os.path.isfile(args.exclude):
parser.error("Value for option --exclude-list is not a valid file")
f = open(args.exclude, 'r')
exclude_list = set([line.rstrip() for line in f.readlines()])
else:
exclude_list = set()
logger.debug("Exclude list: %s" %str(exclude_list))
#### Parse csv, validate tile ID and get tilegeom
tiles = {}
csv = open(csvpath,'r')
for line in csv:
tile = line.rstrip().split(",")
if len(tile) != 9:
logger.warning("funny csv line: %s" %line.strip('\n'))
else:
name = tile[2]
if name != "name":
### Tile csv schema: row, column, name, status, xmin, xmax, ymin, ymax, epsg code
t = mosaic.TileParams(float(tile[4]),float(tile[5]),float(tile[6]),float(tile[7]),int(tile[0]),int(tile[1]),tile[2])
t.status = tile[3]
t.epsg = int(tile[8])
tiles[name] = t
csv.close()
if args.ttile is not None:
if "," in args.ttile:
ttiles = args.ttile.split(",")
else:
ttiles = [args.ttile]
for ttile in ttiles:
if ttile not in tiles:
logger.info("Target tile is not in the tile csv: %s" %ttile)
else:
t = tiles[ttile]
if t.status == "0":
logger.error("Tile status indicates it should not be created: %s, %s" %(ttile,t.status))
else:
HandleTile(t,shp,dstdir,csvpath,args,exclude_list)
else:
keys = tiles.keys()
keys.sort()
for tile in keys:
t = tiles[tile]
if t.status == "1":
HandleTile(t,shp,dstdir,csvpath,args,exclude_list)
