def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"package setsm dems (build mdf and readme files and create archive) in place in the filesystem"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space to build index shps")
#### Optionsl Arguments
parser.add_argument('--mdf-only',
action='store_true',
default=False,
help="build mdf and readme files only, do not archive")
parser.add_argument(
'--lsf',
action='store_true',
default=False,
help="package LSF DEM instead of original DEM. Includes metadata flag."
)
parser.add_argument(
'--filter-dems',
action='store_true',
default=False,
help="filter dems with area < 5.6 sqkm and density < 0.1")
parser.add_argument(
'--force-filter-dems',
action='store_true',
default=False,
help=
"filter dems where tar has already been built with area < 5.6 sqkm and density < 0.1"
)
parser.add_argument('-v',
action='store_true',
default=False,
help="verbose output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument(
"--tasks-per-job",
type=int,
help="number of tasks to bundle into a single job (requires pbs option)"
)
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_package.sh in script root folder)"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
pos_arg_keys = ['src', 'scratch']
#### Parse Arguments
scriptpath = os.path.abspath(sys.argv[0])
args = parser.parse_args()
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
if not os.path.isdir(
args.scratch
) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs
parser.error("Scratch directory does not exist: %s" % args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
if args.tasks_per_job and not args.pbs:
parser.error("jobs-per-task argument requires the pbs option")
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
if args.lsf:
logger.info('Packaging LSF DEMs')
else:
logger.info('Packaging non-LSF DEMs')
j = 0
scenes = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith('.tif'):
logger.debug(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(src)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(src)
elif not os.path.isfile(raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(raster.readme):
scenes.append(src)
elif os.path.isfile(src) and src.endswith('.txt'):
fh = open(src, 'r')
for line in fh.readlines():
sceneid = line.strip()
try:
raster = dem.SetsmDem(sceneid)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(sceneid)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(sceneid)
elif not os.path.isfile(raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(raster.readme):
scenes.append(sceneid)
elif os.path.isdir(src):
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif") and "m_" in f:
srcfp = os.path.join(root, f)
logger.debug(srcfp)
try:
raster = dem.SetsmDem(srcfp)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(srcfp)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(srcfp)
elif not os.path.isfile(
raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(
raster.readme):
scenes.append(srcfp)
else:
logger.error("src must be a directory, a strip dem, or a text file")
scenes = list(set(scenes))
logger.info('Number of src rasters: {}'.format(j))
logger.info('Number of incomplete tasks: {}'.format(len(scenes)))
tm = datetime.now()
job_count = 0
scene_count = 0
scenes_in_job_count = 0
task_queue = []
for srcfp in scenes:
scene_count += 1
srcdir, srcfn = os.path.split(srcfp)
if args.tasks_per_job:
# bundle tasks into text files in the dst dir and pass the text file in as src
scenes_in_job_count += 1
src_txt = os.path.join(
scratch,
'src_dems_{}_{}.txt'.format(tm.strftime("%Y%m%d%H%M%S"),
job_count))
if scenes_in_job_count == 1:
# remove text file if dst already exists
try:
os.remove(src_txt)
except OSError:
pass
if scenes_in_job_count <= args.tasks_per_job:
# add to txt file
fh = open(src_txt, 'a')
fh.write("{}\n".format(srcfp))
fh.close()
if scenes_in_job_count == args.tasks_per_job or scene_count == len(
scenes):
scenes_in_job_count = 0
job_count += 1
task = taskhandler.Task(
'Pkg{:04g}'.format(job_count),
'Pkg{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, src_txt,
scratch), build_archive,
[srcfp, scratch, args])
task_queue.append(task)
else:
job_count += 1
task = taskhandler.Task(
srcfn, 'Pkg{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, scratch),
build_archive, [srcfp, scratch, args])
task_queue.append(task)
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, scratch, task_arg_obj = task.method_arg_list
if not args.dryrun:
task.method(src, scratch, task_arg_obj)
else:
logger.info("No tasks found to process")
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/submit batch image ortho and conversion tasks")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM "
"default is slurm_ortho.py, in script root folder)")
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help='print actions without executing')
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: {}".format(
src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_ortho.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_ortho.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
## Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
#### Verify that dem and ortho_height are not both specified
if args.dem is not None and args.ortho_height is not None:
parser.error(
"--dem and --ortho_height options are mutually exclusive. Please choose only one."
)
#### Test if DEM exists
if args.dem:
if not os.path.isfile(args.dem):
parser.error("DEM does not exist: {}".format(args.dem))
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list1 = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list1 = utils.find_images(src, True, ortho_functions.exts)
else:
image_list1 = [src]
## Group Ikonos
image_list2 = []
for srcfp in image_list1:
srcdir, srcfn = os.path.split(srcfp)
if "IK01" in srcfn and sum(
[b in srcfn for b in ortho_functions.ikMsiBands]) > 0:
for b in ortho_functions.ikMsiBands:
if b in srcfn:
newname = os.path.join(srcdir, srcfn.replace(b, "msi"))
break
image_list2.append(newname)
else:
image_list2.append(srcfp)
image_list = list(set(image_list2))
logger.info('Number of src images: %i', len(image_list))
## Build task queue
i = 0
task_queue = []
for srcfp in image_list:
srcdir, srcfn = os.path.split(srcfp)
dstfp = os.path.join(
dstdir, "{}_{}{}{}{}".format(
os.path.splitext(srcfn)[0], utils.get_bit_depth(args.outtype),
args.stretch, spatial_ref.epsg,
ortho_functions.formats[args.format]))
done = os.path.isfile(dstfp)
if done is False:
i += 1
task = taskhandler.Task(
srcfn, 'Or{:04g}'.format(i), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, dstdir),
ortho_functions.process_image, [srcfp, dstfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: %i', i)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
for task in task_queue:
src, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(src, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
def main():
parser = argparse.ArgumentParser()
#### Set Up Options
parser.add_argument("srcdir", help="source directory or image")
parser.add_argument("dstdir", help="destination directory")
parser.add_argument("epsg", type=int, help="target epsg code")
parser.add_argument(
"-r",
"--resolution",
default=default_res,
type=int,
help="output resolution (default={})".format(default_res))
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_resample.sh in script root folder)"
)
parser.add_argument("--dryrun",
action="store_true",
default=False,
help="print actions without executing")
pos_arg_keys = ['srcdir', 'dstdir', 'epsg']
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
srcpath = os.path.abspath(args.srcdir)
#### Validate Required Arguments
if not os.path.isdir(srcpath) and not os.path.isfile(srcpath):
parser.error('src must be a valid directory or file')
dstdir = os.path.abspath(args.dstdir)
if not os.path.isdir(dstdir):
os.makedirs(dstdir)
## Verify EPSG
test_sr = osr.SpatialReference()
status = test_sr.ImportFromEPSG(args.epsg)
if status != 0:
parser.error(
'EPSG test osr.SpatialReference.ImportFromEPSG returns error code %d'
% status)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_resample.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
task_queue = []
i = 0
logger.info("Searching for SETSM rasters")
if os.path.isfile(srcpath):
if srcpath.endswith('dem.tif'):
raster = srcpath
new_raster = os.path.join(args.dstdir, os.path.basename(raster))
if not os.path.isfile(new_raster):
i += 1
task = taskhandler.Task(
os.path.basename(raster), 'Reproject{:04g}'.format(i),
'python', '{} {} {} {} {}'.format(scriptpath, arg_str_base,
raster, dstdir,
args.epsg),
resample_setsm, [raster, dstdir, args])
task_queue.append(task)
else:
for root, dirs, files in os.walk(srcpath):
for f in files:
if f.endswith('dem.tif'):
raster = os.path.join(root, f)
new_raster = os.path.join(args.dstdir,
os.path.basename(raster))
if not os.path.isfile(new_raster):
i += 1
task = taskhandler.Task(
f, 'Reproject{:04g}'.format(i), 'python',
'{} {} {} {} {}'.format(scriptpath, arg_str_base,
raster, dstdir, args.epsg),
resample_setsm, [raster, dstdir, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: {}'.format(i))
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, dst, task_arg_obj = task.method_arg_list
if not args.dryrun:
#### Set up processing log handler
logfile = os.path.join(
dst,
os.path.splitext(os.path.basename(src))[0] + ".log")
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
task.method(src, dst, task_arg_obj)
else:
logger.info("No tasks found to process")
def main():
parser = argparse.ArgumentParser()
#### Set Up Options
parser.add_argument("src", help="source directory or image")
parser.add_argument("--dstdir", help="dstination directory")
parser.add_argument("-o",
"--overwrite",
action="store_true",
default=False,
help="overwrite existing files if present")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in scheduler submission (PBS default is qsub_resample.sh, SLURM default is slurm_resample.sh)"
)
parser.add_argument("--dryrun",
action="store_true",
default=False,
help="print actions without executing")
pos_arg_keys = ['src']
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
#### Validate Required Arguments
if not os.path.isdir(src) and not os.path.isfile(src):
parser.error('src must be avalid directory or file')
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_resample.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_resample.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
## Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
task_queue = []
i = 0
logger.info("Searching for SETSM rasters")
if os.path.isfile(src):
if src.endswith(('dem.tif', 'matchtag.tif', 'ortho.tif')):
srcfp = src
regfp = srcfp.replace('dem.tif', 'reg.txt').replace(
'matchtag.tif', 'reg.txt').replace('ortho.tif', 'reg.txt')
if not os.path.isfile(regfp):
logger.info("No regfile found for {}".format(src))
else:
if args.dstdir:
dstfp = "{}_reg.tif".format(
os.path.join(
args.dstdir,
os.path.basename(os.path.splitext(srcfp)[0])))
else:
dstfp = "{}_reg.tif".format(os.path.splitext(srcfp)[0])
if not os.path.isfile(dstfp):
i += 1
task = taskhandler.Task(
os.path.basename(srcfp), 'Reg{:04g}'.format(i),
'python', '{} {} {}'.format(scriptpath, arg_str_base,
srcfp), apply_reg,
[srcfp, args])
task_queue.append(task)
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith(('dem.tif', 'matchtag.tif', 'ortho.tif')):
srcfp = os.path.join(root, f)
regfp = srcfp.replace('dem.tif', 'reg.txt').replace(
'matchtag.tif',
'reg.txt').replace('ortho.tif', 'reg.txt')
if not os.path.isfile(regfp):
logger.info("No regfile found for {}".format(srcfp))
else:
if args.dstdir:
dstfp = "{}_reg.tif".format(
os.path.join(
args.dstdir,
os.path.basename(
os.path.splitext(srcfp)[0])))
else:
dstfp = "{}_reg.tif".format(
os.path.splitext(srcfp)[0])
if not os.path.isfile(dstfp):
i += 1
task = taskhandler.Task(
f, 'Reg{:04g}'.format(i), 'python',
'{} {} {}'.format(scriptpath, arg_str_base,
srcfp), apply_reg,
[srcfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: {}'.format(i))
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, task_arg_obj = task.method_arg_list
#### Set up processing log handler
if args.dstdir:
logfile = "{}.log".format(
os.path.join(
args.dstdir,
os.path.basename(os.path.splitext(src)[0])))
else:
logfile = os.path.splitext(src)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
task.method(src, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
else:
logger.info("No tasks found to process")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="divide setsm mosaics into subtiles")
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
pos_arg_keys = ['src']
#### Optionsl Arguments
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--num-rows',
type=int,
default=1,
help="number of subtile rows")
parser.add_argument('--num-cols',
type=int,
default=1,
help="number of subtile columns")
parser.add_argument('--res',
type=int,
default=2,
help="resolution in meters")
parser.add_argument('--tiles',
help="list of tiles to process, comma delimited")
parser.add_argument("--version", help="version string (ex: v1.2)")
parser.add_argument(
"--cutline-loc",
help="directory containing cutline shps indicating areas of bad data")
parser.add_argument('--build-ovr',
action='store_true',
default=False,
help="build overviews")
parser.add_argument(
'--resample',
default="bilinear",
help=
"dem_resampling strategy (default=bilinear). matchtag resampling is always nearest neighbor"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_divide.sh in script root folder)"
)
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
src = os.path.abspath(args.src)
if not os.path.isdir(src) and not os.path.isfile(src):
parser.error("Source directory or file does not exist: %s" % src)
if args.cutline_loc:
if not os.path.isdir(args.cutline_loc):
parser.error("Cutline directory does not exist: {}".format(
args.cutline_loc))
scriptpath = os.path.abspath(sys.argv[0])
## Verify qsubscript
if args.pbs:
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_divide.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.version:
version_str = '_{}'.format(args.version)
else:
version_str = ''
if args.tiles:
tiles = args.tiles.split(',')
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tiles')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
task_queue = []
i = 0
#### ID rasters
logger.info('Identifying DEM mosaics')
if os.path.isfile(src):
logger.info(src)
try:
raster = dem.SetsmTile(src)
except RuntimeError as e:
logger.error(e)
else:
if src.endswith('reg_dem.tif'):
dstfp_list = glob.glob('{}*{}m{}_reg_dem.tif'.format(
src[:-15], args.res, version_str))
else:
dstfp_list = glob.glob('{}*{}m{}_dem.tif'.format(
src[:-12], args.res, version_str))
#### verify that cutlines can be found if requested
if args.cutline_loc:
tile = raster.tile_name
cutline_shp = os.path.join(args.cutline_loc, tile + '_cut.shp')
if not os.path.isfile(cutline_shp):
logger.warning("Cutline shp not found for tile {}".format(
raster.tileid))
if len(dstfp_list) == 0:
i += 1
task = taskhandler.Task(
raster.tileid, 'div_{}'.format(raster.tileid), 'python',
'{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp),
divide_tile, [raster.srcfp, args])
task_queue.append(task)
else:
logger.info("output tile(s) already exist: {}".format(src))
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith(("2m_dem.tif", "2m_reg_dem.tif")):
if args.tiles:
tile = f[:5]
process = True if tile in tiles else False
else:
process = True
if process:
srcfp = os.path.join(root, f)
try:
raster = dem.SetsmTile(srcfp)
except RuntimeError as e:
logger.error(e)
else:
if srcfp.endswith('reg_dem.tif'):
dstfp_list = glob.glob(
'{}*{}m{}_reg_dem.tif'.format(
srcfp[:-15], args.res, version_str))
else:
dstfp_list = glob.glob(
'{}*{}m{}_dem.tif'.format(
srcfp[:-12], args.res, version_str))
if len(dstfp_list) == 0:
logger.info("computing tile: {}".format(srcfp))
#### verify that cutlines can be found if requested
if args.cutline_loc:
tile = raster.tilename
cutline_shp = os.path.join(
args.cutline_loc, tile + '_cut.shp')
if not os.path.isfile(cutline_shp):
logger.warning(
"Cutline shp not found for tile {}"
.format(raster.tileid))
i += 1
task = taskhandler.Task(
raster.tileid,
'div_{}'.format(raster.tileid), 'python',
'{} {} {}'.format(scriptpath, arg_str_base,
raster.srcfp),
divide_tile, [raster.srcfp, args])
#print '{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp)
task_queue.append(task)
else:
logger.info(
"output tile(s) already exist: {}".format(
','.join(dstfp_list)))
logger.info('Number of incomplete tasks: {}'.format(i))
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(src)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
task.method(src, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
else:
logger.info("No tasks found to process")
## Build task queue
i = 0
task_queue = []
for srcfp in image_list:
srcdir, srcfn = os.path.split(srcfp)
dstfp = os.path.join(
dstdir, "%s_%s%s%d%s" %
(os.path.splitext(srcfn)[0], utils.get_bit_depth(
args.outtype), args.stretch, spatial_ref.epsg,
ortho_functions.formats[args.format]))
done = os.path.isfile(dstfp)
if done is False:
i += 1
task = taskhandler.Task(
srcfn, 'Or{:04g}'.format(i), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, dstdir),
ortho_functions.process_image, [srcfp, dstfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: {}'.format(i))
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError, e:
logger.error(e)
else:
## Build task queue
i = 0
task_queue = []
for image_pair in pair_list:
bittype = utils.get_bit_depth(args.outtype)
pansh_dstfp = os.path.join(
dstdir, "{}_{}{}{}_pansh.tif".format(
os.path.splitext(image_pair.mul_srcfn)[0], bittype,
args.stretch, args.epsg))
if not os.path.isfile(pansh_dstfp):
i += 1
task = taskhandler.Task(
image_pair.mul_srcfn, 'Psh{:04g}'.format(i), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base,
image_pair.mul_srcfp, dstdir),
exec_pansharpen, [image_pair, pansh_dstfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: {}'.format(i))
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError, e:
logger.error(e)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"package setsm dems (build mdf and readme files and create archive) in place in the filesystem"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space to build index shps")
#### Optionsl Arguments
parser.add_argument('--log', help="directory for log output")
parser.add_argument(
'--epsg',
type=int,
default=3413,
help="egsg code for output index projection (default epsg:3413)")
parser.add_argument('-v',
action='store_true',
default=False,
help="verbose output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_package.sh in script root folder)"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
if not os.path.isdir(args.scratch) and not os.path.isfile(args.scratch):
parser.error("Source directory or file does not exist: %s" %
args.scratch)
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
pos_arg_keys = ['src', 'scratch']
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
if args.log:
if os.path.isdir(args.log):
tm = datetime.now()
logfile = os.path.join(
args.log, "package_setsm_tiles_{}.log".format(
tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
rasters = []
j = 0
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src):
j += 1
logger.debug(src)
try:
raster = dem.SetsmTile(os.path.join(src))
except RuntimeError as e:
logger.error(e)
else:
if not os.path.isfile(raster.archive) or args.overwrite:
rasters.append(raster)
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif"):
j += 1
logger.debug(os.path.join(root, f))
try:
raster = dem.SetsmTile(os.path.join(root, f))
except RuntimeError as e:
logger.error(e)
else:
if not os.path.isfile(
raster.archive) or args.overwrite:
rasters.append(raster)
rasters = list(set(rasters))
logger.info('Number of src rasters: {}'.format(j))
logger.info('Number of incomplete tasks: {}'.format(len(rasters)))
task_queue = []
logger.info('Packaging DEMs')
for raster in rasters:
# logger.info("[{}/{}] {}".format(i,total,raster.srcfp))
task = taskhandler.Task(
raster.srcfn, 'Pkg_{}'.format(raster.tileid), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, raster.srcfp,
scratch), build_archive,
[raster, scratch, args])
task_queue.append(task)
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
raster, scratch, task_arg_obj = task.method_arg_list
if not args.dryrun:
task.method(raster, scratch, task_arg_obj)
else:
logger.info("No tasks found to process")
def main():
parser = argparse.ArgumentParser()
#### Set Up Options
parser.add_argument("srcdir", help="source directory or image")
parser.add_argument("-c", "--component", choices=suffixes, default='dem',
help="SETSM DEM component to resample")
parser.add_argument("-r", "--resolution", default=default_res, type=int,
help="output resolution (default={})".format(default_res))
parser.add_argument("-o", "--overwrite", action="store_true", default=False,
help="overwrite existing files if present")
parser.add_argument("--pbs", action='store_true', default=False,
help="submit tasks to PBS")
parser.add_argument("--parallel-processes", type=int, default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument("--qsubscript",
help="qsub script to use in PBS submission (default is qsub_resample.sh in script root folder)")
parser.add_argument("--dryrun", action="store_true", default=False,
help="print actions without executing")
pos_arg_keys = ['srcdir']
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
path = os.path.abspath(args.srcdir)
#### Validate Required Arguments
if not os.path.isdir(path) and not os.path.isfile(path):
parser.error('src must be a valid directory or file')
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_resample.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" %qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive")
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
task_queue = []
i=0
logger.info("Searching for SETSM rasters")
if os.path.isfile(path):
if path.endswith('{}.tif'.format(args.component)):
dem = path
low_res_dem = "{}.tif".format(os.path.splitext(dem)[0])
low_res_dem = os.path.join(os.path.dirname(low_res_dem),os.path.basename(low_res_dem.replace("_2m","_{}m".format(args.resolution))))
if dem != low_res_dem and not os.path.isfile(low_res_dem):
i+=1
task = taskhandler.Task(
os.path.basename(dem),
'Resample{:04g}'.format(i),
'python',
'{} {} {}'.format(scriptpath, arg_str_base, dem),
resample_setsm,
[dem, args]
)
task_queue.append(task)
else:
for root,dirs,files in os.walk(path):
for f in files:
if f.endswith('{}.tif'.format(args.component)):
dem = os.path.join(root,f)
low_res_dem = "{}.tif".format(os.path.splitext(dem)[0])
low_res_dem = os.path.join(os.path.dirname(low_res_dem),os.path.basename(low_res_dem.replace("_2m","_{}m".format(args.resolution))))
if dem != low_res_dem and not os.path.isfile(low_res_dem):
i+=1
task = taskhandler.Task(
f,
'Resample{:04g}'.format(i),
'python',
'{} {} {}'.format(scriptpath, arg_str_base, dem),
resample_setsm,
[dem, args]
)
task_queue.append(task)
logger.info('Number of incomplete tasks: {}'.format(i))
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(src)[0]+".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
task.method(src, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
else:
logger.info("No tasks found to process")
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/Submit batch pansharpening in parallel")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_pansharpen.sh, "
"SLURM default is slurm_pansharpen.py, in script root folder)")
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action="store_true",
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: {}".format(
src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
# Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_pansharpen.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_pansharpen.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
### Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
## Check GDAL version (2.1.0 minimum)
gdal_version = gdal.VersionInfo()
try:
if int(gdal_version) < 2010000:
parser.error(
"gdal_pansharpen requires GDAL version 2.1.0 or higher")
except ValueError:
parser.error("Cannot parse GDAL version: {}".format(gdal_version))
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list1 = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list1 = utils.find_images(src, True, ortho_functions.exts)
else:
image_list1 = [src]
pair_list = []
for srcfp in image_list1:
#print(srcfp)
try:
image_pair = ImagePair(srcfp, spatial_ref)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Image: %s, Sensor: %s", image_pair.mul_srcfn,
image_pair.sensor)
pair_list.append(image_pair)
logger.info('Number of src image pairs: %i', len(pair_list))
## Build task queue
i = 0
task_queue = []
for image_pair in pair_list:
bittype = utils.get_bit_depth(args.outtype)
pansh_dstfp = os.path.join(
dstdir, "{}_{}{}{}_pansh.tif".format(
os.path.splitext(image_pair.mul_srcfn)[0], bittype,
args.stretch, args.epsg))
if not os.path.isfile(pansh_dstfp):
i += 1
task = taskhandler.Task(
image_pair.mul_srcfn, 'Psh{:04g}'.format(i), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base,
image_pair.mul_srcfp, dstdir),
exec_pansharpen, [image_pair, pansh_dstfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: %i', i)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
lfh = None
for task in task_queue:
src, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(src, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="Run/Submit batch ndvi calculation in parallel"
)
parser.add_argument("src", help="source image, text file, or directory")
parser.add_argument("dst", help="destination directory")
pos_arg_keys = ["src", "dst"]
parser.add_argument("-t", "--outtype", choices=outtypes, default='Float32',
help="output data type (for Int16, output values are scaled from -1000 to 1000)")
parser.add_argument("-s", "--save-temps", action="store_true", default=False,
help="save temp files")
parser.add_argument("--wd",
help="local working directory for cluster jobs (default is dst dir)")
parser.add_argument("--pbs", action='store_true', default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm", action='store_true', default=False,
help="submit tasks to SLURM")
parser.add_argument("--parallel-processes", type=int, default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument("--qsubscript",
help="submission script to use in PBS/SLURM submission (PBS default is qsub_ndvi.sh, SLURM "
"default is slurm_ndvi.py, in script root folder)")
parser.add_argument("-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun", action="store_true", default=False,
help="print actions without executing")
parser.add_argument("--version", action='version', version="imagery_utils v{}".format(utils.package_version))
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error("Error arg1 is not a recognized file path or file type: {}".format(src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_ndvi.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'slurm_ndvi.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
## Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error("HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive")
#### Set concole logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'pbs', 'slurm', 'parallel_processes', 'dryrun')
arg_str = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list = utils.find_images(src, True, ortho_functions.exts)
else:
image_list = [src]
logger.info('Number of src images: %i', len(image_list))
## Build task queue
i = 0
task_queue = []
for srcfp in image_list:
srcdir, srcfn = os.path.split(srcfp)
bn, ext = os.path.splitext(srcfn)
dstfp = os.path.join(dstdir, bn + '_ndvi.tif')
if not os.path.isfile(dstfp):
i += 1
task = taskhandler.Task(
srcfn,
'NDVI{:04g}'.format(i),
'python',
'{} {} {} {}'.format(scriptpath, arg_str, srcfp, dstdir),
calc_ndvi,
[srcfp, dstfp, args]
)
task_queue.append(task)
logger.info('Number of incomplete tasks: %i', i)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i", task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
for task in task_queue:
srcfp, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s', '%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(srcfp, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
pass
if scenes_in_job_count <= args.tasks_per_job:
# add to txt file
fh = open(src_txt,'a')
fh.write("{}\n".format(srcfp))
fh.close()
if scenes_in_job_count == args.tasks_per_job or scene_count == len(scenes):
scenes_in_job_count=0
job_count+=1
task = taskhandler.Task(
'Dens{:04g}'.format(job_count),
'Dens{:04g}'.format(job_count),
'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, src_txt, scratch),
build_archive,
[srcfp, scratch, args]
)
task_queue.append(task)
else:
job_count += 1
task = taskhandler.Task(
srcfn,
'Dens{:04g}'.format(job_count),
'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, scratch),
build_archive,
[srcfp, scratch, args]
)
else:
if not os.path.isfile(raster.archive) or args.overwrite:
rasters.append(raster)
rasters = list(set(rasters))
logger.info('Number of src rasters: {}'.format(j))
logger.info('Number of incomplete tasks: {}'.format(len(rasters)))
task_queue = []
logger.info('Packaging DEMs')
for raster in rasters:
# logger.info("[{}/{}] {}".format(i,total,raster.srcfp))
task = taskhandler.Task(
raster.srcfn,
'Pkg_{}'.format(raster.tileid),
'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, raster.srcfp, scratch),
build_archive,
[raster, scratch, args]
)
task_queue.append(task)
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(args.parallel_processes)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="Sumbit/run batch mosaic tasks")
parser.add_argument(
"src", help="textfile or directory of input rasters (tif only)")
parser.add_argument("mosaicname",
help="output mosaic name excluding extension")
pos_arg_keys = ["src", "mosaicname"]
parser.add_argument(
"-r",
"--resolution",
nargs=2,
type=float,
help=
"output pixel resolution -- xres yres (default is same as first input file)"
)
parser.add_argument(
"-e",
"--extent",
nargs=4,
type=float,
help=
"extent of output mosaic -- xmin xmax ymin ymax (default is union of all inputs)"
)
parser.add_argument(
"-t",
"--tilesize",
nargs=2,
type=float,
help=
"tile size in coordinate system units -- xsize ysize (default is 40,000 times output "
"resolution)")
parser.add_argument(
"--force-pan-to-multi",
action="store_true",
default=False,
help="if output is multiband, force script to also use 1 band images")
parser.add_argument(
"-b",
"--bands",
type=int,
help=
"number of output bands( default is number of bands in the first image)"
)
parser.add_argument(
"--tday",
help=
"month and day of the year to use as target for image suitability ranking -- 04-05"
)
parser.add_argument(
"--tyear",
help=
"year (or year range) to use as target for image suitability ranking -- 2017 or 2015-2017"
)
parser.add_argument(
"--nosort",
action="store_true",
default=False,
help=
"do not sort images by metadata. script uses the order of the input textfile or directory "
"(first image is first drawn). Not recommended if input is a directory; order will be "
"random")
parser.add_argument(
"--use-exposure",
action="store_true",
default=False,
help="use exposure settings in metadata to inform score")
parser.add_argument(
"--exclude",
help=
"file of file name patterns (text only, no wildcards or regexs) to exclude"
)
parser.add_argument(
"--max-cc",
type=float,
default=0.2,
help="maximum fractional cloud cover (0.0-1.0, default 0.5)")
parser.add_argument(
"--include-all-ms",
action="store_true",
default=False,
help=
"include all multispectral imagery, even if the imagery has differing numbers of bands"
)
parser.add_argument(
"--min-contribution-area",
type=int,
default=20000000,
help=
"minimum area contribution threshold in target projection units (default=20000000). "
"Higher values remove more image slivers from the resulting mosaic")
parser.add_argument(
"--median-remove",
action="store_true",
default=False,
help=
"subtract the median from each input image before forming the mosaic in order to correct "
"for contrast")
parser.add_argument(
"--allow-invalid-geom",
action="store_true",
default=False,
help=
"normally, if 1 or more images has a invalid geometry, a tile will not be created. this "
"option will attempt to create a mosaic with the remaining valid geometries, if any."
)
parser.add_argument(
"--mode",
choices=mosaic.MODES,
default="ALL",
help=
" mode: ALL- all steps (default), SHP- create shapefiles, MOSAIC- create tiled tifs, "
"TEST- create log only")
parser.add_argument("--wd",
help="scratch space (default is mosaic directory)")
parser.add_argument("--component-shp",
action="store_true",
default=False,
help="create shp of all componenet images")
parser.add_argument(
"--cutline-step",
type=int,
default=2,
help="cutline calculator pixel skip interval (default=2)")
parser.add_argument(
"--calc-stats",
action="store_true",
default=False,
help="calculate image stats and record them in the index")
parser.add_argument("--gtiff-compression",
choices=mosaic.GTIFF_COMPRESSIONS,
default="lzw",
help="GTiff compression type. Default=lzw ({})".format(
",".join(mosaic.GTIFF_COMPRESSIONS)))
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_mosaic.sh, SLURM "
"default is slurm_mosaic.py, in script root folder)")
parser.add_argument(
"-l",
help=
"PBS resources requested (mimicks qsub syntax). Use only on HPC systems."
)
parser.add_argument(
"--log",
help="file to log progress (default is <output dir>\{}".format(
default_logfile))
parser.add_argument("--version",
action='version',
version="imagery_utils v{}".format(
utils.package_version))
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
inpath = os.path.abspath(args.src)
mosaicname = os.path.abspath(args.mosaicname)
mosaicname = os.path.splitext(mosaicname)[0]
mosaic_dir = os.path.dirname(mosaicname)
tile_builder_script = os.path.join(os.path.dirname(scriptpath),
'pgc_mosaic_build_tile.py')
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_mosaic.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_mosaic.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
## Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
#### Validate Arguments
if os.path.isfile(inpath):
bTextfile = True
elif os.path.isdir(inpath):
bTextfile = False
else:
parser.error(
"Arg1 is not a valid file path or directory: {}".format(inpath))
#### Validate target day option
if args.tday is not None:
try:
m = int(args.tday.split("-")[0])
d = int(args.tday.split("-")[1])
td = date(2000, m, d)
except ValueError:
parser.error("Target day must be in mm-dd format (i.e 04-05)")
sys.exit(1)
else:
m = 0
d = 0
#### Validate target year/year range option
if args.tyear is not None:
if len(str(args.tyear)) == 4:
## ensure single year is valid
try:
tyear_test = datetime(year=args.tyear, month=1, day=1)
except ValueError:
parser.error("Supplied year {0} is not valid".format(
args.tyear))
sys.exit(1)
elif len(str(args.tyear)) == 9:
if '-' in args.tyear:
## decouple range and build year
yrs = args.tyear.split('-')
yrs_range = range(int(yrs[0]), int(yrs[1]) + 1)
for yy in yrs_range:
try:
tyear_test = datetime(year=yy, month=1, day=1)
except ValueError:
parser.error(
"Supplied year {0} in range {1} is not valid".
format(yy, args.tyear))
sys.exit(1)
else:
parser.error(
"Supplied year range {0} is not valid; should be like: 2015 OR 2015-2017"
.format(args.tyear))
sys.exit(1)
else:
parser.error(
"Supplied year {0} is not valid, or its format is incorrect; should be 4 digits for single "
"year (e.g., 2017), eight digits and dash for range (e.g., 2015-2017)"
.format(args.tyear))
sys.exit(1)
#### Get exclude list if specified
if args.exclude is not None:
if not os.path.isfile(args.exclude):
parser.error("Value for option --exclude-list is not a valid file")
## Build tasks
task_queue = []
#### Create task for mosaic
arg_keys_to_remove = ('l', 'qsubscript', 'pbs', 'slurm')
mos_arg_str = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
cmd = r'{} {} {} {}'.format(scriptpath, mos_arg_str, inpath, mosaicname)
task = taskhandler.Task('Mosaic {}'.format(os.path.basename(mosaicname)),
'Mos{:04g}'.format(1), 'python', cmd)
task_queue.append(task)
#logger.info(task_queue)
if len(task_queue) > 0:
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
task_handler.run_tasks(task_queue)
else:
try:
run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir,
args, pos_arg_keys)
except RuntimeError as e:
logger.error(e)
else:
logger.info("No tasks to process")
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/submit batch image ortho and conversion tasks")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--tasks-per-job",
type=int,
help=
"Number of tasks to bundle into a single job. (requires --pbs or --slurm option) (Warning:"
" a higher number of tasks per job may require modification of default wallclock limit.)"
)
parser.add_argument(
'--scratch',
default=ARGDEF_SCRATCH,
help="Scratch space to build task bundle text files. (default={})".
format(ARGDEF_SCRATCH))
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM "
"default is slurm_ortho.py, in script root folder)")
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help='print actions without executing')
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
scratch = os.path.abspath(args.scratch)
bittype = utils.get_bit_depth(args.outtype)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: {}".format(
src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_ortho.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_ortho.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
## Verify processing options do not conflict
requested_threads = ortho_functions.ARGDEF_CPUS_AVAIL if args.threads == "ALL_CPUS" else args.threads
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
if (args.pbs or args.slurm) and requested_threads > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --threads > 1 are mutually exclusive"
)
if requested_threads < 1:
parser.error(
"--threads count must be positive, nonzero integer or ALL_CPUS")
if args.parallel_processes > 1:
total_proc_count = requested_threads * args.parallel_processes
if total_proc_count > ortho_functions.ARGDEF_CPUS_AVAIL:
parser.error(
"the (threads * number of processes requested) ({0}) exceeds number of available threads "
"({1}); reduce --threads and/or --parallel-processes count".
format(total_proc_count, ortho_functions.ARGDEF_CPUS_AVAIL))
if args.tasks_per_job:
if not (args.pbs or args.slurm):
parser.error(
"--tasks-per-job option requires the (--pbs or --slurm) option"
)
if not os.path.isdir(args.scratch):
print("Creating --scratch directory: {}".format(args.scratch))
os.makedirs(args.scratch)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
#### Verify that dem and ortho_height are not both specified
if args.dem is not None and args.ortho_height is not None:
parser.error(
"--dem and --ortho_height options are mutually exclusive. Please choose only one."
)
#### Test if DEM exists
if args.dem:
if not os.path.isfile(args.dem):
parser.error("DEM does not exist: {}".format(args.dem))
if args.l is None:
if args.dem.endswith('.vrt'):
total_dem_filesz_gb = 0.0
tree = ET.parse(args.dem)
root = tree.getroot()
for sourceFilename in root.iter('SourceFilename'):
dem_filename = sourceFilename.text
if not os.path.isfile(dem_filename):
parser.error(
"VRT DEM component raster does not exist: {}".
format(dem_filename))
dem_filesz_gb = os.path.getsize(
dem_filename) / 1024.0 / 1024 / 1024
total_dem_filesz_gb += dem_filesz_gb
dem_filesz_gb = total_dem_filesz_gb
else:
dem_filesz_gb = os.path.getsize(
args.dem) / 1024.0 / 1024 / 1024
pbs_req_mem_gb = int(max(math.ceil(dem_filesz_gb) + 2, 4))
args.l = 'mem={}gb'.format(pbs_req_mem_gb)
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Handle thread count that exceeds system limits
if requested_threads > ortho_functions.ARGDEF_CPUS_AVAIL:
logger.info(
"threads requested ({0}) exceeds number available on system ({1}), setting thread count to "
"'ALL_CPUS'".format(requested_threads,
ortho_functions.ARGDEF_CPUS_AVAIL))
args.threads = 'ALL_CPUS'
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes', 'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list1 = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list1 = utils.find_images(src, True, ortho_functions.exts)
else:
image_list1 = [src]
## Group Ikonos
image_list2 = []
for srcfp in image_list1:
srcdir, srcfn = os.path.split(srcfp)
if "IK01" in srcfn and sum(
[b in srcfn for b in ortho_functions.ikMsiBands]) > 0:
for b in ortho_functions.ikMsiBands:
if b in srcfn:
newname = os.path.join(srcdir, srcfn.replace(b, "msi"))
break
image_list2.append(newname)
else:
image_list2.append(srcfp)
image_list = list(set(image_list2))
logger.info('Number of src images: %i', len(image_list))
## Build task queue
i = 0
images_to_process = []
for srcfp in image_list:
srcdir, srcfn = os.path.split(srcfp)
dstfp = os.path.join(
dstdir, "{}_{}{}{}{}".format(
os.path.splitext(srcfn)[0], bittype, args.stretch,
spatial_ref.epsg, ortho_functions.formats[args.format]))
done = os.path.isfile(dstfp)
if done is False:
i += 1
images_to_process.append(srcfp)
logger.info('Number of incomplete tasks: %i', i)
if len(images_to_process) == 0:
logger.info("No images found to process")
sys.exit(0)
task_queue = []
if args.tasks_per_job and args.tasks_per_job > 1:
task_srcfp_list = utils.write_task_bundles(images_to_process,
args.tasks_per_job, scratch,
'Or_src')
else:
task_srcfp_list = images_to_process
for job_count, srcfp in enumerate(task_srcfp_list, 1):
srcdir, srcfn = os.path.split(srcfp)
if task_srcfp_list is images_to_process:
dstfp = os.path.join(
dstdir, "{}_{}{}{}{}".format(
os.path.splitext(srcfn)[0], bittype, args.stretch,
spatial_ref.epsg, ortho_functions.formats[args.format]))
else:
dstfp = None
task = taskhandler.Task(
srcfn, 'Or{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(argval2str(scriptpath), arg_str_base,
argval2str(srcfp), argval2str(dstdir)),
ortho_functions.process_image, [srcfp, dstfp, args])
task_queue.append(task)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue, dryrun=args.dryrun)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
for task in task_queue:
src, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(src, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
def run_mosaic(tile_builder_script, inpath, mosaicname, mosaic_dir, args,
pos_arg_keys):
if os.path.isfile(inpath):
bTextfile = True
elif os.path.isdir(inpath):
bTextfile = False
if not os.path.isdir(mosaic_dir):
os.makedirs(mosaic_dir)
## TODO: verify logger woks for both interactive and hpc jobs
#### Configure Logger
if args.log is not None:
logfile = os.path.abspath(args.log)
else:
logfile = os.path.join(mosaic_dir, default_logfile)
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get exclude list if specified
if args.exclude is not None:
if not os.path.isfile(args.exclude):
logger.error("Value for option --exclude-list is not a valid file")
f = open(args.exclude, 'r')
exclude_list = set([line.rstrip() for line in f.readlines()])
else:
exclude_list = set()
#### Get Images
#logger.info("Reading input images")
xs = []
ys = []
image_list = utils.find_images_with_exclude_list(inpath, bTextfile,
mosaic.EXTS, exclude_list)
if len(image_list) == 0:
raise RuntimeError(
"No images found in input file or directory: {}".format(inpath))
# remove duplicate images
image_list_unique = list(set(image_list))
dupes = [x for n, x in enumerate(image_list) if x in image_list[:n]]
if len(dupes) > 0:
logger.info("Removed %i duplicate image paths", len(dupes))
logger.debug("Dupes: %s", dupes)
image_list = image_list_unique
logger.info("%i existing images found", len(image_list))
#### gather image info list
logger.info("Getting image info")
imginfo_list = [mosaic.ImageInfo(image, "IMAGE") for image in image_list]
#### Get mosaic parameters
logger.info("Setting mosaic parameters")
params = mosaic.getMosaicParameters(imginfo_list[0], args)
logger.info("Mosaic parameters: band count=%i, datatype=%s", params.bands,
params.datatype)
logger.info("Mosaic parameters: projection=%s", params.proj)
#### Remove images that do not match ref
logger.info("Applying attribute filter")
imginfo_list2 = mosaic.filterMatchingImages(imginfo_list, params)
if len(imginfo_list2) == 0:
raise RuntimeError(
"No valid images found. Check input filter parameters.")
logger.info("%i of %i images match filter", len(imginfo_list2),
len(image_list))
#### if extent is specified, build tile params and compare extent to input image geom
if args.extent:
imginfo_list3 = mosaic.filter_images_by_geometry(imginfo_list2, params)
#### else set extent after image geoms computed
else:
#### Get geom for each image
imginfo_list3 = []
for iinfo in imginfo_list2:
if iinfo.geom is not None:
xs = xs + iinfo.xs
ys = ys + iinfo.ys
imginfo_list3.append(iinfo)
else: # remove from list if no geom
logger.debug("Null geometry for image: %s", iinfo.srcfn)
params.xmin = min(xs)
params.xmax = max(xs)
params.ymin = min(ys)
params.ymax = max(ys)
poly_wkt = 'POLYGON (( {} {}, {} {}, {} {}, {} {}, {} {} ))'.format(
params.xmin, params.ymin, params.xmin, params.ymax, params.xmax,
params.ymax, params.xmax, params.ymin, params.xmin, params.ymin)
params.extent_geom = ogr.CreateGeometryFromWkt(poly_wkt)
if len(imginfo_list3) == 0:
raise RuntimeError("No images found that intersect mosaic extent")
logger.info(
"Mosaic parameters: resolution %f x %f, tilesize %f x %f, extent %f %f %f %f",
params.xres, params.yres, params.xtilesize, params.ytilesize,
params.xmin, params.xmax, params.ymin, params.ymax)
logger.info("%d of %d input images intersect mosaic extent",
len(imginfo_list3), len(imginfo_list2))
## Sort images by score
logger.info("Reading image metadata and determining sort order")
for iinfo in imginfo_list3:
iinfo.getScore(params)
if not args.nosort:
imginfo_list3.sort(key=lambda x: x.score)
logger.info("Getting Exact Image geometry")
imginfo_list4 = []
all_valid = True
for iinfo in imginfo_list3:
if iinfo.score > 0 or args.nosort:
simplify_tolerance = 2.0 * (
(params.xres + params.yres) / 2.0
) ## 2 * avg(xres, yres), should be 1 for panchromatic mosaics where res = 0.5m
geom, xs1, ys1 = mosaic.GetExactTrimmedGeom(
iinfo.srcfp,
step=args.cutline_step,
tolerance=simplify_tolerance)
if geom is None:
logger.warning(
"%s: geometry could not be determined, verify image is valid",
iinfo.srcfn)
all_valid = False
elif geom.IsEmpty():
logger.warning("%s: geometry is empty", iinfo.srcfn)
all_valid = False
else:
iinfo.geom = geom
tm = datetime.today()
imginfo_list4.append(iinfo)
centroid = geom.Centroid()
logger.info("%s: geometry acquired - centroid: %f, %f",
iinfo.srcfn, centroid.GetX(), centroid.GetY())
#print(geom)
else:
logger.debug("Image has an invalid score: %s --> %i", iinfo.srcfp,
iinfo.score)
if not all_valid:
if not args.allow_invalid_geom:
raise RuntimeError(
"Some source images do not have valid geometries. Cannot proceeed"
)
else:
logger.info(
"--allow-invalid-geom used; mosaic will be created using %i valid images (%i invalid \
images not used.)".format(
len(imginfo_list4),
len(imginfo_list3) - len(imginfo_list4)))
# Get stats if needed
logger.info("Getting image metadata")
for iinfo in imginfo_list4:
logger.info(iinfo.srcfn)
if args.calc_stats or args.median_remove:
iinfo.get_raster_stats(args.calc_stats, args.median_remove)
# Build componenet index
if args.component_shp:
if args.mode == "ALL" or args.mode == "SHP":
contribs = [(iinfo, iinfo.geom) for iinfo in imginfo_list4]
logger.info("Number of contributors: %d", len(contribs))
logger.info("Building component index")
comp_shp = mosaicname + "_components.shp"
if len(contribs) > 0:
if os.path.isfile(comp_shp):
logger.info("Components shapefile already exists: %s",
comp_shp)
else:
build_shp(contribs, comp_shp, args, params)
else:
logger.error("No contributing images")
# Build cutlines index
#### Overlay geoms and remove non-contributors
logger.info("Overlaying images to determine contribution geom")
contribs = mosaic.determine_contributors(imginfo_list4, params.extent_geom,
args.min_contribution_area)
logger.info("Number of contributors: %d", len(contribs))
if args.mode == "ALL" or args.mode == "SHP":
logger.info("Building cutlines index")
shp = mosaicname + "_cutlines.shp"
if len(contribs) > 0:
if os.path.isfile(shp):
logger.info("Cutlines shapefile already exists: %s", shp)
else:
build_shp(contribs, shp, args, params)
else:
logger.error("No contributing images")
## Create tile objects
tiles = []
xtiledim = math.ceil((params.xmax - params.xmin) / params.xtilesize)
ytiledim = math.ceil((params.ymax - params.ymin) / params.ytilesize)
logger.info("Tiles: %d rows, %d columns", ytiledim, xtiledim)
xtdb = len(str(int(xtiledim)))
ytdb = len(str(int(ytiledim)))
i = 1
for x in mosaic.drange(params.xmin, params.xmax,
params.xtilesize): # Columns
if x + params.xtilesize > params.xmax:
x2 = params.xmax
else:
x2 = x + params.xtilesize
j = 1
for y in mosaic.drange(params.ymin, params.ymax,
params.ytilesize): # Rows
if y + params.ytilesize > params.ymax:
y2 = params.ymax
else:
y2 = y + params.ytilesize
tilename = "{}_{}_{}.tif".format(mosaicname,
mosaic.buffernum(j, ytdb),
mosaic.buffernum(i, xtdb))
tile = mosaic.TileParams(x, x2, y, y2, j, i, tilename)
tiles.append(tile)
j += 1
i += 1
#### Write shapefile of tiles
if len(tiles) == 0:
raise RuntimeError("No tile objects created")
if args.mode == "ALL" or args.mode == "SHP":
build_tiles_shp(mosaicname, tiles, params)
## Build tile tasks
task_queue = []
#### Create task for each tile
arg_keys_to_remove = ('l', 'qsubscript', 'parallel_processes', 'log',
'mode', 'extent', 'resolution', 'bands', 'max_cc',
'exclude', 'nosort', 'component_shp', 'cutline_step',
'min_contribution_area', 'calc_stats', 'pbs',
'slurm', 'tday', 'tyear', 'allow_invalid_geom')
tile_arg_str = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
logger.debug("Identifying components of %i subtiles", len(tiles))
i = 0
for t in tiles:
logger.debug("Identifying components of tile %i of %i: %s", i,
len(tiles), os.path.basename(t.name))
#### determine which images in each tile - create geom and query image geoms
logger.debug("Running intersect with imagery")
intersects = []
for iinfo, contrib_geom in contribs:
if contrib_geom.Intersects(t.geom):
if args.median_remove:
## parse median dct into text
median_string = ";".join([
"{}:{}".format(k, v) for k, v in iinfo.median.items()
])
intersects.append("{},{}".format(iinfo.srcfp,
median_string))
else:
intersects.append(iinfo.srcfp)
#### If any images are in the tile, mosaic them
if len(intersects) > 0:
tile_basename = os.path.basename(os.path.splitext(t.name)[0])
logger.info("Number of contributors to subtile %s: %i",
tile_basename, len(intersects))
itpath = os.path.join(mosaic_dir,
tile_basename + "_intersects.txt")
it = open(itpath, "w")
it.write("\n".join(intersects))
it.close()
#### Submit QSUB job
logger.debug("Building mosaicking job for tile: %s",
os.path.basename(t.name))
if not os.path.isfile(t.name):
cmd = r'{} {} -e {} {} {} {} -r {} {} -b {} {} {}'.format(
tile_builder_script, tile_arg_str, t.xmin, t.xmax, t.ymin,
t.ymax, params.xres, params.yres, params.bands, t.name,
itpath)
task = taskhandler.Task(
'Tile {0}'.format(os.path.basename(t.name)),
'Mos{:04g}'.format(i), 'python', cmd)
if args.mode == "ALL" or args.mode == "MOSAIC":
logger.debug(cmd)
task_queue.append(task)
else:
logger.info("Tile already exists: %s",
os.path.basename(t.name))
i += 1
if args.mode == "ALL" or args.mode == "MOSAIC":
logger.info("Submitting Tasks")
#logger.info(task_queue)
if len(task_queue) > 0:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
if task_handler.num_processes > 1:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
task_handler.run_tasks(task_queue)
logger.info("Done")
else:
logger.info("No tasks to process")
dstfp_list = glob.glob('{}*{}m{}_dem.tif'.format(srcfp[:-12], args.res, version_str))
if len(dstfp_list) == 0:
logger.info("computing tile: {}".format(srcfp))
#### verify that cutlines can be found if requested
if args.cutline_loc:
tile = raster.tilename
cutline_shp = os.path.join(args.cutline_loc, tile + '_cut.shp')
if not os.path.isfile(cutline_shp):
logger.warning("Cutline shp not found for tile {}".format(raster.tileid))
i+=1
task = taskhandler.Task(
raster.tileid,
'div_{}'.format(raster.tileid),
'python',
'{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp),
divide_tile,
[raster.srcfp, args]
)
#print '{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp)
task_queue.append(task)
else:
logger.info("output tile(s) already exist: {}".format(','.join(dstfp_list)))
logger.info('Number of incomplete tasks: {}'.format(i))
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
