def __init__(self):
self.epsg = 32629
self.resolution = None
self.rgb = False
self.bgrn = False
self.stretch = 'rf'
self.spatial_ref = utils.SpatialRef(self.epsg)
def test_spatial_ref(self):
sref_np = utils.SpatialRef(self.epsg_npole)
sref_sp = utils.SpatialRef(self.epsg_spole)
with self.assertRaises(RuntimeError) as cm:
utils.SpatialRef(self.epsg_bad_1) # breaks for not being an integer
with self.assertRaises(RuntimeError) as cm:
utils.SpatialRef(self.epsg_bad_2) # break for invalid EPSG code
self.assertTrue(isinstance(sref_np.srs, osgeo.osr.SpatialReference))
self.assertTrue(isinstance(sref_sp.srs, osgeo.osr.SpatialReference))
self.assertTrue(sref_np.proj4, '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs ')
self.assertTrue(sref_sp.proj4, '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs ')
self.assertTrue(sref_np.epsg, 3413)
self.assertTrue(sref_sp.epsg, 3031)
def setUp(self):
self.srcdir = os.path.join(test_dir, 'output')
self.spatial_ref = utils.SpatialRef(3031)
self.dem = os.path.join(test_dir, 'dem', 'ramp_lowres.tif')
self.dem_none = None
self.ortho_height = None
self.wd = None
self.skip_dem_overlap_check = True
self.resolution = None
self.rgb = False
self.bgrn = False
self.stretch = 'rf'
def __init__(self,epsg,stretch):
self.epsg = epsg
self.resolution = None
self.rgb = False
self.bgrn = False
self.skip_warp = False
self.dem = None
self.ortho_height = None
self.resample = 'near'
self.outtype = 'Byte'
self.format = "GTiff"
self.gtiff_compression = "lzw"
self.stretch = stretch
self.spatial_ref = utils.SpatialRef(self.epsg)
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/submit batch image ortho and conversion tasks")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM "
"default is slurm_ortho.py, in script root folder)")
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help='print actions without executing')
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: {}".format(
src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_ortho.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_ortho.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
## Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
#### Verify that dem and ortho_height are not both specified
if args.dem is not None and args.ortho_height is not None:
parser.error(
"--dem and --ortho_height options are mutually exclusive. Please choose only one."
)
#### Test if DEM exists
if args.dem:
if not os.path.isfile(args.dem):
parser.error("DEM does not exist: {}".format(args.dem))
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list1 = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list1 = utils.find_images(src, True, ortho_functions.exts)
else:
image_list1 = [src]
## Group Ikonos
image_list2 = []
for srcfp in image_list1:
srcdir, srcfn = os.path.split(srcfp)
if "IK01" in srcfn and sum(
[b in srcfn for b in ortho_functions.ikMsiBands]) > 0:
for b in ortho_functions.ikMsiBands:
if b in srcfn:
newname = os.path.join(srcdir, srcfn.replace(b, "msi"))
break
image_list2.append(newname)
else:
image_list2.append(srcfp)
image_list = list(set(image_list2))
logger.info('Number of src images: %i', len(image_list))
## Build task queue
i = 0
task_queue = []
for srcfp in image_list:
srcdir, srcfn = os.path.split(srcfp)
dstfp = os.path.join(
dstdir, "{}_{}{}{}{}".format(
os.path.splitext(srcfn)[0], utils.get_bit_depth(args.outtype),
args.stretch, spatial_ref.epsg,
ortho_functions.formats[args.format]))
done = os.path.isfile(dstfp)
if done is False:
i += 1
task = taskhandler.Task(
srcfn, 'Or{:04g}'.format(i), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, dstdir),
ortho_functions.process_image, [srcfp, dstfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: %i', i)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
for task in task_queue:
src, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(src, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
def setUp(self):
self.dem = os.path.join(
os.path.join(test_dir, 'dem', 'ramp_lowres.tif'))
self.srs = utils.SpatialRef(4326)
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/submit batch image ortho and conversion tasks")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM default is slurm_ortho.py, in script root folder)"
)
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help='print actions without executing')
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: %s" %
(src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: %s" % (dstdir))
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_ortho.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_ortho.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
## Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError, e:
parser.error(e)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="build setsm DEM index"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or image")
parser.add_argument('dst', help="destination index dataset (use PG:<config.ini section name>:<layer name> for a postgresql DB")
#### Optional Arguments
parser.add_argument('--mode', choices=MODES.keys(), default='scene',
help="type of items to index {} default=scene".format(MODES.keys()))
parser.add_argument('--config', default=os.path.join(os.path.dirname(sys.argv[0]),'config.ini'),
help="config file (default is config.ini in script dir")
parser.add_argument('--epsg', type=int, default=4326,
help="egsg code for output index projection (default wgs85 geographic epsg:4326)")
parser.add_argument('--dsp-original-res', action='store_true', default=False,
help='write index of downsampled product (dsp) scenes as original resolution (mode=scene only)')
parser.add_argument('--status', help='custom value for status field')
parser.add_argument('--include-registration', action='store_true', default=False,
help='include registration info if present (mode=strip and tile only)')
parser.add_argument('--search-masked', action='store_true', default=False,
help='search for masked and unmasked DEMs (mode=strip only)')
parser.add_argument('--read-json', action='store_true', default=False,
help='search for json files instead of images to populate the index')
parser.add_argument('--write-json', action='store_true', default=False,
help='write results to json files in dst folder')
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--overwrite', action='store_true', default=False,
help="overwrite existing index")
parser.add_argument('--append', action='store_true', default=False,
help="append records to existing index")
parser.add_argument('--check', action='store_true', default=False,
help='verify new records exist in target index (not compatible with --write-json)')
parser.add_argument('--skip-region-lookup', action='store_true', default=False,
help="skip region lookup on danco (used for testing)")
parser.add_argument("--write-pickle", help="store region lookup in a pickle file. skipped if --write-json is used")
parser.add_argument("--read-pickle", help='read region lookup from a pickle file. skipped if --write-json is used')
parser.add_argument("--bp-paths", action='store_true', default=False, help='Use BlackPearl path schema')
parser.add_argument("--tnva-paths", action='store_true', default=False, help='Use Terranova path schema')
parser.add_argument('--project', choices=PROJECTS.keys(), help='project name (required when writing tiles)')
parser.add_argument('--dryrun', action='store_true', default=False, help='run script without inserting records')
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" %args.src)
#src = os.path.abspath(args.src)
src = args.src
dst = args.dst
if args.overwrite and args.append:
parser.error('--append and --overwrite are mutually exclusive')
if args.write_json and args.append:
parser.error('--append cannot be used with the --write-json option')
if (args.write_json or args.read_json) and args.search_masked:
parser.error('--search-masked cannot be used with the --write-json or --read-json options')
if args.mode != 'strip' and args.search_masked:
parser.error('--search-masked applies only to mode=strip')
if args.write_json and args.check:
parser.error('--check cannot be used with the --write-json option')
## Check project
if args.mode == 'tile' and not args.project:
parser.error("--project option is required if when mode=tile")
if args.write_pickle:
if not os.path.isdir(os.path.dirname(args.write_pickle)):
parser.error("Pickle file must be in an existing directory")
if args.read_pickle:
if not os.path.isfile(args.read_pickle):
parser.error("Pickle file must be an existing file")
if args.status and args.bp_paths:
parser.error("--bp-paths sets status field to 'tape' and cannot be used with --status")
if args.tnva_paths and args.bp_paths:
parser.error("--bp-paths and --tnva-paths are incompatible options")
if args.mode != 'scene' and args.dsp_original_res:
parser.error("--dsp-original-res is applicable only when mode = scene")
if args.bp_paths:
db_path_prefix = BP_PATH_PREFIX
elif args.tnva_paths:
db_path_prefix = TNVA_PATH_PREFIX
else:
db_path_prefix = None
#### Set up loggers
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
if args.log:
if os.path.isdir(args.log):
tm = datetime.datetime.now()
logfile = os.path.join(args.log,"index_setsm_{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
## Check path if --write-json is invoked
if args.write_json:
if not os.path.isdir(dst):
parser.error("Destination must be an existing directory with --write-json option")
ogr_driver_str = None
if args.epsg:
logger.warning('--epsg and --dsp-original-res will be ignored with the --write-json option')
## If not writing to JSON, get OGR driver, ds name, and layer name
else:
try:
ogr_driver_str, dst_dsp, dst_lyr = utils.get_source_names2(dst)
except RuntimeError as e:
parser.error(e)
ogrDriver = ogr.GetDriverByName(ogr_driver_str)
if ogrDriver is None:
parser.error("Driver is not available: {}".format(ogr_driver_str))
#### Get Config file contents
try:
config = ConfigParser.SafeConfigParser()
except NameError:
config = ConfigParser.ConfigParser() # ConfigParser() replaces SafeConfigParser() in Python >=3.2
config.read(args.config)
#### Get output DB connection if specified
if ogr_driver_str in ("PostgreSQL"):
section = dst_dsp
if section in config.sections():
conn_info = {
'host':config.get(section,'host'),
'port':config.getint(section,'port'),
'name':config.get(section,'name'),
'schema':config.get(section,'schema'),
'user':config.get(section,'user'),
'pw':config.get(section,'pw'),
}
dst_ds = "PG:host={host} port={port} dbname={name} user={user} password={pw} active_schema={schema}".format(**conn_info)
else:
logger.error('Config.ini file must contain credentials to connect to {}'.format(section))
sys.exit(-1)
#### Set dataset path is SHP or GDB
elif ogr_driver_str in ("ESRI Shapefile","FileGDB"):
dst_ds = dst_dsp
else:
logger.error("Format {} is not supported".format(ogr_driver_str))
## Get pairname-region dict
if args.skip_region_lookup or args.mode == 'tile':
pairs = {}
else:
if args.read_pickle:
logger.info("Fetching region lookup from pickle file")
pairs = pickle.load(open(args.read_pickle, "rb"))
else:
#### Get Danco connection if available
section = 'danco'
if section in config.sections():
danco_conn_info = {
'host':config.get(section,'host'),
'port':config.getint(section,'port'),
'name':config.get(section,'name'),
'schema':config.get(section,'schema'),
'user':config.get(section,'user'),
'pw':config.get(section,'pw'),
}
logger.info("Fetching region lookup from Danco")
pairs = get_pair_region_dict(danco_conn_info)
else:
logger.warning('Config file does not contain credentials to connect to Danco. Region cannot be determined.')
pairs = {}
if len(pairs) == 0:
logger.warning("Cannot get region-pair lookup")
if args.tnva_paths:
logger.error("Region-pair lookup required for --tnva-paths option")
sys.exit()
## Save pickle if selected
if args.write_pickle:
logger.info("Pickling region lookup")
pickle.dump(pairs, open(args.write_pickle, "wb"))
#### Test epsg
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
#### Test if dst table exists
if ogr_driver_str == 'ESRI Shapefile' and os.path.isfile(dst_ds):
if args.overwrite:
logger.info("Removing old index... %s" %os.path.basename(dst_ds))
if not args.dryrun:
ogrDriver.DeleteDataSource(dst_ds)
elif not args.append:
logger.error("Dst shapefile exists. Use the --overwrite or --append options.")
sys.exit(-1)
if ogr_driver_str == 'FileGDB' and os.path.isdir(dst_ds):
ds = ogrDriver.Open(dst_ds,1)
if ds:
for i in range(ds.GetLayerCount()):
lyr = ds.GetLayer(i)
if lyr.GetName() == dst_lyr:
if args.overwrite:
logger.info("Removing old index layer: {}".format(dst_lyr))
del lyr
ds.DeleteLayer(i)
break
elif not args.append:
logger.error("Dst GDB layer exists. Use the --overwrite or --append options.")
sys.exit(-1)
ds = None
## Postgres check - do not overwrite
if ogr_driver_str == 'PostgreSQL':
ds = ogrDriver.Open(dst_ds,1)
if ds:
for i in range(ds.GetLayerCount()):
lyr = ds.GetLayer(i)
if lyr.GetName() == dst_lyr:
if args.overwrite:
logger.info("Removing old index layer: {}".format(dst_lyr))
del lyr
ds.DeleteLayer(i)
break
elif not args.append:
logger.error("Dst DB layer exists. Use the --overwrite or --append options.")
sys.exit(-1)
ds = None
#### ID records
dem_class, suffix, groupid_fld, fld_defs_base, reg_fld_defs = MODES[args.mode]
if args.mode == 'strip' and args.search_masked:
suffix = mask_strip_suffixes + tuple([suffix])
fld_defs = fld_defs_base + reg_fld_defs if args.include_registration else fld_defs_base
src_fps = []
records = []
logger.info('Identifying DEMs')
if os.path.isfile(src):
logger.info(src)
src_fps.append(src)
else:
for root,dirs,files in os.walk(src):
for f in files:
if (f.endswith('.json') and args.read_json) or (f.endswith(suffix)):
logger.debug(os.path.join(root,f))
src_fps.append(os.path.join(root,f))
for src_fp in src_fps:
if args.read_json:
temp_records = read_json(os.path.join(src_fp),args.mode)
records.extend(temp_records)
else:
try:
record = dem_class(src_fp)
record.get_dem_info()
except RuntimeError as e:
logger.error( e )
else:
if args.mode == 'scene' and not os.path.isfile(record.dspinfo) and args.dsp_original_res:
logger.error("Record {} has no Dsp downsample info file: {}, skipping".format(record.id,record.dspinfo))
else:
records.append(record)
total = len(records)
if total == 0:
logger.info("No valid records found")
else:
logger.info("{} records found".format(total))
## Group into strips or tiles for json writing
groups = {}
for record in records:
groupid = getattr(record,groupid_fld)
if groupid in groups:
groups[groupid].append(record)
else:
groups[groupid] = [record]
#### Write index
if args.write_json:
write_to_json(dst, groups, total, args)
else:
write_result = write_to_ogr_dataset(ogr_driver_str, ogrDriver, dst_ds, dst_lyr, groups,
pairs, total, db_path_prefix, fld_defs, args)
if write_result:
sys.exit(0)
else:
sys.exit(-1)
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/submit batch image ortho and conversion tasks")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--tasks-per-job",
type=int,
help=
"Number of tasks to bundle into a single job. (requires --pbs or --slurm option) (Warning:"
" a higher number of tasks per job may require modification of default wallclock limit.)"
)
parser.add_argument(
'--scratch',
default=ARGDEF_SCRATCH,
help="Scratch space to build task bundle text files. (default={})".
format(ARGDEF_SCRATCH))
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_ortho.sh, SLURM "
"default is slurm_ortho.py, in script root folder)")
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help='print actions without executing')
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
scratch = os.path.abspath(args.scratch)
bittype = utils.get_bit_depth(args.outtype)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: {}".format(
src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
## Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_ortho.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_ortho.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
## Verify processing options do not conflict
requested_threads = ortho_functions.ARGDEF_CPUS_AVAIL if args.threads == "ALL_CPUS" else args.threads
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
if (args.pbs or args.slurm) and requested_threads > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --threads > 1 are mutually exclusive"
)
if requested_threads < 1:
parser.error(
"--threads count must be positive, nonzero integer or ALL_CPUS")
if args.parallel_processes > 1:
total_proc_count = requested_threads * args.parallel_processes
if total_proc_count > ortho_functions.ARGDEF_CPUS_AVAIL:
parser.error(
"the (threads * number of processes requested) ({0}) exceeds number of available threads "
"({1}); reduce --threads and/or --parallel-processes count".
format(total_proc_count, ortho_functions.ARGDEF_CPUS_AVAIL))
if args.tasks_per_job:
if not (args.pbs or args.slurm):
parser.error(
"--tasks-per-job option requires the (--pbs or --slurm) option"
)
if not os.path.isdir(args.scratch):
print("Creating --scratch directory: {}".format(args.scratch))
os.makedirs(args.scratch)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
#### Verify that dem and ortho_height are not both specified
if args.dem is not None and args.ortho_height is not None:
parser.error(
"--dem and --ortho_height options are mutually exclusive. Please choose only one."
)
#### Test if DEM exists
if args.dem:
if not os.path.isfile(args.dem):
parser.error("DEM does not exist: {}".format(args.dem))
if args.l is None:
if args.dem.endswith('.vrt'):
total_dem_filesz_gb = 0.0
tree = ET.parse(args.dem)
root = tree.getroot()
for sourceFilename in root.iter('SourceFilename'):
dem_filename = sourceFilename.text
if not os.path.isfile(dem_filename):
parser.error(
"VRT DEM component raster does not exist: {}".
format(dem_filename))
dem_filesz_gb = os.path.getsize(
dem_filename) / 1024.0 / 1024 / 1024
total_dem_filesz_gb += dem_filesz_gb
dem_filesz_gb = total_dem_filesz_gb
else:
dem_filesz_gb = os.path.getsize(
args.dem) / 1024.0 / 1024 / 1024
pbs_req_mem_gb = int(max(math.ceil(dem_filesz_gb) + 2, 4))
args.l = 'mem={}gb'.format(pbs_req_mem_gb)
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Handle thread count that exceeds system limits
if requested_threads > ortho_functions.ARGDEF_CPUS_AVAIL:
logger.info(
"threads requested ({0}) exceeds number available on system ({1}), setting thread count to "
"'ALL_CPUS'".format(requested_threads,
ortho_functions.ARGDEF_CPUS_AVAIL))
args.threads = 'ALL_CPUS'
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes', 'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list1 = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list1 = utils.find_images(src, True, ortho_functions.exts)
else:
image_list1 = [src]
## Group Ikonos
image_list2 = []
for srcfp in image_list1:
srcdir, srcfn = os.path.split(srcfp)
if "IK01" in srcfn and sum(
[b in srcfn for b in ortho_functions.ikMsiBands]) > 0:
for b in ortho_functions.ikMsiBands:
if b in srcfn:
newname = os.path.join(srcdir, srcfn.replace(b, "msi"))
break
image_list2.append(newname)
else:
image_list2.append(srcfp)
image_list = list(set(image_list2))
logger.info('Number of src images: %i', len(image_list))
## Build task queue
i = 0
images_to_process = []
for srcfp in image_list:
srcdir, srcfn = os.path.split(srcfp)
dstfp = os.path.join(
dstdir, "{}_{}{}{}{}".format(
os.path.splitext(srcfn)[0], bittype, args.stretch,
spatial_ref.epsg, ortho_functions.formats[args.format]))
done = os.path.isfile(dstfp)
if done is False:
i += 1
images_to_process.append(srcfp)
logger.info('Number of incomplete tasks: %i', i)
if len(images_to_process) == 0:
logger.info("No images found to process")
sys.exit(0)
task_queue = []
if args.tasks_per_job and args.tasks_per_job > 1:
task_srcfp_list = utils.write_task_bundles(images_to_process,
args.tasks_per_job, scratch,
'Or_src')
else:
task_srcfp_list = images_to_process
for job_count, srcfp in enumerate(task_srcfp_list, 1):
srcdir, srcfn = os.path.split(srcfp)
if task_srcfp_list is images_to_process:
dstfp = os.path.join(
dstdir, "{}_{}{}{}{}".format(
os.path.splitext(srcfn)[0], bittype, args.stretch,
spatial_ref.epsg, ortho_functions.formats[args.format]))
else:
dstfp = None
task = taskhandler.Task(
srcfn, 'Or{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(argval2str(scriptpath), arg_str_base,
argval2str(srcfp), argval2str(dstdir)),
ortho_functions.process_image, [srcfp, dstfp, args])
task_queue.append(task)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue, dryrun=args.dryrun)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
for task in task_queue:
src, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(src, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
def main():
#### Set Up Arguments
parent_parser, pos_arg_keys = ortho_functions.buildParentArgumentParser()
parser = argparse.ArgumentParser(
parents=[parent_parser],
description="Run/Submit batch pansharpening in parallel")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument("--slurm",
action='store_true',
default=False,
help="submit tasks to SLURM")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"submission script to use in PBS/SLURM submission (PBS default is qsub_pansharpen.sh, "
"SLURM default is slurm_pansharpen.py, in script root folder)")
parser.add_argument(
"-l", help="PBS resources requested (mimicks qsub syntax, PBS only)")
parser.add_argument("--dryrun",
action="store_true",
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
dstdir = os.path.abspath(args.dst)
#### Validate Required Arguments
if os.path.isdir(src):
srctype = 'dir'
elif os.path.isfile(src) and os.path.splitext(src)[1].lower() == '.txt':
srctype = 'textfile'
elif os.path.isfile(src) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
elif os.path.isfile(src.replace('msi', 'blu')) and os.path.splitext(
src)[1].lower() in ortho_functions.exts:
srctype = 'image'
else:
parser.error(
"Error arg1 is not a recognized file path or file type: {}".format(
src))
if not os.path.isdir(dstdir):
parser.error("Error arg2 is not a valid file path: {}".format(dstdir))
# Verify qsubscript
if args.pbs or args.slurm:
if args.qsubscript is None:
if args.pbs:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_pansharpen.sh')
if args.slurm:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'slurm_pansharpen.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: {}".format(qsubpath))
### Verify processing options do not conflict
if args.pbs and args.slurm:
parser.error("Options --pbs and --slurm are mutually exclusive")
if (args.pbs or args.slurm) and args.parallel_processes > 1:
parser.error(
"HPC Options (--pbs or --slurm) and --parallel-processes > 1 are mutually exclusive"
)
#### Verify EPSG
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError as e:
parser.error(e)
## Check GDAL version (2.1.0 minimum)
gdal_version = gdal.VersionInfo()
try:
if int(gdal_version) < 2010000:
parser.error(
"gdal_pansharpen requires GDAL version 2.1.0 or higher")
except ValueError:
parser.error("Cannot parse GDAL version: {}".format(gdal_version))
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('l', 'qsubscript', 'dryrun', 'pbs', 'slurm',
'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
## Identify source images
if srctype == 'dir':
image_list1 = utils.find_images(src, False, ortho_functions.exts)
elif srctype == 'textfile':
image_list1 = utils.find_images(src, True, ortho_functions.exts)
else:
image_list1 = [src]
pair_list = []
for srcfp in image_list1:
#print(srcfp)
try:
image_pair = ImagePair(srcfp, spatial_ref)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Image: %s, Sensor: %s", image_pair.mul_srcfn,
image_pair.sensor)
pair_list.append(image_pair)
logger.info('Number of src image pairs: %i', len(pair_list))
## Build task queue
i = 0
task_queue = []
for image_pair in pair_list:
bittype = utils.get_bit_depth(args.outtype)
pansh_dstfp = os.path.join(
dstdir, "{}_{}{}{}_pansh.tif".format(
os.path.splitext(image_pair.mul_srcfn)[0], bittype,
args.stretch, args.epsg))
if not os.path.isfile(pansh_dstfp):
i += 1
task = taskhandler.Task(
image_pair.mul_srcfn, 'Psh{:04g}'.format(i), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base,
image_pair.mul_srcfp, dstdir),
exec_pansharpen, [image_pair, pansh_dstfp, args])
task_queue.append(task)
logger.info('Number of incomplete tasks: %i', i)
## Run tasks
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
l = "-l {}".format(args.l) if args.l else ""
try:
task_handler = taskhandler.PBSTaskHandler(qsubpath, l)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.slurm:
try:
task_handler = taskhandler.SLURMTaskHandler(qsubpath)
except RuntimeError as e:
logger.error(e)
else:
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
try:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
except RuntimeError as e:
logger.error(e)
else:
logger.info("Number of child processes to spawn: %i",
task_handler.num_processes)
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
results = {}
lfh = None
for task in task_queue:
src, dstfp, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(dstfp)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
results[task.name] = task.method(src, dstfp, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
#### remove existing file handler
logger.removeHandler(lfh)
#### Print Images with Errors
for k, v in results.items():
if v != 0:
logger.warning("Failed Image: %s", k)
logger.info("Done")
else:
logger.info("No images found to process")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(description="build setsm DEM index")
#### Positional Arguments
parser.add_argument('src', help="source directory or image")
parser.add_argument(
'dst',
help=
"destination index dataset (use PG:<config.ini section name>:<layer name> for a postgresql DB"
)
#### Optional Arguments
parser.add_argument('--mode',
choices=MODES.keys(),
default='scene',
help="type of items to index {} default=scene".format(
MODES.keys()))
parser.add_argument(
'--config',
default=os.path.join(os.path.dirname(sys.argv[0]), 'config.ini'),
help="config file (default is config.ini in script dir")
parser.add_argument(
'--epsg',
type=int,
default=4326,
help=
"egsg code for output index projection (default wgs85 geographic epsg:4326)"
)
parser.add_argument(
'--read-json',
action='store_true',
default=False,
help='search for json files instead of images to populate the index')
parser.add_argument('--write-json',
action='store_true',
default=False,
help='write results to json files in dst folder')
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument('--append',
action='store_true',
default=False,
help="append records to existing index")
parser.add_argument('--skip-region-lookup',
action='store_true',
default=False,
help="skip region lookup on danco (used for testing)")
parser.add_argument(
"--write-pickle",
help=
"store region lookup in a pickle file. skipped if --write-json is used"
)
parser.add_argument(
"--read-pickle",
help=
'read region lookup from a pickle file. skipped if --write-json is used'
)
parser.add_argument("--bp-paths",
action='store_true',
default=False,
help='Use BlackPearl path schema')
parser.add_argument('--project',
choices=PROJECTS,
help='project name (required when writing tiles)')
parser.add_argument('--dryrun',
action='store_true',
default=False,
help='run script without inserting records')
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
src = os.path.abspath(args.src)
dst = args.dst
if args.overwrite and args.append:
parser.error('--append and --overwrite are mutually exclusive')
if args.write_json and args.append:
parser.error('--append cannot be used with the --write-json option')
## Check project
if args.mode == 'tile' and not args.project:
parser.error("--project option is required if when mode=tile")
if args.write_pickle:
if not os.path.isdir(os.path.dirname(args.write_pickle)):
parser.error("Pickle file must be in an existing directory")
if args.read_pickle:
if not os.path.isfile(args.read_pickle):
parser.error("Pickle file must be an existing file")
if args.bp_paths:
db_path_prefix = BP_PATH_PREFIX
else:
db_path_prefix = None
#### Set up loggers
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
if args.log:
if os.path.isdir(args.log):
tm = datetime.datetime.now()
logfile = os.path.join(
args.log,
"index_setsm_{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
## Check path if --write-json is invoked
if args.write_json:
if not os.path.isdir(dst):
parser.error(
"Destination must be an existing directory with --write-json option"
)
ogr_driver_str = None
if args.epsg:
logger.warning(
'--epsg will be ignored with the --write-json option')
## If not writing to JSON, get OGR driver, ds name, and layer name
else:
try:
ogr_driver_str, dst_dsp, dst_lyr = utils.get_source_names2(dst)
except RuntimeError as e:
parser.error(e)
ogrDriver = ogr.GetDriverByName(ogr_driver_str)
if ogrDriver is None:
parser.error("Driver is not available: {}".format(ogr_driver_str))
#### Get Config file contents
try:
config = ConfigParser.SafeConfigParser()
except NameError:
config = ConfigParser.ConfigParser(
) # ConfigParser() replaces SafeConfigParser() in Python >=3.2
config.read(args.config)
#### Get output DB connection if specified
if ogr_driver_str in ("PostgreSQL"):
section = dst_dsp
if section in config.sections():
conn_info = {
'host': config.get(section, 'host'),
'port': config.getint(section, 'port'),
'name': config.get(section, 'name'),
'schema': config.get(section, 'schema'),
'user': config.get(section, 'user'),
'pw': config.get(section, 'pw'),
}
dst_ds = "PG:host={host} port={port} dbname={name} user={user} password={pw} active_schema={schema}".format(
**conn_info)
else:
logger.error(
'Config.ini file must contain credentials to connect to {}'
.format(section))
sys.exit(-1)
#### Set dataset path is SHP or GDB
elif ogr_driver_str in ("ESRI Shapefile", "FileGDB"):
dst_ds = dst_dsp
else:
logger.error("Format {} is not supported".format(ogr_driver_str))
## Get pairname-region dict
if args.skip_region_lookup or args.mode == 'tile':
pairs = {}
else:
if args.read_pickle:
logger.info("Fetching region lookup from pickle file")
pairs = pickle.load(open(args.read_pickle, "rb"))
else:
#### Get Danco connection if available
section = 'danco'
if section in config.sections():
danco_conn_info = {
'host': config.get(section, 'host'),
'port': config.getint(section, 'port'),
'name': config.get(section, 'name'),
'schema': config.get(section, 'schema'),
'user': config.get(section, 'user'),
'pw': config.get(section, 'pw'),
}
logger.info("Fetching region lookup from Danco")
pairs = get_pair_region_dict(danco_conn_info)
else:
logger.warning(
'Config file does not contain credentials to connect to Danco. Region cannot be determined.'
)
pairs = {}
if len(pairs) == 0:
logger.warning("Cannot get region-pair lookup")
## Save pickle if selected
if args.write_pickle:
logger.info("Pickling region lookup")
pickle.dump(pairs, open(args.write_pickle, "wb"))
#### Test epsg
try:
spatial_ref = utils.SpatialRef(args.epsg)
except RuntimeError, e:
parser.error(e)
#### Test if dst table exists
if ogr_driver_str == 'ESRI Shapefile' and os.path.isfile(dst_ds):
if args.overwrite:
logger.info("Removing old index... %s" %
os.path.basename(dst_ds))
if not args.dryrun:
ogrDriver.DeleteDataSource(dst_ds)
elif not args.append:
logger.error(
"Dst shapefile exists. Use the --overwrite or --append options."
)
sys.exit()
if ogr_driver_str == 'FileGDB' and os.path.isdir(dst_ds):
ds = ogrDriver.Open(dst_ds, 1)
if ds:
for i in range(ds.GetLayerCount()):
lyr = ds.GetLayer(i)
if lyr.GetName() == dst_lyr:
if args.overwrite:
logger.info(
"Removing old index layer: {}".format(dst_lyr))
del lyr
ds.DeleteLayer(i)
break
elif not args.append:
logger.error(
"Dst GDB layer exists. Use the --overwrite or --append options."
)
sys.exit()
ds = None
## Postgres check - do not overwrite
if ogr_driver_str == 'PostgreSQL':
ds = ogrDriver.Open(dst_ds, 1)
if ds:
for i in range(ds.GetLayerCount()):
lyr = ds.GetLayer(i)
if lyr.GetName() == dst_lyr:
if args.overwrite:
logger.info(
"Removing old index layer: {}".format(dst_lyr))
del lyr
ds.DeleteLayer(i)
break
elif not args.append:
logger.error(
"Dst DB layer exists. Use the --overwrite or --append options."
)
sys.exit()
ds = None