def build_archive(src,scratch,args):
logger.info("Calculating density of raster: {}".format(src))
raster = dem.SetsmTile(src)
try:
raster.get_dem_info()
except RuntimeError, e:
logger.error(e)
def build_archive(src, scratch, args):
logger.info("Calculating density of raster: {}".format(src))
raster = dem.SetsmTile(src)
try:
raster.get_dem_info()
except RuntimeError as e:
logger.error(e)
else:
## get raster density if not precomputed
if raster.density is None:
try:
raster.compute_density_and_statistics()
except RuntimeError as e:
logger.warning(e)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"calculate density for setsm tile or subtile DEMS and save to a density.txt file"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space")
#### Optionsl Arguments
parser.add_argument('-v',
action='store_true',
default=False,
help="verbose output")
parser.add_argument(
"--tasks-per-job",
type=int,
help="number of tasks to bundle into a single job (requires pbs option)"
)
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_package.sh in script root folder)"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
pos_arg_keys = ['src', 'scratch']
#### Parse Arguments
scriptpath = os.path.abspath(sys.argv[0])
args = parser.parse_args()
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
if not os.path.isdir(
args.scratch
) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs
parser.error("Scratch directory does not exist: %s" % args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
if args.tasks_per_job and not args.pbs:
parser.error("jobs-per-task argument requires the pbs option")
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
j = 0
scenes = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith('.tif'):
logger.debug(src)
try:
raster = dem.SetsmTile(src)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if not os.path.isfile(raster.density_file):
scenes.append(src)
elif os.path.isfile(src) and src.endswith('.txt'):
fh = open(src, 'r')
for line in fh.readlines():
sceneid = line.strip()
try:
raster = dem.SetsmTile(sceneid)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if not os.path.isfile(raster.density_file):
scenes.append(sceneid)
elif os.path.isdir(src):
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif") and "m_" in f:
srcfp = os.path.join(root, f)
logger.debug(srcfp)
try:
raster = dem.SetsmTile(srcfp)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if not os.path.isfile(raster.density_file):
scenes.append(srcfp)
else:
logger.error("src must be a directory, a strip dem, or a text file")
scenes = list(set(scenes))
logger.info('Number of src rasters: {}'.format(j))
logger.info('Number of incomplete tasks: {}'.format(len(scenes)))
tm = datetime.now()
job_count = 0
scene_count = 0
scenes_in_job_count = 0
task_queue = []
for srcfp in scenes:
scene_count += 1
srcdir, srcfn = os.path.split(srcfp)
if args.tasks_per_job:
# bundle tasks into text files in the dst dir and pass the text file in as src
scenes_in_job_count += 1
src_txt = os.path.join(
scratch,
'src_dems_{}_{}.txt'.format(tm.strftime("%Y%m%d%H%M%S"),
job_count))
if scenes_in_job_count == 1:
# remove text file if dst already exists
try:
os.remove(src_txt)
except OSError:
pass
if scenes_in_job_count <= args.tasks_per_job:
# add to txt file
fh = open(src_txt, 'a')
fh.write("{}\n".format(srcfp))
fh.close()
if scenes_in_job_count == args.tasks_per_job or scene_count == len(
scenes):
scenes_in_job_count = 0
job_count += 1
task = taskhandler.Task(
'Dens{:04g}'.format(job_count),
'Dens{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, src_txt,
scratch), build_archive,
[srcfp, scratch, args])
task_queue.append(task)
else:
job_count += 1
task = taskhandler.Task(
srcfn, 'Dens{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, scratch),
build_archive, [srcfp, scratch, args])
task_queue.append(task)
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, scratch, task_arg_obj = task.method_arg_list
if not args.dryrun:
task.method(src, scratch, task_arg_obj)
else:
logger.info("No tasks found to process")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="divide setsm mosaics into subtiles")
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
pos_arg_keys = ['src']
#### Optionsl Arguments
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--num-rows',
type=int,
default=1,
help="number of subtile rows")
parser.add_argument('--num-cols',
type=int,
default=1,
help="number of subtile columns")
parser.add_argument('--res',
type=int,
default=2,
help="resolution in meters")
parser.add_argument('--tiles',
help="list of tiles to process, comma delimited")
parser.add_argument("--version", help="version string (ex: v1.2)")
parser.add_argument(
"--cutline-loc",
help="directory containing cutline shps indicating areas of bad data")
parser.add_argument('--build-ovr',
action='store_true',
default=False,
help="build overviews")
parser.add_argument(
'--resample',
default="bilinear",
help=
"dem_resampling strategy (default=bilinear). matchtag resampling is always nearest neighbor"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_divide.sh in script root folder)"
)
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
src = os.path.abspath(args.src)
if not os.path.isdir(src) and not os.path.isfile(src):
parser.error("Source directory or file does not exist: %s" % src)
if args.cutline_loc:
if not os.path.isdir(args.cutline_loc):
parser.error("Cutline directory does not exist: {}".format(
args.cutline_loc))
scriptpath = os.path.abspath(sys.argv[0])
## Verify qsubscript
if args.pbs:
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),
'qsub_divide.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.version:
version_str = '_{}'.format(args.version)
else:
version_str = ''
if args.tiles:
tiles = args.tiles.split(',')
#### Set up console logging handler
lso = logging.StreamHandler()
lso.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lso.setFormatter(formatter)
logger.addHandler(lso)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tiles')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
task_queue = []
i = 0
#### ID rasters
logger.info('Identifying DEM mosaics')
if os.path.isfile(src):
logger.info(src)
try:
raster = dem.SetsmTile(src)
except RuntimeError as e:
logger.error(e)
else:
if src.endswith('reg_dem.tif'):
dstfp_list = glob.glob('{}*{}m{}_reg_dem.tif'.format(
src[:-15], args.res, version_str))
else:
dstfp_list = glob.glob('{}*{}m{}_dem.tif'.format(
src[:-12], args.res, version_str))
#### verify that cutlines can be found if requested
if args.cutline_loc:
tile = raster.tile_name
cutline_shp = os.path.join(args.cutline_loc, tile + '_cut.shp')
if not os.path.isfile(cutline_shp):
logger.warning("Cutline shp not found for tile {}".format(
raster.tileid))
if len(dstfp_list) == 0:
i += 1
task = taskhandler.Task(
raster.tileid, 'div_{}'.format(raster.tileid), 'python',
'{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp),
divide_tile, [raster.srcfp, args])
task_queue.append(task)
else:
logger.info("output tile(s) already exist: {}".format(src))
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith(("2m_dem.tif", "2m_reg_dem.tif")):
if args.tiles:
tile = f[:5]
process = True if tile in tiles else False
else:
process = True
if process:
srcfp = os.path.join(root, f)
try:
raster = dem.SetsmTile(srcfp)
except RuntimeError as e:
logger.error(e)
else:
if srcfp.endswith('reg_dem.tif'):
dstfp_list = glob.glob(
'{}*{}m{}_reg_dem.tif'.format(
srcfp[:-15], args.res, version_str))
else:
dstfp_list = glob.glob(
'{}*{}m{}_dem.tif'.format(
srcfp[:-12], args.res, version_str))
if len(dstfp_list) == 0:
logger.info("computing tile: {}".format(srcfp))
#### verify that cutlines can be found if requested
if args.cutline_loc:
tile = raster.tilename
cutline_shp = os.path.join(
args.cutline_loc, tile + '_cut.shp')
if not os.path.isfile(cutline_shp):
logger.warning(
"Cutline shp not found for tile {}"
.format(raster.tileid))
i += 1
task = taskhandler.Task(
raster.tileid,
'div_{}'.format(raster.tileid), 'python',
'{} {} {}'.format(scriptpath, arg_str_base,
raster.srcfp),
divide_tile, [raster.srcfp, args])
#print '{} {} {}'.format(scriptpath, arg_str_base, raster.srcfp)
task_queue.append(task)
else:
logger.info(
"output tile(s) already exist: {}".format(
','.join(dstfp_list)))
logger.info('Number of incomplete tasks: {}'.format(i))
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, task_arg_obj = task.method_arg_list
#### Set up processing log handler
logfile = os.path.splitext(src)[0] + ".log"
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
if not args.dryrun:
task.method(src, task_arg_obj)
#### remove existing file handler
logger.removeHandler(lfh)
else:
logger.info("No tasks found to process")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="calculate density for setsm tile or subtile DEMS and save to a density.txt file"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space")
#### Optionsl Arguments
parser.add_argument('-v', action='store_true', default=False, help="verbose output")
parser.add_argument("--tasks-per-job", type=int, help="number of tasks to bundle into a single job (requires pbs option)")
parser.add_argument("--pbs", action='store_true', default=False,
help="submit tasks to PBS")
parser.add_argument("--parallel-processes", type=int, default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument("--qsubscript",
help="qsub script to use in PBS submission (default is qsub_package.sh in script root folder)")
parser.add_argument('--dryrun', action='store_true', default=False,
help="print actions without executing")
pos_arg_keys = ['src','scratch']
#### Parse Arguments
scriptpath = os.path.abspath(sys.argv[0])
args = parser.parse_args()
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" %args.src)
if not os.path.isdir(args.scratch) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs
parser.error("Scratch directory does not exist: %s" %args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" %qsubpath)
if args.tasks_per_job and not args.pbs:
parser.error("jobs-per-task argument requires the pbs option")
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive")
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes', 'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
j=0
scenes = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith('.tif'):
logger.debug(src)
try:
raster = dem.SetsmTile(src)
except RuntimeError, e:
logger.error( e )
else:
j+=1
if not os.path.isfile(raster.density_file):
scenes.append(src)
try:
raster = dem.SetsmTile(src)
except RuntimeError, e:
logger.error( e )
else:
j+=1
if not os.path.isfile(raster.density_file):
scenes.append(src)
elif os.path.isfile(src) and src.endswith('.txt'):
fh = open(src,'r')
for line in fh.readlines():
sceneid = line.strip()
try:
raster = dem.SetsmTile(sceneid)
except RuntimeError, e:
logger.error( e )
else:
j+=1
if not os.path.isfile(raster.density_file):
scenes.append(sceneid)
elif os.path.isdir(src):
for root,dirs,files in os.walk(src):
for f in files:
if f.endswith("_dem.tif") and "m_" in f:
srcfp = os.path.join(root,f)
logger.debug(srcfp)
try:
raster = dem.SetsmTile(srcfp)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="package setsm dems (build mdf and readme files and create archive) in place in the filesystem"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space to build index shps")
#### Optionsl Arguments
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--epsg', type=int, default=3413,
help="egsg code for output index projection (default epsg:3413)")
parser.add_argument('-v', action='store_true', default=False, help="verbose output")
parser.add_argument('--overwrite', action='store_true', default=False,
help="overwrite existing index")
parser.add_argument("--pbs", action='store_true', default=False,
help="submit tasks to PBS")
parser.add_argument("--parallel-processes", type=int, default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument("--qsubscript",
help="qsub script to use in PBS submission (default is qsub_package.sh in script root folder)")
parser.add_argument('--dryrun', action='store_true', default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" %args.src)
if not os.path.isdir(args.scratch) and not os.path.isfile(args.scratch):
parser.error("Source directory or file does not exist: %s" %args.scratch)
scriptpath = os.path.abspath(sys.argv[0])
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath),'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" %qsubpath)
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error("Options --pbs and --parallel-processes > 1 are mutually exclusive")
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
pos_arg_keys = ['src','scratch']
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes')
arg_str_base = taskhandler.convert_optional_args_to_string(args, pos_arg_keys, arg_keys_to_remove)
if args.log:
if os.path.isdir(args.log):
tm = datetime.now()
logfile = os.path.join(args.log,"package_setsm_tiles_{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
rasters = []
j=0
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src):
j+=1
logger.debug(src)
try:
raster = dem.SetsmTile(os.path.join(src))
except RuntimeError, e:
logger.error( e )
else:
if not os.path.isfile(raster.archive) or args.overwrite:
rasters.append(raster)
try:
raster = dem.SetsmTile(os.path.join(src))
except RuntimeError, e:
logger.error( e )
else:
if not os.path.isfile(raster.archive) or args.overwrite:
rasters.append(raster)
else:
for root,dirs,files in os.walk(src):
for f in files:
if f.endswith("_dem.tif"):
j+=1
logger.debug(os.path.join(root,f))
try:
raster = dem.SetsmTile(os.path.join(root,f))
except RuntimeError, e:
logger.error( e )
else:
if not os.path.isfile(raster.archive) or args.overwrite:
rasters.append(raster)
rasters = list(set(rasters))
logger.info('Number of src rasters: {}'.format(j))
logger.info('Number of incomplete tasks: {}'.format(len(rasters)))
task_queue = []
logger.info('Packaging DEMs')
for raster in rasters:
# logger.info("[{}/{}] {}".format(i,total,raster.srcfp))
task = taskhandler.Task(
logger.info("output tile(s) already exist: {}".format(src))
else:
for root,dirs,files in os.walk(src):
for f in files:
if f.endswith(("2m_dem.tif","2m_reg_dem.tif")):
if args.tiles:
tile = f[:5]
process = True if tile in tiles else False
else:
process = True
if process:
srcfp = os.path.join(root,f)
try:
raster = dem.SetsmTile(srcfp)
except RuntimeError, e:
logger.error( e )
else:
if srcfp.endswith('reg_dem.tif'):
dstfp_list = glob.glob('{}*{}m{}_reg_dem.tif'.format(srcfp[:-15], args.res, version_str))
else:
dstfp_list = glob.glob('{}*{}m{}_dem.tif'.format(srcfp[:-12], args.res, version_str))
if len(dstfp_list) == 0:
logger.info("computing tile: {}".format(srcfp))
#### verify that cutlines can be found if requested
if args.cutline_loc:
tile = raster.tilename
cutline_shp = os.path.join(args.cutline_loc, tile + '_cut.shp')
if not os.path.isfile(cutline_shp):