def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(description="rename setsm strips")
parser.add_argument("srcdir", help="source directory")
parser.add_argument("version", help="version string (ex: v1.2)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
src = args.srcdir
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
rasters = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith(
"_dem.tif") and not os.path.basename(src).startswith("SETSM_"):
logger.debug(src)
try:
raster = dem.SetsmDem(os.path.join(src))
except RuntimeError as e:
logger.error(e)
else:
rasters.append(raster)
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif") and not f.startswith("SETSM_"):
#logger.info(os.path.join(root, f))
try:
raster = dem.SetsmDem(os.path.join(root, f))
except RuntimeError as e:
logger.error(e)
else:
rasters.append(raster)
i = 0
total = len(rasters)
logger.info('Renaming DEMs')
if total > 0:
for raster in rasters:
i += 1
logger.info("[{}/{}] {}".format(i, total, raster.srcfp))
rename(raster, args)
else:
print("No DEMs found requiring processing")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(description="rename setsm strips")
parser.add_argument("srcdir", help="source directory")
parser.add_argument("version", help="version string (ex: v1.2)")
parser.add_argument("--dryrun",
action='store_true',
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
src = args.srcdir
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
rasters = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith(
"_dem.tif") and not os.path.basename(src).startswith("SETSM_"):
logger.debug(src)
try:
raster = dem.SetsmDem(os.path.join(src))
except RuntimeError, e:
logger.error(e)
else:
rasters.append(raster)
def build_archive(src, scratch, args):
logger.info("Calculating density of raster: {}".format(src))
raster = dem.SetsmDem(src)
try:
raster.get_dem_info()
except RuntimeError, e:
logger.error(e)
def build_archive(src, scratch, args):
logger.info("Packaging Raster: {}".format(src))
raster = dem.SetsmDem(src)
dstfp = raster.archive
dstdir, dstfn = os.path.split(raster.archive)
#print dstfn
#print dstfp
try:
raster.get_dem_info()
except RuntimeError, e:
logger.error(e)
def build_archive(src, scratch, args):
logger.info("Calculating density of raster: {}".format(src))
raster = dem.SetsmDem(src)
try:
raster.get_dem_info()
except RuntimeError as e:
logger.error(e)
else:
## get raster density if not precomputed
if raster.density is None:
try:
raster.compute_density_and_statistics()
except RuntimeError as e:
logger.warning(e)
def build_archive(src, scratch, args):
logger.info("Packaging Raster: {}".format(src))
raster = dem.SetsmDem(src)
dstfp = raster.archive
dstdir, dstfn = os.path.split(raster.archive)
#print dstfn
#print dstfp
try:
raster.get_dem_info()
except RuntimeError as e:
logger.error(e)
else:
process = True
## get raster density if not precomputed
if raster.density is None:
try:
raster.compute_density_and_statistics()
except RuntimeError as e:
logger.warning(e)
if args.filter_dems or args.force_filter_dems:
# filter dems with area < 5.5 sqkm and density < .1
area = raster.geom.Area()
# logger.info(raster.density)
if area < 5500000:
logger.info("Raster area {} falls below threshold: {}".format(
area, raster.srcfp))
process = False
elif raster.density < 0.1:
logger.info(
"Raster density {} falls below threshold: {}".format(
raster.density, raster.srcfp))
process = False
if not process:
logger.info('Removing {}'.format(raster.srcfp))
to_remove = glob.glob(raster.srcfp[:-8] + '*')
for f in to_remove:
#logger.info('Removing {}'.format(f))
os.remove(f)
if process:
#### Build mdf
if not os.path.isfile(raster.mdf) or args.overwrite:
if os.path.isfile(raster.mdf):
if not args.dryrun:
os.remove(raster.mdf)
try:
if not args.dryrun:
raster.write_mdf_file(args.lsf)
except RuntimeError as e:
logger.error(e)
#### Build Readme
if not os.path.isfile(raster.readme) or args.overwrite:
if os.path.isfile(raster.readme):
if not args.dryrun:
os.remove(raster.readme)
if not args.dryrun:
raster.write_readme_file()
#### Build Archive
if not args.mdf_only:
if os.path.isfile(dstfp) and args.overwrite is True:
if not args.dryrun:
try:
os.remove(dstfp)
except:
print("Cannot replace archive: %s" % dstfp)
if not os.path.isfile(dstfp):
if args.lsf:
components = (
os.path.basename(raster.srcfp).replace(
"dem.tif", "dem_smooth.tif"), # dem
os.path.basename(raster.matchtag), # matchtag
os.path.basename(raster.mdf), # mdf
os.path.basename(raster.readme), # readme
os.path.basename(raster.browse), # browse
# index shp files
)
else:
components = (
os.path.basename(raster.srcfp), # dem
os.path.basename(raster.matchtag), # matchtag
os.path.basename(raster.mdf), # mdf
os.path.basename(raster.readme), # readme
os.path.basename(raster.browse), # browse
# index shp files
)
optional_components = [
os.path.basename(r) for r in raster.reg_files
] #reg
os.chdir(dstdir)
#logger.info(os.getcwd())
k = 0
existing_components = sum([
int(os.path.isfile(component))
for component in components
])
### check if exists, print
#logger.info(existing_components)
if existing_components == len(components):
## Build index
index = os.path.join(scratch,
raster.stripid + "_index.shp")
## create dem index shp: <strip_id>_index.shp
try:
index_dir, index_lyr = utils.get_source_names(
index)
except RuntimeError as e:
logger.error("{}: {}".format(index, e))
if os.path.isfile(index):
ogrDriver.DeleteDataSource(index)
if not os.path.isfile(index):
ds = ogrDriver.CreateDataSource(index)
if ds is not None:
lyr = ds.CreateLayer(index_lyr, tgt_srs,
ogr.wkbPolygon)
if lyr is not None:
for field_def in utils.DEM_ATTRIBUTE_DEFINITIONS_BASIC:
field = ogr.FieldDefn(
field_def.fname, field_def.ftype)
field.SetWidth(field_def.fwidth)
field.SetPrecision(
field_def.fprecision)
lyr.CreateField(field)
#print raster.stripid
feat = ogr.Feature(lyr.GetLayerDefn())
## Set fields
feat.SetField("DEM_ID", raster.stripid)
feat.SetField("PAIRNAME", raster.pairname)
feat.SetField("SENSOR1", raster.sensor1)
feat.SetField("SENSOR2", raster.sensor2)
feat.SetField(
"ACQDATE1",
raster.acqdate1.strftime("%Y-%m-%d"))
feat.SetField(
"ACQDATE2",
raster.acqdate2.strftime("%Y-%m-%d"))
feat.SetField("CATALOGID1", raster.catid1)
feat.SetField("CATALOGID2", raster.catid2)
feat.SetField("ND_VALUE", raster.ndv)
feat.SetField("DEM_NAME", raster.srcfn)
feat.SetField("ALGM_VER",
raster.algm_version)
res = (raster.xres + raster.yres) / 2.0
feat.SetField("DEM_RES", res)
feat.SetField("DENSITY", raster.density)
#### Set fields if populated (will not be populated if metadata file is not found)
if raster.creation_date:
feat.SetField(
"CR_DATE",
raster.creation_date.strftime(
"%Y-%m-%d"))
## transfrom and write geom
feat.SetField("PROJ4", raster.proj4)
feat.SetField("EPSG", raster.epsg)
src_srs = utils.osr_srs_preserve_axis_order(
osr.SpatialReference())
src_srs.ImportFromWkt(raster.proj)
if raster.geom:
geom = raster.geom.Clone()
transform = osr.CoordinateTransformation(
src_srs, tgt_srs)
geom.Transform(transform)
centroid = geom.Centroid()
feat.SetField("CENT_LAT",
centroid.GetY())
feat.SetField("CENT_LON",
centroid.GetX())
feat.SetGeometry(geom)
else:
logger.error(
'No valid geom found: {}'.format(
raster.srcfp))
#### add new feature to layer
lyr.CreateFeature(feat)
## Close layer and dataset
lyr = None
ds = None
if os.path.isfile(index):
## Create archive
if not args.dryrun:
archive = tarfile.open(
dstfp, "w:gz")
#archive = tarfile.open(dstfp,"w:")
if not os.path.isfile(dstfp):
logger.error(
"Cannot create archive: {}"
.format(dstfn))
## Add components
for component in components:
logger.debug(
"Adding {} to {}".format(
component, dstfn))
k += 1
if "dem_smooth.tif" in component:
arcn = component.replace(
"dem_smooth.tif",
"dem.tif")
else:
arcn = component
if not args.dryrun:
archive.add(component,
arcname=arcn)
## Add optional components
for component in optional_components:
if os.path.isfile(component):
logger.debug(
"Adding {} to {}".format(
component, dstfn))
k += 1
if not args.dryrun:
archive.add(component)
## Add index in subfolder
os.chdir(scratch)
for f in glob.glob(index_lyr + ".*"):
arcn = os.path.join("index", f)
logger.debug(
"Adding {} to {}".format(
f, dstfn))
k += 1
if not args.dryrun:
archive.add(f, arcname=arcn)
os.remove(f)
logger.info(
"Added {} items to archive: {}".
format(k, dstfn))
## Close archive and compress with gz
if not args.dryrun:
try:
archive.close()
except Exception as e:
print(e)
else:
logger.error(
'Cannot create layer: {}'.format(
index_lyr))
else:
logger.error(
"Cannot create index: {}".format(index))
else:
logger.error(
"Cannot remove existing index: {}".format(
index))
else:
logger.error(
"Not enough existing components to make a valid archive: {} ({} found, {} required)"
.format(raster.srcfp, existing_components,
len(components)))
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"package setsm dems (build mdf and readme files and create archive) in place in the filesystem"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space to build index shps")
#### Optionsl Arguments
parser.add_argument('--mdf-only',
action='store_true',
default=False,
help="build mdf and readme files only, do not archive")
parser.add_argument(
'--lsf',
action='store_true',
default=False,
help="package LSF DEM instead of original DEM. Includes metadata flag."
)
parser.add_argument(
'--filter-dems',
action='store_true',
default=False,
help="filter dems with area < 5.6 sqkm and density < 0.1")
parser.add_argument(
'--force-filter-dems',
action='store_true',
default=False,
help=
"filter dems where tar has already been built with area < 5.6 sqkm and density < 0.1"
)
parser.add_argument('-v',
action='store_true',
default=False,
help="verbose output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument(
"--tasks-per-job",
type=int,
help="number of tasks to bundle into a single job (requires pbs option)"
)
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_package.sh in script root folder)"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
pos_arg_keys = ['src', 'scratch']
#### Parse Arguments
scriptpath = os.path.abspath(sys.argv[0])
args = parser.parse_args()
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
if not os.path.isdir(
args.scratch
) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs
parser.error("Scratch directory does not exist: %s" % args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
if args.tasks_per_job and not args.pbs:
parser.error("jobs-per-task argument requires the pbs option")
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
if args.lsf:
logger.info('Packaging LSF DEMs')
else:
logger.info('Packaging non-LSF DEMs')
j = 0
scenes = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith('.tif'):
logger.debug(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(src)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(src)
elif not os.path.isfile(raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(raster.readme):
scenes.append(src)
elif os.path.isfile(src) and src.endswith('.txt'):
fh = open(src, 'r')
for line in fh.readlines():
sceneid = line.strip()
try:
raster = dem.SetsmDem(sceneid)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(sceneid)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(sceneid)
elif not os.path.isfile(raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(raster.readme):
scenes.append(sceneid)
elif os.path.isdir(src):
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif") and "m_" in f:
srcfp = os.path.join(root, f)
logger.debug(srcfp)
try:
raster = dem.SetsmDem(srcfp)
except RuntimeError as e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(srcfp)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(srcfp)
elif not os.path.isfile(
raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(
raster.readme):
scenes.append(srcfp)
else:
logger.error("src must be a directory, a strip dem, or a text file")
scenes = list(set(scenes))
logger.info('Number of src rasters: {}'.format(j))
logger.info('Number of incomplete tasks: {}'.format(len(scenes)))
tm = datetime.now()
job_count = 0
scene_count = 0
scenes_in_job_count = 0
task_queue = []
for srcfp in scenes:
scene_count += 1
srcdir, srcfn = os.path.split(srcfp)
if args.tasks_per_job:
# bundle tasks into text files in the dst dir and pass the text file in as src
scenes_in_job_count += 1
src_txt = os.path.join(
scratch,
'src_dems_{}_{}.txt'.format(tm.strftime("%Y%m%d%H%M%S"),
job_count))
if scenes_in_job_count == 1:
# remove text file if dst already exists
try:
os.remove(src_txt)
except OSError:
pass
if scenes_in_job_count <= args.tasks_per_job:
# add to txt file
fh = open(src_txt, 'a')
fh.write("{}\n".format(srcfp))
fh.close()
if scenes_in_job_count == args.tasks_per_job or scene_count == len(
scenes):
scenes_in_job_count = 0
job_count += 1
task = taskhandler.Task(
'Pkg{:04g}'.format(job_count),
'Pkg{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, src_txt,
scratch), build_archive,
[srcfp, scratch, args])
task_queue.append(task)
else:
job_count += 1
task = taskhandler.Task(
srcfn, 'Pkg{:04g}'.format(job_count), 'python',
'{} {} {} {}'.format(scriptpath, arg_str_base, srcfp, scratch),
build_archive, [srcfp, scratch, args])
task_queue.append(task)
if len(task_queue) > 0:
logger.info("Submitting Tasks")
if args.pbs:
task_handler = taskhandler.PBSTaskHandler(qsubpath)
if not args.dryrun:
task_handler.run_tasks(task_queue)
elif args.parallel_processes > 1:
task_handler = taskhandler.ParallelTaskHandler(
args.parallel_processes)
logger.info("Number of child processes to spawn: {0}".format(
task_handler.num_processes))
if not args.dryrun:
task_handler.run_tasks(task_queue)
else:
for task in task_queue:
src, scratch, task_arg_obj = task.method_arg_list
if not args.dryrun:
task.method(src, scratch, task_arg_obj)
else:
logger.info("No tasks found to process")
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="shelve setsm files by <platform>/<year>/<month>/<date>")
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('dst', help="destination directory")
#### Optionsl Arguments
parser.add_argument('--try-link',
action='store_true',
default=False,
help="try linking instead of copying files")
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
src = os.path.abspath(args.src)
dst = os.path.abspath(args.dst)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
if args.log:
if os.path.isdir(args.log):
tm = datetime.now()
logfile = os.path.join(
args.log,
"shelve_setsm{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
rasters = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src):
logger.info(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError as e:
logger.error(e)
else:
if raster.metapath is not None:
rasters.append(raster)
else:
logger.warning(
"DEM does not include a valid meta.txt and cannot be shelved: {}"
.format(raster.srcfp))
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif"):
logger.debug(os.path.join(root, f))
try:
raster = dem.SetsmDem(os.path.join(root, f))
except RuntimeError as e:
logger.error(e)
else:
if raster.metapath is not None:
rasters.append(raster)
else:
logger.warning(
"DEM does not include a valid meta.txt and cannot be shelved: {}"
.format(raster.srcfp))
logger.info('Shelving DEMs')
total = len(rasters)
i = 0
for raster in rasters:
#### print count/total as progress meter
i += 1
#logger.info("[{} of {}] - {}".format(i,total,raster.stripid))
#### move data
platform = raster.sensor1
year = raster.acqdate1.strftime("%Y")
month = raster.acqdate1.strftime("%m")
day = raster.acqdate1.strftime("%d")
#pair_folder = "{}_{}".format(raster.pairname, raster.creation_date.strftime("%Y%m%d"))
#dst_dir = os.path.join(dst, platform, year, month, day, pair_folder)
dst_dir = os.path.join(dst, platform, year, month, day)
if not os.path.isdir(dst_dir):
if not args.dryrun:
os.makedirs(dst_dir)
for ifp in glob.glob(
os.path.join(raster.srcdir, raster.stripid) + "*"):
ofp = os.path.join(dst_dir, os.path.basename(ifp))
if os.path.isfile(ofp) and args.overwrite:
logger.debug("Copying {} to {}".format(ifp, ofp))
if not args.dryrun:
os.remove(ofp)
if args.try_link:
os.link(ifp, ofp)
else:
shutil.copy2(ifp, ofp)
elif not os.path.isfile(ofp):
logger.debug("Copying {} to {}".format(ifp, ofp))
if not args.dryrun:
if args.try_link:
os.link(ifp, ofp)
else:
shutil.copy2(ifp, ofp)
else:
logger.warning("Cannot copy {} to {}".format(ifp, ofp))
logger.info('Done')
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"shelve setsm files by centroid location relative to a shp index")
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('dst', help="destination directory")
parser.add_argument('shp', help='shp index defining grid scheme')
parser.add_argument('field', help='shp index field with grid name')
#### Optionsl Arguments
parser.add_argument('--try-link',
action='store_true',
default=False,
help="try linking instead of copying files")
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
src = os.path.abspath(args.src)
dst = os.path.abspath(args.dst)
shp = os.path.abspath(args.shp)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
if args.log:
if os.path.isdir(args.log):
tm = datetime.now()
logfile = os.path.join(
args.log,
"shelve_setsm{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
### Open shp, verify field, verify projetion, extract tile geoms
tiles = {}
ds = ogr.Open(shp)
if ds is not None:
lyr = ds.GetLayerByName(os.path.splitext(os.path.basename(shp))[0])
lyr.ResetReading()
src_srs = lyr.GetSpatialRef()
i = lyr.FindFieldIndex(args.field, 1)
if i == -1:
logger.error("Cannot locate field {} in {}".format(
args.field, args.shp))
sys.exit(-1)
shp_srs = lyr.GetSpatialRef()
if shp_srs is None:
logger.error("Shp must have a defined spatial reference")
sys.exit(-1)
for feat in lyr:
tile_name = feat.GetFieldAsString(i)
tile_geom = feat.GetGeometryRef().Clone()
if not tile_name in tiles:
tiles[tile_name] = tile_geom
else:
logger.error(
"Found features with duplicate name: {} - Ignoring 2nd feature"
.format(tile_name))
else:
logger.error("Cannot open {}".format(src))
if len(tiles) == 0:
logger.error("No features found in shp")
else:
rasters = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src):
logger.info(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError as e:
logger.error(e)
else:
if raster.metapath or os.path.isfile(raster.mdf):
rasters.append(raster)
else:
logger.warning(
"DEM does not include a valid meta.txt or mdf.txt, skipping: {}"
.format(raster.srcfp))
else:
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif"):
logger.debug(os.path.join(root, f))
try:
raster = dem.SetsmDem(os.path.join(root, f))
except RuntimeError as e:
logger.error(e)
else:
if raster.metapath or os.path.isfile(raster.mdf):
rasters.append(raster)
else:
logger.warning(
"DEM does not include a valid meta.txt or mdf.txt, skipping: {}"
.format(raster.srcfp))
logger.info('Shelving DEMs')
total = len(rasters)
i = 0
for raster in rasters:
#### print count/total as progress meter
i += 1
#logger.info("[{} of {}] - {}".format(i,total,raster.stripid))
## Convert geom to match shp srs and get centroid
raster.get_metafile_info()
geom_copy = raster.exact_geom.Clone()
srs = utils.osr_srs_preserve_axis_order(osr.SpatialReference())
srs.ImportFromProj4(raster.proj4_meta)
if not shp_srs.IsSame(srs):
ctf = osr.CoordinateTransformation(srs, shp_srs)
geom_copy.Transform(ctf)
centroid = geom_copy.Centroid()
## Run intersection with each tile
tile_overlaps = []
for tile_name, tile_geom in tiles.items():
if centroid.Intersects(tile_geom):
tile_overlaps.append(tile_name)
## Raise an error on multiple intersections or zero intersections
if len(tile_overlaps) == 0:
logger.error(
"raster {} does not intersect the index shp, skipping".
format(raster.srcfn))
elif len(tile_overlaps) > 1:
logger.error(
"raster {} intersects more than one tile ({}), skipping".
format(raster.srcfn, ','.join(tile_overlaps)))
else:
logger.info("{} shelved to tile {}".format(
raster.stripid, tile_overlaps[0]))
dst_dir = os.path.join(dst, tile_overlaps[0])
if not os.path.isdir(dst_dir):
if not args.dryrun:
os.makedirs(dst_dir)
for ifp in glob.glob(
os.path.join(raster.srcdir, raster.stripid) + "*"):
ofp = os.path.join(dst_dir, os.path.basename(ifp))
if os.path.isfile(ofp) and args.overwrite:
logger.debug("Copying {} to {}".format(ifp, ofp))
if not args.dryrun:
os.remove(ofp)
if args.try_link:
os.link(ifp, ofp)
else:
shutil.copy2(ifp, ofp)
elif not os.path.isfile(ofp):
logger.debug("Copying {} to {}".format(ifp, ofp))
if not args.dryrun:
if args.try_link:
os.link(ifp, ofp)
else:
shutil.copy2(ifp, ofp)
else:
logger.debug(
"File already exists, skipping {} to {}".format(
ifp, ofp))
logger.info('Done')
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"calculate density for setsm strip DEMS and save to a density.txt file"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space")
#### Optionsl Arguments
parser.add_argument('-v',
action='store_true',
default=False,
help="verbose output")
parser.add_argument(
"--tasks-per-job",
type=int,
help="number of tasks to bundle into a single job (requires pbs option)"
)
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_package.sh in script root folder)"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
pos_arg_keys = ['src', 'scratch']
#### Parse Arguments
scriptpath = os.path.abspath(sys.argv[0])
args = parser.parse_args()
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
if not os.path.isdir(
args.scratch
) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs
parser.error("Scratch directory does not exist: %s" % args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
if args.tasks_per_job and not args.pbs:
parser.error("jobs-per-task argument requires the pbs option")
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
j = 0
scenes = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith('.tif'):
logger.debug(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError, e:
logger.error(e)
else:
j += 1
if not os.path.isfile(raster.density_file):
scenes.append(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError, e:
logger.error(e)
else:
j += 1
if not os.path.isfile(raster.density_file):
scenes.append(src)
elif os.path.isfile(src) and src.endswith('.txt'):
fh = open(src, 'r')
for line in fh.readlines():
sceneid = line.strip()
try:
raster = dem.SetsmDem(sceneid)
except RuntimeError, e:
logger.error(e)
else:
j += 1
if not os.path.isfile(raster.density_file):
scenes.append(sceneid)
elif os.path.isdir(src):
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith("_dem.tif") and "m_" in f:
srcfp = os.path.join(root, f)
logger.debug(srcfp)
try:
raster = dem.SetsmDem(srcfp)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="shelve setsm files by geocell"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('dst', help="destination directory")
#### Optionsl Arguments
parser.add_argument('--res', choices=['2m','8m'], help="only shelve DEMs of resolution <res> (2 or 8)")
parser.add_argument('--try-link', action='store_true', default=False,
help="try linking instead of copying files")
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--overwrite', action='store_true', default=False,
help="overwrite existing index")
parser.add_argument('--dryrun', action='store_true', default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" %args.src)
src = os.path.abspath(args.src)
dst = os.path.abspath(args.dst)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
if args.log:
if os.path.isdir(args.log):
tm = datetime.now()
logfile = os.path.join(args.log,"shelve_setsm{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
rasters = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src):
logger.info(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError, e:
logger.error( e )
else:
if raster.metapath is not None:
if args.res:
if raster.res == args.res:
rasters.append(raster)
else:
rasters.append(raster)
else:
logger.warning("DEM does not include a valid meta.txt and cannot be shelved: {}".format(raster.srcfp))
if raster.metapath is not None:
if args.res:
if raster.res == args.res:
rasters.append(raster)
else:
rasters.append(raster)
else:
logger.warning("DEM does not include a valid meta.txt and cannot be shelved: {}".format(raster.srcfp))
else:
for root,dirs,files in os.walk(src):
for f in files:
if f.endswith("_dem.tif"):
logger.debug(os.path.join(root,f))
try:
raster = dem.SetsmDem(os.path.join(root,f))
except RuntimeError, e:
logger.error( e )
else:
if raster.metapath is not None:
if args.res:
if raster.res == args.res:
rasters.append(raster)
else:
rasters.append(raster)
else:
logger.warning("DEM does not include a valid meta.txt and cannot be shelved: {}".format(raster.srcfp))
logger.info('Shelving DEMs')
total = len(rasters)
i = 0
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description=
"package setsm dems (build mdf and readme files and create archive) in place in the filesystem"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('scratch', help="scratch space to build index shps")
#### Optionsl Arguments
parser.add_argument('--mdf-only',
action='store_true',
default=False,
help="build mdf and readme files only, do not archive")
parser.add_argument(
'--lsf',
action='store_true',
default=False,
help="package LSF DEM instead of original DEM. Includes metadata flag."
)
parser.add_argument(
'--filter-dems',
action='store_true',
default=False,
help="filter dems with area < 5.6 sqkm and density < 0.1")
parser.add_argument(
'--force-filter-dems',
action='store_true',
default=False,
help=
"filter dems where tar has already been built with area < 5.6 sqkm and density < 0.1"
)
parser.add_argument('-v',
action='store_true',
default=False,
help="verbose output")
parser.add_argument('--overwrite',
action='store_true',
default=False,
help="overwrite existing index")
parser.add_argument(
"--tasks-per-job",
type=int,
help="number of tasks to bundle into a single job (requires pbs option)"
)
parser.add_argument("--pbs",
action='store_true',
default=False,
help="submit tasks to PBS")
parser.add_argument(
"--parallel-processes",
type=int,
default=1,
help="number of parallel processes to spawn (default 1)")
parser.add_argument(
"--qsubscript",
help=
"qsub script to use in PBS submission (default is qsub_package.sh in script root folder)"
)
parser.add_argument('--dryrun',
action='store_true',
default=False,
help="print actions without executing")
pos_arg_keys = ['src', 'scratch']
#### Parse Arguments
scriptpath = os.path.abspath(sys.argv[0])
args = parser.parse_args()
src = os.path.abspath(args.src)
scratch = os.path.abspath(args.scratch)
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" % args.src)
if not os.path.isdir(
args.scratch
) and not args.pbs: #scratch dir may not exist on head node when running jobs via pbs
parser.error("Scratch directory does not exist: %s" % args.scratch)
## Verify qsubscript
if args.qsubscript is None:
qsubpath = os.path.join(os.path.dirname(scriptpath), 'qsub_package.sh')
else:
qsubpath = os.path.abspath(args.qsubscript)
if not os.path.isfile(qsubpath):
parser.error("qsub script path is not valid: %s" % qsubpath)
if args.tasks_per_job and not args.pbs:
parser.error("jobs-per-task argument requires the pbs option")
## Verify processing options do not conflict
if args.pbs and args.parallel_processes > 1:
parser.error(
"Options --pbs and --parallel-processes > 1 are mutually exclusive"
)
if args.v:
log_level = logging.DEBUG
else:
log_level = logging.INFO
lsh = logging.StreamHandler()
lsh.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s',
'%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
#### Get args ready to pass to task handler
arg_keys_to_remove = ('qsubscript', 'dryrun', 'pbs', 'parallel_processes',
'tasks_per_job')
arg_str_base = taskhandler.convert_optional_args_to_string(
args, pos_arg_keys, arg_keys_to_remove)
if args.lsf:
logger.info('Packaging LSF DEMs')
else:
logger.info('Packaging non-LSF DEMs')
j = 0
scenes = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src) and src.endswith('.tif'):
logger.debug(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError, e:
logger.error(e)
else:
j += 1
if args.overwrite or args.force_filter_dems:
scenes.append(src)
elif args.mdf_only:
if (not os.path.isfile(raster.mdf)
or not os.path.isfile(raster.readme)):
scenes.append(src)
elif not os.path.isfile(raster.archive) or not os.path.isfile(
raster.mdf) or not os.path.isfile(raster.readme):
scenes.append(src)
def main():
#### Set Up Arguments
parser = argparse.ArgumentParser(
description="shelve setsm files by centroid location relative to a shp index"
)
#### Positional Arguments
parser.add_argument('src', help="source directory or dem")
parser.add_argument('dst', help="destination directory")
parser.add_argument('shp', help='shp index defining grid scheme')
parser.add_argument('field', help='shp index field with grid name')
#### Optionsl Arguments
parser.add_argument('--try-link', action='store_true', default=False,
help="try linking instead of copying files")
parser.add_argument('--log', help="directory for log output")
parser.add_argument('--overwrite', action='store_true', default=False,
help="overwrite existing index")
parser.add_argument('--dryrun', action='store_true', default=False,
help="print actions without executing")
#### Parse Arguments
args = parser.parse_args()
#### Verify Arguments
if not os.path.isdir(args.src) and not os.path.isfile(args.src):
parser.error("Source directory or file does not exist: %s" %args.src)
src = os.path.abspath(args.src)
dst = os.path.abspath(args.dst)
shp = os.path.abspath(args.shp)
lsh = logging.StreamHandler()
lsh.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lsh.setFormatter(formatter)
logger.addHandler(lsh)
if args.log:
if os.path.isdir(args.log):
tm = datetime.now()
logfile = os.path.join(args.log,"shelve_setsm{}.log".format(tm.strftime("%Y%m%d%H%M%S")))
else:
parser.error('log folder does not exist: {}'.format(args.log))
lfh = logging.FileHandler(logfile)
lfh.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s %(levelname)s- %(message)s','%m-%d-%Y %H:%M:%S')
lfh.setFormatter(formatter)
logger.addHandler(lfh)
### Open shp, verify field, verify projetion, extract tile geoms
tiles = {}
ds = ogr.Open(shp)
if ds is not None:
lyr = ds.GetLayerByName(os.path.splitext(os.path.basename(shp))[0])
lyr.ResetReading()
src_srs = lyr.GetSpatialRef()
i = lyr.FindFieldIndex(args.field,1)
if i == -1:
logger.error("Cannot locate field {} in {}".format(args.field, args.shp))
sys.exit(-1)
shp_srs = lyr.GetSpatialRef()
if shp_srs is None:
logger.error("Shp must have a defined spatial reference")
sys.exit(-1)
for feat in lyr:
tile_name = feat.GetFieldAsString(i)
tile_geom = feat.GetGeometryRef().Clone()
if not tile_name in tiles:
tiles[tile_name] = tile_geom
else:
logger.error("Found features with duplicate name: {} - Ignoring 2nd feature".format(tile_name))
else:
logger.error("Cannot open {}".format(src))
if len(tiles) == 0:
logger.error("No features found in shp")
else:
rasters = []
#### ID rasters
logger.info('Identifying DEMs')
if os.path.isfile(src):
logger.info(src)
try:
raster = dem.SetsmDem(src)
except RuntimeError, e:
logger.error( e )
else:
if raster.metapath or os.path.isfile(raster.mdf):
rasters.append(raster)
else:
logger.warning("DEM does not include a valid meta.txt or mdf.txt, skipping: {}".format(raster.srcfp))
else:
