def reproject_stripmeta(metafile_src, metafile_dst, target_epsg, args):
outmeta_txt = ""
inmeta_fp = open(metafile_src, 'r')
target_res = args.get(ARGSTR_TARGET_RESOLUTION)
# Create output spatial reference and get
# output projection in proj4 format.
spatref_out = osr_srs_preserve_axis_order(osr.SpatialReference())
spatref_out.ImportFromEPSG(target_epsg)
proj4_out = spatref_out.ExportToProj4()
# Get strip projection.
line = inmeta_fp.readline()
while not line.startswith('Strip projection (proj4):') and line != "":
outmeta_txt += line
line = inmeta_fp.readline()
if line == "":
logger.error(
"Projection string cannot be parsed from meta file: {}".format(
metafile_src))
inmeta_fp.close()
return
proj4_in = line.split("'")[1]
outmeta_txt += "Strip projection (proj4): '{}'\n".format(proj4_out)
# Get strip footprint geometry.
line = inmeta_fp.readline()
while not line.startswith('Strip Footprint Vertices') and line != "":
outmeta_txt += line
line = inmeta_fp.readline()
if line == "":
logger.error(
"Footprint vertices cannot be parsed from meta file: {}".format(
metafile_src))
inmeta_fp.close()
return
outmeta_txt += line
if args.get(ARGSTR_CORRECT_META_FP_VERTS):
## Recalculate strip footprint vertices from reprojected strip DEM raster.
from lib import raster_array_tools as rat
for i in range(2):
inmeta_fp.readline()
demfile_dst = metafile_dst.replace(SETSM_META_SUFFIX, STRIP_DEM_SUFFIX)
Z, Y, X = raster_io.extractRasterData(demfile_dst, 'z', 'y', 'x')
fp_vertices = rat.getFPvertices(Z,
X,
Y,
label=-9999,
label_type='nodata',
replicate_matlab=True,
dtype_out_int64_if_equal=True)
del Z, Y, X
x_out, y_out = fp_vertices
else:
## Reproject existing strip footprint vertices.
line = inmeta_fp.readline()
x_in = np.fromstring(line.replace('X:', '').strip(),
dtype=np.float64,
sep=' ')
line = inmeta_fp.readline()
y_in = np.fromstring(line.replace('Y:', '').strip(),
dtype=np.float64,
sep=' ')
wkt_in = raster_io.coordsToWkt(np.array([x_in, y_in]).T)
# Create input footprint geometry with spatial reference.
geom = ogr.Geometry(wkt=wkt_in)
spatref_in = osr_srs_preserve_axis_order(osr.SpatialReference())
spatref_in.ImportFromProj4(proj4_in)
geom.AssignSpatialReference(spatref_in)
# Transform geometry to new spatial reference
# and extract transformed coordinates.
geom.TransformTo(spatref_out)
wkt_out = geom.ExportToWkt()
x_out, y_out = raster_io.wktToCoords(wkt_out).T
# x_out = (np.around(x_out / target_res) * target_res).astype(np.int64)
# y_out = (np.around(y_out / target_res) * target_res).astype(np.int64)
outmeta_txt += "X: {}\n".format(' '.join(
np.array_str(x_out, max_line_width=np.inf).strip()[1:-1].split()))
outmeta_txt += "Y: {}\n".format(' '.join(
np.array_str(y_out, max_line_width=np.inf).strip()[1:-1].split()))
outmeta_txt += inmeta_fp.read()
inmeta_fp.close()
# Write output metadata file.
outmeta_fp = open(metafile_dst, 'w')
outmeta_fp.write(outmeta_txt)
outmeta_fp.close()
def reproject_setsm(src_metafile,
dstdir=None,
target_epsg=None,
target_resolution=None,
args=None):
src_is_strip_metafile = DRE_STRIPMETAFILE.fullmatch(
os.path.basename(src_metafile)).matched
normal_strip_output = (target_resolution in (None, 2)
and not args.get(ARGSTR_ADD_RES_SUFFIX))
if args is None and any([a is None for a in [dstdir, target_epsg]]):
raise script_utils.InvalidArgumentError(
"`args` must be provided if `dstdir` or `target_epsg` is None")
if dstdir is None:
dstdir = args.get(ARGSTR_DSTDIR)
if target_epsg is None:
target_epsg = args.get(ARGSTR_TARGET_EPSG)
if target_resolution is None:
target_resolution = args.get(ARGSTR_TARGET_RESOLUTION)
if not os.path.isfile(src_metafile):
raise script_utils.InvalidArgumentError(
"Source metafile does not exist: {}".format(src_metafile))
dstdir = get_task_dstdir(src_metafile, dstdir)
if not os.path.isdir(dstdir):
if not args.get(ARGSTR_DRYRUN):
os.makedirs(dstdir, exist_ok=True)
if src_is_strip_metafile and normal_strip_output:
src_rasters_to_reproject = [
src_metafile.replace(SETSM_META_SUFFIX, suffix)
for suffix in BUILD_STRIP_AUX_CORE_STRIP_SUFFIXES
]
src_ortho2file = src_metafile.replace(SETSM_META_SUFFIX, 'ortho2.tif')
if os.path.isfile(src_ortho2file):
src_rasters_to_reproject.append(src_ortho2file)
else:
src_rasters_to_reproject = glob.glob(
src_metafile.replace(SETSM_META_SUFFIX, '*.tif'))
commands = []
for rasterfile_src in src_rasters_to_reproject:
rasterfile_dst = os.path.join(dstdir, os.path.basename(rasterfile_src))
rasterfile_dst_uncompressed = None
if args.get(ARGSTR_ADD_RES_SUFFIX):
rasterfile_dst_base, rasterfile_dst_ext = os.path.splitext(
rasterfile_dst)
rasterfile_dst = "{}_{}m{}".format(
rasterfile_dst_base, args.get(ARGSTR_TARGET_RESOLUTION),
rasterfile_dst_ext)
rasterfile_dst_res = (target_resolution if target_resolution
is not None else raster_io.extractRasterData(
rasterfile_src, 'res'))
if rasterfile_src.endswith('dem.tif'):
rasterfile_dst_base, rasterfile_dst_ext = os.path.splitext(
rasterfile_dst)
rasterfile_dst_uncompressed = "{}_uncompressed{}".format(
rasterfile_dst_base, rasterfile_dst_ext)
resample_method = None
for suffix_key, suffix_method in SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP:
if suffix_key in rasterfile_src:
resample_method = suffix_method
break
if resample_method is None:
raise script_utils.DeveloperError(
"No suffix key in `SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP` matches "
"source raster file: {}".format(rasterfile_src))
if rasterfile_src.endswith(('ortho.tif', 'ortho2.tif')):
predictor = 2
else:
predictor = 1
commands.append((
'gdalwarp "{0}" "{1}" -q -tap -t_srs EPSG:{2} -tr {3} {3} -r {4} {5} '
'-overwrite -co TILED=YES -co BIGTIFF=YES -co COMPRESS=LZW -co PREDICTOR={6}'
.format(
rasterfile_src,
rasterfile_dst if rasterfile_dst_uncompressed is None else
rasterfile_dst_uncompressed,
target_epsg,
rasterfile_dst_res,
resample_method,
'-dstnodata 1' if 'bitmask' in rasterfile_src else '',
predictor,
)))
if rasterfile_src.endswith('dem.tif'):
commands.append((
'gdal_calc.py --quiet --overwrite -A "{0}" --outfile="{1}" --calc="round_(A*128.0)/128.0" --NoDataValue=-9999'
' --co TILED=YES --co BIGTIFF=YES --co COMPRESS=LZW --co PREDICTOR=3'
.format(rasterfile_dst_uncompressed, rasterfile_dst)))
elif 'bitmask' in rasterfile_src:
commands.append(
'gdal_edit.py -unsetnodata {}'.format(rasterfile_dst))
for cmd in commands:
print(cmd)
if not args.get(ARGSTR_DRYRUN):
rc, _, _ = script_utils.exec_cmd(cmd)
if rc != 0:
raise script_utils.ExternalError(
"External program call finished with non-zero exit status ({})"
.format(rc))
metafile_src = src_metafile
metafile_dst = os.path.join(dstdir, os.path.basename(metafile_src))
if src_is_strip_metafile and normal_strip_output:
demfile_dst = metafile_dst.replace(SETSM_META_SUFFIX, STRIP_DEM_SUFFIX)
saveStripBrowse(args, demfile_dst, STRIP_DEM_SUFFIX, 'bitmask.tif')
if src_is_strip_metafile:
print(
"Reprojecting strip footprint vertices in metafile to EPSG:{}: {} -> {}"
.format(target_epsg, metafile_src, metafile_dst))
if not args.get(ARGSTR_DRYRUN):
reproject_stripmeta(metafile_src, metafile_dst, target_epsg, args)
else:
print("Simply copying scene metafile: {} -> {}".format(
metafile_src, metafile_dst))
if not args.get(ARGSTR_DRYRUN):
shutil.copyfile(metafile_src, metafile_dst)
def reproject_setsm(src_metafile,
dstdir=None,
target_epsg=None,
target_resolution=None,
args=None):
src_is_strip_metafile = demregex.StripDemFile(
os.path.basename(src_metafile)).match().matched
normal_strip_output = (target_resolution in (None, 2)
and not args.get(ARGSTR_ADD_RES_SUFFIX))
if args is None and any([a is None for a in [dstdir, target_epsg]]):
raise script_utils.InvalidArgumentError(
"`args` must be provided if `dstdir` or `target_epsg` is None")
if dstdir is None:
dstdir = args.get(ARGSTR_DSTDIR)
if target_epsg is None:
target_epsg = args.get(ARGSTR_TARGET_EPSG)
if target_resolution is None:
target_resolution = args.get(ARGSTR_TARGET_RESOLUTION)
if not os.path.isfile(src_metafile):
raise script_utils.InvalidArgumentError(
"Source metafile does not exist: {}".format(src_metafile))
dstdir = get_task_dstdir(src_metafile, dstdir)
if not os.path.isdir(dstdir):
if not args.get(ARGSTR_DRYRUN):
os.makedirs(dstdir, exist_ok=True)
if src_is_strip_metafile and normal_strip_output:
src_rasters_to_reproject = [
src_metafile.replace(SETSM_META_SUFFIX, suffix)
for suffix in BUILD_STRIP_AUX_CORE_STRIP_SUFFIXES
]
src_ortho2file = src_metafile.replace(SETSM_META_SUFFIX, 'ortho2.tif')
if os.path.isfile(src_ortho2file):
src_rasters_to_reproject.append(src_ortho2file)
else:
src_rasters_to_reproject = glob.glob(
src_metafile.replace(SETSM_META_SUFFIX, '*.tif'))
commands = []
for rasterfile_src in src_rasters_to_reproject:
rasterfile_dst = os.path.join(dstdir, os.path.basename(rasterfile_src))
if args.get(ARGSTR_ADD_RES_SUFFIX):
rasterfile_dst_base, rasterfile_dst_ext = os.path.splitext(
rasterfile_dst)
rasterfile_dst = "{}_{}m{}".format(
rasterfile_dst_base, args.get(ARGSTR_TARGET_RESOLUTION),
rasterfile_dst_ext)
rasterfile_dst_res = (target_resolution if target_resolution
is not None else raster_io.extractRasterData(
rasterfile_src, 'res'))
resample_method = None
for suffix_key, suffix_method in SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP:
if suffix_key in rasterfile_src:
resample_method = suffix_method
break
if resample_method is None:
raise script_utils.DeveloperError(
"No suffix key in `SETSM_SUFFIX_KEY_TO_RESAMPLE_METHOD_LOOKUP` matches "
"source raster file: {}".format(rasterfile_src))
commands.append((
'gdalwarp "{0}" "{1}" -q -tap -t_srs EPSG:{2} -tr {3} {3} -r {4} {5} '
'-overwrite -co TILED=YES -co BIGTIFF=IF_SAFER -co COMPRESS=LZW'.
format(rasterfile_src, rasterfile_dst, target_epsg,
rasterfile_dst_res, resample_method,
'-dstnodata 1' if 'bitmask' in rasterfile_src else '')))
if 'bitmask' in rasterfile_src:
commands.append(
'gdal_edit.py -unsetnodata {}'.format(rasterfile_dst))
for cmd in commands:
print(cmd)
if not args.get(ARGSTR_DRYRUN):
script_utils.exec_cmd(cmd)
metafile_src = src_metafile
metafile_dst = os.path.join(dstdir, os.path.basename(metafile_src))
if src_is_strip_metafile and normal_strip_output:
demfile_dst = metafile_dst.replace(SETSM_META_SUFFIX, STRIP_DEM_SUFFIX)
saveStripBrowse(args, demfile_dst, STRIP_DEM_SUFFIX, 'bitmask.tif')
if src_is_strip_metafile:
print(
"Reprojecting strip footprint vertices in metafile to EPSG:{}: {} -> {}"
.format(target_epsg, metafile_src, metafile_dst))
if not args.get(ARGSTR_DRYRUN):
reproject_stripmeta(metafile_src, metafile_dst, target_epsg, args)
else:
print("Simply copying scene metafile: {} -> {}".format(
metafile_src, metafile_dst))
if not args.get(ARGSTR_DRYRUN):
shutil.copyfile(metafile_src, metafile_dst)
def mask_rasters(maskFile, suffix_maskval_dict, args):
import numpy as np
import lib.raster_io as rio
global SRC_SUFFIX_CATCH_ALL
nodata_opt = args.get(ARGSTR_DST_NODATA)
bitmaskSuffix = getBitmaskSuffix(maskFile)
bitmask_is_strip_lsf = (bitmaskSuffix == SUFFIX_STRIP_BITMASK_LSF)
if suffix_maskval_dict is None:
# maskFile_base = maskFile.replace(bitmaskSuffix, '')
# suffix_maskval_dict = {src_rasterFile.replace(maskFile_base, ''): None
# for src_rasterFile in glob.glob(maskFile_base+SRC_SUFFIX_CATCH_ALL)
# if not src_rasterFile.endswith(bitmaskSuffix)}
suffix_maskval_dict = {SRC_SUFFIX_CATCH_ALL: None}
if True in args.get(ARGSTR_EDGE, ARGSTR_WATER, ARGSTR_CLOUD):
# Read in mask raster, then unset bits that will not be used to mask.
mask_select, mask_x, mask_y = rio.extractRasterData(
maskFile, 'z', 'x', 'y')
mask_ones = np.ones_like(mask_select)
if not args.get(ARGSTR_EDGE):
np.bitwise_and(mask_select,
~np.left_shift(mask_ones, MASKCOMP_EDGE_BIT),
out=mask_select)
if not args.get(ARGSTR_WATER):
np.bitwise_and(mask_select,
~np.left_shift(mask_ones, MASKCOMP_WATER_BIT),
out=mask_select)
if not args.get(ARGSTR_CLOUD):
np.bitwise_and(mask_select,
~np.left_shift(mask_ones, MASKCOMP_CLOUD_BIT),
out=mask_select)
del mask_ones
# Convert remaining component bits to a binary boolean mask.
mask_select = mask_select.astype(np.bool)
else:
mask_select = None
# Make list for downsampled mask array "pyramids"
# that may be built on-the-fly.
mask_pyramids = [mask_select]
mask_pyramid_current = mask_pyramids[0]
# Apply mask to source raster images and save results.
for src_suffix, maskval in suffix_maskval_dict.items():
bitmaskSuffix_temp = bitmaskSuffix
if src_suffix.startswith(STRIP_LSF_PREFIX):
if not bitmask_is_strip_lsf:
continue
elif bitmask_is_strip_lsf:
bitmaskSuffix_temp = SUFFIX_STRIP_BITMASK
src_rasterFile = maskFile.replace(bitmaskSuffix_temp, src_suffix)
if '*' in src_rasterFile:
src_rasterFile_list = glob.glob(src_rasterFile)
if len(src_rasterFile_list) == 0:
print("No source rasters found matching filename pattern: {}".
format(src_rasterFile))
continue
maskFile_globbed = None
for src_rasterFile in src_rasterFile_list:
if src_rasterFile.endswith(bitmaskSuffix_temp):
maskFile_globbed = src_rasterFile
break
if maskFile_globbed is not None:
src_rasterFile_list.remove(maskFile_globbed)
else:
src_rasterFile_list = [src_rasterFile]
for src_rasterFile in src_rasterFile_list:
if not os.path.isfile(src_rasterFile):
print(
"Source raster does not exist: {}".format(src_rasterFile))
continue
dst_rasterFile = get_dstFile(src_rasterFile, args)
if os.path.isfile(dst_rasterFile):
print(
"Output raster already exists: {}".format(dst_rasterFile))
if not args.get(ARGSTR_OVERWRITE):
continue
print("Masking source raster ({}) to output raster ({})".format(
src_rasterFile, dst_rasterFile))
# Read in source raster.
dst_array, src_nodataval = rio.extractRasterData(
src_rasterFile, 'array', 'nodata_val')
print("Source NoData value: {}".format(src_nodataval))
# Set masking value to source NoDataVal if necessary.
if maskval is None:
if src_nodataval is None:
print(
"Source raster does not have a set NoData value, "
"so masking value cannot be automatically determined; skipping"
)
continue
else:
maskval = src_nodataval
print("Masking value: {}".format(maskval))
if mask_pyramid_current is not None:
# Apply mask.
if mask_pyramid_current.shape != dst_array.shape:
# See if the required mask pyramid level has already been built.
mask_pyramid_current = None
for arr in mask_pyramids:
if arr.shape == dst_array.shape:
mask_pyramid_current = arr
break
if mask_pyramid_current is None:
# Build the required mask pyramid level and add to pyramids.
import lib.raster_array_tools as rat
mask_pyramid_current = rat.imresize(mask_select,
dst_array.shape,
interp='nearest')
mask_pyramids.append(mask_pyramid_current)
dst_array[mask_pyramid_current] = maskval
# Handle nodata options.
if nodata_opt == ARGCHO_DST_NODATA_SAME:
dst_nodataval = src_nodataval
elif nodata_opt == ARGCHO_DST_NODATA_ADD:
dst_nodataval = maskval if src_nodataval is None else src_nodataval
elif nodata_opt == ARGCHO_DST_NODATA_SWITCH:
dst_nodataval = maskval
elif nodata_opt == ARGCHO_DST_NODATA_CONVERT:
if src_nodataval is not None:
dst_nodata = ((dst_array == src_nodataval)
if not np.isnan(src_nodataval) else
np.isnan(dst_array))
dst_array[dst_nodata] = maskval
dst_nodataval = maskval
elif nodata_opt == ARGCHO_DST_NODATA_UNSET:
dst_nodataval = None
print("Output NoData value: {}".format(dst_nodataval))
# Save output masked raster.
rio.saveArrayAsTiff(dst_array,
dst_rasterFile,
nodata_val=dst_nodataval,
like_raster=src_rasterFile,
dtype_out=dst_array.dtype)