def getFP(demFile):
demFile = findTestFile(demFile)
Z, X, Y = rat.extractRasterData(demFile, 'z', 'x', 'y')
fp_vertices = rat.getFPvertices(Z, Y, X, label=-9999, label_type='nodata')
num = len(fp_vertices[0])
test_str = ("""demFile: {}
Strip Footprint Vertices
X: {}
Y: {}
""".format(
demFile,
str(fp_vertices[0]).replace(',', '')[1:-1],
str(fp_vertices[1]).replace(',', '')[1:-1],
))
return num, test_str
def main():
parser = argparse.ArgumentParser(description=(
"Make sure matchtags contain only zeros and ones."
"\nPrint filenames of all invalid matchtags and "
"store results in a text file."))
parser.add_argument('src',
help="Path to directory containing *_matchtag.tif files.")
parser.add_argument('-o', '--out', default=os.path.join(os.getcwd(), 'check_match_results.txt'),
help="File path of results text file (default is './check_match_results.txt').")
# Parse and validate arguments.
args = parser.parse_args()
srcdir = os.path.abspath(args.src)
outfile = os.path.abspath(args.out)
outdir = os.path.dirname(outfile)
if not os.path.isdir(srcdir):
parser.error("src must be a directory")
if os.path.isfile(outfile):
parser.error("out file already exists")
if not os.path.isdir(os.path.dirname(outdir)):
print("Creating directory for output results file: {}".format(outdir))
os.makedirs(outdir)
matchFiles = glob.glob(os.path.join(srcdir, '*_matchtag.tif'))
results_fp = open(outfile, 'w')
for m in matchFiles:
array = extractRasterData(m, 'array')
if np.any(array > 1):
affected_fname = os.path.basename(m)
print(affected_fname)
results_fp.write(affected_fname+'\n')
results_fp.close()
print("Done!")
def saveDBP(demFile):
demFile = findTestFile(demFile)
demSuffix = getDemSuffix(demFile)
shapefileFile = demFile.replace(demSuffix, 'dem_boundary.shp')
Z, X, Y, proj_ref = rat.extractRasterData(demFile, 'z', 'x', 'y',
'proj_ref')
poly = rat.getDataBoundariesPoly(Z, X, Y, nodata_val=-9999)
if not poly:
raise TestingError("Failed to create data cluster boundaries polygon")
driver = ogr.GetDriverByName("ESRI Shapefile")
ds = driver.CreateDataSource(shapefileFile)
srs = osr.SpatialReference()
srs.ImportFromWkt(proj_ref)
layer = ds.CreateLayer("data_clusters", srs, ogr.wkbPolygon)
feature = ogr.Feature(layer.GetLayerDefn())
feature.SetGeometry(poly)
layer.CreateFeature(feature)
# Dereference datasource to initiate write to disk.
ds = None
print("'{}' saved".format(shapefileFile))
def diff_strips(demFile1, demFile2, diff_demFile, save_match):
# TODO: Write docstring.
# Construct filenames.
demSuffix1 = getDemSuffix(demFile1)
demSuffix2 = getDemSuffix(demFile2)
diff_demFile_root, diff_demFile_ext = os.path.splitext(diff_demFile)
matchFile1 = selectBestMatchtag(demFile1)
matchFile2 = selectBestMatchtag(demFile2)
metaFile1 = demFile1.replace(demSuffix1, 'mdf.txt')
metaFile2 = demFile2.replace(demSuffix2, 'mdf.txt')
regFile1 = demFile1.replace(demSuffix1, 'reg.txt')
regFile2 = demFile2.replace(demSuffix2, 'reg.txt')
diff_matchFile = '{}_matchtag{}'.format(diff_demFile_root,
diff_demFile_ext)
diff_metaFile = '{}_meta.txt'.format(diff_demFile_root, diff_demFile_ext)
# Read georeferenced strip geometries.
x1, y1, spatref1 = rat.extractRasterData(demFile1, 'x', 'y', 'spat_ref')
x2, y2, spatref2 = rat.extractRasterData(demFile2, 'x', 'y', 'spat_ref')
# Make sure strips have same projection.
if spatref2.IsSame(spatref1) != 1:
raise SpatialRefError(
"Base strip '{}' spatial reference ({}) mismatch with "
"compare strip spatial reference ({})".format(
demFile1, spatref1.ExportToWkt(), spatref2.ExportToWkt()))
spat_ref = spatref1
# Find area of overlap.
z1_c0, z1_r0 = 0, 0
z1_c1, z1_r1 = None, None
z2_c0, z2_r0 = 0, 0
z2_c1, z2_r1 = None, None
try:
if x1[0] < x2[0]:
overlap_ind = np.where(x1 == x2[0])[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z1_c0 = overlap_ind[0]
else:
overlap_ind = np.where(x1[0] == x2)[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z2_c0 = overlap_ind[0]
if x1[-1] > x2[-1]:
overlap_ind = np.where(x1 == x2[-1])[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z1_c1 = overlap_ind[0] + 1
else:
overlap_ind = np.where(x1[-1] == x2)[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z2_c1 = overlap_ind[0] + 1
if y1[0] > y2[0]:
overlap_ind = np.where(y1 == y2[0])[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z1_r0 = overlap_ind[0]
else:
overlap_ind = np.where(y1[0] == y2)[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z2_r0 = overlap_ind[0]
if y1[-1] < y2[-1]:
overlap_ind = np.where(y1 == y2[-1])[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z1_r1 = overlap_ind[0] + 1
else:
overlap_ind = np.where(y1[-1] == y2)[0]
if overlap_ind.size == 0:
raise NoOverlapError("")
z2_r1 = overlap_ind[0] + 1
except NoOverlapError:
raise NoOverlapError("Strip geometries do not overlap")
if save_match:
# Load matchtag data into arrays.
print("Loading matchtag data")
m1 = rat.extractRasterData(matchFile1, 'array')
m2 = rat.extractRasterData(matchFile2, 'array')
m1 = m1[z1_r0:z1_r1, z1_c0:z1_c1]
m2 = m2[z2_r0:z2_r1, z2_c0:z2_c1]
r0, r1, c0, c1 = crop_strip(m1, m2, method='data_density')
# del m1, m2
# Load DEM data into arrays.
print("Loading raster data")
z1 = rat.extractRasterData(demFile1, 'z')
z2 = rat.extractRasterData(demFile2, 'z')
z1[z1 == -9999] = np.nan
z2[z2 == -9999] = np.nan
# Crop arrays to area of overlap.
x1 = x1[z1_c0:z1_c1]
y1 = y1[z1_r0:z1_r1]
x2 = x2[z2_c0:z2_c1]
y2 = y2[z2_r0:z2_r1]
z1 = z1[z1_r0:z1_r1, z1_c0:z1_c1]
z2 = z2[z2_r0:z2_r1, z2_c0:z2_c1]
# Crop arrays further to decrease memory use.
if 'r0' not in vars():
# r0, r1, c0, c1 = crop_strip(z1, method='center')
r0, r1, c0, c1 = crop_strip(rat.getDataArray(z1, np.nan),
rat.getDataArray(z2, np.nan),
method='data_density')
x1_crop = x1[c0:c1]
y1_crop = y1[r0:r1]
x2_crop = x2[c0:c1]
y2_crop = y2[r0:r1]
z1_crop = z1[r0:r1, c0:c1]
z2_crop = z2[r0:r1, c0:c1]
# Get initial guess of translation vector to
# hopefully speed up coregistration...
trans1 = get_trans_vector(regFile1)
trans2 = get_trans_vector(regFile2)
trans_guess = trans2 - trans1
# Coregister the two DEMs.
print("Beginning coregistration")
_, trans, _, rmse = coregisterdems(x1_crop,
y1_crop,
z1_crop,
x2_crop,
y2_crop,
z2_crop,
trans_guess=trans_guess)
dz, dx, dy = trans
# Interpolate comparison DEM to reference DEM.
print("Interpolating dem2 to dem1")
z2i = rat.interp2_gdal(x2 - dx, y2 - dy, z2 - dz, x1, y1, 'linear')
del z2
# Difference DEMs and save result.
print("Saving difference DEM")
z_diff = z2i - z1
z_diff[np.isnan(z_diff)] = -9999
del z1, z2i
rat.saveArrayAsTiff(z_diff,
diff_demFile,
x1,
y1,
spat_ref,
nodata_val=-9999,
dtype_out='float32')
print("Extracting footprint vertices for metadata")
fp_vertices = rat.getFPvertices(z_diff,
x1,
y1,
label=-9999,
label_type='nodata',
replicate_matlab=True,
dtype_out_int64_if_equal=True)
del z_diff
if save_match:
if 'm2' not in vars():
print("Loading match2")
m2 = rat.extractRasterData(matchFile2, 'array').astype(np.float32)
m2 = m2[z2_r0:z2_r1, z2_c0:z2_c1]
elif m2.dtype != np.float32:
m2 = m2.astype(np.float32)
print("Interpolating match2 to match1")
m2i = rat.interp2_gdal(x2 - dx, y2 - dy, m2, x1, y1, 'nearest')
del m2
m2i[np.isnan(m2i)] = 0 # convert back to uint8
m2i = m2i.astype(np.bool)
if 'm1' not in vars():
print("Loading match1")
m1 = rat.extractRasterData(matchFile1, 'array').astype(np.bool)
m1 = m1[z1_r0:z1_r1, z1_c0:z1_c1]
elif m1.dtype != np.bool:
m1 = m1.astype(np.bool)
print("Saving difference matchtag")
m_diff = (m1 & m2i)
del m1
rat.saveArrayAsTiff(m_diff,
diff_matchFile,
x1,
y1,
spat_ref,
nodata_val=0,
dtype_out='uint8')
del m_diff
# Write metadata for difference image.
proj4 = spat_ref.ExportToProj4()
time = datetime.today().strftime("%d-%b-%Y %H:%M:%S")
writeDiffMeta(diff_metaFile, demFile1, demFile2, trans, rmse, proj4,
fp_vertices, time)
def mask_rasters(maskFile, suffix_maskval_dict, args):
import numpy as np
import lib.raster_array_tools as rat
global SRC_SUFFIX_CATCH_ALL
nodata_opt = args.get(ARGSTR_DST_NODATA)
bitmaskSuffix = getBitmaskSuffix(maskFile)
bitmask_is_strip_lsf = (bitmaskSuffix == SUFFIX_STRIP_BITMASK_LSF)
if suffix_maskval_dict is None:
# maskFile_base = maskFile.replace(bitmaskSuffix, '')
# suffix_maskval_dict = {src_rasterFile.replace(maskFile_base, ''): None
# for src_rasterFile in glob.glob(maskFile_base+SRC_SUFFIX_CATCH_ALL)
# if not src_rasterFile.endswith(bitmaskSuffix)}
suffix_maskval_dict = {SRC_SUFFIX_CATCH_ALL: None}
if True in args.get(ARGSTR_EDGE, ARGSTR_WATER, ARGSTR_CLOUD):
# Read in mask raster, then unset bits that will not be used to mask.
mask_select, mask_x, mask_y = rat.extractRasterData(
maskFile, 'z', 'x', 'y')
mask_ones = np.ones_like(mask_select)
if not args.get(ARGSTR_EDGE):
np.bitwise_and(mask_select,
~np.left_shift(mask_ones, MASKCOMP_EDGE_BIT),
out=mask_select)
if not args.get(ARGSTR_WATER):
np.bitwise_and(mask_select,
~np.left_shift(mask_ones, MASKCOMP_WATER_BIT),
out=mask_select)
if not args.get(ARGSTR_CLOUD):
np.bitwise_and(mask_select,
~np.left_shift(mask_ones, MASKCOMP_CLOUD_BIT),
out=mask_select)
del mask_ones
# Convert remaining component bits to a binary boolean mask.
mask_select = mask_select.astype(np.bool)
else:
mask_select = None
# Make list for downsampled mask array "pyramids"
# that may be built on-the-fly.
mask_pyramids = [mask_select]
mask_pyramid_current = mask_pyramids[0]
# Apply mask to source raster images and save results.
for src_suffix, maskval in suffix_maskval_dict.items():
bitmaskSuffix_temp = bitmaskSuffix
if src_suffix.startswith(STRIP_LSF_PREFIX):
if not bitmask_is_strip_lsf:
continue
elif bitmask_is_strip_lsf:
bitmaskSuffix_temp = SUFFIX_STRIP_BITMASK
src_rasterFile = maskFile.replace(bitmaskSuffix_temp, src_suffix)
if '*' in src_rasterFile:
src_rasterFile_list = glob.glob(src_rasterFile)
if len(src_rasterFile_list) == 0:
print("No source rasters found matching filename pattern: {}".
format(src_rasterFile))
continue
maskFile_globbed = None
for src_rasterFile in src_rasterFile_list:
if src_rasterFile.endswith(bitmaskSuffix_temp):
maskFile_globbed = src_rasterFile
break
if maskFile_globbed is not None:
src_rasterFile_list.remove(maskFile_globbed)
else:
src_rasterFile_list = [src_rasterFile]
for src_rasterFile in src_rasterFile_list:
if not os.path.isfile(src_rasterFile):
print(
"Source raster does not exist: {}".format(src_rasterFile))
continue
dst_rasterFile = get_dstFile(src_rasterFile, args)
if os.path.isfile(dst_rasterFile):
print(
"Output raster already exists: {}".format(dst_rasterFile))
if not args.get(ARGSTR_OVERWRITE):
continue
print("Masking source raster ({}) to output raster ({})".format(
src_rasterFile, dst_rasterFile))
# Read in source raster.
dst_array, src_nodataval = rat.extractRasterData(
src_rasterFile, 'array', 'nodata_val')
print("Source NoData value: {}".format(src_nodataval))
# Set masking value to source NoDataVal if necessary.
if maskval is None:
if src_nodataval is None:
print(
"Source raster does not have a set NoData value, "
"so masking value cannot be automatically determined; skipping"
)
continue
else:
maskval = src_nodataval
print("Masking value: {}".format(maskval))
if mask_pyramid_current is not None:
# Apply mask.
if mask_pyramid_current.shape != dst_array.shape:
# See if the required mask pyramid level has already been built.
mask_pyramid_current = None
for arr in mask_pyramids:
if arr.shape == dst_array.shape:
mask_pyramid_current = arr
break
if mask_pyramid_current is None:
# Build the required mask pyramid level and add to pyramids.
mask_pyramid_current = rat.imresize(mask_select,
dst_array.shape,
interp='nearest')
mask_pyramids.append(mask_pyramid_current)
dst_array[mask_pyramid_current] = maskval
# Handle nodata options.
if nodata_opt == ARGCHO_DST_NODATA_SAME:
dst_nodataval = src_nodataval
elif nodata_opt == ARGCHO_DST_NODATA_ADD:
dst_nodataval = maskval if src_nodataval is None else src_nodataval
elif nodata_opt == ARGCHO_DST_NODATA_SWITCH:
dst_nodataval = maskval
elif nodata_opt == ARGCHO_DST_NODATA_CONVERT:
if src_nodataval is not None:
dst_nodata = ((dst_array == src_nodataval)
if not np.isnan(src_nodataval) else
np.isnan(dst_array))
dst_array[dst_nodata] = maskval
dst_nodataval = maskval
elif nodata_opt == ARGCHO_DST_NODATA_UNSET:
dst_nodataval = None
print("Output NoData value: {}".format(dst_nodataval))
# Save output masked raster.
rat.saveArrayAsTiff(dst_array,
dst_rasterFile,
nodata_val=dst_nodataval,
like_raster=src_rasterFile)
def readRasterZ(rasterFile='testRaster_ml.tif'):
return rat.extractRasterData(findTestFile(rasterFile), 'z')