def validate_supplied_lat_lon(params: dict) -> None:
"""
Function to validate that the user supplied lat/lon values sit within image bounds
"""
lon, lat = params[cf.REFX], params[cf.REFY]
if lon == -1 or lat == -1:
return
xmin, ymin, xmax, ymax = prepifg_helper.get_analysis_extent(
crop_opt=params[cf.IFG_CROP_OPT],
rasters=[
prepifg_helper.dem_or_ifg(p.sampled_path)
for p in params[cf.INTERFEROGRAM_FILES]
],
xlooks=params[cf.IFG_LKSX],
ylooks=params[cf.IFG_LKSY],
user_exts=(params[cf.IFG_XFIRST], params[cf.IFG_YFIRST],
params[cf.IFG_XLAST], params[cf.IFG_YLAST]))
msg = "Supplied {} value is outside the bounds of the interferogram data"
lat_lon_txt = ''
if (lon < xmin) or (lon > xmax):
lat_lon_txt += 'longitude'
if (lat < ymin) or (lat > ymax):
lat_lon_txt += ' and latitude' if lat_lon_txt else 'latitude'
if lat_lon_txt:
raise RefPixelError(msg.format(lat_lon_txt))
def main(params):
"""
Main workflow function for preparing interferograms for PyRate.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume ifg_paths is ordered correcly
# pylint: disable=too-many-branches
shared.mpi_vs_multiprocess_logging("prepifg", params)
ifg_paths = params[cf.INTERFEROGRAM_FILES]
if params[cf.DEM_FILE] is not None: # optional DEM conversion
ifg_paths.append(params[cf.DEM_FILE_PATH])
if params[cf.COH_MASK]:
ifg_paths.extend(params[cf.COHERENCE_FILE_PATHS])
shared.mkdir_p(params[cf.OUT_DIR]) # create output dir
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST])
xlooks, ylooks, crop = cf.transform_params(params)
ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in ifg_paths]
exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts)
process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank]
do_prepifg(process_ifgs_paths, exts, params)
mpiops.comm.barrier()
log.info("Finished prepifg")
def do_prepifg(gtiff_paths: List[str], params: dict) -> None:
"""
Prepare interferograms by applying multilooking/cropping operations.
:param list gtiff_paths: List of full-res geotiffs
:param dict params: Parameters dictionary corresponding to config file
"""
# pylint: disable=expression-not-assigned
parallel = params[cf.PARALLEL]
if mpiops.size > 1:
parallel = False
for f in gtiff_paths:
if not os.path.isfile(f):
raise FileNotFoundError("Can not find geotiff: " + str(f) + ". Ensure you have converted your "
"interferograms to geotiffs.")
ifgs = [prepifg_helper.dem_or_ifg(p) for p in gtiff_paths]
xlooks, ylooks, crop = cf.transform_params(params)
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST], params[cf.IFG_XLAST], params[cf.IFG_YLAST])
exts = prepifg_helper.get_analysis_extent(crop, ifgs, xlooks, ylooks, user_exts=user_exts)
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
if params[cf.LARGE_TIFS]:
log.info("Using gdal system calls to process prepifg")
ifg = ifgs[0]
res_str = [xlooks * ifg.x_step, ylooks * ifg.y_step]
res_str = ' '.join([str(e) for e in res_str])
if parallel:
Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(
delayed(__prepifg_system)(
crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks) for gtiff_path in gtiff_paths
)
else:
for gtiff_path in gtiff_paths:
__prepifg_system(crop, exts, gtiff_path, params, res_str, thresh, xlooks, ylooks)
else:
if parallel:
Parallel(n_jobs=params[cf.PROCESSES], verbose=50)(
delayed(_prepifg_multiprocessing)(p, xlooks, ylooks, exts, thresh, crop, params) for p in gtiff_paths
)
else:
for gtiff_path in gtiff_paths:
_prepifg_multiprocessing(gtiff_path, xlooks, ylooks, exts, thresh, crop, params)
def do_prepifg(multi_paths: List[MultiplePaths],
exts: Tuple[float, float, float, float], params: dict) -> None:
"""
Prepare interferograms by applying multilooking/cropping operations.
"""
# pylint: disable=expression-not-assigned
parallel = params[C.PARALLEL]
if mpiops.size > 1:
parallel = False
for f in multi_paths:
if not os.path.isfile(f.converted_path):
raise FileNotFoundError("Can not find geotiff: " + str(f) +
". Ensure you have converted your "
"interferograms to geotiffs.")
if params[C.LARGE_TIFS]:
log.info("Using gdal system calls to execute 'prepifg' step")
ifg = prepifg_helper.dem_or_ifg(multi_paths[0].converted_path)
ifg.open()
xlooks, ylooks = params[C.IFG_LKSX], params[C.IFG_LKSY]
res_str = [xlooks * ifg.x_step, ylooks * ifg.y_step]
res_str = ' '.join([str(e) for e in res_str])
if parallel:
Parallel(n_jobs=params[C.PROCESSES], verbose=50)(
delayed(__prepifg_system)(exts, gtiff_path, params, res_str)
for gtiff_path in multi_paths)
else:
for m_path in multi_paths:
__prepifg_system(exts, m_path, params, res_str)
else:
if parallel:
Parallel(n_jobs=params[C.PROCESSES], verbose=50)(
delayed(_prepifg_multiprocessing)(p, exts, params)
for p in multi_paths)
else:
for m_path in multi_paths:
_prepifg_multiprocessing(m_path, exts, params)
mpiops.comm.barrier()
def do_prepifg(gtiff_paths, params):
"""
Prepare interferograms by applying multilooking/cropping operations.
:param list gtiff_paths: List of full-res geotiffs
:param dict params: Parameters dictionary corresponding to config file
"""
# pylint: disable=expression-not-assigned
log.info("Preparing interferograms by cropping/multilooking")
parallel = params[cf.PARALLEL]
for f in gtiff_paths:
if not os.path.isfile(f):
raise Exception(
"Can not find geotiff: " + str(f) +
". Ensure you have converted your interferograms to geotiffs.")
ifgs = [prepifg_helper.dem_or_ifg(p) for p in gtiff_paths]
xlooks, ylooks, crop = cf.transform_params(params)
user_exts = (params[cf.IFG_XFIRST], params[cf.IFG_YFIRST],
params[cf.IFG_XLAST], params[cf.IFG_YLAST])
exts = prepifg_helper.get_analysis_extent(crop,
ifgs,
xlooks,
ylooks,
user_exts=user_exts)
log.debug("Extents (xmin, ymin, xmax, ymax): " + str(exts))
thresh = params[cf.NO_DATA_AVERAGING_THRESHOLD]
if parallel:
Parallel(n_jobs=params[cf.PROCESSES],
verbose=50)(delayed(_prepifg_multiprocessing)(
p, xlooks, ylooks, exts, thresh, crop, params)
for p in gtiff_paths)
else:
[
_prepifg_multiprocessing(p, xlooks, ylooks, exts, thresh, crop,
params) for p in gtiff_paths
]
def __prepifg_system(crop, exts, gtiff, params, res, thresh, xlooks, ylooks):
p, c, l = _prepifg_multiprocessing(gtiff, xlooks, ylooks, exts, thresh, crop, params)
log.info("Multilooking {p} into {l}".format(p=p, l=l))
extents = ' '.join([str(e) for e in exts])
if isinstance(prepifg_helper.dem_or_ifg(p), shared.DEM):
check_call('gdalwarp {co} -te\t{extents}\t-tr\t{res}\t-r\taverage \t{p}\t{l}\n'.format(
co=COMMON_OPTIONS, extents=extents, res=res, p=p, l=l), shell=True)
__update_meta_data(p, c, l)
return
p_unset = Path(params[cf.OUT_DIR]).joinpath(Path(p).name).with_suffix('.unset.tif')
# change nodataval from zero, also leave input geotifs unchanged if one supplies conv2tif output/geotifs
check_call('gdal_translate {co} -a_nodata nan\t{p}\t{q}'.format(co=COMMON_OPTIONS, p=p, q=p_unset),
shell=True)
# calculate nan-fraction
# TODO: use output options and datatypes to reduce size of the next two tifs
nan_frac = Path(l).with_suffix('.nanfrac.tif')
nan_frac_avg = Path(l).with_suffix('.nanfrac.avg.tif')
corrected_p = Path(p_unset).with_suffix('.corrected.tif')
if c is not None:
# find all the nans
log.info(f"applying coherence + nodata masking on {p}")
check_call(f'{GDAL_CALC} {COMMON_OPTIONS2} -A {p_unset} -B {c} --outfile={nan_frac}\t'
f'--calc=\"logical_or((B<{params[cf.COH_THRESH]}), isclose(A,0,atol=0.000001))\"\t'
f'--NoDataValue=nan', shell=True)
# coh masking
check_call(f'{GDAL_CALC} {COMMON_OPTIONS2} --overwrite -A {p_unset} -B {c}\t'
f'--calc=\"A*(B>={params[cf.COH_THRESH]})'
f'-99999*logical_or((B<{params[cf.COH_THRESH]}),isclose(A,0,atol=0.000001))\"\t'
f'--outfile={corrected_p}\t'
f'--NoDataValue=nan', shell=True)
else:
log.info(f"applying nodata masking on {p}")
check_call(f'{GDAL_CALC} {COMMON_OPTIONS2} --overwrite -A {p_unset}\t'
f'--calc=\"isclose(A,0,atol=0.000001)\"\t'
f'--outfile={nan_frac}\t'
f'--NoDataValue=nan', shell=True)
check_call(f'{GDAL_CALC} {COMMON_OPTIONS2} --overwrite -A {p_unset}\t'
f'--calc=\"A - 99999 *isclose(A, 0, atol=0.000001)\"\t'
f'--outfile={corrected_p}\t'
f'--NoDataValue=nan', shell=True)
# crop resample/average multilooking of nan-fraction
check_call('gdalwarp {co} -te\t{extents}\t-tr\t{res}\t-r\taverage\t{p}\t{out_file}'.format(
co=COMMON_OPTIONS, extents=extents, res=res, p=nan_frac, out_file=nan_frac_avg), shell=True)
# crop resample/average multilooking of raster
check_call('gdalwarp {co} -te\t{extents}\t-tr\t{res}\t-r\taverage \t{p}\t{l}'.format(
co=COMMON_OPTIONS, extents=extents, res=res, p=corrected_p, l=l), shell=True)
check_call(f'{GDAL_CALC} {COMMON_OPTIONS2} --overwrite -A {nan_frac_avg}\t-B {l}\t'
f'--calc=\"B*(A < {thresh}) -99999*(A >= {thresh})\"\t'
f'--outfile={l}\t'
f'--NoDataValue=nan', shell=True)
__update_meta_data(p_unset.as_posix(), c, l)
# clean up
nan_frac_avg.unlink()
nan_frac.unlink()
corrected_p.unlink()
p_unset.unlink()
def crop_resample_average(input_tif,
extents,
new_res,
output_file,
thresh,
out_driver_type='GTiff',
match_pyrate=False,
hdr=None,
coherence_path=None,
coherence_thresh=None):
"""
Crop, resample, and average a geotiff image.
:param str input_tif: Path to input geotiff to resample/crop
:param tuple extents: Cropping extents (xfirst, yfirst, xlast, ylast)
:param list new_res: [xres, yres] resolution of output image
:param str output_file: Path to output resampled/cropped geotiff
:param float thresh: NaN fraction threshold
:param str out_driver_type: The output driver; `MEM` or `GTiff` (optional)
:param bool match_pyrate: Match Legacy output (optional)
:param dict hdr: dictionary of metadata
:return: resampled_average: output cropped and resampled image
:rtype: ndarray
:return: out_ds: destination gdal dataset object
:rtype: gdal.Dataset
"""
dst_ds, _, _, _ = _crop_resample_setup(extents,
input_tif,
new_res,
output_file,
out_bands=2,
dst_driver_type='MEM')
# make a temporary copy of the dst_ds for PyRate style prepifg
tmp_ds = gdal.GetDriverByName('MEM').CreateCopy('', dst_ds) \
if (match_pyrate and new_res[0]) else None
src_ds, src_ds_mem = _setup_source(input_tif)
if coherence_path and coherence_thresh:
coherence_raster = prepifg_helper.dem_or_ifg(coherence_path)
coherence_raster.open()
coherence_ds = coherence_raster.dataset
coherence_masking(src_ds_mem, coherence_ds, coherence_thresh)
elif coherence_path and not coherence_thresh:
raise ValueError(f"Coherence file provided without a coherence "
f"threshold. Please ensure you provide 'cohthresh' "
f"in your config if coherence masking is enabled.")
resampled_average, src_ds_mem = \
gdal_average(dst_ds, src_ds, src_ds_mem, thresh)
src_dtype = src_ds_mem.GetRasterBand(1).DataType
src_gt = src_ds_mem.GetGeoTransform()
# required to match Legacy output
if tmp_ds:
_alignment(input_tif, new_res, resampled_average, src_ds_mem, src_gt,
tmp_ds)
# grab metadata from existing geotiff
gt = dst_ds.GetGeoTransform()
wkt = dst_ds.GetProjection()
# TEST HERE IF EXISTING FILE HAS PYRATE METADATA. IF NOT ADD HERE
if not ifc.DATA_TYPE in dst_ds.GetMetadata() and hdr is not None:
md = shared.collate_metadata(hdr)
else:
md = dst_ds.GetMetadata()
# update metadata for output
for k, v in md.items():
if k == ifc.DATA_TYPE:
# update data type metadata
if v == ifc.ORIG and coherence_path:
md.update({ifc.DATA_TYPE: ifc.COHERENCE})
elif v == ifc.ORIG and not coherence_path:
md.update({ifc.DATA_TYPE: ifc.MULTILOOKED})
elif v == ifc.DEM:
md.update({ifc.DATA_TYPE: ifc.MLOOKED_DEM})
elif v == ifc.INCIDENCE:
md.update({ifc.DATA_TYPE: ifc.MLOOKED_INC})
elif v == ifc.COHERENCE and coherence_path:
pass
elif v == ifc.MULTILOOKED and coherence_path:
md.update({ifc.DATA_TYPE: ifc.COHERENCE})
elif v == ifc.MULTILOOKED and not coherence_path:
pass
else:
raise TypeError('Data Type metadata not recognised')
# In-memory GDAL driver doesn't support compression so turn it off.
creation_opts = ['compress=packbits'] if out_driver_type != 'MEM' else []
out_ds = shared.gdal_dataset(output_file,
dst_ds.RasterXSize,
dst_ds.RasterYSize,
driver=out_driver_type,
bands=1,
dtype=src_dtype,
metadata=md,
crs=wkt,
geotransform=gt,
creation_opts=creation_opts)
shared.write_geotiff(resampled_average, out_ds, np.nan)
return resampled_average, out_ds
def main(params):
"""
Main workflow function for preparing interferograms for PyRate.
:param dict params: Parameters dictionary read in from the config file
"""
# TODO: looks like ifg_paths are ordered according to ifg list
# This probably won't be a problem because input list won't be reordered
# and the original gamma generated list is ordered) this may not affect
# the important pyrate stuff anyway, but might affect gen_thumbs.py.
# Going to assume ifg_paths is ordered correcly
# pylint: disable=too-many-branches
shared.mpi_vs_multiprocess_logging("prepifg", params)
ifg_paths = params[C.INTERFEROGRAM_FILES]
if params[C.DEM_FILE] is not None: # optional DEM conversion
ifg_paths.append(params[C.DEM_FILE_PATH])
if params[C.COH_FILE_LIST] is not None:
ifg_paths.extend(params[C.COHERENCE_FILE_PATHS])
if params[C.COH_FILE_LIST] is None and params[C.COH_MASK]:
raise FileNotFoundError(
"Cannot apply coherence masking: no coherence file list "
"supplied (parameter 'cohfilelist')")
shared.mkdir_p(params[C.OUT_DIR]) # create output dir
user_exts = (params[C.IFG_XFIRST], params[C.IFG_YFIRST],
params[C.IFG_XLAST], params[C.IFG_YLAST])
xlooks, ylooks, crop = transform_params(params)
ifgs = [prepifg_helper.dem_or_ifg(p.converted_path) for p in ifg_paths]
exts = prepifg_helper.get_analysis_extent(crop,
ifgs,
xlooks,
ylooks,
user_exts=user_exts)
ifg0 = ifgs[0]
ifg0.open()
transform = ifg0.dataset.GetGeoTransform()
process_ifgs_paths = np.array_split(ifg_paths, mpiops.size)[mpiops.rank]
do_prepifg(process_ifgs_paths, exts, params)
if params[C.LT_FILE] is not None:
log.info("Calculating and writing geometry files")
__write_geometry_files(params, exts, transform,
ifg_paths[0].sampled_path)
else:
log.info("Skipping geometry calculations: Lookup table not provided")
if params[C.COH_FILE_LIST] is not None:
log.info("Calculating and writing coherence statistics")
mpiops.run_once(__calc_coherence_stats, params,
ifg_paths[0].sampled_path)
else:
log.info("Skipping coherence file statistics computation.")
log.info("Finished 'prepifg' step")
ifg0.close()
