def _save_stack(ifgs_dict, params, tiles, out_type):
"""
Save stacking outputs
"""
log.info('Merging PyRate outputs {}'.format(out_type))
gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt']
epochlist = ifgs_dict['epochlist']
ifgs = [v for v in ifgs_dict.values() if isinstance(v, PrereadIfg)]
dest = os.path.join(params[cf.OUT_DIR], out_type + ".tif")
md[ifc.EPOCH_DATE] = epochlist.dates
if out_type == 'stack_rate':
md[ifc.DATA_TYPE] = ifc.STACKRATE
elif out_type == 'stack_error':
md[ifc.DATA_TYPE] = ifc.STACKERROR
else:
md[ifc.DATA_TYPE] = ifc.STACKSAMP
rate = np.zeros(shape=ifgs[0].shape, dtype=np.float32)
for t in tiles:
rate_file = os.path.join(params[cf.TMPDIR],
out_type + '_' + str(t.index) + '.npy')
rate_file = Path(rate_file)
rate_tile = np.load(file=rate_file)
rate[t.top_left_y:t.bottom_right_y,
t.top_left_x:t.bottom_right_x] = rate_tile
shared.write_output_geotiff(md, gt, wkt, rate, dest, np.nan)
npy_rate_file = os.path.join(params[cf.OUT_DIR], out_type + '.npy')
np.save(file=npy_rate_file, arr=rate)
log.debug('Finished PyRate merging {}'.format(out_type))
def _write_dem_errors(ifg_paths: list, params: dict,
preread_ifgs: dict) -> None:
"""
Convenience function for writing DEM error (one file) and DEM error correction for each IFG to disc
:param ifg_paths: List of interferogram class objects.
:param params: Dictionary of PyRate configuration parameters.
:param preread_ifgs: Dictionary of interferogram metadata.
"""
tiles = params[C.TILES]
# re-assemble tiles and save into dem_error dir
shape = preread_ifgs[ifg_paths[0]].shape
# save dem error as geotiff file in out directory
gt, md, wkt = shared.get_geotiff_header_info(ifg_paths[0])
md[ifc.DATA_TYPE] = ifc.DEM_ERROR
dem_error = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type='dem_error')
dem_error_file = os.path.join(params[C.DEM_ERROR_DIR], 'dem_error.tif')
shared.remove_file_if_exists(dem_error_file)
shared.write_output_geotiff(md, gt, wkt, dem_error, dem_error_file, np.nan)
# read the average bperp vals for each ifg and each tile
bperp = np.empty(shape=(len(tiles), len(ifg_paths)), dtype=np.float64)
for t in tiles:
bperp_file = Path(
join(params[C.TMPDIR], 'bperp_avg_' + str(t.index) + '.npy'))
arr = np.load(file=bperp_file)
bperp[t.index, :] = arr
# loop over all ifgs
idx = 0
for ifg_path in ifg_paths:
ifg = Ifg(ifg_path)
ifg.open()
shared.nan_and_mm_convert(ifg,
params) # ensure we have phase data in mm
# read dem error correction file from tmpdir
dem_error_correction_ifg = assemble_tiles(
shape,
params[C.TMPDIR],
tiles,
out_type='dem_error_correction',
index=idx)
# calculate average bperp value across all tiles for the ifg
bperp_val = np.nanmean(bperp[:, idx])
dem_error_correction_on_disc = MultiplePaths.dem_error_path(
ifg.data_path, params)
np.save(file=dem_error_correction_on_disc,
arr=dem_error_correction_ifg)
idx += 1
# subtract DEM error from the ifg
ifg.phase_data -= dem_error_correction_ifg
_save_dem_error_corrected_phase(ifg, bperp_val)
def __save_geom_files(ifg_path, dest, array, out_type):
"""
Convenience function to save geometry geotiff files
"""
log.debug('Saving PyRate outputs {}'.format(out_type))
gt, md, wkt = shared.get_geotiff_header_info(ifg_path)
md[ifc.DATA_TYPE] = out_type_md_dict[out_type]
shared.remove_file_if_exists(dest)
log.info(f"Writing geotiff: {dest}")
shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan)
def _save_merged_files(ifgs_dict,
outdir,
array,
out_type,
index=None,
savenpy=None):
"""
Convenience function to save PyRate geotiff and numpy array files
"""
log.debug('Saving PyRate outputs {}'.format(out_type))
gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt']
epochlist = ifgs_dict['epochlist']
if out_type in ('tsincr', 'tscuml'):
epoch = epochlist.dates[index + 1]
dest = join(outdir, out_type + "_" + str(epoch) + ".tif")
npy_file = join(outdir, out_type + "_" + str(epoch) + ".npy")
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = index + 1
md[ifc.EPOCH_DATE] = epoch
else:
dest = join(outdir, out_type + ".tif")
npy_file = join(outdir, out_type + '.npy')
md[ifc.EPOCH_DATE] = [d.strftime('%Y-%m-%d') for d in epochlist.dates]
if out_type == 'stack_rate':
md[ifc.DATA_TYPE] = ifc.STACKRATE
elif out_type == 'stack_error':
md[ifc.DATA_TYPE] = ifc.STACKERROR
elif out_type == 'stack_samples':
md[ifc.DATA_TYPE] = ifc.STACKSAMP
elif out_type == 'linear_rate':
md[ifc.DATA_TYPE] = ifc.LINRATE
elif out_type == 'linear_error':
md[ifc.DATA_TYPE] = ifc.LINERROR
elif out_type == 'linear_samples':
md[ifc.DATA_TYPE] = ifc.LINSAMP
elif out_type == 'linear_intercept':
md[ifc.DATA_TYPE] = ifc.LINICPT
elif out_type == 'linear_rsquared':
md[ifc.DATA_TYPE] = ifc.LINRSQ
elif out_type == 'tsincr':
md[ifc.DATA_TYPE] = ifc.INCR
elif out_type == 'tscuml':
md[ifc.DATA_TYPE] = ifc.CUML
else:
log.warning('Output type "{}" not recognised'.format(out_type))
shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan)
if savenpy:
np.save(file=npy_file, arr=array)
log.debug('Finished saving {}'.format(out_type))
def write_timeseries_geotiff(ifgs, params, tsincr, pr_type):
# setup metadata for writing into result files
gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path)
epochlist = algorithm.get_epochs(ifgs)[0]
for i in range(tsincr.shape[2]):
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
md['SEQUENCE_POSITION'] = i + 1 # sequence position
data = tsincr[:, :, i]
dest = join(params[cf.OUT_DIR],
pr_type + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = pr_type
write_output_geotiff(md, gt, wkt, data, dest, np.nan)
def write_stackrate_tifs(ifgs, params, res):
rate, error, samples = res
gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path)
epochlist = algorithm.get_epochs(ifgs)[0]
dest = join(params[cf.OUT_DIR], "stack_rate.tif")
md[ifc.EPOCH_DATE] = epochlist.dates
md[ifc.DATA_TYPE] = ifc.STACKRATE
write_output_geotiff(md, gt, wkt, rate, dest, np.nan)
dest = join(params[cf.OUT_DIR], "stack_error.tif")
md[ifc.DATA_TYPE] = ifc.STACKERROR
write_output_geotiff(md, gt, wkt, error, dest, np.nan)
dest = join(params[cf.OUT_DIR], "stack_samples.tif")
md[ifc.DATA_TYPE] = ifc.STACKSAMP
write_output_geotiff(md, gt, wkt, samples, dest, np.nan)
write_stackrate_numpy_files(error, rate, samples, params)
def write_linrate_tifs(ifgs, params, res):
# log.info('Writing linrate results')
rate, error, samples = res
gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path)
epochlist = algorithm.get_epochs(ifgs)[0]
dest = join(params[cf.OUT_DIR], "linrate.tif")
md[ifc.EPOCH_DATE] = epochlist.dates
md[ifc.DATA_TYPE] = ifc.LINRATE
write_output_geotiff(md, gt, wkt, rate, dest, np.nan)
dest = join(params[cf.OUT_DIR], "linerror.tif")
md[ifc.DATA_TYPE] = ifc.LINERROR
write_output_geotiff(md, gt, wkt, error, dest, np.nan)
dest = join(params[cf.OUT_DIR], "linsamples.tif")
md[ifc.DATA_TYPE] = ifc.LINSAMP
write_output_geotiff(md, gt, wkt, samples, dest, np.nan)
write_linrate_numpy_files(error, rate, samples, params)
def _merge_timeseries(rows, cols, params):
"""
Merge time series output
"""
xlks, _, crop = cf.transform_params(params)
base_unw_paths = []
for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"):
if "dem" not in str(p):
base_unw_paths.append(str(p))
if "tif" in base_unw_paths[0].split(".")[1]:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
for i, dest_tif in enumerate(dest_tifs):
dest_tifs[i] = dest_tif.replace("_tif", "")
else:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
output_dir = params[cf.TMPDIR]
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(output_dir, 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
# metadata and projections
gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt']
epochlist = ifgs['epochlist']
ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)]
tiles = shared.get_tiles(dest_tifs[0], rows, cols)
# load the first tsincr file to determine the number of time series tifs
tsincr_file = os.path.join(output_dir, 'tsincr_0.npy')
tsincr = np.load(file=tsincr_file)
# pylint: disable=no-member
no_ts_tifs = tsincr.shape[2]
# we create 2 x no_ts_tifs as we are splitting tsincr and tscuml
# to all processes.
process_tifs = mpiops.array_split(range(2 * no_ts_tifs))
# depending on nvelpar, this will not fit in memory
# e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory
# 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB
# the double for loop helps us overcome the memory limit
log.info('Process {} writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
for i in process_tifs:
tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
if i < no_ts_tifs:
for n, t in enumerate(tiles):
_assemble_tiles(i, n, t, tscum_g, output_dir, 'tscuml')
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
dest = os.path.join(
params[cf.OUT_DIR],
'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.CUML
shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan)
else:
tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
i %= no_ts_tifs
for n, t in enumerate(tiles):
_assemble_tiles(i, n, t, tsincr_g, output_dir, 'tsincr')
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
dest = os.path.join(
params[cf.OUT_DIR],
'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.INCR
shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest, np.nan)
mpiops.comm.barrier()
log.debug('Process {} finished writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs),
no_ts_tifs * 2))
def __save_merged_files(ifgs_dict,
params,
array,
out_type,
index=None,
savenpy=None):
"""
Convenience function to save PyRate geotiff and numpy array files
"""
outdir = params[C.OUT_DIR]
ts_dir = params[C.TIMESERIES_DIR]
vel_dir = params[C.VELOCITY_DIR]
log.debug('Saving PyRate outputs {}'.format(out_type))
gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt']
epochlist = ifgs_dict['epochlist']
if out_type in ('tsincr', 'tscuml'):
epoch = epochlist.dates[index + 1]
dest = join(ts_dir, out_type + "_" + str(epoch) + ".tif")
npy_file = join(ts_dir, out_type + "_" + str(epoch) + ".npy")
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = index + 1
md[ifc.EPOCH_DATE] = epoch
else:
dest = join(vel_dir, out_type + ".tif")
npy_file = join(vel_dir, out_type + '.npy')
md[ifc.EPOCH_DATE] = [d.strftime('%Y-%m-%d') for d in epochlist.dates]
md[ifc.DATA_TYPE] = out_type_md_dict[out_type]
if out_type in los_projection_out_types: # apply LOS projection and sign conversion for these outputs
incidence_path = Path(
Configuration.geometry_files(params)['incidence_angle'])
if incidence_path.exists(): # We can do LOS projection
if params[C.LOS_PROJECTION] == ifc.LINE_OF_SIGHT:
log.info(
f"Retaining Line-of-sight signal projection for file {dest}"
)
elif params[C.LOS_PROJECTION] != ifc.LINE_OF_SIGHT:
log.info(
f"Projecting Line-of-sight signal into "
f"{ifc.LOS_PROJECTION_OPTION[params[C.LOS_PROJECTION]]} for file {dest}"
)
incidence = shared.Geometry(incidence_path)
incidence.open()
array /= los_projection_divisors[params[C.LOS_PROJECTION]](
incidence.data) * params[C.SIGNAL_POLARITY]
md[C.LOS_PROJECTION.upper()] = ifc.LOS_PROJECTION_OPTION[params[
C.LOS_PROJECTION]]
md[C.SIGNAL_POLARITY.upper()] = params[C.SIGNAL_POLARITY]
if out_type in error_out_types: # add n-sigma value to error file metadata
# TODO: move the multiplication of nsigma and error data here?
md[C.VELERROR_NSIG.upper()] = params[C.VELERROR_NSIG]
log.info(
f"Saving file {dest} with {params[C.VELERROR_NSIG]}-sigma uncertainties"
)
shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan)
# clear the extra metadata so it does not mess up other images
if C.LOS_PROJECTION.upper() in md:
md.pop(C.LOS_PROJECTION.upper())
if C.SIGNAL_POLARITY.upper() in md:
md.pop(C.SIGNAL_POLARITY.upper())
if C.VELERROR_NSIG.upper() in md:
md.pop(C.VELERROR_NSIG.upper())
if savenpy:
np.save(file=npy_file, arr=array)
log.debug('Finished saving {}'.format(out_type))
