def _save_stack(ifgs_dict, params, tiles, out_type): """ Save stacking outputs """ log.info('Merging PyRate outputs {}'.format(out_type)) gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt'] epochlist = ifgs_dict['epochlist'] ifgs = [v for v in ifgs_dict.values() if isinstance(v, PrereadIfg)] dest = os.path.join(params[cf.OUT_DIR], out_type + ".tif") md[ifc.EPOCH_DATE] = epochlist.dates if out_type == 'stack_rate': md[ifc.DATA_TYPE] = ifc.STACKRATE elif out_type == 'stack_error': md[ifc.DATA_TYPE] = ifc.STACKERROR else: md[ifc.DATA_TYPE] = ifc.STACKSAMP rate = np.zeros(shape=ifgs[0].shape, dtype=np.float32) for t in tiles: rate_file = os.path.join(params[cf.TMPDIR], out_type + '_' + str(t.index) + '.npy') rate_file = Path(rate_file) rate_tile = np.load(file=rate_file) rate[t.top_left_y:t.bottom_right_y, t.top_left_x:t.bottom_right_x] = rate_tile shared.write_output_geotiff(md, gt, wkt, rate, dest, np.nan) npy_rate_file = os.path.join(params[cf.OUT_DIR], out_type + '.npy') np.save(file=npy_rate_file, arr=rate) log.debug('Finished PyRate merging {}'.format(out_type))
def _write_dem_errors(ifg_paths: list, params: dict, preread_ifgs: dict) -> None: """ Convenience function for writing DEM error (one file) and DEM error correction for each IFG to disc :param ifg_paths: List of interferogram class objects. :param params: Dictionary of PyRate configuration parameters. :param preread_ifgs: Dictionary of interferogram metadata. """ tiles = params[C.TILES] # re-assemble tiles and save into dem_error dir shape = preread_ifgs[ifg_paths[0]].shape # save dem error as geotiff file in out directory gt, md, wkt = shared.get_geotiff_header_info(ifg_paths[0]) md[ifc.DATA_TYPE] = ifc.DEM_ERROR dem_error = assemble_tiles(shape, params[C.TMPDIR], tiles, out_type='dem_error') dem_error_file = os.path.join(params[C.DEM_ERROR_DIR], 'dem_error.tif') shared.remove_file_if_exists(dem_error_file) shared.write_output_geotiff(md, gt, wkt, dem_error, dem_error_file, np.nan) # read the average bperp vals for each ifg and each tile bperp = np.empty(shape=(len(tiles), len(ifg_paths)), dtype=np.float64) for t in tiles: bperp_file = Path( join(params[C.TMPDIR], 'bperp_avg_' + str(t.index) + '.npy')) arr = np.load(file=bperp_file) bperp[t.index, :] = arr # loop over all ifgs idx = 0 for ifg_path in ifg_paths: ifg = Ifg(ifg_path) ifg.open() shared.nan_and_mm_convert(ifg, params) # ensure we have phase data in mm # read dem error correction file from tmpdir dem_error_correction_ifg = assemble_tiles( shape, params[C.TMPDIR], tiles, out_type='dem_error_correction', index=idx) # calculate average bperp value across all tiles for the ifg bperp_val = np.nanmean(bperp[:, idx]) dem_error_correction_on_disc = MultiplePaths.dem_error_path( ifg.data_path, params) np.save(file=dem_error_correction_on_disc, arr=dem_error_correction_ifg) idx += 1 # subtract DEM error from the ifg ifg.phase_data -= dem_error_correction_ifg _save_dem_error_corrected_phase(ifg, bperp_val)
def __save_geom_files(ifg_path, dest, array, out_type): """ Convenience function to save geometry geotiff files """ log.debug('Saving PyRate outputs {}'.format(out_type)) gt, md, wkt = shared.get_geotiff_header_info(ifg_path) md[ifc.DATA_TYPE] = out_type_md_dict[out_type] shared.remove_file_if_exists(dest) log.info(f"Writing geotiff: {dest}") shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan)
def _save_merged_files(ifgs_dict, outdir, array, out_type, index=None, savenpy=None): """ Convenience function to save PyRate geotiff and numpy array files """ log.debug('Saving PyRate outputs {}'.format(out_type)) gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt'] epochlist = ifgs_dict['epochlist'] if out_type in ('tsincr', 'tscuml'): epoch = epochlist.dates[index + 1] dest = join(outdir, out_type + "_" + str(epoch) + ".tif") npy_file = join(outdir, out_type + "_" + str(epoch) + ".npy") # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = index + 1 md[ifc.EPOCH_DATE] = epoch else: dest = join(outdir, out_type + ".tif") npy_file = join(outdir, out_type + '.npy') md[ifc.EPOCH_DATE] = [d.strftime('%Y-%m-%d') for d in epochlist.dates] if out_type == 'stack_rate': md[ifc.DATA_TYPE] = ifc.STACKRATE elif out_type == 'stack_error': md[ifc.DATA_TYPE] = ifc.STACKERROR elif out_type == 'stack_samples': md[ifc.DATA_TYPE] = ifc.STACKSAMP elif out_type == 'linear_rate': md[ifc.DATA_TYPE] = ifc.LINRATE elif out_type == 'linear_error': md[ifc.DATA_TYPE] = ifc.LINERROR elif out_type == 'linear_samples': md[ifc.DATA_TYPE] = ifc.LINSAMP elif out_type == 'linear_intercept': md[ifc.DATA_TYPE] = ifc.LINICPT elif out_type == 'linear_rsquared': md[ifc.DATA_TYPE] = ifc.LINRSQ elif out_type == 'tsincr': md[ifc.DATA_TYPE] = ifc.INCR elif out_type == 'tscuml': md[ifc.DATA_TYPE] = ifc.CUML else: log.warning('Output type "{}" not recognised'.format(out_type)) shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan) if savenpy: np.save(file=npy_file, arr=array) log.debug('Finished saving {}'.format(out_type))
def write_timeseries_geotiff(ifgs, params, tsincr, pr_type): # setup metadata for writing into result files gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] for i in range(tsincr.shape[2]): md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] md['SEQUENCE_POSITION'] = i + 1 # sequence position data = tsincr[:, :, i] dest = join(params[cf.OUT_DIR], pr_type + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = pr_type write_output_geotiff(md, gt, wkt, data, dest, np.nan)
def write_stackrate_tifs(ifgs, params, res): rate, error, samples = res gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] dest = join(params[cf.OUT_DIR], "stack_rate.tif") md[ifc.EPOCH_DATE] = epochlist.dates md[ifc.DATA_TYPE] = ifc.STACKRATE write_output_geotiff(md, gt, wkt, rate, dest, np.nan) dest = join(params[cf.OUT_DIR], "stack_error.tif") md[ifc.DATA_TYPE] = ifc.STACKERROR write_output_geotiff(md, gt, wkt, error, dest, np.nan) dest = join(params[cf.OUT_DIR], "stack_samples.tif") md[ifc.DATA_TYPE] = ifc.STACKSAMP write_output_geotiff(md, gt, wkt, samples, dest, np.nan) write_stackrate_numpy_files(error, rate, samples, params)
def write_linrate_tifs(ifgs, params, res): # log.info('Writing linrate results') rate, error, samples = res gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path) epochlist = algorithm.get_epochs(ifgs)[0] dest = join(params[cf.OUT_DIR], "linrate.tif") md[ifc.EPOCH_DATE] = epochlist.dates md[ifc.DATA_TYPE] = ifc.LINRATE write_output_geotiff(md, gt, wkt, rate, dest, np.nan) dest = join(params[cf.OUT_DIR], "linerror.tif") md[ifc.DATA_TYPE] = ifc.LINERROR write_output_geotiff(md, gt, wkt, error, dest, np.nan) dest = join(params[cf.OUT_DIR], "linsamples.tif") md[ifc.DATA_TYPE] = ifc.LINSAMP write_output_geotiff(md, gt, wkt, samples, dest, np.nan) write_linrate_numpy_files(error, rate, samples, params)
def _merge_timeseries(rows, cols, params): """ Merge time series output """ xlks, _, crop = cf.transform_params(params) base_unw_paths = [] for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"): if "dem" not in str(p): base_unw_paths.append(str(p)) if "tif" in base_unw_paths[0].split(".")[1]: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) for i, dest_tif in enumerate(dest_tifs): dest_tifs[i] = dest_tif.replace("_tif", "") else: dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks) output_dir = params[cf.TMPDIR] # load previously saved prepread_ifgs dict preread_ifgs_file = join(output_dir, 'preread_ifgs.pk') ifgs = cp.load(open(preread_ifgs_file, 'rb')) # metadata and projections gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt'] epochlist = ifgs['epochlist'] ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)] tiles = shared.get_tiles(dest_tifs[0], rows, cols) # load the first tsincr file to determine the number of time series tifs tsincr_file = os.path.join(output_dir, 'tsincr_0.npy') tsincr = np.load(file=tsincr_file) # pylint: disable=no-member no_ts_tifs = tsincr.shape[2] # we create 2 x no_ts_tifs as we are splitting tsincr and tscuml # to all processes. process_tifs = mpiops.array_split(range(2 * no_ts_tifs)) # depending on nvelpar, this will not fit in memory # e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory # 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB # the double for loop helps us overcome the memory limit log.info('Process {} writing {} timeseries tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2)) for i in process_tifs: tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32) if i < no_ts_tifs: for n, t in enumerate(tiles): _assemble_tiles(i, n, t, tscum_g, output_dir, 'tscuml') md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = i + 1 dest = os.path.join( params[cf.OUT_DIR], 'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = ifc.CUML shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan) else: tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32) i %= no_ts_tifs for n, t in enumerate(tiles): _assemble_tiles(i, n, t, tsincr_g, output_dir, 'tsincr') md[ifc.EPOCH_DATE] = epochlist.dates[i + 1] # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = i + 1 dest = os.path.join( params[cf.OUT_DIR], 'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif") md[ifc.DATA_TYPE] = ifc.INCR shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest, np.nan) mpiops.comm.barrier() log.debug('Process {} finished writing {} timeseries tifs of ' 'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
def __save_merged_files(ifgs_dict, params, array, out_type, index=None, savenpy=None): """ Convenience function to save PyRate geotiff and numpy array files """ outdir = params[C.OUT_DIR] ts_dir = params[C.TIMESERIES_DIR] vel_dir = params[C.VELOCITY_DIR] log.debug('Saving PyRate outputs {}'.format(out_type)) gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt'] epochlist = ifgs_dict['epochlist'] if out_type in ('tsincr', 'tscuml'): epoch = epochlist.dates[index + 1] dest = join(ts_dir, out_type + "_" + str(epoch) + ".tif") npy_file = join(ts_dir, out_type + "_" + str(epoch) + ".npy") # sequence position; first time slice is #0 md['SEQUENCE_POSITION'] = index + 1 md[ifc.EPOCH_DATE] = epoch else: dest = join(vel_dir, out_type + ".tif") npy_file = join(vel_dir, out_type + '.npy') md[ifc.EPOCH_DATE] = [d.strftime('%Y-%m-%d') for d in epochlist.dates] md[ifc.DATA_TYPE] = out_type_md_dict[out_type] if out_type in los_projection_out_types: # apply LOS projection and sign conversion for these outputs incidence_path = Path( Configuration.geometry_files(params)['incidence_angle']) if incidence_path.exists(): # We can do LOS projection if params[C.LOS_PROJECTION] == ifc.LINE_OF_SIGHT: log.info( f"Retaining Line-of-sight signal projection for file {dest}" ) elif params[C.LOS_PROJECTION] != ifc.LINE_OF_SIGHT: log.info( f"Projecting Line-of-sight signal into " f"{ifc.LOS_PROJECTION_OPTION[params[C.LOS_PROJECTION]]} for file {dest}" ) incidence = shared.Geometry(incidence_path) incidence.open() array /= los_projection_divisors[params[C.LOS_PROJECTION]]( incidence.data) * params[C.SIGNAL_POLARITY] md[C.LOS_PROJECTION.upper()] = ifc.LOS_PROJECTION_OPTION[params[ C.LOS_PROJECTION]] md[C.SIGNAL_POLARITY.upper()] = params[C.SIGNAL_POLARITY] if out_type in error_out_types: # add n-sigma value to error file metadata # TODO: move the multiplication of nsigma and error data here? md[C.VELERROR_NSIG.upper()] = params[C.VELERROR_NSIG] log.info( f"Saving file {dest} with {params[C.VELERROR_NSIG]}-sigma uncertainties" ) shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan) # clear the extra metadata so it does not mess up other images if C.LOS_PROJECTION.upper() in md: md.pop(C.LOS_PROJECTION.upper()) if C.SIGNAL_POLARITY.upper() in md: md.pop(C.SIGNAL_POLARITY.upper()) if C.VELERROR_NSIG.upper() in md: md.pop(C.VELERROR_NSIG.upper()) if savenpy: np.save(file=npy_file, arr=array) log.debug('Finished saving {}'.format(out_type))