def test_mixed_metadata_raises(self):
# change config to 5 ifgs
self.params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(p, self.params) for p in self.small_tifs[:5]
]
for p in self.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.sampled_path
# correct reference phase for some of the ifgs
ref_phase_est_wrapper(self.params)
for ifg in self.ifgs:
ifg.open()
# change config to all ifgs
self.params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(p, self.params) for p in self.small_tifs
]
for p in self.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.sampled_path
# now it should raise exception if we want to correct refernece phase again on all of them
with pytest.raises(CorrectionStatusError):
ref_phase_est_wrapper(self.params)
def test_orb_error(self, orbfit_method, orbfit_degrees):
self.params[cf.ORBITAL_FIT_METHOD] = orbfit_method
self.params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
multi_paths = self.params[cf.INTERFEROGRAM_FILES]
self.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
remove_orbital_error(self.ifg_paths, self.params)
# test_orb_errors_written
orb_error_files = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths]
assert all(p.exists() for p in orb_error_files)
last_mod_times = np.array([os.stat(o).st_mtime for o in orb_error_files])
# run orbit removal again
remove_orbital_error(self.ifg_paths, self.params)
orb_error_files2 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths]
# if files are written again - times will change
last_mod_times_2 = np.array([os.stat(o).st_mtime for o in orb_error_files2])
# test_orb_error_reused_if_params_unchanged
assert all(a == b for a, b in zip(last_mod_times, last_mod_times_2))
# change one of the params
_degrees = set(cf.ORB_DEGREE_NAMES.keys())
_degrees.discard(orbfit_degrees)
# test_orb_errors_recalculated_if_params_change
self.params[cf.ORBITAL_FIT_DEGREE] = _degrees.pop()
remove_orbital_error(self.ifg_paths, self.params)
orb_error_files3 = [MultiplePaths.orb_error_path(i, self.params) for i in self.ifg_paths]
last_mod_times_3 = np.array([os.stat(o).st_mtime for o in orb_error_files3])
assert all(a != b for a, b in zip(last_mod_times, last_mod_times_3))
def test_multilooked_tiffs_converted_to_unw_are_same(self):
# Get multilooked geotiffs
geotiffs = list(set(self.dest_paths))
geotiffs = [g for g in geotiffs if 'dem' not in g]
# Convert back to .unw
dest_unws = []
for g in set(geotiffs):
dest_unw = os.path.join(self.params[cf.OUT_DIR],
Path(g).stem + '.unw')
shared.write_unw_from_data_or_geotiff(geotif_or_data=g,
dest_unw=dest_unw,
ifg_proc=1)
dest_unws.append(dest_unw)
dest_unws_ = []
for d in dest_unws:
dest_unws_.append(MultiplePaths(d, self.params))
# Convert back to tiff
new_geotiffs_ = conv2tif.do_geotiff(dest_unws_, self.params)
new_geotiffs = [gt for gt, b in new_geotiffs_]
# Ensure original multilooked geotiffs and
# unw back to geotiff are the same
geotiffs.sort()
new_geotiffs.sort()
for g, u in zip(geotiffs, new_geotiffs):
g_ds = gdal.Open(g)
u_gs = gdal.Open(u)
np.testing.assert_array_almost_equal(u_gs.ReadAsArray(),
g_ds.ReadAsArray())
u_gs = None
g_ds = None
def _ts_to_ifgs(tsincr, preread_ifgs, params):
"""
Function that converts an incremental displacement time series into
interferometric phase observations. Used to re-construct an interferogram
network from a time series.
:param ndarray tsincr: incremental time series array of size
(ifg.shape, nepochs-1)
:param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances
:return: None, interferograms are saved to disk
"""
log.debug('Reconstructing interferometric observations from time series')
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
_, n = mpiops.run_once(get_epochs, ifgs)
index_first, index_second = n[:len(ifgs)], n[len(ifgs):]
num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs)))
num_ifgs_tuples = [(int(num), ifg) for num, ifg in num_ifgs_tuples]
for i, ifg in num_ifgs_tuples:
aps_correction_on_disc = MultiplePaths.aps_error_path(
ifg.tmp_path, params)
phase = np.sum(tsincr[:, :, index_first[i]:index_second[i]], axis=2)
np.save(file=aps_correction_on_disc, arr=phase)
_save_aps_corrected_phase(ifg.tmp_path, phase)
def test_multilooked_tiffs_converted_to_unw_are_same(self):
# Get multilooked geotiffs
geotiffs = self.dest_paths
# Convert back to .unw
dest_unws = []
for g in geotiffs:
dest_unw = os.path.join(self.params[cf.OUT_DIR],
os.path.splitext(g)[0] + '.unw')
shared.write_unw_from_data_or_geotiff(geotif_or_data=g,
dest_unw=dest_unw,
ifg_proc=1)
dest_unws.append(dest_unw)
dest_unws_ = [
MultiplePaths(self.params[cf.OUT_DIR], b, self.params[cf.IFG_LKSX],
self.params[cf.IFG_CROP_OPT]) for b in dest_unws
]
# Convert back to tiff
new_geotiffs_ = conv2tif.do_geotiff(dest_unws_, self.params)
new_geotiffs = [gt for gt, b in new_geotiffs_]
# Ensure original multilooked geotiffs and
# unw back to geotiff are the same
for g, u in zip(geotiffs, new_geotiffs):
g_ds = gdal.Open(g)
u_gs = gdal.Open(u)
np.testing.assert_array_almost_equal(u_gs.ReadAsArray(),
g_ds.ReadAsArray())
u_gs = None
g_ds = None
def setup_method(self):
self.params = Configuration(common.TEST_CONF_GAMMA).__dict__
self.tmp_dir = tempfile.mkdtemp()
self.params[cf.OUT_DIR] = self.tmp_dir
self.params[cf.REF_EST_METHOD] = 1
self.params[cf.PARALLEL] = False
self.params[cf.TMPDIR] = self.tmp_dir
common.copytree(common.SML_TEST_TIF, self.tmp_dir)
self.small_tifs = glob.glob(os.path.join(self.tmp_dir, "*.tif"))
for s in self.small_tifs:
os.chmod(s, 0o644)
self.ifgs = common.small_data_setup(self.tmp_dir, is_dir=True)
self.params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(p, self.params) for p in self.small_tifs
]
for p in self.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.sampled_path
for ifg in self.ifgs:
ifg.close()
self.params[cf.REFX], self.params[cf.REFY] = -1, -1
self.params[cf.REFNX], self.params[cf.REFNY] = 10, 10
self.params[cf.REF_CHIP_SIZE], self.params[cf.REF_MIN_FRAC] = 21, 0.5
self.params['rows'], self.params['cols'] = 3, 2
self.params[cf.REF_PIXEL_FILE] = Configuration.ref_pixel_path(
self.params)
correct._update_params_with_tiles(self.params)
correct.ref_pixel_calc_wrapper(self.params)
def __check_and_apply_demerrors_found_on_disc(ifg_paths: list,
params: dict) -> bool:
"""
Convenience function to check if DEM error correction files have already been produced in a previous run
:param ifg_paths: List of interferogram class objects.
:param params: Dictionary of PyRate configuration parameters.
:return: bool value: True if dem error files found on disc, otherwise False.
"""
saved_dem_err_paths = [
MultiplePaths.dem_error_path(ifg_path, params)
for ifg_path in ifg_paths
]
for d, i in zip(saved_dem_err_paths, ifg_paths):
if d.exists():
dem_corr = np.load(d)
if isinstance(i, str):
# are paths
ifg = Ifg(i)
ifg.open()
else:
ifg = i
ifg.phase_data -= dem_corr
# set geotiff meta tag and save phase to file
# TODO: calculate avg bperp and add to metadata even for reused DEM error correction
_save_dem_error_corrected_phase(ifg)
return all(d.exists() for d in saved_dem_err_paths)
def spatio_temporal_filter(params: dict) -> None:
"""
Applies a spatio-temporal filter to remove the atmospheric phase screen
(APS) and saves the corrected interferograms. Firstly the incremental
time series is computed using the SVD method, before a cascade of temporal
then spatial Gaussian filters is applied. The resulting APS corrections are
saved to disc before being subtracted from each interferogram.
:param params: Dictionary of PyRate configuration parameters.
"""
if params[C.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[C.TILES]
preread_ifgs = params[C.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_paths = [MultiplePaths.aps_error_path(i, params) for i in ifg_paths]
if all(a.exists() for a in aps_paths):
log.warning('Reusing APS errors from previous run')
_apply_aps_correction(ifg_paths, aps_paths, params)
return
# obtain the incremental time series using SVD
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
# get lists of epochs and ifgs
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
epochlist = mpiops.run_once(get_epochs, ifgs)[0]
# first perform temporal high pass filter
ts_hp = temporal_high_pass_filter(tsincr, epochlist, params)
# second perform spatial low pass filter to obtain APS correction in ts domain
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
ts_aps = spatial_low_pass_filter(ts_hp, ifg, params)
ifg.close()
# construct APS corrections for each ifg
_make_aps_corrections(ts_aps, ifgs, params)
# apply correction to ifgs and save ifgs to disc.
_apply_aps_correction(ifg_paths, aps_paths, params)
# update/save the phase_data in the tiled numpy files
shared.save_numpy_phase(ifg_paths, params)
def _write_dem_errors(ifg_paths: list, params: dict,
preread_ifgs: dict) -> None:
"""
Convenience function for writing DEM error (one file) and DEM error correction for each IFG to disc
:param ifg_paths: List of interferogram class objects.
:param params: Dictionary of PyRate configuration parameters.
:param preread_ifgs: Dictionary of interferogram metadata.
"""
tiles = params[C.TILES]
# re-assemble tiles and save into dem_error dir
shape = preread_ifgs[ifg_paths[0]].shape
# save dem error as geotiff file in out directory
gt, md, wkt = shared.get_geotiff_header_info(ifg_paths[0])
md[ifc.DATA_TYPE] = ifc.DEM_ERROR
dem_error = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type='dem_error')
dem_error_file = os.path.join(params[C.DEM_ERROR_DIR], 'dem_error.tif')
shared.remove_file_if_exists(dem_error_file)
shared.write_output_geotiff(md, gt, wkt, dem_error, dem_error_file, np.nan)
# read the average bperp vals for each ifg and each tile
bperp = np.empty(shape=(len(tiles), len(ifg_paths)), dtype=np.float64)
for t in tiles:
bperp_file = Path(
join(params[C.TMPDIR], 'bperp_avg_' + str(t.index) + '.npy'))
arr = np.load(file=bperp_file)
bperp[t.index, :] = arr
# loop over all ifgs
idx = 0
for ifg_path in ifg_paths:
ifg = Ifg(ifg_path)
ifg.open()
shared.nan_and_mm_convert(ifg,
params) # ensure we have phase data in mm
# read dem error correction file from tmpdir
dem_error_correction_ifg = assemble_tiles(
shape,
params[C.TMPDIR],
tiles,
out_type='dem_error_correction',
index=idx)
# calculate average bperp value across all tiles for the ifg
bperp_val = np.nanmean(bperp[:, idx])
dem_error_correction_on_disc = MultiplePaths.dem_error_path(
ifg.data_path, params)
np.save(file=dem_error_correction_on_disc,
arr=dem_error_correction_ifg)
idx += 1
# subtract DEM error from the ifg
ifg.phase_data -= dem_error_correction_ifg
_save_dem_error_corrected_phase(ifg, bperp_val)
def independent_orbital_correction(ifg, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using an offset
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
degree = params[cf.ORBITAL_FIT_DEGREE]
offset = params[cf.ORBFIT_OFFSET]
orbfit_correction_on_disc = MultiplePaths.orb_error_path(
ifg.data_path, params)
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
if orbfit_correction_on_disc.exists():
log.info(
f'Reusing already computed orbital fit correction for {ifg.data_path}'
)
orbital_correction = np.load(file=orbfit_correction_on_disc)
else:
# vectorise, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
dm = get_design_matrix(ifg, degree, offset)
# filter NaNs out before getting model
clean_dm = dm[~isnan(vphase)]
data = vphase[~isnan(vphase)]
model = lstsq(clean_dm, data)[0] # first arg is the model params
# calculate forward model & morph back to 2D
if offset:
fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]),
ifg.phase_data.shape)
else:
fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape)
if not orbfit_correction_on_disc.parent.exists():
shared.mkdir_p(orbfit_correction_on_disc.parent)
offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb))
orbital_correction = fullorb - offset_removal
# dump to disc
np.save(file=orbfit_correction_on_disc, arr=orbital_correction)
# subtract orbital error from the ifg
ifg.phase_data -= orbital_correction
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
ifg.close()
def setup_class(cls):
# testing constants2
cls.BASE_DIR = tempfile.mkdtemp()
cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out')
cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem')
cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif')
try:
# copy source data (treat as prepifg already run)
os.makedirs(cls.BASE_OUT_DIR)
for path in glob.glob(join(common.SML_TEST_TIF, '*')):
dest = join(cls.BASE_OUT_DIR, os.path.basename(path))
shutil.copy(path, dest)
os.chmod(dest, 0o660)
os.makedirs(cls.BASE_DEM_DIR)
orig_dem = common.SML_TEST_DEM_TIF
os.symlink(orig_dem, cls.BASE_DEM_FILE)
os.chdir(cls.BASE_DIR)
# Turn off validation because we're in a different working dir
# and relative paths in config won't be work.
params = config.get_config_params(common.TEST_CONF_ROIPAC)
params['correct'] = [
'orbfit', 'refphase', 'mst', 'apscorrect', 'maxvar'
]
params[cf.OUT_DIR] = cls.BASE_OUT_DIR
params[cf.PROCESSOR] = 0 # roipac
params[cf.APS_CORRECTION] = 0
paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif'))
paths = sorted(paths)
params[cf.PARALLEL] = False
params[cf.ORBFIT_OFFSET] = True
params[cf.TEMP_MLOOKED_DIR] = cls.BASE_OUT_DIR.join(
cf.TEMP_MLOOKED_DIR)
params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(p, params) for p in paths
]
for p in params[cf.INTERFEROGRAM_FILES]: # cheat
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.converted_path
params["rows"], params["cols"] = 2, 2
params[cf.REF_PIXEL_FILE] = Configuration.ref_pixel_path(params)
Path(params[cf.OUT_DIR]).joinpath(cf.APS_ERROR_DIR).mkdir(
exist_ok=True, parents=True)
Path(params[cf.OUT_DIR]).joinpath(cf.MST_DIR).mkdir(exist_ok=True,
parents=True)
correct.correct_ifgs(params)
if not hasattr(cls, 'ifgs'):
cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR)
except:
# revert working dir & avoid paths busting other tests
os.chdir(CURRENT_DIR)
raise
def test_need_at_least_two_ifgs(self):
self.params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(p, self.params) for p in self.small_tifs[:1]
]
for p in self.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
p.tmp_sampled_path = p.sampled_path
with pytest.raises(ReferencePhaseError):
ref_phase_est_wrapper(self.params)
def setup_class(cls):
roipac_params = Configuration(TEST_CONF_ROIPAC).__dict__
from copy import deepcopy
params = deepcopy(roipac_params)
shared.mkdir_p(params[C.TMPDIR])
params[C.REF_EST_METHOD] = 2
conv2tif.main(params)
params = deepcopy(roipac_params)
prepifg.main(params)
params = deepcopy(roipac_params)
base_ifg_paths = [
c.unwrapped_path for c in params[C.INTERFEROGRAM_FILES]
]
dest_paths = [c.converted_path for c in params[C.INTERFEROGRAM_FILES]]
params[C.INTERFEROGRAM_FILES] = [
MultiplePaths(d, params) for d in dest_paths
]
for p in params[C.INTERFEROGRAM_FILES]: # hack
p.sampled_path = p.converted_path
for i in dest_paths:
Path(i).chmod(
0o664
) # assign write permission as conv2tif output is readonly
ifgs = common.pre_prepare_ifgs(dest_paths, params)
correct._copy_mlooked(params)
correct._update_params_with_tiles(params)
correct._create_ifg_dict(params)
pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
params[C.ORBFIT_OFFSET] = True
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
for p in params[C.INTERFEROGRAM_FILES]: # hack
p.tmp_sampled_path = p.sampled_path
_, cls.ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
ifgs[0].open()
r_dist = RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
cls.maxvar = [
cvd(i,
params,
r_dist,
calc_alpha=True,
save_acg=True,
write_vals=True)[0] for i in dest_paths
]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
for ifg in ifgs:
ifg.close()
cls.params = params
def __run_once(self):
dem_files = [
MultiplePaths.dem_error_path(i, self.params)
for i in self.ifg_paths
]
correct._copy_mlooked(self.params)
correct._update_params_with_tiles(self.params)
correct._create_ifg_dict(self.params)
save_numpy_phase(self.ifg_paths, self.params)
dem_error_calc_wrapper(self.params)
assert all(m.exists() for m in dem_files)
return [os.stat(o).st_mtime for o in dem_files]
def test_aps_error_files_on_disc(self, slpfmethod, slpfcutoff, slpforder):
self.params[cf.SLPF_METHOD] = slpfmethod
self.params[cf.SLPF_CUTOFF] = slpfcutoff
self.params[cf.SLPF_ORDER] = slpforder
wrap_spatio_temporal_filter(self.params)
# test_orb_errors_written
aps_error_files = [
MultiplePaths.aps_error_path(i, self.params)
for i in self.ifg_paths
]
assert all(p.exists() for p in aps_error_files)
last_mod_times = [os.stat(o).st_mtime for o in aps_error_files]
phase_prev = [i.phase_data for i in self.ifgs]
# run aps error removal again
wrap_spatio_temporal_filter(self.params)
aps_error_files2 = [
MultiplePaths.aps_error_path(i, self.params)
for i in self.ifg_paths
]
# if files are written again - times will change
last_mod_times_2 = [os.stat(o).st_mtime for o in aps_error_files2]
# test_aps_error_reused_if_params_unchanged
assert all(a == b for a, b in zip(last_mod_times, last_mod_times_2))
phase_now = [i.phase_data for i in self.ifgs]
# run aps error correction once mroe
wrap_spatio_temporal_filter(self.params)
aps_error_files3 = [
MultiplePaths.aps_error_path(i, self.params)
for i in self.ifg_paths
]
last_mod_times_3 = [os.stat(o).st_mtime for o in aps_error_files3]
assert all(a == b for a, b in zip(last_mod_times, last_mod_times_3))
phase_again = [i.phase_data for i in self.ifgs]
np.testing.assert_array_equal(phase_prev, phase_now)
np.testing.assert_array_equal(phase_prev, phase_again)
def prepare_ifgs(raster_data_paths,
crop_opt,
xlooks,
ylooks,
headers,
params,
thresh=0.5,
user_exts=None,
write_to_disc=True):
"""
Wrapper function to prepare a sequence of interferogram files for
PyRate analysis. See prepifg.prepare_ifg() for full description of
inputs and returns.
Note: function need refining for crop options
:param list raster_data_paths: List of interferogram file paths
:param int crop_opt: Crop option
:param int xlooks: Number of multi-looks in x; 5 is 5 times smaller, 1 is no change
:param int ylooks: Number of multi-looks in y
:param float thresh: see thresh in prepare_ifgs()
:param tuple user_exts: Tuple of user defined georeferenced extents for
new file: (xfirst, yfirst, xlast, ylast)cropping coordinates
:param bool write_to_disc: Write new data to disk
:return: resampled_data: output cropped and resampled image
:rtype: ndarray
:return: out_ds: destination gdal dataset object
:rtype: List[gdal.Dataset]
"""
if xlooks != ylooks:
log.warning('X and Y multi-look factors are not equal')
# use metadata check to check whether it's a dem or ifg
rasters = [dem_or_ifg(r) for r in raster_data_paths]
exts = get_analysis_extent(crop_opt, rasters, xlooks, ylooks, user_exts)
out_paths = []
for r, t in zip(raster_data_paths, rasters):
if isinstance(t, DEM):
input_type = InputTypes.DEM
else:
input_type = InputTypes.IFG
out_path = MultiplePaths(r, params, input_type).sampled_path
out_paths.append(out_path)
return [
prepare_ifg(d, xlooks, ylooks, exts, thresh, crop_opt, h,
write_to_disc, p)
for d, h, p in zip(raster_data_paths, headers, out_paths)
]
def _remove_network_orb_error(coefs, dm, ifg, ids, offset, params):
"""
remove network orbital error from input interferograms
"""
saved_orb_err_path = MultiplePaths.orb_error_path(ifg.data_path, params)
orb = dm.dot(coefs[ids[ifg.second]] - coefs[ids[ifg.first]])
orb = orb.reshape(ifg.shape)
# offset estimation
if offset:
# bring all ifgs to same base level
orb -= nanmedian(np.ravel(ifg.phase_data - orb))
# subtract orbital error from the ifg
ifg.phase_data -= orb
# save orb error on disc
np.save(file=saved_orb_err_path, arr=orb)
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
def test_orbital_correction_legacy_equality(self):
from pyrate import correct
from pyrate.configuration import MultiplePaths
multi_paths = [MultiplePaths(p, params=self.params) for p in self.ifg_paths]
for m in multi_paths: # cheat
m.sampled_path = m.converted_path
self.params[cf.INTERFEROGRAM_FILES] = multi_paths
self.params['rows'], self.params['cols'] = 2, 3
Path(self.BASE_DIR).joinpath('tmpdir').mkdir(exist_ok=True, parents=True)
correct._copy_mlooked(self.params)
correct._update_params_with_tiles(self.params)
correct._create_ifg_dict(self.params)
correct._copy_mlooked(self.params)
pyrate.core.orbital.orb_fit_calc_wrapper(self.params)
onlyfiles = [f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method1_')]
count = 0
for i, f in enumerate(onlyfiles):
ifg_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f), delimiter=',')
for k, j in enumerate([m.tmp_sampled_path for m in multi_paths]):
ifg = Ifg(j)
ifg.open()
print(f)
print(j)
if os.path.basename(j).split('_ifg.')[0] == os.path.basename(f).split(
'_orb_planar_1lks_method1_geo_')[1].split('.')[0]:
count += 1
# all numbers equal
np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2)
# means must also be equal
assert np.nanmean(ifg_data) == pytest.approx(np.nanmean(ifg.phase_data), abs=1e-2)
# number of nans must equal
assert np.sum(np.isnan(ifg_data)) == np.sum(np.isnan(ifg.phase_data))
ifg.close()
# ensure that we have expected number of matches
assert count == len(self.ifg_paths)
def __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
saved_orb_err_paths = [
MultiplePaths.orb_error_path(ifg_path, params)
for ifg_path in ifg_paths
]
for p, i in zip(saved_orb_err_paths, ifg_paths):
if p.exists():
orb = np.load(p)
if isinstance(i, str):
# are paths
ifg = Ifg(i)
ifg.open(readonly=False)
shared.nan_and_mm_convert(ifg, params)
else:
ifg = i
ifg.phase_data -= orb
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
return all(p.exists() for p in saved_orb_err_paths)
def _remove_network_orb_error(coefs, dm, ifg, ids, offset, params):
"""
remove network orbital error from input interferograms
"""
saved_orb_err_path = MultiplePaths.orb_error_path(ifg.data_path, params)
orb = dm.dot(coefs[ids[ifg.second]] - coefs[ids[ifg.first]])
orb = orb.reshape(ifg.shape)
# Estimate the offset of the interferogram as the median of ifg minus model
# Only needed if reference phase correction has already been applied?
if offset:
# brings all ifgs to same reference level
orb -= nanmedian(np.ravel(ifg.phase_data - orb))
# subtract orbital error from the ifg
ifg.phase_data -= orb
# save orb error on disc
np.save(file=saved_orb_err_path, arr=orb)
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg, params)
def test_orbital_correction_legacy_equality(self):
from pyrate import process
from pyrate.configuration import MultiplePaths
multi_paths = [MultiplePaths(self.BASE_DIR, p) for p in self.ifg_paths]
for m in multi_paths: # cheat
m.sampled_path = m.converted_path
process._orb_fit_calc(multi_paths, self.params)
onlyfiles = [
f for f in os.listdir(SML_TEST_LEGACY_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method1_')
]
count = 0
for i, f in enumerate(onlyfiles):
ifg_data = np.genfromtxt(os.path.join(SML_TEST_LEGACY_ORBITAL_DIR,
f),
delimiter=',')
for k, j in enumerate(self.ifg_paths):
ifg = Ifg(j)
ifg.open()
if os.path.basename(j).split('_unw.')[0] == os.path.basename(
f).split('_orb_planar_1lks_method1_')[1].split('.')[0]:
count += 1
# all numbers equal
np.testing.assert_array_almost_equal(ifg_data,
ifg.phase_data,
decimal=2)
# means must also be equal
self.assertAlmostEqual(np.nanmean(ifg_data),
np.nanmean(ifg.phase_data),
places=2)
# number of nans must equal
self.assertEqual(np.sum(np.isnan(ifg_data)),
np.sum(np.isnan(ifg.phase_data)))
ifg.close()
# ensure that we have expected number of matches
self.assertEqual(count, len(self.ifg_paths))
def setUpClass(cls):
# testing constants2
cls.BASE_DIR = tempfile.mkdtemp()
cls.BASE_OUT_DIR = join(cls.BASE_DIR, 'out')
cls.BASE_DEM_DIR = join(cls.BASE_DIR, 'dem')
cls.BASE_DEM_FILE = join(cls.BASE_DEM_DIR, 'roipac_test_trimmed.tif')
try:
# copy source data (treat as prepifg already run)
os.makedirs(cls.BASE_OUT_DIR)
for path in glob.glob(join(common.SML_TEST_TIF, '*')):
dest = join(cls.BASE_OUT_DIR, os.path.basename(path))
shutil.copy(path, dest)
os.chmod(dest, 0o660)
os.makedirs(cls.BASE_DEM_DIR)
orig_dem = common.SML_TEST_DEM_TIF
os.symlink(orig_dem, cls.BASE_DEM_FILE)
os.chdir(cls.BASE_DIR)
# Turn off validation because we're in a different working dir
# and relative paths in config won't be work.
params = config.get_config_params(common.TEST_CONF_ROIPAC)
params[cf.OUT_DIR] = cls.BASE_OUT_DIR
params[cf.PROCESSOR] = 0 # roipac
params[cf.APS_CORRECTION] = 0
paths = glob.glob(join(cls.BASE_OUT_DIR, 'geo_*-*.tif'))
params[cf.PARALLEL] = False
params[cf.INTERFEROGRAM_FILES] = [
MultiplePaths(cls.BASE_OUT_DIR, p) for p in paths
]
for p in params[cf.INTERFEROGRAM_FILES]: # cheat
p.sampled_path = p.converted_path
process.process_ifgs(sorted(paths), params, 2, 2)
if not hasattr(cls, 'ifgs'):
cls.ifgs = get_ifgs(out_dir=cls.BASE_OUT_DIR)
except:
# revert working dir & avoid paths busting other tests
os.chdir(CURRENT_DIR)
raise
def wrap_spatio_temporal_filter(params):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[cf.TILES]
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path
for ifg_path in params[cf.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_error_files_on_disc = [
MultiplePaths.aps_error_path(i, params) for i in ifg_paths
]
if all(a.exists() for a in aps_error_files_on_disc):
log.warning("Reusing APS errors from previous run!!!")
for ifg_path, a in mpiops.array_split(
list(zip(ifg_paths, aps_error_files_on_disc))):
phase = np.load(a)
_save_aps_corrected_phase(ifg_path, phase)
else:
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
def _make_aps_corrections(ts_aps: np.ndarray, ifgs: List[Ifg],
params: dict) -> None:
"""
Function to convert the time series APS filter output into interferometric
phase corrections and save them to disc.
:param ts_aps: Incremental APS time series array.
:param ifgs: List of Ifg class objects.
:param params: Dictionary of PyRate configuration parameters.
"""
log.debug('Reconstructing interferometric observations from time series')
# get first and second image indices
_, n = mpiops.run_once(get_epochs, ifgs)
index_first, index_second = n[:len(ifgs)], n[len(ifgs):]
num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs)))
for i, ifg in [(int(num), ifg) for num, ifg in num_ifgs_tuples]:
# sum time slice data from first to second epoch
ifg_aps = np.sum(ts_aps[:, :, index_first[i]:index_second[i]], axis=2)
aps_error_on_disc = MultiplePaths.aps_error_path(ifg.tmp_path, params)
np.save(file=aps_error_on_disc, arr=ifg_aps) # save APS as numpy array
mpiops.comm.barrier()
def setup_class(cls):
# change to orbital error correction method 2
cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.BASE_DIR = cls.params[cf.OUT_DIR]
cls.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
cls.params[cf.ORBITAL_FIT_LOOKS_X] = 1
cls.params[cf.ORBITAL_FIT_LOOKS_Y] = 1
cls.params[cf.ORBFIT_OFFSET] = True
cls.params[cf.OUT_DIR] = cls.BASE_DIR
data_paths = [os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()]
cls.new_data_paths = [os.path.join(cls.BASE_DIR, os.path.basename(d)) for d in data_paths]
cls.params[cf.INTERFEROGRAM_FILES] = [MultiplePaths(file_name=d, params=cls.params) for d in data_paths]
for p in cls.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
# copy the files from the dir into temp dir
for d in data_paths:
d_copy = os.path.join(cls.BASE_DIR, os.path.basename(d))
shutil.copy(d, d_copy)
os.chmod(d_copy, 0o660)
cls.headers = [roipac.roipac_header(i, cls.params) for i in cls.new_data_paths]
cls.orb_error_dir = Path(cls.params[cf.OUT_DIR]).joinpath(ORB_ERROR_DIR)
cls.orb_error_dir.mkdir(parents=True, exist_ok=True)
def independent_orbital_correction(ifg_path, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
log.debug(f"Orbital correction of {ifg_path}")
degree = params[C.ORBITAL_FIT_DEGREE]
offset = params[C.ORBFIT_OFFSET]
intercept = params[C.ORBFIT_INTERCEPT]
xlooks = params[C.ORBITAL_FIT_LOOKS_X]
ylooks = params[C.ORBITAL_FIT_LOOKS_Y]
ifg0 = shared.Ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path
# get full-resolution design matrix
fullres_dm = get_design_matrix(ifg0, degree, intercept=intercept)
ifg = shared.dem_or_ifg(ifg_path) if isinstance(ifg_path,
str) else ifg_path
multi_path = MultiplePaths(ifg.data_path, params)
orb_on_disc = MultiplePaths.orb_error_path(ifg.data_path, params)
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
fullres_ifg = ifg # keep a backup
fullres_phase = fullres_ifg.phase_data
if orb_on_disc.exists():
log.info(
f'Reusing already computed orbital fit correction: {orb_on_disc}')
orbital_correction = np.load(file=orb_on_disc)
else:
# Multi-look the ifg data if either X or Y is greater than 1
if (xlooks > 1) or (ylooks > 1):
exts, _, _ = __extents_from_params(params)
mlooked = _create_mlooked_dataset(multi_path, ifg.data_path, exts,
params)
ifg = Ifg(mlooked) # multi-looked Ifg object
ifg.initialize()
shared.nan_and_mm_convert(ifg, params)
# vectorise phase data, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
# compute design matrix for multi-looked data
mlooked_dm = get_design_matrix(ifg, degree, intercept=intercept)
# invert to obtain the correction image (forward model) at full-res
orbital_correction = __orb_correction(fullres_dm,
mlooked_dm,
fullres_phase,
vphase,
offset=offset)
# save correction to disc
if not orb_on_disc.parent.exists():
shared.mkdir_p(orb_on_disc.parent)
np.save(file=orb_on_disc, arr=orbital_correction)
# subtract orbital correction from the full-res ifg
fullres_ifg.phase_data -= orbital_correction
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(fullres_ifg, params)
def test_coherence_files_not_converted():
# define constants
NO_DATA_VALUE = 0
driver = gdal.GetDriverByName('GTiff')
# create a sample gdal dataset
# sample gdal dataset
sample_gdal_filename = "dataset_01122000.tif"
options = ['PROFILE=GeoTIFF']
sample_gdal_dataset = driver.Create(sample_gdal_filename, 5, 5, 1, gdal.GDT_Float32, options=options)
srs = osr.SpatialReference()
wkt_projection = srs.ExportToWkt()
sample_gdal_dataset.SetProjection(wkt_projection)
sample_gdal_band = sample_gdal_dataset.GetRasterBand(1)
sample_gdal_band.SetNoDataValue(NO_DATA_VALUE)
sample_gdal_band.WriteArray(np.arange(25).reshape(5, 5))
sample_gdal_dataset.SetMetadataItem(ifc.FIRST_DATE, '2019-10-20')
sample_gdal_dataset.SetMetadataItem(ifc.SECOND_DATE, '2019-11-01')
sample_gdal_dataset.SetMetadataItem(ifc.PYRATE_WAVELENGTH_METRES, '10.05656')
sample_gdal_dataset.FlushCache()
sample_gdal_dataset = None
ifg = Ifg(sample_gdal_filename)
ifg.open()
# create a coherence mask dataset
tmpdir = tempfile.mkdtemp()
out_dir = Path(tmpdir) # we won't be creating any output coherence mask files as there are already GeoTIFFs
params = common.min_params(out_dir)
coherence_mask_filename = MultiplePaths(Path("mask_dataset_01122000-02122000.tif").as_posix(), params)
coherence_mask_dataset = driver.Create(coherence_mask_filename.converted_path, 5, 5, 1, gdal.GDT_Float32)
srs = osr.SpatialReference()
wkt_projection = srs.ExportToWkt()
coherence_mask_dataset.SetProjection(wkt_projection)
coherence_mask_band = coherence_mask_dataset.GetRasterBand(1)
coherence_mask_band.SetNoDataValue(NO_DATA_VALUE)
arr = np.arange(0, 75, 3).reshape(5, 5) / 100.0
arr[3, 4] = 0.25 # insert some random lower than threshold number
arr[4, 2] = 0.20 # insert some random lower than threshold number
coherence_mask_band.WriteArray(arr)
# del the tmp handler datasets created
del coherence_mask_dataset
# create an artificial masked dataset
expected_result_array = np.nan_to_num(
np.array(
[
[np.nan, np.nan, np.nan, np.nan, np.nan],
[np.nan, np.nan, np.nan, np.nan, np.nan],
[10.0, 11.0, 12.0, 13.0, 14.0],
[15.0, 16.0, 17.0, 18.0, np.nan],
[20.0, 21.0, np.nan, 23.0, 24.0],
]
)
)
# use the gdal_python.coherence_masking to find the actual mask dataset
coherence_thresh = 0.3
gdal_python.coherence_masking(ifg.dataset, coherence_mask_filename.converted_path, coherence_thresh)
sample_gdal_array = np.nan_to_num(ifg.phase_data)
# compare the artificial masked and actual masked datasets
np.testing.assert_array_equal(sample_gdal_array, expected_result_array)
# del the tmp datasets created
os.remove(coherence_mask_filename.converted_path)
ifg.close()
os.remove(sample_gdal_filename)
def mlooked_path(path, params, input_type):
m = MultiplePaths(path, params=params, input_type=input_type)
return m.sampled_path
def setUpClass(cls):
rate_types = ['stack_rate', 'stack_error', 'stack_samples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
from pyrate.configuration import Configuration
# change the required params
params = Configuration(cls.test_conf).__dict__
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSES] = 4
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
rows, cols = params["rows"], params["cols"]
# xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST]))
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
# dest_paths are tifs that have been geotif converted and multilooked
cls.converted_paths = [b.converted_path for b in multi_paths]
cls.sampled_paths = [b.sampled_path for b in multi_paths]
from copy import copy
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols)
ifgs = common.small_data_setup()
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs(
cls.sampled_paths, params, rows, cols)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.PROCESSES] = 1
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
cls.converted_paths_s = [b.converted_path for b in multi_paths]
cls.sampled_paths_s = [b.sampled_path for b in multi_paths]
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs(
cls.sampled_paths_s, params, rows, cols)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
