def test_single_workflow(gamma_conf):
check_call(f"mpirun -n 4 pyrate workflow -f {gamma_conf}", shell=True)
params = Configuration(gamma_conf).__dict__
log_file_name = 'pyrate.log.' + 'workflow'
files = list(Path(params[cf.OUT_DIR]).glob(log_file_name + '.*'))
assert len(files) == 1
# ref pixel file generated
ref_pixel_file = params[cf.REF_PIXEL_FILE]
assert Path(ref_pixel_file).exists()
ref_pixel = np.load(ref_pixel_file)
np.testing.assert_array_equal(ref_pixel, [38, 58])
# assert orbfit exists on disc
from pyrate.core import shared
looked_files = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]]
ifgs = [shared.Ifg(ifg) for ifg in looked_files]
orbfits_on_disc = [
Path(params[cf.OUT_DIR], cf.ORB_ERROR_DIR,
Path(ifg.data_path).stem + '_orbfit.npy') for ifg in ifgs
]
assert all(orbfits_on_disc)
shutil.rmtree(params[cf.OUT_DIR])
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.tmp_dir = tempfile.mkdtemp()
common.copytree(common.SML_TEST_TIF, cls.tmp_dir)
tifs = glob.glob(os.path.join(cls.tmp_dir, "*.tif"))
for t in tifs:
os.chmod(t, 0o644)
small_ifgs = common.small_data_setup(datafiles=tifs)
ifg_paths = [i.data_path for i in small_ifgs]
cls.ifg_ret = common.pre_prepare_ifgs(ifg_paths, params=params)
for i in cls.ifg_ret:
i.close()
nan_conversion = params[C.NAN_CONVERSION]
# prepare a second set
cls.tmp_dir2 = tempfile.mkdtemp()
common.copytree(common.SML_TEST_TIF, cls.tmp_dir2)
tifs = glob.glob(os.path.join(cls.tmp_dir2, "*.tif"))
for t in tifs:
os.chmod(t, 0o644)
small_ifgs = common.small_data_setup(datafiles=tifs)
ifg_paths = [i.data_path for i in small_ifgs]
cls.ifgs = [shared.Ifg(p) for p in ifg_paths]
for i in cls.ifgs:
i.open(readonly=False)
if nan_conversion: # nan conversion happens here in networkx mst
i.nodata_value = params[C.NO_DATA_VALUE]
i.convert_to_nans()
if not i.mm_converted:
i.convert_to_mm()
i.close()
def test_single_workflow(gamma_or_mexicoa_conf):
if MPI_INSTALLED:
check_call(f"mpirun -n 4 pyrate workflow -f {gamma_or_mexicoa_conf}",
shell=True)
else:
check_call(f"pyrate workflow -f {gamma_or_mexicoa_conf}", shell=True)
params = Configuration(gamma_or_mexicoa_conf).__dict__
log_file_name = 'pyrate.log.' + 'workflow'
files = list(Path(params[C.OUT_DIR]).glob(log_file_name + '.*'))
assert len(files) == 1
# ref pixel file generated
ref_pixel_file = params[C.REF_PIXEL_FILE]
assert Path(ref_pixel_file).exists()
ref_pixel = np.load(ref_pixel_file)
if gamma_or_mexicoa_conf == MEXICO_CROPA_CONF:
np.testing.assert_array_equal(ref_pixel, [42, 2])
for f in C.GEOMETRY_OUTPUT_TYPES:
assert Path(params[C.GEOMETRY_DIR]).joinpath(f + '.tif').exists()
else:
np.testing.assert_array_equal(ref_pixel, [38, 58])
# assert orbfit exists on disc
from pyrate.core import shared
looked_files = [p.sampled_path for p in params[C.INTERFEROGRAM_FILES]]
ifgs = [shared.Ifg(ifg) for ifg in looked_files]
orbfits_on_disc = [
Path(params[C.OUT_DIR], C.ORB_ERROR_DIR,
Path(ifg.data_path).stem + '_orbfit.npy') for ifg in ifgs
]
assert all(orbfits_on_disc)
shutil.rmtree(params[C.OUT_DIR])
def get_ifgs(out_dir, _open=True):
paths = glob.glob(join(out_dir, 'geo_*-*_unw.tif'))
ifgs = [shared.Ifg(p) for p in paths]
assert len(ifgs) == 17, 'Got %s' % ifgs
if _open:
for i in ifgs:
i.open(readonly=False)
return ifgs
def cvd(ifg_path,
params,
r_dist,
calc_alpha=False,
write_vals=False,
save_acg=False):
"""
Calculate the 1D covariance function of an entire interferogram as the
radial average of its 2D autocorrelation.
:param str ifg_path: An interferogram file path. OR
:param dict params: Dictionary of configuration parameters
:param ndarray r_dist: Array of distance values from the image centre
(See Rdist class for more details)
:param bool calc_alpha: If True calculate alpha
:param bool write_vals: If True write maxvar and alpha values to
interferogram metadata
:param bool save_acg: If True write autocorrelation and radial distance
data to numpy array file on disk
:return: maxvar: The maximum variance (at zero lag)
:rtype: float
:return: alpha: the exponential length-scale of decay factor
:rtype: float
"""
ifg = shared.Ifg(ifg_path)
ifg.open()
shared.nan_and_mm_convert(ifg, params)
# calculate 2D auto-correlation of image using the
# spectral method (Wiener-Khinchin theorem)
if ifg.nan_converted: # if nancoverted earlier, convert nans back to 0's
phase = where(isnan(ifg.phase_data), 0, ifg.phase_data)
else:
phase = ifg.phase_data
maxvar, alpha = cvd_from_phase(phase,
ifg,
r_dist,
calc_alpha,
save_acg=save_acg,
params=params)
if write_vals:
ifg.add_metadata(**{
ifc.PYRATE_MAXVAR: str(maxvar),
ifc.PYRATE_ALPHA: str(alpha)
})
ifg.close()
return maxvar, alpha
def test_metadata(self):
refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(
self.params_chipsize_15)
for i in self.ifg_paths:
ifg = shared.Ifg(i)
ifg.open(readonly=True)
md = ifg.meta_data
for k, v in zip([
ifc.PYRATE_REFPIX_X, ifc.PYRATE_REFPIX_Y,
ifc.PYRATE_REFPIX_LAT, ifc.PYRATE_REFPIX_LON,
ifc.PYRATE_MEAN_REF_AREA, ifc.PYRATE_STDDEV_REF_AREA
], [str(refx), str(refy), 0, 0, 0, 0]):
assert k in md # metadata present
# assert values
ifg.close()
def setup_method(cls):
cls.conf = common.TEST_CONF_GAMMA
params = Configuration(cls.conf).__dict__
conv2tif.main(params)
params = Configuration(cls.conf).__dict__
prepifg.main(params)
cls.params = Configuration(cls.conf).__dict__
correct._copy_mlooked(cls.params)
correct._update_params_with_tiles(cls.params)
correct._create_ifg_dict(cls.params)
multi_paths = cls.params[cf.INTERFEROGRAM_FILES]
cls.ifg_paths = [p.tmp_sampled_path for p in multi_paths]
cls.ifgs = [shared.Ifg(i) for i in cls.ifg_paths]
for i in cls.ifgs:
i.open()
shared.save_numpy_phase(cls.ifg_paths, cls.params)
correct.mst_calc_wrapper(cls.params)
def remove_orbital_error(ifgs: List, params: dict) -> None:
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
"""
mpiops.run_once(__orb_params_check, params)
ifg_paths = [i.data_path
for i in ifgs] if isinstance(ifgs[0], Ifg) else ifgs
method = params[cf.ORBITAL_FIT_METHOD]
# mlooking is not necessary for independent correction in a computational sense
# can use multiple procesing if write_to_disc=True
if method == INDEPENDENT_METHOD:
log.info('Calculating orbital correction using independent method')
#TODO: implement multi-looking for independent orbit method
if params[cf.ORBITAL_FIT_LOOKS_X] > 1 or params[
cf.ORBITAL_FIT_LOOKS_Y] > 1:
log.warning(
'Multi-looking is not applied in independent orbit method')
ifgs = [shared.Ifg(p)
for p in ifg_paths] if isinstance(ifgs[0], str) else ifgs
process_ifgs = mpiops.array_split(ifgs)
for ifg in process_ifgs:
independent_orbital_correction(ifg, params=params)
elif method == NETWORK_METHOD:
log.info('Calculating orbital correction using network method')
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MAIN_PROCESS:
mlooked = __create_multilooked_dataset_for_network_correction(
params)
_validate_mlooked(mlooked, ifg_paths)
network_orbital_correction(ifg_paths, params, mlooked)
else:
raise OrbitalError("Unrecognised orbital correction method")
def independent_orbital_correction(ifg, degree, offset, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using an offset
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
ifg = shared.Ifg(ifg) if isinstance(ifg, str) else ifg
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
# vectorise, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
dm = get_design_matrix(ifg, degree, offset)
# filter NaNs out before getting model
clean_dm = dm[~isnan(vphase)]
data = vphase[~isnan(vphase)]
model = lstsq(clean_dm, data)[0] # first arg is the model params
# calculate forward model & morph back to 2D
if offset:
fullorb = np.reshape(np.dot(dm[:, :-1], model[:-1]),
ifg.phase_data.shape)
else:
fullorb = np.reshape(np.dot(dm, model), ifg.phase_data.shape)
offset_removal = nanmedian(np.ravel(ifg.phase_data - fullorb))
# subtract orbital error from the ifg
ifg.phase_data -= (fullorb - offset_removal)
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(ifg)
if ifg.open():
ifg.close()
def network_orbital_correction(ifg_paths,
params,
m_ifgs: Optional[List] = None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifg_paths: List of Ifg class objects reduced to a minimum spanning
tree network
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using offsets
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects
(sequence must be multilooked versions of 'ifgs' arg)
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
offset = params[cf.ORBFIT_OFFSET]
degree = params[cf.ORBITAL_FIT_DEGREE]
preread_ifgs = params[cf.PREREAD_IFGS]
# all orbit corrections available?
if isinstance(ifg_paths[0], str):
if __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
log.warning("Reusing orbfit errors from previous run!!!")
return
# all corrections are available in numpy files already saved - return
ifgs = [shared.Ifg(i) for i in ifg_paths]
else: # alternate test paths # TODO: improve
ifgs = ifg_paths
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = first_second_ids(get_all_epochs(temp_ifgs))
else:
ids = first_second_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
ifg = ifgs[0]
dm = get_design_matrix(ifg, degree, offset=False)
for i in ifg_paths:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset, params)
def network_orbital_correction(ifg_paths,
params,
m_ifgs: Optional[List] = None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifg_paths: List of Ifg class objects reduced to a minimum spanning tree network
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
offset = params[C.ORBFIT_OFFSET]
degree = params[C.ORBITAL_FIT_DEGREE]
preread_ifgs = params[C.PREREAD_IFGS]
intercept = params[C.ORBFIT_INTERCEPT]
scale = params[C.ORBFIT_SCALE]
# all orbit corrections available?
if isinstance(ifg_paths[0], str):
if __check_and_apply_orberrors_found_on_disc(ifg_paths, params):
log.warning("Reusing orbfit errors from previous run!!!")
return
# all corrections are available in numpy files already saved - return
ifgs = [shared.Ifg(i) for i in ifg_paths]
else: # alternate test paths # TODO: improve
ifgs = ifg_paths
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = first_second_ids(get_all_epochs(temp_ifgs))
else:
ids = first_second_ids(get_all_epochs(ifgs))
nepochs = len(set(ids))
# call the actual inversion routine
coefs = calc_network_orb_correction(src_ifgs,
degree,
scale,
nepochs,
intercept=intercept)
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifg_paths[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg,
degree,
intercept=intercept,
scale=scale)
temp_ifg.close()
else:
ifg = ifgs[0]
dm = get_design_matrix(ifg, degree, intercept=intercept, scale=scale)
for i in ifg_paths:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset, params)
def independent_orbital_correction(ifg_path, params):
"""
Calculates and removes an orbital error surface from a single independent
interferogram.
Warning: This will write orbital error corrected phase_data to the ifg.
:param Ifg class instance ifg: the interferogram to be corrected
:param dict params: dictionary of configuration parameters
:return: None - interferogram phase data is updated and saved to disk
"""
log.debug(f"Orbital correction of {ifg_path}")
degree = params[C.ORBITAL_FIT_DEGREE]
offset = params[C.ORBFIT_OFFSET]
intercept = params[C.ORBFIT_INTERCEPT]
xlooks = params[C.ORBITAL_FIT_LOOKS_X]
ylooks = params[C.ORBITAL_FIT_LOOKS_Y]
ifg0 = shared.Ifg(ifg_path) if isinstance(ifg_path, str) else ifg_path
# get full-resolution design matrix
fullres_dm = get_design_matrix(ifg0, degree, intercept=intercept)
ifg = shared.dem_or_ifg(ifg_path) if isinstance(ifg_path,
str) else ifg_path
multi_path = MultiplePaths(ifg.data_path, params)
orb_on_disc = MultiplePaths.orb_error_path(ifg.data_path, params)
if not ifg.is_open:
ifg.open()
shared.nan_and_mm_convert(ifg, params)
fullres_ifg = ifg # keep a backup
fullres_phase = fullres_ifg.phase_data
if orb_on_disc.exists():
log.info(
f'Reusing already computed orbital fit correction: {orb_on_disc}')
orbital_correction = np.load(file=orb_on_disc)
else:
# Multi-look the ifg data if either X or Y is greater than 1
if (xlooks > 1) or (ylooks > 1):
exts, _, _ = __extents_from_params(params)
mlooked = _create_mlooked_dataset(multi_path, ifg.data_path, exts,
params)
ifg = Ifg(mlooked) # multi-looked Ifg object
ifg.initialize()
shared.nan_and_mm_convert(ifg, params)
# vectorise phase data, keeping NODATA
vphase = reshape(ifg.phase_data, ifg.num_cells)
# compute design matrix for multi-looked data
mlooked_dm = get_design_matrix(ifg, degree, intercept=intercept)
# invert to obtain the correction image (forward model) at full-res
orbital_correction = __orb_correction(fullres_dm,
mlooked_dm,
fullres_phase,
vphase,
offset=offset)
# save correction to disc
if not orb_on_disc.parent.exists():
shared.mkdir_p(orb_on_disc.parent)
np.save(file=orb_on_disc, arr=orbital_correction)
# subtract orbital correction from the full-res ifg
fullres_ifg.phase_data -= orbital_correction
# set orbfit meta tag and save phase to file
_save_orbital_error_corrected_phase(fullres_ifg, params)
