def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params):
"""
xxxx
:param grid: List of tuples (y, x) corresponding reference pixel grids
:param half_patch_size: Patch size in pixels corresponding to reference pixel grids
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return xxxx
"""
log.info('Saving ref pixel blocks')
outdir = params[cf.TMPDIR]
for pth in ifg_paths:
ifg = Ifg(pth)
ifg.open(readonly=True)
ifg.nodata_value = params[cf.NO_DATA_VALUE]
ifg.convert_to_nans()
ifg.convert_to_mm()
for y, x in grid:
data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1,
x - half_patch_size:x + half_patch_size + 1]
data_file = join(
outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format(
b=os.path.basename(pth).split('.')[0], y=y, x=x))
np.save(file=data_file, arr=data)
ifg.close()
log.info('Saved ref pixel blocks')
def _phase_sum(ifg_paths, params):
"""
Save phase data and phase sum used in the reference phase estimation
"""
p_paths = mpiops.array_split(ifg_paths)
ifg = Ifg(p_paths[0])
ifg.open(readonly=True)
shape = ifg.shape
phs_sum = np.zeros(shape=shape, dtype=np.float64)
ifg.close()
for d in p_paths:
ifg = Ifg(d)
ifg.open()
ifg.nodata_value = params[cf.NO_DATA_VALUE]
phs_sum += ifg.phase_data
ifg.close()
if mpiops.rank == MASTER_PROCESS:
phase_sum_all = phs_sum
# loop is better for memory
for i in range(1, mpiops.size): # pragma: no cover
phs_sum = np.zeros(shape=shape, dtype=np.float64)
mpiops.comm.Recv(phs_sum, source=i, tag=i)
phase_sum_all += phs_sum
comp = np.isnan(phase_sum_all) # this is the same as in Matlab
comp = np.ravel(comp, order='F') # this is the same as in Matlab
else: # pragma: no cover
comp = None
mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank)
comp = mpiops.comm.bcast(comp, root=0)
return comp
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = vcm_module.RDist(ifg)()
ifg.close()
return r_dist
def test_multilook(self):
"""Test resampling method using a scaling factor of 4"""
scale = 4 # assumes square cells
self.ifgs.append(DEM(SML_TEST_DEM_TIF))
self.ifg_paths = [i.data_path for i in self.ifgs]
cext = self._custom_extents_tuple()
xlooks = ylooks = scale
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
thresh=1.0,
user_exts=cext)
for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]):
i = Ifg(ipath)
i.open()
self.assertEqual(i.dataset.RasterXSize, 20 / scale)
self.assertEqual(i.dataset.RasterYSize, 28 / scale)
# verify resampling
path = join(PREP_TEST_TIF, "%s.tif" % n)
ds = gdal.Open(path)
src_data = ds.GetRasterBand(2).ReadAsArray()
exp_resample = multilooking(src_data, scale, scale, thresh=0)
self.assertEqual(exp_resample.shape, (7, 5))
assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray())
ds = None
i.close()
os.remove(ipath)
# verify DEM has been correctly processed
# ignore output values as resampling has already been tested for phase
exp_dem_path = join(SML_TEST_DEM_DIR,
'roipac_test_trimmed_4rlks_3cr.tif')
self.assertTrue(exists(exp_dem_path))
orignal_dem = DEM(SML_TEST_DEM_TIF)
orignal_dem.open()
dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType
orignal_dem.close()
dem = DEM(exp_dem_path)
dem.open()
# test multilooked dem is of the same datatype as the original dem tif
self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType)
self.assertEqual(dem.dataset.RasterXSize, 20 / scale)
self.assertEqual(dem.dataset.RasterYSize, 28 / scale)
data = dem.height_band.ReadAsArray()
self.assertTrue(data.ptp() != 0)
# close ifgs
dem.close()
for i in self.ifgs:
i.close()
os.remove(exp_dem_path)
def _inner(ifg_path):
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_ph = rpe.est_ref_phs_method2(phase_data, half_chip_size, refpx,
refpy, thresh)
phase_data -= ref_ph
md = ifg.meta_data
md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_ph
def _inner(ifg_path):
""" convenient inner loop """
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_phase = rpe.est_ref_phs_method1(phase_data, comp)
phase_data -= ref_phase
md = ifg.meta_data
md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_phase
def _save_aps_corrected_phase(ifg_path, phase):
"""
Save (update) interferogram metadata and phase data after
spatio-temporal filter (APS) correction.
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data
)]
# set aps tags after aps error correction
ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED)
ifg.write_modified_phase()
ifg.close()
def test_nodata(self):
"""Verify NODATA value copied correctly (amplitude band not copied)"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
# NB: amplitude band doesn't have a NODATA value
self.assertTrue(
isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue()))
ifg.close()
for i in self.ifgs:
i.close()
def get_tiles(ifg_path, rows, cols):
"""
Break up the interferograms into tiles based on user supplied rows and columns.
:param ifg_path: List of destination tifs
:param rows: Number of rows to break each interferogram into
:param cols: Number of columns to break each interferogram into
:return tiles: List of shared.Tile instances
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=True)
tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols)
ifg.close()
return tiles
def test_nans(self):
"""Verify NaNs replace 0 in the multilooked phase band"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
phase = ifg.phase_band.ReadAsArray()
self.assertFalse((phase == 0).any())
self.assertTrue((isnan(phase)).any())
ifg.close()
self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo
self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo
for i in self.ifgs:
i.close()
def _ref_pixel_calc(ifg_paths, params):
"""
Wrapper for reference pixel calculation
"""
# unlikely, but possible the refpixel can be (0,0)
# check if there is a pre-specified reference pixel coord
refx = params[cf.REFX]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx > ifg.ncols - 1:
msg = ('Supplied reference pixel X coordinate is greater than '
'the number of ifg columns: {}').format(refx)
raise ValueError(msg)
refy = params[cf.REFY]
if refy > ifg.nrows - 1:
msg = ('Supplied reference pixel Y coordinate is greater than '
'the number of ifg rows: {}').format(refy)
raise ValueError(msg)
if refx <= 0 or refy <= 0: # if either zero or negative
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = refpixel.ref_pixel_setup(ifg_paths,
params)
process_grid = mpiops.array_split(grid)
refpixel.save_ref_pixel_blocks(process_grid, half_patch_size,
ifg_paths, params)
mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size,
ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MASTER_PROCESS:
mean_sds = np.hstack(mean_sds)
refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid)
log.info('Selected reference pixel coordinate: '
'({}, {})'.format(refx, refy))
else: # pragma: no cover
log.info('Reusing reference pixel from config file: '
'({}, {})'.format(refx, refy))
ifg.close()
return refx, refy
def test_min_extents(self):
"""Test ifgcropopt=1 crops datasets to min extents."""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
ifg = Ifg(self.exp_files[0])
ifg.open()
# output files should have same extents
# NB: also verifies gdalwarp correctly copies geotransform across
# NB: expected data copied from gdalinfo output
gt = ifg.dataset.GetGeoTransform()
exp_gt = (150.911666666, 0.000833333, 0, -34.172499999, 0,
-0.000833333)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[0], self.exp_files[4]])
ifg.close()
for i in self.ifgs:
i.close()
def test_orbital_correction_matlab_equality(self):
from pyrate.scripts import run_pyrate
run_pyrate._orb_fit_calc(self.ifg_paths, self.params)
onlyfiles = [
f for f in os.listdir(SML_TEST_MATLAB_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method1_')
]
count = 0
for i, f in enumerate(onlyfiles):
ifg_data = np.genfromtxt(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR,
f),
delimiter=',')
for k, j in enumerate(self.ifg_paths):
ifg = Ifg(j)
ifg.open()
if os.path.basename(j).split('_unw.')[0] == \
os.path.basename(f).split(
'_orb_planar_1lks_method1_')[1].split('.')[0]:
count += 1
# all numbers equal
np.testing.assert_array_almost_equal(ifg_data,
ifg.phase_data,
decimal=2)
# means must also be equal
self.assertAlmostEqual(np.nanmean(ifg_data),
np.nanmean(ifg.phase_data),
places=2)
# number of nans must equal
self.assertEqual(np.sum(np.isnan(ifg_data)),
np.sum(np.isnan(ifg.phase_data)))
ifg.close()
# ensure that we have expected number of matches
self.assertEqual(count, len(self.ifg_paths))
def test_custom_extents(self):
xlooks = ylooks = 1
cext = self._custom_extents_tuple()
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
user_exts=cext)
ifg = Ifg(self.exp_files[2])
ifg.open()
gt = ifg.dataset.GetGeoTransform()
exp_gt = (cext.xfirst, self.xs, 0, cext.yfirst, 0, self.ys)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[2], self.exp_files[6]])
# close ifgs
ifg.close()
for i in self.ifgs:
i.close()
def test_write(self):
base = TEMPDIR
src = self.ifg.data_path
dest = join(base, basename(self.ifg.data_path))
# shutil.copy needs to copy writeable permission from src
os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH |
S_IRUSR | S_IWUSR)
shutil.copy(src, dest)
os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert
i = Ifg(dest)
i.open()
i.phase_data[0, 1:] = nan
i.write_modified_phase()
del i
# reopen to ensure data/nans can be read back out
i = Ifg(dest)
i.open(readonly=True)
assert_array_equal(True, isnan(i.phase_data[0, 1:]))
i.close()
os.remove(dest)
def test_default_max_extents(self):
"""Test ifgcropopt=2 crops datasets to max bounding box extents."""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MAXIMUM_CROP, xlooks, ylooks)
for f in [self.exp_files[1], self.exp_files[5]]:
self.assertTrue(exists(f), msg="Output files not created")
# output files should have same extents
# NB: also verifies gdalwarp correctly copies geotransform across
ifg = Ifg(self.exp_files[1])
ifg.open()
gt = ifg.dataset.GetGeoTransform()
# copied from gdalinfo output
exp_gt = (150.91, 0.000833333, 0, -34.17, 0, -0.000833333)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[1], self.exp_files[5]])
ifg.close()
for i in self.ifgs:
i.close()
def _wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
Required due to differences between Matlab and Python MST
implementations.
"""
if not params[cf.APSEST]:
log.info('APS correction not required.')
return
# perform some checks on existing ifgs
log.info('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, preread_ifgs,
ifc.PYRATE_APS_ERROR):
log.info('Finished APS correction')
return # return if True condition returned
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
ifg.close()
def ref_pixel_calc(ifg_paths, params):
"""
Reference pixel calculation setup.
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return refx: Reference pixel x-coordinate
:return refy: Reference pixel y-coordinate
"""
# unlikely, but possible the refpixel can be (0,0)
# check if there is a pre-specified reference pixel coord
refx = params[cf.REFX]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx > ifg.ncols - 1:
msg = ('Supplied reference pixel X coordinate is greater than '
'the number of ifg columns: {}').format(refx)
raise ValueError(msg)
refy = params[cf.REFY]
if refy > ifg.nrows - 1:
msg = ('Supplied reference pixel Y coordinate is greater than '
'the number of ifg rows: {}').format(refy)
raise ValueError(msg)
if refx <= 0 or refy <= 0: # if either zero or negative
log.info('Searching for best reference pixel location')
refy, refx = find_ref_pixel(ifg_paths, params)
log.info('Selected reference pixel coordinate: '
'({}, {})'.format(refx, refy))
else: # pragma: no cover
log.info('Reusing reference pixel from config file: '
'({}, {})'.format(refx, refy))
ifg.close()
return refx, refy
def network_correction(ifgs,
degree,
offset,
params,
m_ifgs=None,
preread_ifgs=None):
"""
Calculates orbital correction model, removing this from the interferograms.
.. warn:: This will write orbital error corrected phase_data in the interferograms.
:param ifgs: Interferograms reduced to a minimum tree from prior
MST calculations
:param degree: PLANAR, QUADRATIC or PART_CUBIC
:param offset: True to calculate the model using offsets
:param params: Parameter dictionary
:param m_ifgs: Multi-looked orbfit interferograms (sequence must be mlooked
versions of 'ifgs' arg)
:param preread_ifgs: Parameters dict corresponding to config file
:return xxxx
"""
# pylint: disable=too-many-locals, too-many-arguments
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = master_slave_ids(get_all_epochs(temp_ifgs))
else:
ids = master_slave_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifgs[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
dm = get_design_matrix(ifgs[0], degree, offset=False)
# TODO: remove this import from tests suite
from tests.common import MockIfg
for i in ifgs:
# open if not Ifg instance
if not (isinstance(i, Ifg)
or isinstance(i, MockIfg)): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_networkx_error(coefs, dm, i, ids, offset)
def network_orbital_correction(ifgs,
degree,
offset,
params,
m_ifgs=None,
preread_ifgs=None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifgs: List of Ifg class objects reduced to a minimum spanning
tree network
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using offsets
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects
(sequence must be multilooked versions of 'ifgs' arg)
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = master_slave_ids(get_all_epochs(temp_ifgs))
else:
ids = master_slave_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifgs[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
dm = get_design_matrix(ifgs[0], degree, offset=False)
for i in ifgs:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset)