def _phase_sum(ifg_paths, params):
"""
Save phase data and phase sum used in the reference phase estimation
"""
p_paths = mpiops.array_split(ifg_paths)
ifg = Ifg(p_paths[0])
ifg.open(readonly=True)
shape = ifg.shape
phs_sum = np.zeros(shape=shape, dtype=np.float64)
ifg.close()
for d in p_paths:
ifg = Ifg(d)
ifg.open()
ifg.nodata_value = params[cf.NO_DATA_VALUE]
phs_sum += ifg.phase_data
ifg.close()
if mpiops.rank == MASTER_PROCESS:
phase_sum_all = phs_sum
# loop is better for memory
for i in range(1, mpiops.size): # pragma: no cover
phs_sum = np.zeros(shape=shape, dtype=np.float64)
mpiops.comm.Recv(phs_sum, source=i, tag=i)
phase_sum_all += phs_sum
comp = np.isnan(phase_sum_all) # this is the same as in Matlab
comp = np.ravel(comp, order='F') # this is the same as in Matlab
else: # pragma: no cover
comp = None
mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank)
comp = mpiops.comm.bcast(comp, root=0)
return comp
def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params):
"""
xxxx
:param grid: List of tuples (y, x) corresponding reference pixel grids
:param half_patch_size: Patch size in pixels corresponding to reference pixel grids
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return xxxx
"""
log.info('Saving ref pixel blocks')
outdir = params[cf.TMPDIR]
for pth in ifg_paths:
ifg = Ifg(pth)
ifg.open(readonly=True)
ifg.nodata_value = params[cf.NO_DATA_VALUE]
ifg.convert_to_nans()
ifg.convert_to_mm()
for y, x in grid:
data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1,
x - half_patch_size:x + half_patch_size + 1]
data_file = join(
outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format(
b=os.path.basename(pth).split('.')[0], y=y, x=x))
np.save(file=data_file, arr=data)
ifg.close()
log.info('Saved ref pixel blocks')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = vcm_module.RDist(ifg)()
ifg.close()
return r_dist
def test_multilook(self):
"""Test resampling method using a scaling factor of 4"""
scale = 4 # assumes square cells
self.ifgs.append(DEM(SML_TEST_DEM_TIF))
self.ifg_paths = [i.data_path for i in self.ifgs]
cext = self._custom_extents_tuple()
xlooks = ylooks = scale
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
thresh=1.0,
user_exts=cext)
for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]):
i = Ifg(ipath)
i.open()
self.assertEqual(i.dataset.RasterXSize, 20 / scale)
self.assertEqual(i.dataset.RasterYSize, 28 / scale)
# verify resampling
path = join(PREP_TEST_TIF, "%s.tif" % n)
ds = gdal.Open(path)
src_data = ds.GetRasterBand(2).ReadAsArray()
exp_resample = multilooking(src_data, scale, scale, thresh=0)
self.assertEqual(exp_resample.shape, (7, 5))
assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray())
ds = None
i.close()
os.remove(ipath)
# verify DEM has been correctly processed
# ignore output values as resampling has already been tested for phase
exp_dem_path = join(SML_TEST_DEM_DIR,
'roipac_test_trimmed_4rlks_3cr.tif')
self.assertTrue(exists(exp_dem_path))
orignal_dem = DEM(SML_TEST_DEM_TIF)
orignal_dem.open()
dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType
orignal_dem.close()
dem = DEM(exp_dem_path)
dem.open()
# test multilooked dem is of the same datatype as the original dem tif
self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType)
self.assertEqual(dem.dataset.RasterXSize, 20 / scale)
self.assertEqual(dem.dataset.RasterYSize, 28 / scale)
data = dem.height_band.ReadAsArray()
self.assertTrue(data.ptp() != 0)
# close ifgs
dem.close()
for i in self.ifgs:
i.close()
os.remove(exp_dem_path)
def _inner(ifg_path):
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_ph = rpe.est_ref_phs_method2(phase_data, half_chip_size, refpx,
refpy, thresh)
phase_data -= ref_ph
md = ifg.meta_data
md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_ph
def _inner(ifg_path):
""" convenient inner loop """
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_phase = rpe.est_ref_phs_method1(phase_data, comp)
phase_data -= ref_phase
md = ifg.meta_data
md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_phase
def _save_aps_corrected_phase(ifg_path, phase):
"""
Save (update) interferogram metadata and phase data after
spatio-temporal filter (APS) correction.
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data
)]
# set aps tags after aps error correction
ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED)
ifg.write_modified_phase()
ifg.close()
def test_nodata(self):
"""Verify NODATA value copied correctly (amplitude band not copied)"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
# NB: amplitude band doesn't have a NODATA value
self.assertTrue(
isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue()))
ifg.close()
for i in self.ifgs:
i.close()
def get_tiles(ifg_path, rows, cols):
"""
Break up the interferograms into tiles based on user supplied rows and columns.
:param ifg_path: List of destination tifs
:param rows: Number of rows to break each interferogram into
:param cols: Number of columns to break each interferogram into
:return tiles: List of shared.Tile instances
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=True)
tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols)
ifg.close()
return tiles
def ref_pixel_setup(ifgs_or_paths, params):
"""
Sets up the grid for reference pixel computation and saves numpy files
to disk for later use during ref pixel computation.
:param list ifgs_or_paths: List of interferogram filenames or Ifg objects
:param dict params: Dictionary of configuration parameters
:return: half_patch_size: size of patch
:rtype: float
:return: thresh
:rtype: float
:return: list(product(ysteps, xsteps))
:rtype: list
"""
log.info('Setting up ref pixel computation')
refnx, refny, chipsize, min_frac = params[cf.REFNX], \
params[cf.REFNY], \
params[cf.REF_CHIP_SIZE], \
params[cf.REF_MIN_FRAC]
if len(ifgs_or_paths) < 1:
msg = 'Reference pixel search requires 2+ interferograms'
raise RefPixelError(msg)
if isinstance(ifgs_or_paths[0], str):
head = Ifg(ifgs_or_paths[0])
head.open(readonly=True)
else:
head = ifgs_or_paths[0]
# sanity check inputs
_validate_chipsize(chipsize, head)
_validate_minimum_fraction(min_frac)
_validate_search_win(refnx, refny, chipsize, head)
# pre-calculate useful amounts
half_patch_size = chipsize // 2
chipsize = half_patch_size * 2 + 1
thresh = min_frac * chipsize * chipsize
# do window searches across dataset, central pixel of stack with smallest
# mean is the reference pixel
rows, cols = head.shape
ysteps = _step(rows, refny, half_patch_size)
xsteps = _step(cols, refnx, half_patch_size)
log.info('Ref pixel setup finished')
return half_patch_size, thresh, list(product(ysteps, xsteps))
def test_nans(self):
"""Verify NaNs replace 0 in the multilooked phase band"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
phase = ifg.phase_band.ReadAsArray()
self.assertFalse((phase == 0).any())
self.assertTrue((isnan(phase)).any())
ifg.close()
self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo
self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo
for i in self.ifgs:
i.close()
def _ref_pixel_calc(ifg_paths, params):
"""
Wrapper for reference pixel calculation
"""
# unlikely, but possible the refpixel can be (0,0)
# check if there is a pre-specified reference pixel coord
refx = params[cf.REFX]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx > ifg.ncols - 1:
msg = ('Supplied reference pixel X coordinate is greater than '
'the number of ifg columns: {}').format(refx)
raise ValueError(msg)
refy = params[cf.REFY]
if refy > ifg.nrows - 1:
msg = ('Supplied reference pixel Y coordinate is greater than '
'the number of ifg rows: {}').format(refy)
raise ValueError(msg)
if refx <= 0 or refy <= 0: # if either zero or negative
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = refpixel.ref_pixel_setup(ifg_paths,
params)
process_grid = mpiops.array_split(grid)
refpixel.save_ref_pixel_blocks(process_grid, half_patch_size,
ifg_paths, params)
mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size,
ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MASTER_PROCESS:
mean_sds = np.hstack(mean_sds)
refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid)
log.info('Selected reference pixel coordinate: '
'({}, {})'.format(refx, refy))
else: # pragma: no cover
log.info('Reusing reference pixel from config file: '
'({}, {})'.format(refx, refy))
ifg.close()
return refx, refy
def test_min_extents(self):
"""Test ifgcropopt=1 crops datasets to min extents."""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
ifg = Ifg(self.exp_files[0])
ifg.open()
# output files should have same extents
# NB: also verifies gdalwarp correctly copies geotransform across
# NB: expected data copied from gdalinfo output
gt = ifg.dataset.GetGeoTransform()
exp_gt = (150.911666666, 0.000833333, 0, -34.172499999, 0,
-0.000833333)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[0], self.exp_files[4]])
ifg.close()
for i in self.ifgs:
i.close()
def test_orbital_correction_matlab_equality(self):
from pyrate.scripts import run_pyrate
run_pyrate._orb_fit_calc(self.ifg_paths, self.params)
onlyfiles = [
f for f in os.listdir(SML_TEST_MATLAB_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method1_')
]
count = 0
for i, f in enumerate(onlyfiles):
ifg_data = np.genfromtxt(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR,
f),
delimiter=',')
for k, j in enumerate(self.ifg_paths):
ifg = Ifg(j)
ifg.open()
if os.path.basename(j).split('_unw.')[0] == \
os.path.basename(f).split(
'_orb_planar_1lks_method1_')[1].split('.')[0]:
count += 1
# all numbers equal
np.testing.assert_array_almost_equal(ifg_data,
ifg.phase_data,
decimal=2)
# means must also be equal
self.assertAlmostEqual(np.nanmean(ifg_data),
np.nanmean(ifg.phase_data),
places=2)
# number of nans must equal
self.assertEqual(np.sum(np.isnan(ifg_data)),
np.sum(np.isnan(ifg.phase_data)))
ifg.close()
# ensure that we have expected number of matches
self.assertEqual(count, len(self.ifg_paths))
def test_write(self):
base = TEMPDIR
src = self.ifg.data_path
dest = join(base, basename(self.ifg.data_path))
# shutil.copy needs to copy writeable permission from src
os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH |
S_IRUSR | S_IWUSR)
shutil.copy(src, dest)
os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert
i = Ifg(dest)
i.open()
i.phase_data[0, 1:] = nan
i.write_modified_phase()
del i
# reopen to ensure data/nans can be read back out
i = Ifg(dest)
i.open(readonly=True)
assert_array_equal(True, isnan(i.phase_data[0, 1:]))
i.close()
os.remove(dest)
def test_custom_extents(self):
xlooks = ylooks = 1
cext = self._custom_extents_tuple()
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
user_exts=cext)
ifg = Ifg(self.exp_files[2])
ifg.open()
gt = ifg.dataset.GetGeoTransform()
exp_gt = (cext.xfirst, self.xs, 0, cext.yfirst, 0, self.ys)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[2], self.exp_files[6]])
# close ifgs
ifg.close()
for i in self.ifgs:
i.close()
def test_default_max_extents(self):
"""Test ifgcropopt=2 crops datasets to max bounding box extents."""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MAXIMUM_CROP, xlooks, ylooks)
for f in [self.exp_files[1], self.exp_files[5]]:
self.assertTrue(exists(f), msg="Output files not created")
# output files should have same extents
# NB: also verifies gdalwarp correctly copies geotransform across
ifg = Ifg(self.exp_files[1])
ifg.open()
gt = ifg.dataset.GetGeoTransform()
# copied from gdalinfo output
exp_gt = (150.91, 0.000833333, 0, -34.17, 0, -0.000833333)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[1], self.exp_files[5]])
ifg.close()
for i in self.ifgs:
i.close()
def _wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
Required due to differences between Matlab and Python MST
implementations.
"""
if not params[cf.APSEST]:
log.info('APS correction not required.')
return
# perform some checks on existing ifgs
log.info('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, preread_ifgs,
ifc.PYRATE_APS_ERROR):
log.info('Finished APS correction')
return # return if True condition returned
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
ifg.close()
def ref_pixel_calc(ifg_paths, params):
"""
Reference pixel calculation setup.
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return refx: Reference pixel x-coordinate
:return refy: Reference pixel y-coordinate
"""
# unlikely, but possible the refpixel can be (0,0)
# check if there is a pre-specified reference pixel coord
refx = params[cf.REFX]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx > ifg.ncols - 1:
msg = ('Supplied reference pixel X coordinate is greater than '
'the number of ifg columns: {}').format(refx)
raise ValueError(msg)
refy = params[cf.REFY]
if refy > ifg.nrows - 1:
msg = ('Supplied reference pixel Y coordinate is greater than '
'the number of ifg rows: {}').format(refy)
raise ValueError(msg)
if refx <= 0 or refy <= 0: # if either zero or negative
log.info('Searching for best reference pixel location')
refy, refx = find_ref_pixel(ifg_paths, params)
log.info('Selected reference pixel coordinate: '
'({}, {})'.format(refx, refy))
else: # pragma: no cover
log.info('Reusing reference pixel from config file: '
'({}, {})'.format(refx, refy))
ifg.close()
return refx, refy
class IfgIOTests(unittest.TestCase):
def setUp(self):
self.ifg = Ifg(join(SML_TEST_TIF, 'geo_070709-070813_unw.tif'))
def test_open(self):
self.assertTrue(self.ifg.dataset is None)
self.assertTrue(self.ifg.is_open is False)
self.ifg.open(readonly=True)
self.assertTrue(self.ifg.dataset is not None)
self.assertTrue(self.ifg.is_open is True)
self.assertTrue(isinstance(self.ifg.dataset, Dataset))
# ensure open cannot be called twice
self.assertRaises(RasterException, self.ifg.open, True)
def test_open_ifg_from_dataset(self):
"""
Test showing open() can not be used for Ifg created with
gdal.Dataset object as Dataset has already been read in
"""
paths = [self.ifg.data_path]
mlooked_phase_data = prepifg.prepare_ifgs(paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=2,
ylooks=2,
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_phase_data]
self.assertRaises(RasterException, mlooked[0].open)
def test_write(self):
base = TEMPDIR
src = self.ifg.data_path
dest = join(base, basename(self.ifg.data_path))
# shutil.copy needs to copy writeable permission from src
os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH |
S_IRUSR | S_IWUSR)
shutil.copy(src, dest)
os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert
i = Ifg(dest)
i.open()
i.phase_data[0, 1:] = nan
i.write_modified_phase()
del i
# reopen to ensure data/nans can be read back out
i = Ifg(dest)
i.open(readonly=True)
assert_array_equal(True, isnan(i.phase_data[0, 1:]))
i.close()
os.remove(dest)
def test_readonly_permission_failure(self):
# ensure failure if opening R/O permission file as writeable/GA_Update
self.assertRaises(IOError, self.ifg.open, False)
def test_write_fails_on_readonly(self):
# check readonly status is same before
# and after open() for readonly file
self.assertTrue(self.ifg.is_read_only)
self.ifg.open(readonly=True)
self.assertTrue(self.ifg.is_read_only)
self.assertRaises(IOError, self.ifg.write_modified_phase)
def test_phase_band_unopened_ifg(self):
try:
_ = self.ifg.phase_band
self.fail("Should not be able to access band without open dataset")
except RasterException:
pass
def test_nan_fraction_unopened(self):
try:
# NB: self.assertRaises doesn't work here (as it is a property?)
_ = self.ifg.nan_fraction
self.fail("Shouldn't be able to "
"call nan_fraction() with unopened Ifg")
except RasterException:
pass
def test_phase_data_properties(self):
# Use raw GDAL to isolate raster reading from Ifg functionality
ds = Open(self.ifg.data_path)
data = ds.GetRasterBand(1).ReadAsArray()
del ds
self.ifg.open()
# test full array and row by row access
assert_array_equal(data, self.ifg.phase_data)
for y, row in enumerate(self.ifg.phase_rows):
assert_array_equal(data[y], row)
# test the data is cached if changed
crd = (5,4)
orig = self.ifg.phase_data[crd]
self.ifg.phase_data[crd] *= 2
nv = self.ifg.phase_data[crd] # pull new value out again
self.assertEqual(nv, 2 * orig)
def network_correction(ifgs,
degree,
offset,
params,
m_ifgs=None,
preread_ifgs=None):
"""
Calculates orbital correction model, removing this from the interferograms.
.. warn:: This will write orbital error corrected phase_data in the interferograms.
:param ifgs: Interferograms reduced to a minimum tree from prior
MST calculations
:param degree: PLANAR, QUADRATIC or PART_CUBIC
:param offset: True to calculate the model using offsets
:param params: Parameter dictionary
:param m_ifgs: Multi-looked orbfit interferograms (sequence must be mlooked
versions of 'ifgs' arg)
:param preread_ifgs: Parameters dict corresponding to config file
:return xxxx
"""
# pylint: disable=too-many-locals, too-many-arguments
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = master_slave_ids(get_all_epochs(temp_ifgs))
else:
ids = master_slave_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifgs[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
dm = get_design_matrix(ifgs[0], degree, offset=False)
# TODO: remove this import from tests suite
from tests.common import MockIfg
for i in ifgs:
# open if not Ifg instance
if not (isinstance(i, Ifg)
or isinstance(i, MockIfg)): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_networkx_error(coefs, dm, i, ids, offset)
class SingleDesignMatrixTests(unittest.TestCase):
"""
Tests to verify correctness of basic planar & quadratic design matrices or
DMs. This class serves two purposes, ensuring the independent method DMs are
produced correctly. Secondly, these indivdual DMs are subsets of the larger
DM 'grid' required for the networked orbital correction method.
"""
def setUp(self):
# faked cell sizes
self.xs = 0.75
self.ys = 0.8
self.ifg = Ifg(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif'))
self.ifg.open()
self.ifg.nodata_value = 0
self.m = MockIfg(self.ifg, 3, 4)
self.m.x_size = self.xs
self.m.y_size = self.ys
# tests for planar model
def test_create_planar_dm(self):
offset = False
act = get_design_matrix(self.m, PLANAR, offset)
self.assertEqual(act.shape, (self.m.num_cells, 2))
exp = unittest_dm(self.m, INDEPENDENT_METHOD, PLANAR, offset)
assert_array_equal(act, exp)
def test_create_planar_dm_offsets(self):
offset = True
act = get_design_matrix(self.m, PLANAR, offset)
self.assertEqual(act.shape, (self.m.num_cells, 3))
exp = unittest_dm(self.m, INDEPENDENT_METHOD, PLANAR, offset)
assert_array_almost_equal(act, exp)
# tests for quadratic model
def test_create_quadratic_dm(self):
offset = False
act = get_design_matrix(self.m, QUADRATIC, offset)
self.assertEqual(act.shape, (self.m.num_cells, 5))
exp = unittest_dm(self.m, INDEPENDENT_METHOD, QUADRATIC, offset)
assert_array_equal(act, exp)
def test_create_quadratic_dm_offsets(self):
offset = True
act = get_design_matrix(self.m, QUADRATIC, offset)
self.assertEqual(act.shape, (self.m.num_cells, 6))
exp = unittest_dm(self.m, INDEPENDENT_METHOD, QUADRATIC, offset)
assert_array_equal(act, exp)
# tests for partial cubic model
def test_create_partcubic_dm(self):
offset = False
act = get_design_matrix(self.m, PART_CUBIC, offset)
self.assertEqual(act.shape, (self.m.num_cells, 6))
exp = unittest_dm(self.m, INDEPENDENT_METHOD, PART_CUBIC, offset)
assert_array_equal(act, exp)
def test_create_partcubic_dm_offsets(self):
offset = True
act = get_design_matrix(self.m, PART_CUBIC, offset)
self.assertEqual(act.shape, (self.m.num_cells, 7))
exp = unittest_dm(self.m, INDEPENDENT_METHOD, PART_CUBIC, offset)
assert_array_equal(act, exp)
# tests for unittest_dm() assuming network method
def test_create_planar_dm_network(self):
# networked method planar version should not have offsets col
ncol_exp = 2
exp = unittest_dm(self.m, NETWORK_METHOD, PLANAR, False)
self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
exp2 = unittest_dm(self.m, NETWORK_METHOD, PLANAR, True)
self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
assert_array_equal(exp, exp2)
def test_create_quadratic_dm_network(self):
# quadratic version with networked method does not have offsets col
ncol_exp = 5
exp = unittest_dm(self.m, NETWORK_METHOD, QUADRATIC, False)
self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
exp2 = unittest_dm(self.m, NETWORK_METHOD, QUADRATIC, True)
self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
assert_array_equal(exp, exp2)
def test_create_partcubic_dm_network(self):
# partial cubic version with networked method does not have offsets col
ncol_exp = 6
exp = unittest_dm(self.m, NETWORK_METHOD, PART_CUBIC, False)
self.assertEqual(exp.shape, (self.m.num_cells, ncol_exp))
exp2 = unittest_dm(self.m, NETWORK_METHOD, PART_CUBIC, True)
self.assertEqual(exp2.shape, (self.m.num_cells, ncol_exp))
assert_array_equal(exp, exp2)
def network_orbital_correction(ifgs,
degree,
offset,
params,
m_ifgs=None,
preread_ifgs=None):
"""
This algorithm implements a network inversion to determine orbital
corrections for a set of interferograms forming a connected network.
Warning: This will write orbital error corrected phase_data to the ifgs.
:param list ifgs: List of Ifg class objects reduced to a minimum spanning
tree network
:param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC)
:param bool offset: True to calculate the model using offsets
:param dict params: dictionary of configuration parameters
:param list m_ifgs: list of multilooked Ifg class objects
(sequence must be multilooked versions of 'ifgs' arg)
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
# pylint: disable=too-many-locals, too-many-arguments
src_ifgs = ifgs if m_ifgs is None else m_ifgs
src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst
vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs])
vphase = squeeze(vphase)
B = get_network_design_matrix(src_ifgs, degree, offset)
# filter NaNs out before getting model
B = B[~isnan(vphase)]
orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)])
ncoef = _get_num_params(degree)
if preread_ifgs:
temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values()
ids = master_slave_ids(get_all_epochs(temp_ifgs))
else:
ids = master_slave_ids(get_all_epochs(ifgs))
coefs = [
orbparams[i:i + ncoef] for i in range(0,
len(set(ids)) * ncoef, ncoef)
]
# create full res DM to expand determined coefficients into full res
# orbital correction (eg. expand coarser model to full size)
if preread_ifgs:
temp_ifg = Ifg(ifgs[0]) # ifgs here are paths
temp_ifg.open()
dm = get_design_matrix(temp_ifg, degree, offset=False)
temp_ifg.close()
else:
dm = get_design_matrix(ifgs[0], degree, offset=False)
for i in ifgs:
# open if not Ifg instance
if isinstance(i, str): # pragma: no cover
# are paths
i = Ifg(i)
i.open(readonly=False)
shared.nan_and_mm_convert(i, params)
_remove_network_orb_error(coefs, dm, i, ids, offset)
class IfgTests(unittest.TestCase):
"""Unit tests for the Ifg/interferogram class."""
def setUp(self):
self.ifg = Ifg(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif'))
self.ifg.open()
self.ifg.nodata_value = 0
def test_headers_as_attr(self):
for a in ['ncols', 'nrows', 'x_first', 'x_step',
'y_first', 'y_step', 'wavelength', 'master', 'slave']:
self.assertTrue(getattr(self.ifg, a) is not None)
def test_convert_to_nans(self):
self.ifg.convert_to_nans()
self.assertTrue(self.ifg.nan_converted)
def test_xylast(self):
# ensure the X|Y_LAST header element has been created
self.assertAlmostEqual(self.ifg.x_last, 150.9491667)
self.assertAlmostEqual(self.ifg.y_last, -34.23)
def test_num_cells(self):
# test cell size from header elements
data = self.ifg.phase_band.ReadAsArray()
ys, xs = data.shape
exp_ncells = ys * xs
self.assertEqual(exp_ncells, self.ifg.num_cells)
def test_shape(self):
self.assertEqual(self.ifg.shape, self.ifg.phase_data.shape)
def test_nan_count(self):
num_nan = 0
for row in self.ifg.phase_data:
for v in row:
if isnan(v):
num_nan += 1
if self.ifg.nan_converted:
self.assertEqual(num_nan, self.ifg.nan_count)
else:
self.assertEqual(num_nan, 0)
def test_phase_band(self):
data = self.ifg.phase_band.ReadAsArray()
self.assertEqual(data.shape, (72, 47) )
def test_nan_fraction(self):
# NB: source data lacks 0 -> NaN conversion
data = self.ifg.phase_data
data = where(data == 0, nan, data) # fake 0 -> nan for the count below
# manually count # nan cells
nans = 0
ys, xs = data.shape
for y, x in product(range(ys), range(xs)):
if isnan(data[y, x]):
nans += 1
del data
num_cells = float(ys * xs)
self.assertTrue(nans > 0)
self.assertTrue(nans <= num_cells)
self.assertEqual(nans / num_cells, self.ifg.nan_fraction)
def test_xy_size(self):
self.assertFalse(self.ifg.ncols is None)
self.assertFalse(self.ifg.nrows is None)
# test with tolerance from base 90m cell
# within 2% of cells over small?
self.assertTrue(self.ifg.y_size > 88.0)
self.assertTrue(self.ifg.y_size < 92.0, 'Got %s' % self.ifg.y_size)
width = 76.9 # from nearby Pirate coords
self.assertTrue(self.ifg.x_size > 0.97 * width) # ~3% tolerance
self.assertTrue(self.ifg.x_size < 1.03 * width)
def test_centre_latlong(self):
lat_exp = self.ifg.y_first + \
(int(self.ifg.nrows / 2) * self.ifg.y_step)
long_exp = self.ifg.x_first + \
(int(self.ifg.ncols / 2) * self.ifg.x_step)
self.assertEqual(lat_exp, self.ifg.lat_centre)
self.assertEqual(long_exp, self.ifg.long_centre)
def test_centre_cell(self):
self.assertEqual(self.ifg.x_centre, 23)
self.assertEqual(self.ifg.y_centre, 36)
def test_time_span(self):
self.assertAlmostEqual(self.ifg.time_span, 0.287474332649)
def test_wavelength(self):
self.assertEqual(self.ifg.wavelength, 0.0562356424)