def setUp(self):
from tests.common import small_data_setup
self.ifgs = small_data_setup()
self.ifg_paths = [i.data_path for i in self.ifgs]
prepare_ifgs(self.ifg_paths, crop_opt=1, xlooks=1, ylooks=1)
looks_paths = [
mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths
]
self.ifgs_with_nan = [Ifg(i) for i in looks_paths]
for ifg in self.ifgs_with_nan:
ifg.open()
def _save_aps_corrected_phase(ifg_path, phase):
"""
Save (update) interferogram metadata and phase data after
spatio-temporal filter (APS) correction.
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data
)]
# set aps tags after aps error correction
ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED)
ifg.write_modified_phase()
ifg.close()
def prepare_ifgs_without_phase(ifg_paths, params):
ifgs = [Ifg(p) for p in ifg_paths]
for i in ifgs:
if not i.is_open:
i.open(readonly=False)
nan_conversion = params[cf.NAN_CONVERSION]
if nan_conversion: # nan conversion happens here in networkx mst
# if not ifg.nan_converted:
i.nodata_value = params[cf.NO_DATA_VALUE]
i.convert_to_nans()
return ifgs
def small5_ifgs():
"""Convenience func to return a subset of 5 linked Ifgs from the testdata"""
BASE_DIR = tempfile.mkdtemp()
data_paths = [os.path.join(SML_TEST_TIF, p) for p in IFMS5.split()]
new_data_paths = [
os.path.join(BASE_DIR, os.path.basename(d)) for d in data_paths
]
for d in data_paths:
shutil.copy(d, os.path.join(BASE_DIR, os.path.basename(d)))
return [Ifg(p) for p in new_data_paths]
def ref_pixel_setup(ifgs_or_paths, params):
"""
Sets up the grid for reference pixel computation.
Also saves numpy files for later use during reference pixel computation.
:param ifgs_or_paths: xxxx
:param params: xxxx
:return xxxx
"""
log.info('Setting up ref pixel computation')
refnx, refny, chipsize, min_frac = params[cf.REFNX], \
params[cf.REFNY], \
params[cf.REF_CHIP_SIZE], \
params[cf.REF_MIN_FRAC]
if len(ifgs_or_paths) < 1:
msg = 'Reference pixel search requires 2+ interferograms'
raise RefPixelError(msg)
if isinstance(ifgs_or_paths[0], str):
head = Ifg(ifgs_or_paths[0])
head.open(readonly=True)
else:
head = ifgs_or_paths[0]
# sanity check inputs
_validate_chipsize(chipsize, head)
_validate_minimum_fraction(min_frac)
_validate_search_win(refnx, refny, chipsize, head)
# pre-calculate useful amounts
half_patch_size = chipsize // 2
chipsize = half_patch_size * 2 + 1
thresh = min_frac * chipsize * chipsize
# do window searches across dataset, central pixel of stack with smallest
# mean is the reference pixel
rows, cols = head.shape
ysteps = step(rows, refny, half_patch_size)
xsteps = step(cols, refnx, half_patch_size)
log.info('Ref pixel setup finished')
return half_patch_size, thresh, list(product(ysteps, xsteps))
def _inner(ifg_path):
"""
Convenient inner loop
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_phase = rpe._est_ref_phs_method1(phase_data, comp)
phase_data -= ref_phase
md = ifg.meta_data
md[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_phase
def test_open_ifg_from_dataset(self):
"""
Test showing open() can not be used for Ifg created with
gdal.Dataset object as Dataset has already been read in
"""
paths = [self.ifg.data_path]
mlooked_phase_data = prepifg.prepare_ifgs(paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=2,
ylooks=2,
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_phase_data]
self.assertRaises(RasterException, mlooked[0].open)
def _inner(ifg_path):
"""
Convenient inner loop
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_ph = rpe._est_ref_phs_method2(phase_data, half_chip_size, refpx,
refpy, thresh)
phase_data -= ref_ph
md = ifg.meta_data
md[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_ph
def remove_orbital_error(ifgs, params, preread_ifgs=None):
"""
Wrapper for orbital error removal functionality.
:param ifgs: List of interferograms or interferogram paths
:param params: Dict corresponding to config parameters
:param preread_ifgs: Dict containing information regarding MPI jobs (optional)
:return xxxx
"""
log.info('Calculating orbital error correction')
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required')
return
if preread_ifgs: # don't check except for mpi tests
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
# perform some general error/sanity checks
if mpiops.rank == 0:
_check_orbital_ifgs(preread_ifgs)
ifg_paths = [i.data_path for i in ifgs] \
if isinstance(ifgs[0], Ifg) else ifgs
mlooked = None
# mlooking is not necessary for independent correction
# can use multiple procesing if write_to_disc=True
if params[cf.ORBITAL_FIT_METHOD] == 2:
mlooked_dataset = prepifg.prepare_ifgs(
ifg_paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=params[cf.ORBITAL_FIT_LOOKS_X],
ylooks=params[cf.ORBITAL_FIT_LOOKS_Y],
thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD],
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_dataset]
for m in mlooked:
m.initialize()
m.nodata_value = params[cf.NO_DATA_VALUE]
m.convert_to_nans()
m.convert_to_mm()
orbital_correction(ifgs,
params,
mlooked=mlooked,
preread_ifgs=preread_ifgs)
def dem_or_ifg(data_path):
"""
Whether tif is a dem or an interferogram
:param: data_path: xxxxx
:return xxxx
"""
ds = gdal.Open(data_path)
md = ds.GetMetadata()
if 'DATE' in md: # ifg
return Ifg(data_path)
else:
return DEM(data_path)
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = vcm_module.RDist(ifg)()
ifg.close()
return r_dist
def dem_or_ifg(data_path):
"""
Returns an Ifg or DEM class object from input geotiff file.
:param str data_path: file path name
:return: Interferogram or DEM object from input file
:rtype: Ifg or DEM class object
"""
ds = gdal.Open(data_path)
md = ds.GetMetadata()
if 'DATE' in md: # ifg
return Ifg(data_path)
else:
return DEM(data_path)
def data_setup(datafiles):
"""
xxxxxxxxxxx.
:param datafiles: xxxx
:return Interferogram objects for the files in the small_test data directory. The input phase data are in radians
(not converted to mm).
"""
datafiles.sort()
ifgs = [Ifg(i) for i in datafiles]
for i in ifgs:
i.open()
return ifgs
def test_same_size_multilooking(self):
ifgs = same_exts_ifgs()
ifg_data_paths = [d.data_path for d in ifgs]
xlooks = ylooks = 2
prepare_ifgs(ifg_data_paths, ALREADY_SAME_SIZE, xlooks, ylooks)
looks_paths = [
mlooked_path(d, looks=xlooks, crop_out=ALREADY_SAME_SIZE)
for d in ifg_data_paths
]
mlooked = [Ifg(i) for i in looks_paths]
for m in mlooked:
m.open()
self.assertEqual(len(mlooked), 2)
for ifg in mlooked:
self.assertAlmostEqual(ifg.x_step, xlooks * self.xs)
self.assertAlmostEqual(ifg.x_step, ylooks * self.xs)
def parse(self, string):
"""
override of :py:meth:`DictParam.parse`.
"""
dct = super(RasterParam, self).parse(string)
raster_type = dct['type']
path = dct['path']
if raster_type == 'DEM':
return DEM(path)
elif raster_type == 'Ifg':
return Ifg(path)
elif raster_type == 'Incidence':
return Incidence(path)
else: # pragma: no cover
raise luigi.parameter.UnknownParameterException(
'rasterBase must be an inscance DEM, '
'Ifg or Incidence is valid')
def small_data_setup(datafiles=None, is_dir=False):
"""Returns Ifg objs for the files in the small test dir
input phase data is in radians; these ifgs are in radians - not converted to mm"""
if is_dir:
datafiles = glob.glob(join(datafiles, "*.tif"))
else:
if datafiles:
for i, d in enumerate(datafiles):
datafiles[i] = os.path.join(SML_TEST_TIF, d)
else:
datafiles = glob.glob(join(SML_TEST_TIF, "*.tif"))
datafiles.sort()
ifgs = [Ifg(i) for i in datafiles]
for i in ifgs:
i.open()
i.nodata_value = 0
return ifgs
def remove_orbital_error(ifgs, params, preread_ifgs=None):
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
:param list ifgs: List of interferograms class objects
:param dict params: Dictionary containing configuration parameters
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
ifg_paths = [i.data_path for i in ifgs] \
if isinstance(ifgs[0], Ifg) else ifgs
mlooked = None
# mlooking is not necessary for independent correction
# can use multiple procesing if write_to_disc=True
if params[cf.ORBITAL_FIT_METHOD] == NETWORK_METHOD:
mlooked_dataset = prepifg.prepare_ifgs(
ifg_paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=params[cf.ORBITAL_FIT_LOOKS_X],
ylooks=params[cf.ORBITAL_FIT_LOOKS_Y],
thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD],
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_dataset]
for m in mlooked:
m.initialize()
m.nodata_value = params[cf.NO_DATA_VALUE]
m.convert_to_nans()
m.convert_to_mm()
_orbital_correction(ifgs,
params,
mlooked=mlooked,
preread_ifgs=preread_ifgs)
def test_nodata(self):
"""Verify NODATA value copied correctly (amplitude band not copied)"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
# NB: amplitude band doesn't have a NODATA value
self.assertTrue(
isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue()))
ifg.close()
for i in self.ifgs:
i.close()
def get_tiles(ifg_path, rows, cols):
"""
Break up the interferograms into tiles based on user supplied rows and columns.
:param ifg_path: List of destination tifs
:param rows: Number of rows to break each interferogram into
:param cols: Number of columns to break each interferogram into
:return tiles: List of shared.Tile instances
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=True)
tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols)
ifg.close()
return tiles
def test_nans(self):
"""Verify NaNs replace 0 in the multilooked phase band"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
phase = ifg.phase_band.ReadAsArray()
self.assertFalse((phase == 0).any())
self.assertTrue((isnan(phase)).any())
ifg.close()
self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo
self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo
for i in self.ifgs:
i.close()
def test_get_epochs(self):
def str2date(s):
segs = s[:4], s[4:6], s[6:] # year, month, day
return date(*[int(sg) for sg in segs])
raw_date = ['20060619', '20060828', '20061002', '20061106', '20061211',
'20070115', '20070219', '20070326', '20070430', '20070604',
'20070709', '20070813', '20070917']
exp_dates = [str2date(d) for d in raw_date]
exp_repeat = [1, 1, 3, 3, 4, 3, 3, 3, 3, 3, 3, 2, 2]
exp_spans = [0, 0.1916, 0.2875, 0.3833, 0.4791, 0.5749, 0.6708, 0.7666,
0.8624, 0.9582, 1.0541, 1.1499, 1.2457]
ifms = join(SML_TEST_TIF, "ifms_17")
ifgs = [Ifg(join(SML_TEST_TIF, p)) for p in parse_namelist(ifms)]
for i in ifgs:
i.open()
epochs = get_epochs(ifgs)[0]
self.assertTrue((exp_dates == epochs.dates).all())
self.assertTrue((exp_repeat == epochs.repeat).all())
assert_array_almost_equal(exp_spans, epochs.spans, decimal=4)
def _ref_pixel_calc(ifg_paths, params):
"""
Wrapper for reference pixel calculation
"""
# unlikely, but possible the refpixel can be (0,0)
# check if there is a pre-specified reference pixel coord
refx = params[cf.REFX]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx > ifg.ncols - 1:
msg = ('Supplied reference pixel X coordinate is greater than '
'the number of ifg columns: {}').format(refx)
raise ValueError(msg)
refy = params[cf.REFY]
if refy > ifg.nrows - 1:
msg = ('Supplied reference pixel Y coordinate is greater than '
'the number of ifg rows: {}').format(refy)
raise ValueError(msg)
if refx <= 0 or refy <= 0: # if either zero or negative
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = refpixel.ref_pixel_setup(ifg_paths,
params)
process_grid = mpiops.array_split(grid)
refpixel.save_ref_pixel_blocks(process_grid, half_patch_size,
ifg_paths, params)
mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size,
ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MASTER_PROCESS:
mean_sds = np.hstack(mean_sds)
refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid)
log.info('Selected reference pixel coordinate: '
'({}, {})'.format(refx, refy))
else: # pragma: no cover
log.info('Reusing reference pixel from config file: '
'({}, {})'.format(refx, refy))
ifg.close()
return refx, refy
def test_min_extents(self):
"""Test ifgcropopt=1 crops datasets to min extents."""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
ifg = Ifg(self.exp_files[0])
ifg.open()
# output files should have same extents
# NB: also verifies gdalwarp correctly copies geotransform across
# NB: expected data copied from gdalinfo output
gt = ifg.dataset.GetGeoTransform()
exp_gt = (150.911666666, 0.000833333, 0, -34.172499999, 0,
-0.000833333)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[0], self.exp_files[4]])
ifg.close()
for i in self.ifgs:
i.close()
def test_orbital_correction_matlab_equality(self):
from pyrate.scripts import run_pyrate
run_pyrate._orb_fit_calc(self.ifg_paths, self.params)
onlyfiles = [
f for f in os.listdir(SML_TEST_MATLAB_ORBITAL_DIR)
if os.path.isfile(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f))
and f.endswith('.csv') and f.__contains__('_method1_')
]
count = 0
for i, f in enumerate(onlyfiles):
ifg_data = np.genfromtxt(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR,
f),
delimiter=',')
for k, j in enumerate(self.ifg_paths):
ifg = Ifg(j)
ifg.open()
if os.path.basename(j).split('_unw.')[0] == \
os.path.basename(f).split(
'_orb_planar_1lks_method1_')[1].split('.')[0]:
count += 1
# all numbers equal
np.testing.assert_array_almost_equal(ifg_data,
ifg.phase_data,
decimal=2)
# means must also be equal
self.assertAlmostEqual(np.nanmean(ifg_data),
np.nanmean(ifg.phase_data),
places=2)
# number of nans must equal
self.assertEqual(np.sum(np.isnan(ifg_data)),
np.sum(np.isnan(ifg.phase_data)))
ifg.close()
# ensure that we have expected number of matches
self.assertEqual(count, len(self.ifg_paths))
def test_default_max_extents(self):
"""Test ifgcropopt=2 crops datasets to max bounding box extents."""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MAXIMUM_CROP, xlooks, ylooks)
for f in [self.exp_files[1], self.exp_files[5]]:
self.assertTrue(exists(f), msg="Output files not created")
# output files should have same extents
# NB: also verifies gdalwarp correctly copies geotransform across
ifg = Ifg(self.exp_files[1])
ifg.open()
gt = ifg.dataset.GetGeoTransform()
# copied from gdalinfo output
exp_gt = (150.91, 0.000833333, 0, -34.17, 0, -0.000833333)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[1], self.exp_files[5]])
ifg.close()
for i in self.ifgs:
i.close()
def test_custom_extents(self):
xlooks = ylooks = 1
cext = self._custom_extents_tuple()
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
user_exts=cext)
ifg = Ifg(self.exp_files[2])
ifg.open()
gt = ifg.dataset.GetGeoTransform()
exp_gt = (cext.xfirst, self.xs, 0, cext.yfirst, 0, self.ys)
for i, j in zip(gt, exp_gt):
self.assertAlmostEqual(i, j)
self.assert_geotransform_equal([self.exp_files[2], self.exp_files[6]])
# close ifgs
ifg.close()
for i in self.ifgs:
i.close()
def ref_pixel_calc(ifg_paths, params):
"""
Reference pixel calculation setup.
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return refx: Reference pixel x-coordinate
:return refy: Reference pixel y-coordinate
"""
# unlikely, but possible the refpixel can be (0,0)
# check if there is a pre-specified reference pixel coord
refx = params[cf.REFX]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
if refx > ifg.ncols - 1:
msg = ('Supplied reference pixel X coordinate is greater than '
'the number of ifg columns: {}').format(refx)
raise ValueError(msg)
refy = params[cf.REFY]
if refy > ifg.nrows - 1:
msg = ('Supplied reference pixel Y coordinate is greater than '
'the number of ifg rows: {}').format(refy)
raise ValueError(msg)
if refx <= 0 or refy <= 0: # if either zero or negative
log.info('Searching for best reference pixel location')
refy, refx = find_ref_pixel(ifg_paths, params)
log.info('Selected reference pixel coordinate: '
'({}, {})'.format(refx, refy))
else: # pragma: no cover
log.info('Reusing reference pixel from config file: '
'({}, {})'.format(refx, refy))
ifg.close()
return refx, refy
def _wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
Required due to differences between Matlab and Python MST
implementations.
"""
if not params[cf.APSEST]:
log.info('APS correction not required.')
return
# perform some checks on existing ifgs
log.info('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, preread_ifgs,
ifc.PYRATE_APS_ERROR):
log.info('Finished APS correction')
return # return if True condition returned
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter
ifg.open()
spatio_temporal_filter(tsincr, ifg, params, preread_ifgs)
ifg.close()
def phase_sum(ifg_paths, params):
"""
Save phase data and phs_sum used in the reference phase estimation.
:param ifg_paths: List of paths to interferograms
:param params: Config dictionary
:return xxxx
"""
p_paths = mpiops.array_split(ifg_paths)
ifg = Ifg(p_paths[0])
ifg.open(readonly=True)
shape = ifg.shape
phs_sum = np.zeros(shape=shape, dtype=np.float64)
ifg.close()
for d in p_paths:
ifg = Ifg(d)
ifg.open()
ifg.nodata_value = params[cf.NO_DATA_VALUE]
phs_sum += ifg.phase_data
ifg.close()
if mpiops.rank == MASTER_PROCESS:
phase_sum_all = phs_sum
# loop is better for memory
for i in range(1, mpiops.size): # pragma: no cover
phs_sum = np.zeros(shape=shape, dtype=np.float64)
mpiops.comm.Recv(phs_sum, source=i, tag=i)
phase_sum_all += phs_sum
comp = np.isnan(phase_sum_all) # this is the same as in Matlab
comp = np.ravel(comp, order='F') # this is the same as in Matlab
else: # pragma: no cover
comp = None
mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank)
comp = mpiops.comm.bcast(comp, root=0)
return comp
def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params):
"""
xxxx
:param grid: List of tuples (y, x) corresponding reference pixel grids
:param half_patch_size: Patch size in pixels corresponding to reference pixel grids
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return xxxx
"""
log.info('Saving ref pixel blocks')
outdir = params[cf.TMPDIR]
for pth in ifg_paths:
ifg = Ifg(pth)
ifg.open(readonly=True)
ifg.nodata_value = params[cf.NO_DATA_VALUE]
ifg.convert_to_nans()
ifg.convert_to_mm()
for y, x in grid:
data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1,
x - half_patch_size:x + half_patch_size + 1]
data_file = join(
outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format(
b=os.path.basename(pth).split('.')[0], y=y, x=x))
np.save(file=data_file, arr=data)
ifg.close()
log.info('Saved ref pixel blocks')
