def _phase_sum(ifg_paths, params):
"""
Save phase data and phase sum used in the reference phase estimation
"""
p_paths = mpiops.array_split(ifg_paths)
ifg = Ifg(p_paths[0])
ifg.open(readonly=True)
shape = ifg.shape
phs_sum = np.zeros(shape=shape, dtype=np.float64)
ifg.close()
for d in p_paths:
ifg = Ifg(d)
ifg.open()
ifg.nodata_value = params[cf.NO_DATA_VALUE]
phs_sum += ifg.phase_data
ifg.close()
if mpiops.rank == MASTER_PROCESS:
phase_sum_all = phs_sum
# loop is better for memory
for i in range(1, mpiops.size): # pragma: no cover
phs_sum = np.zeros(shape=shape, dtype=np.float64)
mpiops.comm.Recv(phs_sum, source=i, tag=i)
phase_sum_all += phs_sum
comp = np.isnan(phase_sum_all) # this is the same as in Matlab
comp = np.ravel(comp, order='F') # this is the same as in Matlab
else: # pragma: no cover
comp = None
mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank)
comp = mpiops.comm.bcast(comp, root=0)
return comp
def test_gdal_python_vs_old_prepifg_prep2(self):
for i in range(10):
thresh = 0.5
x_looks = y_looks = 2
res = 2
extents = [10, 0, 20, 10]
extents_str = [str(e) for e in extents]
data = np.array(np.random.randint(0, 3, size=(10, 10)),
dtype=np.float32)
# create the self.temp_tiff
self.manipulation(data, self.temp_tif, self.md)
# only band 1 is resapled in warp_old
averaged_and_resampled, _ = gdalwarp.crop_resample_average(
self.temp_tif,
extents, [res, -res],
self.out_tif,
thresh,
match_pirate=True)
ifg = Ifg(self.temp_tif)
# only band 1 is resampled in warp_old
data, self.old_prepifg_path = warp_old(ifg,
x_looks,
y_looks,
extents_str, [res, -res],
thresh=thresh,
crop_out=4,
verbose=False)
np.testing.assert_array_almost_equal(data,
averaged_and_resampled,
decimal=4)
def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params):
"""
xxxx
:param grid: List of tuples (y, x) corresponding reference pixel grids
:param half_patch_size: Patch size in pixels corresponding to reference pixel grids
:param ifg_paths: List of interferogram paths
:param params: Parameters dictionary corresponding to config file
:return xxxx
"""
log.info('Saving ref pixel blocks')
outdir = params[cf.TMPDIR]
for pth in ifg_paths:
ifg = Ifg(pth)
ifg.open(readonly=True)
ifg.nodata_value = params[cf.NO_DATA_VALUE]
ifg.convert_to_nans()
ifg.convert_to_mm()
for y, x in grid:
data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1,
x - half_patch_size:x + half_patch_size + 1]
data_file = join(
outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format(
b=os.path.basename(pth).split('.')[0], y=y, x=x))
np.save(file=data_file, arr=data)
ifg.close()
log.info('Saved ref pixel blocks')
def diff_exts_ifgs():
"""Returns pair of test Ifgs with different extents"""
bases = ['geo_060619-061002.tif', 'geo_070326-070917.tif']
random_dir = tempfile.mkdtemp()
for p in bases:
shutil.copy(src=os.path.join(PREP_TEST_TIF, p),
dst=os.path.join(random_dir, p))
return [Ifg(join(random_dir, p)) for p in bases], random_dir
def _data_setup(datafiles):
"""Returns Ifg objs for the files in the interferogram list"""
datafiles.sort()
ifgs = [Ifg(i) for i in datafiles]
for i in ifgs:
i.open()
return ifgs
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = vcm_module.RDist(ifg)()
ifg.close()
return r_dist
def test_multilook(self):
"""Test resampling method using a scaling factor of 4"""
scale = 4 # assumes square cells
self.ifgs.append(DEM(SML_TEST_DEM_TIF))
self.ifg_paths = [i.data_path for i in self.ifgs]
cext = self._custom_extents_tuple()
xlooks = ylooks = scale
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
thresh=1.0,
user_exts=cext)
for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]):
i = Ifg(ipath)
i.open()
self.assertEqual(i.dataset.RasterXSize, 20 / scale)
self.assertEqual(i.dataset.RasterYSize, 28 / scale)
# verify resampling
path = join(PREP_TEST_TIF, "%s.tif" % n)
ds = gdal.Open(path)
src_data = ds.GetRasterBand(2).ReadAsArray()
exp_resample = multilooking(src_data, scale, scale, thresh=0)
self.assertEqual(exp_resample.shape, (7, 5))
assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray())
ds = None
i.close()
os.remove(ipath)
# verify DEM has been correctly processed
# ignore output values as resampling has already been tested for phase
exp_dem_path = join(SML_TEST_DEM_DIR,
'roipac_test_trimmed_4rlks_3cr.tif')
self.assertTrue(exists(exp_dem_path))
orignal_dem = DEM(SML_TEST_DEM_TIF)
orignal_dem.open()
dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType
orignal_dem.close()
dem = DEM(exp_dem_path)
dem.open()
# test multilooked dem is of the same datatype as the original dem tif
self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType)
self.assertEqual(dem.dataset.RasterXSize, 20 / scale)
self.assertEqual(dem.dataset.RasterYSize, 28 / scale)
data = dem.height_band.ReadAsArray()
self.assertTrue(data.ptp() != 0)
# close ifgs
dem.close()
for i in self.ifgs:
i.close()
os.remove(exp_dem_path)
def test_gdal_python_vs_old_prepifg_prep(self):
for i in range(10):
data = np.array(np.random.randint(0, 3, size=(10, 10)),
dtype=np.float32)
src_ds = gdal.GetDriverByName('GTiff').Create(
self.temp_tif, 10, 10, 2, gdalconst.GDT_Float32)
src_ds.GetRasterBand(1).WriteArray(data)
src_ds.GetRasterBand(1).SetNoDataValue(0)
nan_matrix = where(data == 0, nan, data)
src_ds.GetRasterBand(2).WriteArray(np.isnan(nan_matrix))
src_ds.GetRasterBand(2).SetNoDataValue(-100)
src_ds.SetGeoTransform([10, 1, 0, 10, 0, -1])
dst_ds = gdal.GetDriverByName('MEM').Create(
'', 5, 5, 2, gdalconst.GDT_Float32)
dst_ds.SetGeoTransform([10, 2, 0, 10, 0, -2])
for k, v in self.md.items():
src_ds.SetMetadataItem(k, v)
dst_ds.SetMetadataItem(k, v)
src_ds.FlushCache()
gdal.ReprojectImage(src_ds, dst_ds, '', '', gdal.GRA_Average)
nan_frac = dst_ds.GetRasterBand(2).ReadAsArray()
avg = dst_ds.GetRasterBand(1).ReadAsArray()
thresh = 0.5
avg[nan_frac >= thresh] = np.nan
ifg = Ifg(self.temp_tif)
x_looks = y_looks = 2
res = 2
resolution = [res, -res]
# minX, minY, maxX, maxY = extents
# adfGeoTransform[0] /* top left x */
# adfGeoTransform[1] /* w-e pixel resolution */
# adfGeoTransform[2] /* 0 */
# adfGeoTransform[3] /* top left y */
# adfGeoTransform[4] /* 0 */
# adfGeoTransform[5] /* n-s pixel resolution (negative value) */
extents = [str(e) for e in [10, 0, 20, 10]]
# only band 1 is resapled in warp_old
data, self.old_prepifg_path = warp_old(ifg,
x_looks,
y_looks,
extents,
resolution,
thresh=thresh,
crop_out=4,
verbose=False)
np.testing.assert_array_equal(data, avg)
def setUp(self):
from tests.common import small_data_setup
self.ifgs = small_data_setup()
self.ifg_paths = [i.data_path for i in self.ifgs]
prepare_ifgs(self.ifg_paths, crop_opt=1, xlooks=1, ylooks=1)
looks_paths = [
mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths
]
self.ifgs_with_nan = [Ifg(i) for i in looks_paths]
for ifg in self.ifgs_with_nan:
ifg.open()
def setUp(self):
# faked cell sizes
self.xs = 0.75
self.ys = 0.8
self.ifg = Ifg(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif'))
self.ifg.open()
self.ifg.nodata_value = 0
self.m = MockIfg(self.ifg, 3, 4)
self.m.x_size = self.xs
self.m.y_size = self.ys
def prepare_ifgs_without_phase(ifg_paths, params):
ifgs = [Ifg(p) for p in ifg_paths]
for i in ifgs:
if not i.is_open:
i.open(readonly=False)
nan_conversion = params[cf.NAN_CONVERSION]
if nan_conversion: # nan conversion happens here in networkx mst
# if not ifg.nan_converted:
i.nodata_value = params[cf.NO_DATA_VALUE]
i.convert_to_nans()
return ifgs
def pre_prepare_ifgs(ifg_paths, params):
"""
Open ifg for reading
"""
ifgs = [Ifg(p) for p in ifg_paths]
for i in ifgs:
if not i.is_open:
i.open(readonly=False)
nan_and_mm_convert(i, params)
log.info('Opened ifg for reading')
return ifgs
def small5_ifgs():
"""Convenience func to return a subset of 5 linked Ifgs from the testdata"""
BASE_DIR = tempfile.mkdtemp()
data_paths = [os.path.join(SML_TEST_TIF, p) for p in IFMS5.split()]
new_data_paths = [
os.path.join(BASE_DIR, os.path.basename(d)) for d in data_paths
]
for d in data_paths:
shutil.copy(d, os.path.join(BASE_DIR, os.path.basename(d)))
return [Ifg(p) for p in new_data_paths]
def _inner(ifg_path):
""" convenient inner loop """
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_phase = rpe.est_ref_phs_method1(phase_data, comp)
phase_data -= ref_phase
md = ifg.meta_data
md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_phase
def _inner(ifg_path):
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
phase_data = ifg.phase_data
ref_ph = rpe.est_ref_phs_method2(phase_data, half_chip_size, refpx,
refpy, thresh)
phase_data -= ref_ph
md = ifg.meta_data
md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase(data=phase_data)
ifg.close()
return ref_ph
def test_open_ifg_from_dataset(self):
"""
Test showing open() can not be used for Ifg created with
gdal.Dataset object as Dataset has already been read in
"""
paths = [self.ifg.data_path]
mlooked_phase_data = prepifg.prepare_ifgs(paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=2,
ylooks=2,
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_phase_data]
self.assertRaises(RasterException, mlooked[0].open)
def _save_aps_corrected_phase(ifg_path, phase):
"""
Save (update) interferogram metadata and phase data after
spatio-temporal filter (APS) correction.
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=False)
ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data
)]
# set aps tags after aps error correction
ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED)
ifg.write_modified_phase()
ifg.close()
def test_nodata(self):
"""Verify NODATA value copied correctly (amplitude band not copied)"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
# NB: amplitude band doesn't have a NODATA value
self.assertTrue(
isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue()))
ifg.close()
for i in self.ifgs:
i.close()
def dem_or_ifg(data_path):
"""
Whether tif is a dem or an interferogram
:param: data_path: xxxxx
:return xxxx
"""
ds = gdal.Open(data_path)
md = ds.GetMetadata()
if 'DATE' in md: # ifg
return Ifg(data_path)
else:
return DEM(data_path)
def test_write(self):
base = TEMPDIR
src = self.ifg.data_path
dest = join(base, basename(self.ifg.data_path))
# shutil.copy needs to copy writeable permission from src
os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH |
S_IRUSR | S_IWUSR)
shutil.copy(src, dest)
os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert
i = Ifg(dest)
i.open()
i.phase_data[0, 1:] = nan
i.write_modified_phase()
del i
# reopen to ensure data/nans can be read back out
i = Ifg(dest)
i.open(readonly=True)
assert_array_equal(True, isnan(i.phase_data[0, 1:]))
i.close()
os.remove(dest)
def remove_orbital_error(ifgs, params, preread_ifgs=None):
"""
Wrapper for orbital error removal functionality.
:param ifgs: List of interferograms or interferogram paths
:param params: Dict corresponding to config parameters
:param preread_ifgs: Dict containing information regarding MPI jobs (optional)
:return xxxx
"""
log.info('Calculating orbital error correction')
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required')
return
if preread_ifgs: # don't check except for mpi tests
# remove non ifg keys
_ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']]
# perform some general error/sanity checks
if mpiops.rank == 0:
_check_orbital_ifgs(preread_ifgs)
ifg_paths = [i.data_path for i in ifgs] \
if isinstance(ifgs[0], Ifg) else ifgs
mlooked = None
# mlooking is not necessary for independent correction
# can use multiple procesing if write_to_disc=True
if params[cf.ORBITAL_FIT_METHOD] == 2:
mlooked_dataset = prepifg.prepare_ifgs(
ifg_paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=params[cf.ORBITAL_FIT_LOOKS_X],
ylooks=params[cf.ORBITAL_FIT_LOOKS_Y],
thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD],
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_dataset]
for m in mlooked:
m.initialize()
m.nodata_value = params[cf.NO_DATA_VALUE]
m.convert_to_nans()
m.convert_to_mm()
orbital_correction(ifgs,
params,
mlooked=mlooked,
preread_ifgs=preread_ifgs)
def get_tiles(ifg_path, rows, cols):
"""
Break up the interferograms into tiles based on user supplied rows and columns.
:param ifg_path: List of destination tifs
:param rows: Number of rows to break each interferogram into
:param cols: Number of columns to break each interferogram into
:return tiles: List of shared.Tile instances
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=True)
tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols)
ifg.close()
return tiles
def data_setup(datafiles):
"""
xxxxxxxxxxx.
:param datafiles: xxxx
:return Interferogram objects for the files in the small_test data directory. The input phase data are in radians
(not converted to mm).
"""
datafiles.sort()
ifgs = [Ifg(i) for i in datafiles]
for i in ifgs:
i.open()
return ifgs
def dem_or_ifg(data_path):
"""
Returns an Ifg or DEM class object from input geotiff file.
:param str data_path: file path name
:return: Interferogram or DEM object from input file
:rtype: Ifg or DEM class object
"""
ds = gdal.Open(data_path)
md = ds.GetMetadata()
if 'DATE' in md: # ifg
return Ifg(data_path)
else:
return DEM(data_path)
def ref_pixel_setup(ifgs_or_paths, params):
"""
Sets up the grid for reference pixel computation and saves numpy files
to disk for later use during ref pixel computation.
:param list ifgs_or_paths: List of interferogram filenames or Ifg objects
:param dict params: Dictionary of configuration parameters
:return: half_patch_size: size of patch
:rtype: float
:return: thresh
:rtype: float
:return: list(product(ysteps, xsteps))
:rtype: list
"""
log.info('Setting up ref pixel computation')
refnx, refny, chipsize, min_frac = params[cf.REFNX], \
params[cf.REFNY], \
params[cf.REF_CHIP_SIZE], \
params[cf.REF_MIN_FRAC]
if len(ifgs_or_paths) < 1:
msg = 'Reference pixel search requires 2+ interferograms'
raise RefPixelError(msg)
if isinstance(ifgs_or_paths[0], str):
head = Ifg(ifgs_or_paths[0])
head.open(readonly=True)
else:
head = ifgs_or_paths[0]
# sanity check inputs
_validate_chipsize(chipsize, head)
_validate_minimum_fraction(min_frac)
_validate_search_win(refnx, refny, chipsize, head)
# pre-calculate useful amounts
half_patch_size = chipsize // 2
chipsize = half_patch_size * 2 + 1
thresh = min_frac * chipsize * chipsize
# do window searches across dataset, central pixel of stack with smallest
# mean is the reference pixel
rows, cols = head.shape
ysteps = _step(rows, refny, half_patch_size)
xsteps = _step(cols, refnx, half_patch_size)
log.info('Ref pixel setup finished')
return half_patch_size, thresh, list(product(ysteps, xsteps))
def test_nans(self):
"""Verify NaNs replace 0 in the multilooked phase band"""
xlooks = ylooks = 1
prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks)
for ex in [self.exp_files[0], self.exp_files[4]]:
ifg = Ifg(ex)
ifg.open()
phase = ifg.phase_band.ReadAsArray()
self.assertFalse((phase == 0).any())
self.assertTrue((isnan(phase)).any())
ifg.close()
self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo
self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo
for i in self.ifgs:
i.close()
def test_same_size_multilooking(self):
ifgs = same_exts_ifgs()
ifg_data_paths = [d.data_path for d in ifgs]
xlooks = ylooks = 2
prepare_ifgs(ifg_data_paths, ALREADY_SAME_SIZE, xlooks, ylooks)
looks_paths = [
mlooked_path(d, looks=xlooks, crop_out=ALREADY_SAME_SIZE)
for d in ifg_data_paths
]
mlooked = [Ifg(i) for i in looks_paths]
for m in mlooked:
m.open()
self.assertEqual(len(mlooked), 2)
for ifg in mlooked:
self.assertAlmostEqual(ifg.x_step, xlooks * self.xs)
self.assertAlmostEqual(ifg.x_step, ylooks * self.xs)
def parse(self, string):
"""
override of :py:meth:`DictParam.parse`.
"""
dct = super(RasterParam, self).parse(string)
raster_type = dct['type']
path = dct['path']
if raster_type == 'DEM':
return DEM(path)
elif raster_type == 'Ifg':
return Ifg(path)
elif raster_type == 'Incidence':
return Incidence(path)
else: # pragma: no cover
raise luigi.parameter.UnknownParameterException(
'rasterBase must be an inscance DEM, '
'Ifg or Incidence is valid')
def small_data_setup(datafiles=None, is_dir=False):
"""Returns Ifg objs for the files in the small test dir
input phase data is in radians; these ifgs are in radians - not converted to mm"""
if is_dir:
datafiles = glob.glob(join(datafiles, "*.tif"))
else:
if datafiles:
for i, d in enumerate(datafiles):
datafiles[i] = os.path.join(SML_TEST_TIF, d)
else:
datafiles = glob.glob(join(SML_TEST_TIF, "*.tif"))
datafiles.sort()
ifgs = [Ifg(i) for i in datafiles]
for i in ifgs:
i.open()
i.nodata_value = 0
return ifgs
def remove_orbital_error(ifgs, params, preread_ifgs=None):
"""
Wrapper function for PyRate orbital error removal functionality.
NB: the ifg data is modified in situ, rather than create intermediate
files. The network method assumes the given ifgs have already been reduced
to a minimum spanning tree network.
:param list ifgs: List of interferograms class objects
:param dict params: Dictionary containing configuration parameters
:param dict preread_ifgs: Dictionary containing information specifically
for MPI jobs (optional)
:return: None - interferogram phase data is updated and saved to disk
"""
ifg_paths = [i.data_path for i in ifgs] \
if isinstance(ifgs[0], Ifg) else ifgs
mlooked = None
# mlooking is not necessary for independent correction
# can use multiple procesing if write_to_disc=True
if params[cf.ORBITAL_FIT_METHOD] == NETWORK_METHOD:
mlooked_dataset = prepifg.prepare_ifgs(
ifg_paths,
crop_opt=prepifg.ALREADY_SAME_SIZE,
xlooks=params[cf.ORBITAL_FIT_LOOKS_X],
ylooks=params[cf.ORBITAL_FIT_LOOKS_Y],
thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD],
write_to_disc=False)
mlooked = [Ifg(m[1]) for m in mlooked_dataset]
for m in mlooked:
m.initialize()
m.nodata_value = params[cf.NO_DATA_VALUE]
m.convert_to_nans()
m.convert_to_mm()
_orbital_correction(ifgs,
params,
mlooked=mlooked,
preread_ifgs=preread_ifgs)