def test_output_datatype(self):
"""Test resampling method using a scaling factor of 4"""
scale = 4 # assumes square cells
self.ifgs.append(DEM(SML_TEST_DEM_TIF))
self.ifg_paths = [i.data_path for i in self.ifgs] + [SML_TEST_DEM_TIF]
self.headers.append(SML_TEST_DEM_HDR)
cext = self._custom_extents_tuple()
xlooks = ylooks = scale
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
thresh=1.0,
user_exts=cext,
headers=self.headers)
for i in self.ifg_paths:
mlooked_ifg = mlooked_path(i, xlooks, CUSTOM_CROP)
ds1 = DEM(mlooked_ifg)
ds1.open()
ds2 = DEM(i)
ds2.open()
self.assertEqual(
ds1.dataset.GetRasterBand(1).DataType,
ds2.dataset.GetRasterBand(1).DataType)
ds1 = ds2 = None
def test_output_datatype(self):
"""Test resampling method using a scaling factor of 4"""
scale = 4 # assumes square cells
self.ifgs.append(DEM(SML_TEST_DEM_TIF))
ifg_paths = [i.data_path for i in self.ifgs]
data_types = [InputTypes.IFG] * len(self.ifg_paths)
ifg_paths.append(SML_TEST_DEM_TIF)
data_types.append(InputTypes.DEM)
self.headers.append(SML_TEST_DEM_HDR)
cext = self._custom_extents_tuple()
xlooks = ylooks = scale
prepare_ifgs(ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
thresh=1.0,
user_exts=cext,
headers=self.headers,
params=self.params)
self.params[C.IFG_LKSX] = xlooks
self.params[C.IFG_LKSY] = ylooks
self.params[C.IFG_CROP_OPT] = CUSTOM_CROP
for i, t in zip(ifg_paths, data_types):
mlooked_ifg = mlooked_path(i, self.params, input_type=t)
ds1 = DEM(mlooked_ifg)
ds1.open()
ds2 = DEM(i)
ds2.open()
self.assertEqual(
ds1.dataset.GetRasterBand(1).DataType,
ds2.dataset.GetRasterBand(1).DataType)
ds1 = ds2 = None
def test_multilook(self):
"""Test resampling method using a scaling factor of 4"""
scale = 4 # assumes square cells
self.ifgs.append(DEM(SML_TEST_DEM_TIF))
self.ifg_paths = [i.data_path for i in self.ifgs]
# append the dem header
self.headers.append(SML_TEST_DEM_HDR)
cext = self._custom_extents_tuple()
xlooks = ylooks = scale
prepare_ifgs(self.ifg_paths,
CUSTOM_CROP,
xlooks,
ylooks,
thresh=1.0,
user_exts=cext,
headers=self.headers,
params=self.params)
for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]):
i = Ifg(ipath)
i.open()
self.assertEqual(i.dataset.RasterXSize, 20 / scale)
self.assertEqual(i.dataset.RasterYSize, 28 / scale)
# verify resampling
path = join(PREP_TEST_TIF,
"geo_%s.tif" % Path(ipath).name.split('_ifg')[0])
ds = gdal.Open(path)
src_data = ds.GetRasterBand(2).ReadAsArray()
exp_resample = multilooking(src_data, scale, scale, thresh=0)
self.assertEqual(exp_resample.shape, (7, 5))
assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray())
ds = None
i.close()
os.remove(ipath)
# verify DEM has been correctly processed
# ignore output values as resampling has already been tested for phase
exp_dem_path = join(self.params[C.GEOMETRY_DIR], 'dem.tif')
self.assertTrue(exists(exp_dem_path))
orignal_dem = DEM(SML_TEST_DEM_TIF)
orignal_dem.open()
dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType
orignal_dem.close()
dem = DEM(exp_dem_path)
dem.open()
# test multilooked dem is of the same datatype as the original dem tif
self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType)
self.assertEqual(dem.dataset.RasterXSize, 20 / scale)
self.assertEqual(dem.dataset.RasterYSize, 28 / scale)
data = dem.data
self.assertTrue(data.ptp() != 0)
# close ifgs
dem.close()
for i in self.ifgs:
i.close()
os.remove(exp_dem_path)
def test_calc_dem_errors(self):
# validate output of current version of the code with saved files from an independent test run
# only the reference phase and dem_error are used in this test
# saved dem_error.tif (expected)
dem_error_tif_exp = join(dem_error_path, 'dem_error.tif')
dem = DEM(dem_error_tif_exp)
dem_error_exp = dem.data
# run relevant parts of the 'correct' step
correct._copy_mlooked(self.params)
correct._update_params_with_tiles(self.params)
correct._create_ifg_dict(self.params)
save_numpy_phase(self.ifg_paths, self.params)
# subtract the reference phase to enable comparison with a 'normal' pyrate run
ref_phase_est_wrapper(self.params)
dem_error_calc_wrapper(self.params)
# dem_error.tif from this run (result)
dem_error_tif_res = join(self.params[C.DEM_ERROR_DIR], 'dem_error.tif')
dem = DEM(dem_error_tif_res)
dem_error_res = dem.data
# check equality
np.testing.assert_allclose(dem_error_exp, dem_error_res)
# ifg correction files in subdirectory out/dem_error/
# three different ifgs:
# ifg1 -> short_baseline_ifg: 20180106-20180319 (ca. 3 m)
# ifg2 -> long_baseline_ifg: 20180130-20180412(ca. 108 m)
# ifg3 -> medium_baseline_ifg: 20180412-20180518 (ca. 48 m)
# load saved files
dem_error_ifg1_path = join(dem_error_path,
'20180106-20180319_ifg_20_dem_error.npy')
dem_error_ifg1_exp = np.load(dem_error_ifg1_path)
dem_error_ifg2_path = join(dem_error_path,
'20180130-20180412_ifg_20_dem_error.npy')
dem_error_ifg2_exp = np.load(dem_error_ifg2_path)
dem_error_ifg3_path = join(dem_error_path,
'20180412-20180518_ifg_20_dem_error.npy')
dem_error_ifg3_exp = np.load(dem_error_ifg3_path)
# load correction values saved from this run (result)
dem_error_ifg1_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath(
'20180106-20180319_ifg_20_dem_error.npy')
dem_error_ifg1_res = np.load(dem_error_ifg1_path)
dem_error_ifg2_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath(
'20180130-20180412_ifg_20_dem_error.npy')
dem_error_ifg2_res = np.load(dem_error_ifg2_path)
dem_error_ifg3_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath(
'20180412-20180518_ifg_20_dem_error.npy')
dem_error_ifg3_res = np.load(dem_error_ifg3_path)
# check equality
np.testing.assert_allclose(dem_error_ifg1_exp, dem_error_ifg1_res)
np.testing.assert_allclose(dem_error_ifg2_exp, dem_error_ifg2_res)
np.testing.assert_allclose(dem_error_ifg3_exp, dem_error_ifg3_res)
def test_conv2tif_file_types(tempdir, gamma_conf):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[cf.COH_MASK] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
params_s = configuration.Configuration(output_conf).__dict__
conv2tif.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif'))
coh_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_coh.tif'))
dem_file = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0]
# assert coherence and ifgs have correct metadata
for i in itertools.chain(*[ifg_files, coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_ifg.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
shutil.rmtree(tdir)
def __write_geometry_files(params: dict, exts: Tuple[float, float, float,
float], transform,
ifg_path: str) -> None:
"""
Calculate geometry and save to geotiff files using the information in the
first interferogram in the stack, i.e.:
- rdc_azimuth.tif (azimuth radar coordinate at each pixel)
- rdc_range.tif (range radar coordinate at each pixel)
- azimuth_angle.tif (satellite azimuth angle at each pixel)
- incidence_angle.tif (incidence angle at each pixel)
- look_angle.tif (look angle at each pixel)
"""
ifg = Ifg(ifg_path)
ifg.open(readonly=True)
# calculate per-pixel lon/lat
lon, lat = geometry.get_lonlat_coords(ifg)
# not currently implemented for ROIPAC data which breaks some tests
# if statement can be deleted once ROIPAC is deprecated from PyRate
if ifg.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC':
log.warning("Geometry calculations are not implemented for ROI_PAC")
return
# get geometry information and save radar coordinates and angles to tif files
# using metadata of the first image in the stack
# get pixel values of crop (needed to crop lookup table file)
# pixel extent of cropped area (original IFG input)
xmin, ymax = convert_geographic_coordinate_to_pixel_value(
exts[0], exts[1], transform)
xmax, ymin = convert_geographic_coordinate_to_pixel_value(
exts[2], exts[3], transform)
# xmin, xmax: columns of crop
# ymin, ymax: rows of crop
# calculate per-pixel radar coordinates
az, rg = geometry.calc_radar_coords(ifg, params, xmin, xmax, ymin, ymax)
# Read height data from DEM
dem_file = params[C.DEM_FILE_PATH].sampled_path
dem = DEM(dem_file)
# calculate per-pixel look angle (also calculates and saves incidence and azimuth angles)
lk_ang, inc_ang, az_ang, rg_dist = geometry.calc_pixel_geometry(
ifg, rg, lon.data, lat.data, dem.data)
# save radar coordinates and angles to geotiff files
combinations = zip([az, rg, lk_ang, inc_ang, az_ang, rg_dist],
C.GEOMETRY_OUTPUT_TYPES)
shared.iterable_split(__parallelly_write, combinations, params, ifg_path)
def dem_or_ifg(data_path):
"""
Returns an Ifg or DEM class object from input geotiff file.
:param str data_path: file path name
:return: Interferogram or DEM object from input file
:rtype: Ifg or DEM class object
"""
ds = gdal.Open(data_path)
md = ds.GetMetadata()
if ifc.MASTER_DATE in md: # ifg
return Ifg(data_path)
else:
return DEM(data_path)
def setup_class(cls):
cls.params = Configuration(common.MEXICO_CROPA_CONF).__dict__
# run prepifg
prepifg.main(cls.params)
# copy IFGs to temp folder
correct._copy_mlooked(cls.params)
# read radar azimuth, range and dem tif files
geom_files = Configuration.geometry_files(cls.params)
rdc_az_file = geom_files['rdc_azimuth']
geom_az = Geometry(rdc_az_file)
cls.az = geom_az.data
rdc_rg_file = geom_files['rdc_range']
geom_rg = Geometry(rdc_rg_file)
cls.rg = geom_rg.data
dem_file = join(cls.params[C.GEOMETRY_DIR], 'dem.tif')
dem_data = DEM(dem_file)
cls.dem = dem_data.data
# calc bperp using pyrate funcs
cls.pbperp = cls.pyrate_bperp()
def gen_color_file(input_file):
fp, temp_file = tempfile.mkstemp(suffix='.txt')
dem = DEM(input_file)
dem.open()
phase_data = dem.height_band.ReadAsArray()
max_ph = np.nanmax(phase_data)
min_ph = np.nanmin(phase_data)
range_ph = max_ph - min_ph
colors = ['black', 'blue', 'yellow', 'orange', 'red', 'white']
with open(temp_file, 'w') as f:
for i, c in enumerate(colors[:-1]):
f.write(
str(int(min_ph + (i + 1) * range_ph / len(colors))) + ' ' + c +
'\n')
f.write(
str(int(max_ph - range_ph / len(colors))) + ' ' + colors[-1] +
'\n')
os.close(fp)
return temp_file
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[cf.COH_MASK] = coh_mask
params[cf.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
# reread params from config
params_s = Configuration(output_conf).__dict__
prepifg.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif'))
assert len(ifg_files) == 17
mlooked_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg_1rlks_1cr.tif'))
assert len(mlooked_files) == 17
coh_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_cc_coh.tif'))
mlooked_coh_files = list(
Path(tdir.joinpath(
params_s[cf.OUT_DIR])).glob('*_cc_coh_1rlks_1cr.tif'))
if coh_mask:
assert len(coh_files) == 17
assert len(mlooked_coh_files) == 17
dem_file = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0]
mlooked_dem_file = list(
Path(tdir.joinpath(
params_s[cf.OUT_DIR])).glob('*_dem_1rlks_1cr.tif'))[0]
import itertools
# assert coherence and ifgs have correct metadata
for i in itertools.chain(
*[ifg_files, mlooked_files, coh_files, mlooked_coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_ifg.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
continue
if i.name.endswith('_cc_coh_1rlks_1cr.tif'):
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH
continue
if i.name.endswith('_ifg_1rlks_1cr.tif'):
if coh_mask:
assert md[ifc.DATA_TYPE] == ifc.COHERENCE
else:
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
dem = DEM(mlooked_dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM
shutil.rmtree(tdir)
def _process_dem_error_per_tile(tile: Tile, params: dict) -> None:
"""
Convenience function for processing DEM error in tiles
:param tile: pyrate.core.shared.Tile Class object.
:param params: Dictionary of PyRate configuration parameters.
"""
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
ifg0_path = ifg_paths[0]
ifg0 = Ifg(ifg0_path)
ifg0.open(readonly=True)
# read lon and lat values of multi-looked ifg (first ifg only)
lon, lat = geometry.get_lonlat_coords(ifg0)
# read azimuth and range coords and DEM from tif files generated in prepifg
geom_files = Configuration.geometry_files(params)
rdc_az_file = geom_files['rdc_azimuth']
geom_az = Geometry(rdc_az_file)
rdc_rg_file = geom_files['rdc_range']
geom_rg = Geometry(rdc_rg_file)
dem_file = params[C.DEM_FILE_PATH].sampled_path
dem = DEM(dem_file)
preread_ifgs = params[C.PREREAD_IFGS]
threshold = params[C.DE_PTHR]
ifg_parts = [
shared.IfgPart(p, tile, preread_ifgs, params) for p in ifg_paths
]
lon_parts = lon(tile)
lat_parts = lat(tile)
az_parts = geom_az(tile)
rg_parts = geom_rg(tile)
dem_parts = dem(tile)
log.debug(
f"Calculating per-pixel baseline for tile {tile.index} during DEM error correction"
)
bperp, look_angle, range_dist = _calculate_bperp_wrapper(
ifg_paths, az_parts, rg_parts, lat_parts, lon_parts, dem_parts)
log.debug(
f"Calculating DEM error for tile {tile.index} during DEM error correction"
)
# mst_tile = np.load(Configuration.mst_path(params, tile.index))
# calculate the DEM error estimate and the correction values for each IFG
# current implementation uses the look angle and range distance matrix of the primary SLC in the last IFG
# todo: check the impact of using the same information from another SLC
dem_error, dem_error_correction, _ = calc_dem_errors(
ifg_parts, bperp, look_angle, range_dist, threshold)
# dem_error contains the estimated DEM error for each pixel (i.e. the topographic change relative to the DEM)
# size [row, col]
# dem_error_correction contains the correction value for each interferogram
# size [num_ifg, row, col]
# save tiled data in tmpdir
np.save(file=os.path.join(params[C.TMPDIR],
'dem_error_{}.npy'.format(tile.index)),
arr=dem_error)
# swap the axes of 3D array to fit the style used in function assemble_tiles
tmp_array = np.moveaxis(dem_error_correction, 0, -1)
# new dimension is [row, col, num_ifg]
# save tiled data into tmpdir
np.save(file=os.path.join(
params[C.TMPDIR], 'dem_error_correction_{}.npy'.format(tile.index)),
arr=tmp_array)
# Calculate and save the average perpendicular baseline for the tile
bperp_avg = np.nanmean(bperp, axis=(1, 2), dtype=np.float64)
np.save(file=os.path.join(params[C.TMPDIR],
'bperp_avg_{}.npy'.format(tile.index)),
arr=bperp_avg)
def test_is_dem(self):
self.ras = DEM(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif'))
self.assertFalse(hasattr(self.ras, 'datum'))
def setUp(self):
self.ras = DEM(SML_TEST_DEM_TIF)
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[C.COH_MASK] = coh_mask
params[C.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
# reread params from config
params_s = Configuration(output_conf).__dict__
prepifg.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_unw.tif'))
assert len(ifg_files) == 17
mlooked_files = list(
Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_ifg.tif'))
assert len(mlooked_files) == 17
coh_files = list(
Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_cc.tif'))
mlooked_coh_files = list(
Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_coh.tif'))
if coh_mask:
assert len(coh_files) == 17
assert len(mlooked_coh_files) == 17
dem_file = list(
Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('*_dem.tif'))[0]
mlooked_dem_file = list(
Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('dem.tif'))[0]
import itertools
# assert coherence and ifgs have correct metadata
for i in itertools.chain(
*[ifg_files, mlooked_files, coh_files, mlooked_coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_unw.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
assert ifc.IFG_LKSX not in md
assert ifc.IFG_LKSY not in md
assert ifc.IFG_CROP not in md
continue
if i.name.endswith('_cc.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
assert ifc.IFG_LKSX not in md
assert ifc.IFG_LKSY not in md
assert ifc.IFG_CROP not in md
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH
assert md[ifc.IFG_LKSX] == '1'
assert md[ifc.IFG_LKSY] == '1'
assert md[ifc.IFG_CROP] == '1'
continue
if i.name.endswith('_ifg.tif'):
if coh_mask:
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_COH_MASKED_IFG
assert md[ifc.IFG_LKSX] == '1'
assert md[ifc.IFG_LKSY] == '1'
assert md[ifc.IFG_CROP] == '1'
else:
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED
assert md[ifc.IFG_LKSX] == '1'
assert md[ifc.IFG_LKSY] == '1'
assert md[ifc.IFG_CROP] == '1'
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
assert ifc.IFG_LKSX not in md
assert ifc.IFG_LKSY not in md
assert ifc.IFG_CROP not in md
dem = DEM(mlooked_dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM
assert ifc.IFG_LKSX in md
assert ifc.IFG_LKSY in md
assert ifc.IFG_CROP in md
shutil.rmtree(tdir)
def setup_method(self):
self.ras = DEM(SML_TEST_DEM_TIF)