def test_conv2tif_file_types(tempdir, gamma_conf): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[cf.COH_MASK] = 1 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) params_s = configuration.Configuration(output_conf).__dict__ conv2tif.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif')) coh_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_coh.tif')) dem_file = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0] # assert coherence and ifgs have correct metadata for i in itertools.chain(*[ifg_files, coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_ifg.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.COH continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM shutil.rmtree(tdir)
def test_output_datatype(self): """Test resampling method using a scaling factor of 4""" scale = 4 # assumes square cells self.ifgs.append(DEM(SML_TEST_DEM_TIF)) ifg_paths = [i.data_path for i in self.ifgs] data_types = [InputTypes.IFG] * len(self.ifg_paths) ifg_paths.append(SML_TEST_DEM_TIF) data_types.append(InputTypes.DEM) self.headers.append(SML_TEST_DEM_HDR) cext = self._custom_extents_tuple() xlooks = ylooks = scale prepare_ifgs(ifg_paths, CUSTOM_CROP, xlooks, ylooks, thresh=1.0, user_exts=cext, headers=self.headers, params=self.params) self.params[C.IFG_LKSX] = xlooks self.params[C.IFG_LKSY] = ylooks self.params[C.IFG_CROP_OPT] = CUSTOM_CROP for i, t in zip(ifg_paths, data_types): mlooked_ifg = mlooked_path(i, self.params, input_type=t) ds1 = DEM(mlooked_ifg) ds1.open() ds2 = DEM(i) ds2.open() self.assertEqual( ds1.dataset.GetRasterBand(1).DataType, ds2.dataset.GetRasterBand(1).DataType) ds1 = ds2 = None
def test_output_datatype(self): """Test resampling method using a scaling factor of 4""" scale = 4 # assumes square cells self.ifgs.append(DEM(SML_TEST_DEM_TIF)) self.ifg_paths = [i.data_path for i in self.ifgs] + [SML_TEST_DEM_TIF] self.headers.append(SML_TEST_DEM_HDR) cext = self._custom_extents_tuple() xlooks = ylooks = scale prepare_ifgs(self.ifg_paths, CUSTOM_CROP, xlooks, ylooks, thresh=1.0, user_exts=cext, headers=self.headers) for i in self.ifg_paths: mlooked_ifg = mlooked_path(i, xlooks, CUSTOM_CROP) ds1 = DEM(mlooked_ifg) ds1.open() ds2 = DEM(i) ds2.open() self.assertEqual( ds1.dataset.GetRasterBand(1).DataType, ds2.dataset.GetRasterBand(1).DataType) ds1 = ds2 = None
def test_calc_dem_errors(self): # validate output of current version of the code with saved files from an independent test run # only the reference phase and dem_error are used in this test # saved dem_error.tif (expected) dem_error_tif_exp = join(dem_error_path, 'dem_error.tif') dem = DEM(dem_error_tif_exp) dem_error_exp = dem.data # run relevant parts of the 'correct' step correct._copy_mlooked(self.params) correct._update_params_with_tiles(self.params) correct._create_ifg_dict(self.params) save_numpy_phase(self.ifg_paths, self.params) # subtract the reference phase to enable comparison with a 'normal' pyrate run ref_phase_est_wrapper(self.params) dem_error_calc_wrapper(self.params) # dem_error.tif from this run (result) dem_error_tif_res = join(self.params[C.DEM_ERROR_DIR], 'dem_error.tif') dem = DEM(dem_error_tif_res) dem_error_res = dem.data # check equality np.testing.assert_allclose(dem_error_exp, dem_error_res) # ifg correction files in subdirectory out/dem_error/ # three different ifgs: # ifg1 -> short_baseline_ifg: 20180106-20180319 (ca. 3 m) # ifg2 -> long_baseline_ifg: 20180130-20180412(ca. 108 m) # ifg3 -> medium_baseline_ifg: 20180412-20180518 (ca. 48 m) # load saved files dem_error_ifg1_path = join(dem_error_path, '20180106-20180319_ifg_20_dem_error.npy') dem_error_ifg1_exp = np.load(dem_error_ifg1_path) dem_error_ifg2_path = join(dem_error_path, '20180130-20180412_ifg_20_dem_error.npy') dem_error_ifg2_exp = np.load(dem_error_ifg2_path) dem_error_ifg3_path = join(dem_error_path, '20180412-20180518_ifg_20_dem_error.npy') dem_error_ifg3_exp = np.load(dem_error_ifg3_path) # load correction values saved from this run (result) dem_error_ifg1_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180106-20180319_ifg_20_dem_error.npy') dem_error_ifg1_res = np.load(dem_error_ifg1_path) dem_error_ifg2_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180130-20180412_ifg_20_dem_error.npy') dem_error_ifg2_res = np.load(dem_error_ifg2_path) dem_error_ifg3_path = Path(self.params[C.DEM_ERROR_DIR]).joinpath( '20180412-20180518_ifg_20_dem_error.npy') dem_error_ifg3_res = np.load(dem_error_ifg3_path) # check equality np.testing.assert_allclose(dem_error_ifg1_exp, dem_error_ifg1_res) np.testing.assert_allclose(dem_error_ifg2_exp, dem_error_ifg2_res) np.testing.assert_allclose(dem_error_ifg3_exp, dem_error_ifg3_res)
def gen_color_file(input_file): fp, temp_file = tempfile.mkstemp(suffix='.txt') dem = DEM(input_file) dem.open() phase_data = dem.height_band.ReadAsArray() max_ph = np.nanmax(phase_data) min_ph = np.nanmin(phase_data) range_ph = max_ph - min_ph colors = ['black', 'blue', 'yellow', 'orange', 'red', 'white'] with open(temp_file, 'w') as f: for i, c in enumerate(colors[:-1]): f.write( str(int(min_ph + (i + 1) * range_ph / len(colors))) + ' ' + c + '\n') f.write( str(int(max_ph - range_ph / len(colors))) + ' ' + colors[-1] + '\n') os.close(fp) return temp_file
def __write_geometry_files(params: dict, exts: Tuple[float, float, float, float], transform, ifg_path: str) -> None: """ Calculate geometry and save to geotiff files using the information in the first interferogram in the stack, i.e.: - rdc_azimuth.tif (azimuth radar coordinate at each pixel) - rdc_range.tif (range radar coordinate at each pixel) - azimuth_angle.tif (satellite azimuth angle at each pixel) - incidence_angle.tif (incidence angle at each pixel) - look_angle.tif (look angle at each pixel) """ ifg = Ifg(ifg_path) ifg.open(readonly=True) # calculate per-pixel lon/lat lon, lat = geometry.get_lonlat_coords(ifg) # not currently implemented for ROIPAC data which breaks some tests # if statement can be deleted once ROIPAC is deprecated from PyRate if ifg.meta_data[ifc.PYRATE_INSAR_PROCESSOR] == 'ROIPAC': log.warning("Geometry calculations are not implemented for ROI_PAC") return # get geometry information and save radar coordinates and angles to tif files # using metadata of the first image in the stack # get pixel values of crop (needed to crop lookup table file) # pixel extent of cropped area (original IFG input) xmin, ymax = convert_geographic_coordinate_to_pixel_value( exts[0], exts[1], transform) xmax, ymin = convert_geographic_coordinate_to_pixel_value( exts[2], exts[3], transform) # xmin, xmax: columns of crop # ymin, ymax: rows of crop # calculate per-pixel radar coordinates az, rg = geometry.calc_radar_coords(ifg, params, xmin, xmax, ymin, ymax) # Read height data from DEM dem_file = params[C.DEM_FILE_PATH].sampled_path dem = DEM(dem_file) # calculate per-pixel look angle (also calculates and saves incidence and azimuth angles) lk_ang, inc_ang, az_ang, rg_dist = geometry.calc_pixel_geometry( ifg, rg, lon.data, lat.data, dem.data) # save radar coordinates and angles to geotiff files combinations = zip([az, rg, lk_ang, inc_ang, az_ang, rg_dist], C.GEOMETRY_OUTPUT_TYPES) shared.iterable_split(__parallelly_write, combinations, params, ifg_path)
def dem_or_ifg(data_path): """ Returns an Ifg or DEM class object from input geotiff file. :param str data_path: file path name :return: Interferogram or DEM object from input file :rtype: Ifg or DEM class object """ ds = gdal.Open(data_path) md = ds.GetMetadata() if ifc.MASTER_DATE in md: # ifg return Ifg(data_path) else: return DEM(data_path)
class TestDEMTests: 'Unit tests to verify operations on GeoTIFF format DEMs' def setup_method(self): self.ras = DEM(SML_TEST_DEM_TIF) def test_create_raster(self): # validate header path assert os.path.exists(self.ras.data_path) def test_headers_as_attr(self): self.ras.open() attrs = ['ncols', 'nrows', 'x_first', 'x_step', 'y_first', 'y_step'] # TODO: are 'projection' and 'datum' attrs needed? for a in attrs: assert getattr(self.ras, a) is not None def test_is_dem(self): self.ras = DEM(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif')) assert ~hasattr(self.ras, 'datum') def test_open(self): assert self.ras.dataset is None self.ras.open() assert self.ras.dataset is not None assert isinstance(self.ras.dataset, Dataset) # ensure open cannot be called twice with pytest.raises(RasterException): self.ras.open() def test_band_fails_with_unopened_raster(self): # test accessing bands with open and unopened datasets with pytest.raises(RasterException): self.ras.height_band def test_band_read_with_open_raster(self): self.ras.open() data = self.ras.height_band.ReadAsArray() assert data.shape == (72, 47)
def setup_class(cls): cls.params = Configuration(common.MEXICO_CROPA_CONF).__dict__ # run prepifg prepifg.main(cls.params) # copy IFGs to temp folder correct._copy_mlooked(cls.params) # read radar azimuth, range and dem tif files geom_files = Configuration.geometry_files(cls.params) rdc_az_file = geom_files['rdc_azimuth'] geom_az = Geometry(rdc_az_file) cls.az = geom_az.data rdc_rg_file = geom_files['rdc_range'] geom_rg = Geometry(rdc_rg_file) cls.rg = geom_rg.data dem_file = join(cls.params[C.GEOMETRY_DIR], 'dem.tif') dem_data = DEM(dem_file) cls.dem = dem_data.data # calc bperp using pyrate funcs cls.pbperp = cls.pyrate_bperp()
class DEMTests(unittest.TestCase): 'Unit tests to verify operations on GeoTIFF format DEMs' def setUp(self): self.ras = DEM(SML_TEST_DEM_TIF) def test_create_raster(self): # validate header path self.assertTrue(os.path.exists(self.ras.data_path)) def test_headers_as_attr(self): self.ras.open() attrs = ['ncols', 'nrows', 'x_first', 'x_step', 'y_first', 'y_step'] # TODO: are 'projection' and 'datum' attrs needed? for a in attrs: self.assertTrue(getattr(self.ras, a) is not None) def test_is_dem(self): self.ras = DEM(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif')) self.assertFalse(hasattr(self.ras, 'datum')) def test_open(self): self.assertTrue(self.ras.dataset is None) self.ras.open() self.assertTrue(self.ras.dataset is not None) self.assertTrue(isinstance(self.ras.dataset, Dataset)) # ensure open cannot be called twice self.assertRaises(RasterException, self.ras.open) def test_band(self): # test accessing bands with open and unopened datasets try: _ = self.ras.height_band self.fail("Should not be able to access band without open dataset") except RasterException: pass self.ras.open() data = self.ras.height_band.ReadAsArray() self.assertEqual(data.shape, (72, 47))
def test_multilook(self): """Test resampling method using a scaling factor of 4""" scale = 4 # assumes square cells self.ifgs.append(DEM(SML_TEST_DEM_TIF)) self.ifg_paths = [i.data_path for i in self.ifgs] # append the dem header self.headers.append(SML_TEST_DEM_HDR) cext = self._custom_extents_tuple() xlooks = ylooks = scale prepare_ifgs(self.ifg_paths, CUSTOM_CROP, xlooks, ylooks, thresh=1.0, user_exts=cext, headers=self.headers) for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]): i = Ifg(ipath) i.open() self.assertEqual(i.dataset.RasterXSize, 20 / scale) self.assertEqual(i.dataset.RasterYSize, 28 / scale) # verify resampling path = join(PREP_TEST_TIF, "%s.tif" % n) ds = gdal.Open(path) src_data = ds.GetRasterBand(2).ReadAsArray() exp_resample = multilooking(src_data, scale, scale, thresh=0) self.assertEqual(exp_resample.shape, (7, 5)) assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray()) ds = None i.close() os.remove(ipath) # verify DEM has been correctly processed # ignore output values as resampling has already been tested for phase exp_dem_path = join(SML_TEST_DEM_DIR, 'roipac_test_trimmed_4rlks_3cr.tif') self.assertTrue(exists(exp_dem_path)) orignal_dem = DEM(SML_TEST_DEM_TIF) orignal_dem.open() dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType orignal_dem.close() dem = DEM(exp_dem_path) dem.open() # test multilooked dem is of the same datatype as the original dem tif self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType) self.assertEqual(dem.dataset.RasterXSize, 20 / scale) self.assertEqual(dem.dataset.RasterYSize, 28 / scale) data = dem.height_band.ReadAsArray() self.assertTrue(data.ptp() != 0) # close ifgs dem.close() for i in self.ifgs: i.close() os.remove(exp_dem_path)
def _process_dem_error_per_tile(tile: Tile, params: dict) -> None: """ Convenience function for processing DEM error in tiles :param tile: pyrate.core.shared.Tile Class object. :param params: Dictionary of PyRate configuration parameters. """ ifg_paths = [ ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES] ] ifg0_path = ifg_paths[0] ifg0 = Ifg(ifg0_path) ifg0.open(readonly=True) # read lon and lat values of multi-looked ifg (first ifg only) lon, lat = geometry.get_lonlat_coords(ifg0) # read azimuth and range coords and DEM from tif files generated in prepifg geom_files = Configuration.geometry_files(params) rdc_az_file = geom_files['rdc_azimuth'] geom_az = Geometry(rdc_az_file) rdc_rg_file = geom_files['rdc_range'] geom_rg = Geometry(rdc_rg_file) dem_file = params[C.DEM_FILE_PATH].sampled_path dem = DEM(dem_file) preread_ifgs = params[C.PREREAD_IFGS] threshold = params[C.DE_PTHR] ifg_parts = [ shared.IfgPart(p, tile, preread_ifgs, params) for p in ifg_paths ] lon_parts = lon(tile) lat_parts = lat(tile) az_parts = geom_az(tile) rg_parts = geom_rg(tile) dem_parts = dem(tile) log.debug( f"Calculating per-pixel baseline for tile {tile.index} during DEM error correction" ) bperp, look_angle, range_dist = _calculate_bperp_wrapper( ifg_paths, az_parts, rg_parts, lat_parts, lon_parts, dem_parts) log.debug( f"Calculating DEM error for tile {tile.index} during DEM error correction" ) # mst_tile = np.load(Configuration.mst_path(params, tile.index)) # calculate the DEM error estimate and the correction values for each IFG # current implementation uses the look angle and range distance matrix of the primary SLC in the last IFG # todo: check the impact of using the same information from another SLC dem_error, dem_error_correction, _ = calc_dem_errors( ifg_parts, bperp, look_angle, range_dist, threshold) # dem_error contains the estimated DEM error for each pixel (i.e. the topographic change relative to the DEM) # size [row, col] # dem_error_correction contains the correction value for each interferogram # size [num_ifg, row, col] # save tiled data in tmpdir np.save(file=os.path.join(params[C.TMPDIR], 'dem_error_{}.npy'.format(tile.index)), arr=dem_error) # swap the axes of 3D array to fit the style used in function assemble_tiles tmp_array = np.moveaxis(dem_error_correction, 0, -1) # new dimension is [row, col, num_ifg] # save tiled data into tmpdir np.save(file=os.path.join( params[C.TMPDIR], 'dem_error_correction_{}.npy'.format(tile.index)), arr=tmp_array) # Calculate and save the average perpendicular baseline for the tile bperp_avg = np.nanmean(bperp, axis=(1, 2), dtype=np.float64) np.save(file=os.path.join(params[C.TMPDIR], 'bperp_avg_{}.npy'.format(tile.index)), arr=bperp_avg)
def test_is_dem(self): self.ras = DEM(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif')) self.assertFalse(hasattr(self.ras, 'datum'))
def setUp(self): self.ras = DEM(SML_TEST_DEM_TIF)
def setup_method(self): self.ras = DEM(SML_TEST_DEM_TIF)
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[cf.COH_MASK] = coh_mask params[cf.PARALLEL] = 0 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) cf.write_config_file(params=params, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ conv2tif.main(params_s) # reread params from config params_s = Configuration(output_conf).__dict__ prepifg.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif')) assert len(ifg_files) == 17 mlooked_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg_1rlks_1cr.tif')) assert len(mlooked_files) == 17 coh_files = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_cc_coh.tif')) mlooked_coh_files = list( Path(tdir.joinpath( params_s[cf.OUT_DIR])).glob('*_cc_coh_1rlks_1cr.tif')) if coh_mask: assert len(coh_files) == 17 assert len(mlooked_coh_files) == 17 dem_file = list( Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0] mlooked_dem_file = list( Path(tdir.joinpath( params_s[cf.OUT_DIR])).glob('*_dem_1rlks_1cr.tif'))[0] import itertools # assert coherence and ifgs have correct metadata for i in itertools.chain( *[ifg_files, mlooked_files, coh_files, mlooked_coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_ifg.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.COH continue if i.name.endswith('_cc_coh_1rlks_1cr.tif'): assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH continue if i.name.endswith('_ifg_1rlks_1cr.tif'): if coh_mask: assert md[ifc.DATA_TYPE] == ifc.COHERENCE else: assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM dem = DEM(mlooked_dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM shutil.rmtree(tdir)
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask): tdir = Path(tempdir()) params = manipulate_test_conf(gamma_conf, tdir) params[C.COH_MASK] = coh_mask params[C.PARALLEL] = 0 output_conf_file = 'conf.conf' output_conf = tdir.joinpath(output_conf_file) pyrate.configuration.write_config_file(params=params, output_conf_file=output_conf) params_s = Configuration(output_conf).__dict__ conv2tif.main(params_s) # reread params from config params_s = Configuration(output_conf).__dict__ prepifg.main(params_s) ifg_files = list( Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_unw.tif')) assert len(ifg_files) == 17 mlooked_files = list( Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_ifg.tif')) assert len(mlooked_files) == 17 coh_files = list( Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_cc.tif')) mlooked_coh_files = list( Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_coh.tif')) if coh_mask: assert len(coh_files) == 17 assert len(mlooked_coh_files) == 17 dem_file = list( Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('*_dem.tif'))[0] mlooked_dem_file = list( Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('dem.tif'))[0] import itertools # assert coherence and ifgs have correct metadata for i in itertools.chain( *[ifg_files, mlooked_files, coh_files, mlooked_coh_files]): ifg = Ifg(i) ifg.open() md = ifg.meta_data if i.name.endswith('_unw.tif'): assert md[ifc.DATA_TYPE] == ifc.ORIG assert ifc.IFG_LKSX not in md assert ifc.IFG_LKSY not in md assert ifc.IFG_CROP not in md continue if i.name.endswith('_cc.tif'): assert md[ifc.DATA_TYPE] == ifc.COH assert ifc.IFG_LKSX not in md assert ifc.IFG_LKSY not in md assert ifc.IFG_CROP not in md continue if i.name.endswith('_coh.tif'): assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH assert md[ifc.IFG_LKSX] == '1' assert md[ifc.IFG_LKSY] == '1' assert md[ifc.IFG_CROP] == '1' continue if i.name.endswith('_ifg.tif'): if coh_mask: assert md[ifc.DATA_TYPE] == ifc.MLOOKED_COH_MASKED_IFG assert md[ifc.IFG_LKSX] == '1' assert md[ifc.IFG_LKSY] == '1' assert md[ifc.IFG_CROP] == '1' else: assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED assert md[ifc.IFG_LKSX] == '1' assert md[ifc.IFG_LKSY] == '1' assert md[ifc.IFG_CROP] == '1' continue # assert dem has correct metadata dem = DEM(dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.DEM assert ifc.IFG_LKSX not in md assert ifc.IFG_LKSY not in md assert ifc.IFG_CROP not in md dem = DEM(mlooked_dem_file.as_posix()) dem.open() md = dem.dataset.GetMetadata() assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM assert ifc.IFG_LKSX in md assert ifc.IFG_LKSY in md assert ifc.IFG_CROP in md shutil.rmtree(tdir)