def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params): """ xxxx :param grid: List of tuples (y, x) corresponding reference pixel grids :param half_patch_size: Patch size in pixels corresponding to reference pixel grids :param ifg_paths: List of interferogram paths :param params: Parameters dictionary corresponding to config file :return xxxx """ log.info('Saving ref pixel blocks') outdir = params[cf.TMPDIR] for pth in ifg_paths: ifg = Ifg(pth) ifg.open(readonly=True) ifg.nodata_value = params[cf.NO_DATA_VALUE] ifg.convert_to_nans() ifg.convert_to_mm() for y, x in grid: data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1, x - half_patch_size:x + half_patch_size + 1] data_file = join( outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format( b=os.path.basename(pth).split('.')[0], y=y, x=x)) np.save(file=data_file, arr=data) ifg.close() log.info('Saved ref pixel blocks')
def _phase_sum(ifg_paths, params): """ Save phase data and phase sum used in the reference phase estimation """ p_paths = mpiops.array_split(ifg_paths) ifg = Ifg(p_paths[0]) ifg.open(readonly=True) shape = ifg.shape phs_sum = np.zeros(shape=shape, dtype=np.float64) ifg.close() for d in p_paths: ifg = Ifg(d) ifg.open() ifg.nodata_value = params[cf.NO_DATA_VALUE] phs_sum += ifg.phase_data ifg.close() if mpiops.rank == MASTER_PROCESS: phase_sum_all = phs_sum # loop is better for memory for i in range(1, mpiops.size): # pragma: no cover phs_sum = np.zeros(shape=shape, dtype=np.float64) mpiops.comm.Recv(phs_sum, source=i, tag=i) phase_sum_all += phs_sum comp = np.isnan(phase_sum_all) # this is the same as in Matlab comp = np.ravel(comp, order='F') # this is the same as in Matlab else: # pragma: no cover comp = None mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank) comp = mpiops.comm.bcast(comp, root=0) return comp
def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = vcm_module.RDist(ifg)() ifg.close() return r_dist
def test_multilook(self): """Test resampling method using a scaling factor of 4""" scale = 4 # assumes square cells self.ifgs.append(DEM(SML_TEST_DEM_TIF)) self.ifg_paths = [i.data_path for i in self.ifgs] cext = self._custom_extents_tuple() xlooks = ylooks = scale prepare_ifgs(self.ifg_paths, CUSTOM_CROP, xlooks, ylooks, thresh=1.0, user_exts=cext) for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]): i = Ifg(ipath) i.open() self.assertEqual(i.dataset.RasterXSize, 20 / scale) self.assertEqual(i.dataset.RasterYSize, 28 / scale) # verify resampling path = join(PREP_TEST_TIF, "%s.tif" % n) ds = gdal.Open(path) src_data = ds.GetRasterBand(2).ReadAsArray() exp_resample = multilooking(src_data, scale, scale, thresh=0) self.assertEqual(exp_resample.shape, (7, 5)) assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray()) ds = None i.close() os.remove(ipath) # verify DEM has been correctly processed # ignore output values as resampling has already been tested for phase exp_dem_path = join(SML_TEST_DEM_DIR, 'roipac_test_trimmed_4rlks_3cr.tif') self.assertTrue(exists(exp_dem_path)) orignal_dem = DEM(SML_TEST_DEM_TIF) orignal_dem.open() dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType orignal_dem.close() dem = DEM(exp_dem_path) dem.open() # test multilooked dem is of the same datatype as the original dem tif self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType) self.assertEqual(dem.dataset.RasterXSize, 20 / scale) self.assertEqual(dem.dataset.RasterYSize, 28 / scale) data = dem.height_band.ReadAsArray() self.assertTrue(data.ptp() != 0) # close ifgs dem.close() for i in self.ifgs: i.close() os.remove(exp_dem_path)
def _inner(ifg_path): ifg = Ifg(ifg_path) ifg.open(readonly=False) phase_data = ifg.phase_data ref_ph = rpe.est_ref_phs_method2(phase_data, half_chip_size, refpx, refpy, thresh) phase_data -= ref_ph md = ifg.meta_data md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase(data=phase_data) ifg.close() return ref_ph
def _inner(ifg_path): """ convenient inner loop """ ifg = Ifg(ifg_path) ifg.open(readonly=False) phase_data = ifg.phase_data ref_phase = rpe.est_ref_phs_method1(phase_data, comp) phase_data -= ref_phase md = ifg.meta_data md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase(data=phase_data) ifg.close() return ref_phase
def _save_aps_corrected_phase(ifg_path, phase): """ Save (update) interferogram metadata and phase data after spatio-temporal filter (APS) correction. """ ifg = Ifg(ifg_path) ifg.open(readonly=False) ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data )] # set aps tags after aps error correction ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED) ifg.write_modified_phase() ifg.close()
def test_nodata(self): """Verify NODATA value copied correctly (amplitude band not copied)""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) for ex in [self.exp_files[0], self.exp_files[4]]: ifg = Ifg(ex) ifg.open() # NB: amplitude band doesn't have a NODATA value self.assertTrue( isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue())) ifg.close() for i in self.ifgs: i.close()
def get_tiles(ifg_path, rows, cols): """ Break up the interferograms into tiles based on user supplied rows and columns. :param ifg_path: List of destination tifs :param rows: Number of rows to break each interferogram into :param cols: Number of columns to break each interferogram into :return tiles: List of shared.Tile instances """ ifg = Ifg(ifg_path) ifg.open(readonly=True) tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols) ifg.close() return tiles
def test_nans(self): """Verify NaNs replace 0 in the multilooked phase band""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) for ex in [self.exp_files[0], self.exp_files[4]]: ifg = Ifg(ex) ifg.open() phase = ifg.phase_band.ReadAsArray() self.assertFalse((phase == 0).any()) self.assertTrue((isnan(phase)).any()) ifg.close() self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo for i in self.ifgs: i.close()
def _ref_pixel_calc(ifg_paths, params): """ Wrapper for reference pixel calculation """ # unlikely, but possible the refpixel can be (0,0) # check if there is a pre-specified reference pixel coord refx = params[cf.REFX] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) if refx > ifg.ncols - 1: msg = ('Supplied reference pixel X coordinate is greater than ' 'the number of ifg columns: {}').format(refx) raise ValueError(msg) refy = params[cf.REFY] if refy > ifg.nrows - 1: msg = ('Supplied reference pixel Y coordinate is greater than ' 'the number of ifg rows: {}').format(refy) raise ValueError(msg) if refx <= 0 or refy <= 0: # if either zero or negative log.info('Searching for best reference pixel location') half_patch_size, thresh, grid = refpixel.ref_pixel_setup(ifg_paths, params) process_grid = mpiops.array_split(grid) refpixel.save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params) mean_sds = mpiops.comm.gather(mean_sds, root=0) if mpiops.rank == MASTER_PROCESS: mean_sds = np.hstack(mean_sds) refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid) log.info('Selected reference pixel coordinate: ' '({}, {})'.format(refx, refy)) else: # pragma: no cover log.info('Reusing reference pixel from config file: ' '({}, {})'.format(refx, refy)) ifg.close() return refx, refy
def test_min_extents(self): """Test ifgcropopt=1 crops datasets to min extents.""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) ifg = Ifg(self.exp_files[0]) ifg.open() # output files should have same extents # NB: also verifies gdalwarp correctly copies geotransform across # NB: expected data copied from gdalinfo output gt = ifg.dataset.GetGeoTransform() exp_gt = (150.911666666, 0.000833333, 0, -34.172499999, 0, -0.000833333) for i, j in zip(gt, exp_gt): self.assertAlmostEqual(i, j) self.assert_geotransform_equal([self.exp_files[0], self.exp_files[4]]) ifg.close() for i in self.ifgs: i.close()
def test_orbital_correction_matlab_equality(self): from pyrate.scripts import run_pyrate run_pyrate._orb_fit_calc(self.ifg_paths, self.params) onlyfiles = [ f for f in os.listdir(SML_TEST_MATLAB_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method1_') ] count = 0 for i, f in enumerate(onlyfiles): ifg_data = np.genfromtxt(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.ifg_paths): ifg = Ifg(j) ifg.open() if os.path.basename(j).split('_unw.')[0] == \ os.path.basename(f).split( '_orb_planar_1lks_method1_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2) # means must also be equal self.assertAlmostEqual(np.nanmean(ifg_data), np.nanmean(ifg.phase_data), places=2) # number of nans must equal self.assertEqual(np.sum(np.isnan(ifg_data)), np.sum(np.isnan(ifg.phase_data))) ifg.close() # ensure that we have expected number of matches self.assertEqual(count, len(self.ifg_paths))
def test_custom_extents(self): xlooks = ylooks = 1 cext = self._custom_extents_tuple() prepare_ifgs(self.ifg_paths, CUSTOM_CROP, xlooks, ylooks, user_exts=cext) ifg = Ifg(self.exp_files[2]) ifg.open() gt = ifg.dataset.GetGeoTransform() exp_gt = (cext.xfirst, self.xs, 0, cext.yfirst, 0, self.ys) for i, j in zip(gt, exp_gt): self.assertAlmostEqual(i, j) self.assert_geotransform_equal([self.exp_files[2], self.exp_files[6]]) # close ifgs ifg.close() for i in self.ifgs: i.close()
def test_write(self): base = TEMPDIR src = self.ifg.data_path dest = join(base, basename(self.ifg.data_path)) # shutil.copy needs to copy writeable permission from src os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH | S_IRUSR | S_IWUSR) shutil.copy(src, dest) os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert i = Ifg(dest) i.open() i.phase_data[0, 1:] = nan i.write_modified_phase() del i # reopen to ensure data/nans can be read back out i = Ifg(dest) i.open(readonly=True) assert_array_equal(True, isnan(i.phase_data[0, 1:])) i.close() os.remove(dest)
def test_default_max_extents(self): """Test ifgcropopt=2 crops datasets to max bounding box extents.""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MAXIMUM_CROP, xlooks, ylooks) for f in [self.exp_files[1], self.exp_files[5]]: self.assertTrue(exists(f), msg="Output files not created") # output files should have same extents # NB: also verifies gdalwarp correctly copies geotransform across ifg = Ifg(self.exp_files[1]) ifg.open() gt = ifg.dataset.GetGeoTransform() # copied from gdalinfo output exp_gt = (150.91, 0.000833333, 0, -34.17, 0, -0.000833333) for i, j in zip(gt, exp_gt): self.assertAlmostEqual(i, j) self.assert_geotransform_equal([self.exp_files[1], self.exp_files[5]]) ifg.close() for i in self.ifgs: i.close()
def _wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs): """ A wrapper for the spatio-temporal filter so it can be tested. See docstring for spatio_temporal_filter. Required due to differences between Matlab and Python MST implementations. """ if not params[cf.APSEST]: log.info('APS correction not required.') return # perform some checks on existing ifgs log.info('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, preread_ifgs, ifc.PYRATE_APS_ERROR): log.info('Finished APS correction') return # return if True condition returned tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() spatio_temporal_filter(tsincr, ifg, params, preread_ifgs) ifg.close()
def ref_pixel_calc(ifg_paths, params): """ Reference pixel calculation setup. :param ifg_paths: List of interferogram paths :param params: Parameters dictionary corresponding to config file :return refx: Reference pixel x-coordinate :return refy: Reference pixel y-coordinate """ # unlikely, but possible the refpixel can be (0,0) # check if there is a pre-specified reference pixel coord refx = params[cf.REFX] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) if refx > ifg.ncols - 1: msg = ('Supplied reference pixel X coordinate is greater than ' 'the number of ifg columns: {}').format(refx) raise ValueError(msg) refy = params[cf.REFY] if refy > ifg.nrows - 1: msg = ('Supplied reference pixel Y coordinate is greater than ' 'the number of ifg rows: {}').format(refy) raise ValueError(msg) if refx <= 0 or refy <= 0: # if either zero or negative log.info('Searching for best reference pixel location') refy, refx = find_ref_pixel(ifg_paths, params) log.info('Selected reference pixel coordinate: ' '({}, {})'.format(refx, refy)) else: # pragma: no cover log.info('Reusing reference pixel from config file: ' '({}, {})'.format(refx, refy)) ifg.close() return refx, refy
def network_correction(ifgs, degree, offset, params, m_ifgs=None, preread_ifgs=None): """ Calculates orbital correction model, removing this from the interferograms. .. warn:: This will write orbital error corrected phase_data in the interferograms. :param ifgs: Interferograms reduced to a minimum tree from prior MST calculations :param degree: PLANAR, QUADRATIC or PART_CUBIC :param offset: True to calculate the model using offsets :param params: Parameter dictionary :param m_ifgs: Multi-looked orbfit interferograms (sequence must be mlooked versions of 'ifgs' arg) :param preread_ifgs: Parameters dict corresponding to config file :return xxxx """ # pylint: disable=too-many-locals, too-many-arguments src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs]) vphase = squeeze(vphase) B = get_network_design_matrix(src_ifgs, degree, offset) # filter NaNs out before getting model B = B[~isnan(vphase)] orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)]) ncoef = _get_num_params(degree) if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = master_slave_ids(get_all_epochs(temp_ifgs)) else: ids = master_slave_ids(get_all_epochs(ifgs)) coefs = [ orbparams[i:i + ncoef] for i in range(0, len(set(ids)) * ncoef, ncoef) ] # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifgs[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, offset=False) temp_ifg.close() else: dm = get_design_matrix(ifgs[0], degree, offset=False) # TODO: remove this import from tests suite from tests.common import MockIfg for i in ifgs: # open if not Ifg instance if not (isinstance(i, Ifg) or isinstance(i, MockIfg)): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_networkx_error(coefs, dm, i, ids, offset)
def network_orbital_correction(ifgs, degree, offset, params, m_ifgs=None, preread_ifgs=None): """ This algorithm implements a network inversion to determine orbital corrections for a set of interferograms forming a connected network. Warning: This will write orbital error corrected phase_data to the ifgs. :param list ifgs: List of Ifg class objects reduced to a minimum spanning tree network :param str degree: model to fit (PLANAR / QUADRATIC / PART_CUBIC) :param bool offset: True to calculate the model using offsets :param dict params: dictionary of configuration parameters :param list m_ifgs: list of multilooked Ifg class objects (sequence must be multilooked versions of 'ifgs' arg) :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ # pylint: disable=too-many-locals, too-many-arguments src_ifgs = ifgs if m_ifgs is None else m_ifgs src_ifgs = mst.mst_from_ifgs(src_ifgs)[3] # use networkx mst vphase = vstack([i.phase_data.reshape((i.num_cells, 1)) for i in src_ifgs]) vphase = squeeze(vphase) B = get_network_design_matrix(src_ifgs, degree, offset) # filter NaNs out before getting model B = B[~isnan(vphase)] orbparams = dot(pinv(B, 1e-6), vphase[~isnan(vphase)]) ncoef = _get_num_params(degree) if preread_ifgs: temp_ifgs = OrderedDict(sorted(preread_ifgs.items())).values() ids = master_slave_ids(get_all_epochs(temp_ifgs)) else: ids = master_slave_ids(get_all_epochs(ifgs)) coefs = [ orbparams[i:i + ncoef] for i in range(0, len(set(ids)) * ncoef, ncoef) ] # create full res DM to expand determined coefficients into full res # orbital correction (eg. expand coarser model to full size) if preread_ifgs: temp_ifg = Ifg(ifgs[0]) # ifgs here are paths temp_ifg.open() dm = get_design_matrix(temp_ifg, degree, offset=False) temp_ifg.close() else: dm = get_design_matrix(ifgs[0], degree, offset=False) for i in ifgs: # open if not Ifg instance if isinstance(i, str): # pragma: no cover # are paths i = Ifg(i) i.open(readonly=False) shared.nan_and_mm_convert(i, params) _remove_network_orb_error(coefs, dm, i, ids, offset)