def setUp(self): from tests.common import small_data_setup self.ifgs = small_data_setup() self.ifg_paths = [i.data_path for i in self.ifgs] prepare_ifgs(self.ifg_paths, crop_opt=1, xlooks=1, ylooks=1) looks_paths = [ mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths ] self.ifgs_with_nan = [Ifg(i) for i in looks_paths] for ifg in self.ifgs_with_nan: ifg.open()
def _save_aps_corrected_phase(ifg_path, phase): """ Save (update) interferogram metadata and phase data after spatio-temporal filter (APS) correction. """ ifg = Ifg(ifg_path) ifg.open(readonly=False) ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data )] # set aps tags after aps error correction ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED) ifg.write_modified_phase() ifg.close()
def prepare_ifgs_without_phase(ifg_paths, params): ifgs = [Ifg(p) for p in ifg_paths] for i in ifgs: if not i.is_open: i.open(readonly=False) nan_conversion = params[cf.NAN_CONVERSION] if nan_conversion: # nan conversion happens here in networkx mst # if not ifg.nan_converted: i.nodata_value = params[cf.NO_DATA_VALUE] i.convert_to_nans() return ifgs
def small5_ifgs(): """Convenience func to return a subset of 5 linked Ifgs from the testdata""" BASE_DIR = tempfile.mkdtemp() data_paths = [os.path.join(SML_TEST_TIF, p) for p in IFMS5.split()] new_data_paths = [ os.path.join(BASE_DIR, os.path.basename(d)) for d in data_paths ] for d in data_paths: shutil.copy(d, os.path.join(BASE_DIR, os.path.basename(d))) return [Ifg(p) for p in new_data_paths]
def ref_pixel_setup(ifgs_or_paths, params): """ Sets up the grid for reference pixel computation. Also saves numpy files for later use during reference pixel computation. :param ifgs_or_paths: xxxx :param params: xxxx :return xxxx """ log.info('Setting up ref pixel computation') refnx, refny, chipsize, min_frac = params[cf.REFNX], \ params[cf.REFNY], \ params[cf.REF_CHIP_SIZE], \ params[cf.REF_MIN_FRAC] if len(ifgs_or_paths) < 1: msg = 'Reference pixel search requires 2+ interferograms' raise RefPixelError(msg) if isinstance(ifgs_or_paths[0], str): head = Ifg(ifgs_or_paths[0]) head.open(readonly=True) else: head = ifgs_or_paths[0] # sanity check inputs _validate_chipsize(chipsize, head) _validate_minimum_fraction(min_frac) _validate_search_win(refnx, refny, chipsize, head) # pre-calculate useful amounts half_patch_size = chipsize // 2 chipsize = half_patch_size * 2 + 1 thresh = min_frac * chipsize * chipsize # do window searches across dataset, central pixel of stack with smallest # mean is the reference pixel rows, cols = head.shape ysteps = step(rows, refny, half_patch_size) xsteps = step(cols, refnx, half_patch_size) log.info('Ref pixel setup finished') return half_patch_size, thresh, list(product(ysteps, xsteps))
def _inner(ifg_path): """ Convenient inner loop """ ifg = Ifg(ifg_path) ifg.open(readonly=False) phase_data = ifg.phase_data ref_phase = rpe._est_ref_phs_method1(phase_data, comp) phase_data -= ref_phase md = ifg.meta_data md[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase(data=phase_data) ifg.close() return ref_phase
def test_open_ifg_from_dataset(self): """ Test showing open() can not be used for Ifg created with gdal.Dataset object as Dataset has already been read in """ paths = [self.ifg.data_path] mlooked_phase_data = prepifg.prepare_ifgs(paths, crop_opt=prepifg.ALREADY_SAME_SIZE, xlooks=2, ylooks=2, write_to_disc=False) mlooked = [Ifg(m[1]) for m in mlooked_phase_data] self.assertRaises(RasterException, mlooked[0].open)
def _inner(ifg_path): """ Convenient inner loop """ ifg = Ifg(ifg_path) ifg.open(readonly=False) phase_data = ifg.phase_data ref_ph = rpe._est_ref_phs_method2(phase_data, half_chip_size, refpx, refpy, thresh) phase_data -= ref_ph md = ifg.meta_data md[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase(data=phase_data) ifg.close() return ref_ph
def remove_orbital_error(ifgs, params, preread_ifgs=None): """ Wrapper for orbital error removal functionality. :param ifgs: List of interferograms or interferogram paths :param params: Dict corresponding to config parameters :param preread_ifgs: Dict containing information regarding MPI jobs (optional) :return xxxx """ log.info('Calculating orbital error correction') if not params[cf.ORBITAL_FIT]: log.info('Orbital correction not required') return if preread_ifgs: # don't check except for mpi tests # remove non ifg keys _ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']] # perform some general error/sanity checks if mpiops.rank == 0: _check_orbital_ifgs(preread_ifgs) ifg_paths = [i.data_path for i in ifgs] \ if isinstance(ifgs[0], Ifg) else ifgs mlooked = None # mlooking is not necessary for independent correction # can use multiple procesing if write_to_disc=True if params[cf.ORBITAL_FIT_METHOD] == 2: mlooked_dataset = prepifg.prepare_ifgs( ifg_paths, crop_opt=prepifg.ALREADY_SAME_SIZE, xlooks=params[cf.ORBITAL_FIT_LOOKS_X], ylooks=params[cf.ORBITAL_FIT_LOOKS_Y], thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD], write_to_disc=False) mlooked = [Ifg(m[1]) for m in mlooked_dataset] for m in mlooked: m.initialize() m.nodata_value = params[cf.NO_DATA_VALUE] m.convert_to_nans() m.convert_to_mm() orbital_correction(ifgs, params, mlooked=mlooked, preread_ifgs=preread_ifgs)
def dem_or_ifg(data_path): """ Whether tif is a dem or an interferogram :param: data_path: xxxxx :return xxxx """ ds = gdal.Open(data_path) md = ds.GetMetadata() if 'DATE' in md: # ifg return Ifg(data_path) else: return DEM(data_path)
def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = vcm_module.RDist(ifg)() ifg.close() return r_dist
def dem_or_ifg(data_path): """ Returns an Ifg or DEM class object from input geotiff file. :param str data_path: file path name :return: Interferogram or DEM object from input file :rtype: Ifg or DEM class object """ ds = gdal.Open(data_path) md = ds.GetMetadata() if 'DATE' in md: # ifg return Ifg(data_path) else: return DEM(data_path)
def data_setup(datafiles): """ xxxxxxxxxxx. :param datafiles: xxxx :return Interferogram objects for the files in the small_test data directory. The input phase data are in radians (not converted to mm). """ datafiles.sort() ifgs = [Ifg(i) for i in datafiles] for i in ifgs: i.open() return ifgs
def test_same_size_multilooking(self): ifgs = same_exts_ifgs() ifg_data_paths = [d.data_path for d in ifgs] xlooks = ylooks = 2 prepare_ifgs(ifg_data_paths, ALREADY_SAME_SIZE, xlooks, ylooks) looks_paths = [ mlooked_path(d, looks=xlooks, crop_out=ALREADY_SAME_SIZE) for d in ifg_data_paths ] mlooked = [Ifg(i) for i in looks_paths] for m in mlooked: m.open() self.assertEqual(len(mlooked), 2) for ifg in mlooked: self.assertAlmostEqual(ifg.x_step, xlooks * self.xs) self.assertAlmostEqual(ifg.x_step, ylooks * self.xs)
def parse(self, string): """ override of :py:meth:`DictParam.parse`. """ dct = super(RasterParam, self).parse(string) raster_type = dct['type'] path = dct['path'] if raster_type == 'DEM': return DEM(path) elif raster_type == 'Ifg': return Ifg(path) elif raster_type == 'Incidence': return Incidence(path) else: # pragma: no cover raise luigi.parameter.UnknownParameterException( 'rasterBase must be an inscance DEM, ' 'Ifg or Incidence is valid')
def small_data_setup(datafiles=None, is_dir=False): """Returns Ifg objs for the files in the small test dir input phase data is in radians; these ifgs are in radians - not converted to mm""" if is_dir: datafiles = glob.glob(join(datafiles, "*.tif")) else: if datafiles: for i, d in enumerate(datafiles): datafiles[i] = os.path.join(SML_TEST_TIF, d) else: datafiles = glob.glob(join(SML_TEST_TIF, "*.tif")) datafiles.sort() ifgs = [Ifg(i) for i in datafiles] for i in ifgs: i.open() i.nodata_value = 0 return ifgs
def remove_orbital_error(ifgs, params, preread_ifgs=None): """ Wrapper function for PyRate orbital error removal functionality. NB: the ifg data is modified in situ, rather than create intermediate files. The network method assumes the given ifgs have already been reduced to a minimum spanning tree network. :param list ifgs: List of interferograms class objects :param dict params: Dictionary containing configuration parameters :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ ifg_paths = [i.data_path for i in ifgs] \ if isinstance(ifgs[0], Ifg) else ifgs mlooked = None # mlooking is not necessary for independent correction # can use multiple procesing if write_to_disc=True if params[cf.ORBITAL_FIT_METHOD] == NETWORK_METHOD: mlooked_dataset = prepifg.prepare_ifgs( ifg_paths, crop_opt=prepifg.ALREADY_SAME_SIZE, xlooks=params[cf.ORBITAL_FIT_LOOKS_X], ylooks=params[cf.ORBITAL_FIT_LOOKS_Y], thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD], write_to_disc=False) mlooked = [Ifg(m[1]) for m in mlooked_dataset] for m in mlooked: m.initialize() m.nodata_value = params[cf.NO_DATA_VALUE] m.convert_to_nans() m.convert_to_mm() _orbital_correction(ifgs, params, mlooked=mlooked, preread_ifgs=preread_ifgs)
def test_nodata(self): """Verify NODATA value copied correctly (amplitude band not copied)""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) for ex in [self.exp_files[0], self.exp_files[4]]: ifg = Ifg(ex) ifg.open() # NB: amplitude band doesn't have a NODATA value self.assertTrue( isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue())) ifg.close() for i in self.ifgs: i.close()
def get_tiles(ifg_path, rows, cols): """ Break up the interferograms into tiles based on user supplied rows and columns. :param ifg_path: List of destination tifs :param rows: Number of rows to break each interferogram into :param cols: Number of columns to break each interferogram into :return tiles: List of shared.Tile instances """ ifg = Ifg(ifg_path) ifg.open(readonly=True) tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols) ifg.close() return tiles
def test_nans(self): """Verify NaNs replace 0 in the multilooked phase band""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) for ex in [self.exp_files[0], self.exp_files[4]]: ifg = Ifg(ex) ifg.open() phase = ifg.phase_band.ReadAsArray() self.assertFalse((phase == 0).any()) self.assertTrue((isnan(phase)).any()) ifg.close() self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo for i in self.ifgs: i.close()
def test_get_epochs(self): def str2date(s): segs = s[:4], s[4:6], s[6:] # year, month, day return date(*[int(sg) for sg in segs]) raw_date = ['20060619', '20060828', '20061002', '20061106', '20061211', '20070115', '20070219', '20070326', '20070430', '20070604', '20070709', '20070813', '20070917'] exp_dates = [str2date(d) for d in raw_date] exp_repeat = [1, 1, 3, 3, 4, 3, 3, 3, 3, 3, 3, 2, 2] exp_spans = [0, 0.1916, 0.2875, 0.3833, 0.4791, 0.5749, 0.6708, 0.7666, 0.8624, 0.9582, 1.0541, 1.1499, 1.2457] ifms = join(SML_TEST_TIF, "ifms_17") ifgs = [Ifg(join(SML_TEST_TIF, p)) for p in parse_namelist(ifms)] for i in ifgs: i.open() epochs = get_epochs(ifgs)[0] self.assertTrue((exp_dates == epochs.dates).all()) self.assertTrue((exp_repeat == epochs.repeat).all()) assert_array_almost_equal(exp_spans, epochs.spans, decimal=4)
def _ref_pixel_calc(ifg_paths, params): """ Wrapper for reference pixel calculation """ # unlikely, but possible the refpixel can be (0,0) # check if there is a pre-specified reference pixel coord refx = params[cf.REFX] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) if refx > ifg.ncols - 1: msg = ('Supplied reference pixel X coordinate is greater than ' 'the number of ifg columns: {}').format(refx) raise ValueError(msg) refy = params[cf.REFY] if refy > ifg.nrows - 1: msg = ('Supplied reference pixel Y coordinate is greater than ' 'the number of ifg rows: {}').format(refy) raise ValueError(msg) if refx <= 0 or refy <= 0: # if either zero or negative log.info('Searching for best reference pixel location') half_patch_size, thresh, grid = refpixel.ref_pixel_setup(ifg_paths, params) process_grid = mpiops.array_split(grid) refpixel.save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params) mean_sds = refpixel._ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params) mean_sds = mpiops.comm.gather(mean_sds, root=0) if mpiops.rank == MASTER_PROCESS: mean_sds = np.hstack(mean_sds) refy, refx = mpiops.run_once(refpixel.find_min_mean, mean_sds, grid) log.info('Selected reference pixel coordinate: ' '({}, {})'.format(refx, refy)) else: # pragma: no cover log.info('Reusing reference pixel from config file: ' '({}, {})'.format(refx, refy)) ifg.close() return refx, refy
def test_min_extents(self): """Test ifgcropopt=1 crops datasets to min extents.""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) ifg = Ifg(self.exp_files[0]) ifg.open() # output files should have same extents # NB: also verifies gdalwarp correctly copies geotransform across # NB: expected data copied from gdalinfo output gt = ifg.dataset.GetGeoTransform() exp_gt = (150.911666666, 0.000833333, 0, -34.172499999, 0, -0.000833333) for i, j in zip(gt, exp_gt): self.assertAlmostEqual(i, j) self.assert_geotransform_equal([self.exp_files[0], self.exp_files[4]]) ifg.close() for i in self.ifgs: i.close()
def test_orbital_correction_matlab_equality(self): from pyrate.scripts import run_pyrate run_pyrate._orb_fit_calc(self.ifg_paths, self.params) onlyfiles = [ f for f in os.listdir(SML_TEST_MATLAB_ORBITAL_DIR) if os.path.isfile(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f)) and f.endswith('.csv') and f.__contains__('_method1_') ] count = 0 for i, f in enumerate(onlyfiles): ifg_data = np.genfromtxt(os.path.join(SML_TEST_MATLAB_ORBITAL_DIR, f), delimiter=',') for k, j in enumerate(self.ifg_paths): ifg = Ifg(j) ifg.open() if os.path.basename(j).split('_unw.')[0] == \ os.path.basename(f).split( '_orb_planar_1lks_method1_')[1].split('.')[0]: count += 1 # all numbers equal np.testing.assert_array_almost_equal(ifg_data, ifg.phase_data, decimal=2) # means must also be equal self.assertAlmostEqual(np.nanmean(ifg_data), np.nanmean(ifg.phase_data), places=2) # number of nans must equal self.assertEqual(np.sum(np.isnan(ifg_data)), np.sum(np.isnan(ifg.phase_data))) ifg.close() # ensure that we have expected number of matches self.assertEqual(count, len(self.ifg_paths))
def test_default_max_extents(self): """Test ifgcropopt=2 crops datasets to max bounding box extents.""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MAXIMUM_CROP, xlooks, ylooks) for f in [self.exp_files[1], self.exp_files[5]]: self.assertTrue(exists(f), msg="Output files not created") # output files should have same extents # NB: also verifies gdalwarp correctly copies geotransform across ifg = Ifg(self.exp_files[1]) ifg.open() gt = ifg.dataset.GetGeoTransform() # copied from gdalinfo output exp_gt = (150.91, 0.000833333, 0, -34.17, 0, -0.000833333) for i, j in zip(gt, exp_gt): self.assertAlmostEqual(i, j) self.assert_geotransform_equal([self.exp_files[1], self.exp_files[5]]) ifg.close() for i in self.ifgs: i.close()
def test_custom_extents(self): xlooks = ylooks = 1 cext = self._custom_extents_tuple() prepare_ifgs(self.ifg_paths, CUSTOM_CROP, xlooks, ylooks, user_exts=cext) ifg = Ifg(self.exp_files[2]) ifg.open() gt = ifg.dataset.GetGeoTransform() exp_gt = (cext.xfirst, self.xs, 0, cext.yfirst, 0, self.ys) for i, j in zip(gt, exp_gt): self.assertAlmostEqual(i, j) self.assert_geotransform_equal([self.exp_files[2], self.exp_files[6]]) # close ifgs ifg.close() for i in self.ifgs: i.close()
def ref_pixel_calc(ifg_paths, params): """ Reference pixel calculation setup. :param ifg_paths: List of interferogram paths :param params: Parameters dictionary corresponding to config file :return refx: Reference pixel x-coordinate :return refy: Reference pixel y-coordinate """ # unlikely, but possible the refpixel can be (0,0) # check if there is a pre-specified reference pixel coord refx = params[cf.REFX] ifg = Ifg(ifg_paths[0]) ifg.open(readonly=True) if refx > ifg.ncols - 1: msg = ('Supplied reference pixel X coordinate is greater than ' 'the number of ifg columns: {}').format(refx) raise ValueError(msg) refy = params[cf.REFY] if refy > ifg.nrows - 1: msg = ('Supplied reference pixel Y coordinate is greater than ' 'the number of ifg rows: {}').format(refy) raise ValueError(msg) if refx <= 0 or refy <= 0: # if either zero or negative log.info('Searching for best reference pixel location') refy, refx = find_ref_pixel(ifg_paths, params) log.info('Selected reference pixel coordinate: ' '({}, {})'.format(refx, refy)) else: # pragma: no cover log.info('Reusing reference pixel from config file: ' '({}, {})'.format(refx, refy)) ifg.close() return refx, refy
def _wrap_spatio_temporal_filter(ifg_paths, params, tiles, preread_ifgs): """ A wrapper for the spatio-temporal filter so it can be tested. See docstring for spatio_temporal_filter. Required due to differences between Matlab and Python MST implementations. """ if not params[cf.APSEST]: log.info('APS correction not required.') return # perform some checks on existing ifgs log.info('Checking APS correction status') if mpiops.run_once(shared.check_correction_status, preread_ifgs, ifc.PYRATE_APS_ERROR): log.info('Finished APS correction') return # return if True condition returned tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles) ifg = Ifg(ifg_paths[0]) # just grab any for parameters in slpfilter ifg.open() spatio_temporal_filter(tsincr, ifg, params, preread_ifgs) ifg.close()
def phase_sum(ifg_paths, params): """ Save phase data and phs_sum used in the reference phase estimation. :param ifg_paths: List of paths to interferograms :param params: Config dictionary :return xxxx """ p_paths = mpiops.array_split(ifg_paths) ifg = Ifg(p_paths[0]) ifg.open(readonly=True) shape = ifg.shape phs_sum = np.zeros(shape=shape, dtype=np.float64) ifg.close() for d in p_paths: ifg = Ifg(d) ifg.open() ifg.nodata_value = params[cf.NO_DATA_VALUE] phs_sum += ifg.phase_data ifg.close() if mpiops.rank == MASTER_PROCESS: phase_sum_all = phs_sum # loop is better for memory for i in range(1, mpiops.size): # pragma: no cover phs_sum = np.zeros(shape=shape, dtype=np.float64) mpiops.comm.Recv(phs_sum, source=i, tag=i) phase_sum_all += phs_sum comp = np.isnan(phase_sum_all) # this is the same as in Matlab comp = np.ravel(comp, order='F') # this is the same as in Matlab else: # pragma: no cover comp = None mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank) comp = mpiops.comm.bcast(comp, root=0) return comp
def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params): """ xxxx :param grid: List of tuples (y, x) corresponding reference pixel grids :param half_patch_size: Patch size in pixels corresponding to reference pixel grids :param ifg_paths: List of interferogram paths :param params: Parameters dictionary corresponding to config file :return xxxx """ log.info('Saving ref pixel blocks') outdir = params[cf.TMPDIR] for pth in ifg_paths: ifg = Ifg(pth) ifg.open(readonly=True) ifg.nodata_value = params[cf.NO_DATA_VALUE] ifg.convert_to_nans() ifg.convert_to_mm() for y, x in grid: data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1, x - half_patch_size:x + half_patch_size + 1] data_file = join( outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format( b=os.path.basename(pth).split('.')[0], y=y, x=x)) np.save(file=data_file, arr=data) ifg.close() log.info('Saved ref pixel blocks')