def _phase_sum(ifg_paths, params): """ Save phase data and phase sum used in the reference phase estimation """ p_paths = mpiops.array_split(ifg_paths) ifg = Ifg(p_paths[0]) ifg.open(readonly=True) shape = ifg.shape phs_sum = np.zeros(shape=shape, dtype=np.float64) ifg.close() for d in p_paths: ifg = Ifg(d) ifg.open() ifg.nodata_value = params[cf.NO_DATA_VALUE] phs_sum += ifg.phase_data ifg.close() if mpiops.rank == MASTER_PROCESS: phase_sum_all = phs_sum # loop is better for memory for i in range(1, mpiops.size): # pragma: no cover phs_sum = np.zeros(shape=shape, dtype=np.float64) mpiops.comm.Recv(phs_sum, source=i, tag=i) phase_sum_all += phs_sum comp = np.isnan(phase_sum_all) # this is the same as in Matlab comp = np.ravel(comp, order='F') # this is the same as in Matlab else: # pragma: no cover comp = None mpiops.comm.Send(phs_sum, dest=0, tag=mpiops.rank) comp = mpiops.comm.bcast(comp, root=0) return comp
def test_gdal_python_vs_old_prepifg_prep2(self): for i in range(10): thresh = 0.5 x_looks = y_looks = 2 res = 2 extents = [10, 0, 20, 10] extents_str = [str(e) for e in extents] data = np.array(np.random.randint(0, 3, size=(10, 10)), dtype=np.float32) # create the self.temp_tiff self.manipulation(data, self.temp_tif, self.md) # only band 1 is resapled in warp_old averaged_and_resampled, _ = gdalwarp.crop_resample_average( self.temp_tif, extents, [res, -res], self.out_tif, thresh, match_pirate=True) ifg = Ifg(self.temp_tif) # only band 1 is resampled in warp_old data, self.old_prepifg_path = warp_old(ifg, x_looks, y_looks, extents_str, [res, -res], thresh=thresh, crop_out=4, verbose=False) np.testing.assert_array_almost_equal(data, averaged_and_resampled, decimal=4)
def save_ref_pixel_blocks(grid, half_patch_size, ifg_paths, params): """ xxxx :param grid: List of tuples (y, x) corresponding reference pixel grids :param half_patch_size: Patch size in pixels corresponding to reference pixel grids :param ifg_paths: List of interferogram paths :param params: Parameters dictionary corresponding to config file :return xxxx """ log.info('Saving ref pixel blocks') outdir = params[cf.TMPDIR] for pth in ifg_paths: ifg = Ifg(pth) ifg.open(readonly=True) ifg.nodata_value = params[cf.NO_DATA_VALUE] ifg.convert_to_nans() ifg.convert_to_mm() for y, x in grid: data = ifg.phase_data[y - half_patch_size:y + half_patch_size + 1, x - half_patch_size:x + half_patch_size + 1] data_file = join( outdir, 'ref_phase_data_{b}_{y}_{x}.npy'.format( b=os.path.basename(pth).split('.')[0], y=y, x=x)) np.save(file=data_file, arr=data) ifg.close() log.info('Saved ref pixel blocks')
def diff_exts_ifgs(): """Returns pair of test Ifgs with different extents""" bases = ['geo_060619-061002.tif', 'geo_070326-070917.tif'] random_dir = tempfile.mkdtemp() for p in bases: shutil.copy(src=os.path.join(PREP_TEST_TIF, p), dst=os.path.join(random_dir, p)) return [Ifg(join(random_dir, p)) for p in bases], random_dir
def _data_setup(datafiles): """Returns Ifg objs for the files in the interferogram list""" datafiles.sort() ifgs = [Ifg(i) for i in datafiles] for i in ifgs: i.open() return ifgs
def _get_r_dist(ifg_path): """ Get RDIst class object """ ifg = Ifg(ifg_path) ifg.open() r_dist = vcm_module.RDist(ifg)() ifg.close() return r_dist
def test_multilook(self): """Test resampling method using a scaling factor of 4""" scale = 4 # assumes square cells self.ifgs.append(DEM(SML_TEST_DEM_TIF)) self.ifg_paths = [i.data_path for i in self.ifgs] cext = self._custom_extents_tuple() xlooks = ylooks = scale prepare_ifgs(self.ifg_paths, CUSTOM_CROP, xlooks, ylooks, thresh=1.0, user_exts=cext) for n, ipath in enumerate([self.exp_files[3], self.exp_files[7]]): i = Ifg(ipath) i.open() self.assertEqual(i.dataset.RasterXSize, 20 / scale) self.assertEqual(i.dataset.RasterYSize, 28 / scale) # verify resampling path = join(PREP_TEST_TIF, "%s.tif" % n) ds = gdal.Open(path) src_data = ds.GetRasterBand(2).ReadAsArray() exp_resample = multilooking(src_data, scale, scale, thresh=0) self.assertEqual(exp_resample.shape, (7, 5)) assert_array_almost_equal(exp_resample, i.phase_band.ReadAsArray()) ds = None i.close() os.remove(ipath) # verify DEM has been correctly processed # ignore output values as resampling has already been tested for phase exp_dem_path = join(SML_TEST_DEM_DIR, 'roipac_test_trimmed_4rlks_3cr.tif') self.assertTrue(exists(exp_dem_path)) orignal_dem = DEM(SML_TEST_DEM_TIF) orignal_dem.open() dem_dtype = orignal_dem.dataset.GetRasterBand(1).DataType orignal_dem.close() dem = DEM(exp_dem_path) dem.open() # test multilooked dem is of the same datatype as the original dem tif self.assertEqual(dem_dtype, dem.dataset.GetRasterBand(1).DataType) self.assertEqual(dem.dataset.RasterXSize, 20 / scale) self.assertEqual(dem.dataset.RasterYSize, 28 / scale) data = dem.height_band.ReadAsArray() self.assertTrue(data.ptp() != 0) # close ifgs dem.close() for i in self.ifgs: i.close() os.remove(exp_dem_path)
def test_gdal_python_vs_old_prepifg_prep(self): for i in range(10): data = np.array(np.random.randint(0, 3, size=(10, 10)), dtype=np.float32) src_ds = gdal.GetDriverByName('GTiff').Create( self.temp_tif, 10, 10, 2, gdalconst.GDT_Float32) src_ds.GetRasterBand(1).WriteArray(data) src_ds.GetRasterBand(1).SetNoDataValue(0) nan_matrix = where(data == 0, nan, data) src_ds.GetRasterBand(2).WriteArray(np.isnan(nan_matrix)) src_ds.GetRasterBand(2).SetNoDataValue(-100) src_ds.SetGeoTransform([10, 1, 0, 10, 0, -1]) dst_ds = gdal.GetDriverByName('MEM').Create( '', 5, 5, 2, gdalconst.GDT_Float32) dst_ds.SetGeoTransform([10, 2, 0, 10, 0, -2]) for k, v in self.md.items(): src_ds.SetMetadataItem(k, v) dst_ds.SetMetadataItem(k, v) src_ds.FlushCache() gdal.ReprojectImage(src_ds, dst_ds, '', '', gdal.GRA_Average) nan_frac = dst_ds.GetRasterBand(2).ReadAsArray() avg = dst_ds.GetRasterBand(1).ReadAsArray() thresh = 0.5 avg[nan_frac >= thresh] = np.nan ifg = Ifg(self.temp_tif) x_looks = y_looks = 2 res = 2 resolution = [res, -res] # minX, minY, maxX, maxY = extents # adfGeoTransform[0] /* top left x */ # adfGeoTransform[1] /* w-e pixel resolution */ # adfGeoTransform[2] /* 0 */ # adfGeoTransform[3] /* top left y */ # adfGeoTransform[4] /* 0 */ # adfGeoTransform[5] /* n-s pixel resolution (negative value) */ extents = [str(e) for e in [10, 0, 20, 10]] # only band 1 is resapled in warp_old data, self.old_prepifg_path = warp_old(ifg, x_looks, y_looks, extents, resolution, thresh=thresh, crop_out=4, verbose=False) np.testing.assert_array_equal(data, avg)
def setUp(self): from tests.common import small_data_setup self.ifgs = small_data_setup() self.ifg_paths = [i.data_path for i in self.ifgs] prepare_ifgs(self.ifg_paths, crop_opt=1, xlooks=1, ylooks=1) looks_paths = [ mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths ] self.ifgs_with_nan = [Ifg(i) for i in looks_paths] for ifg in self.ifgs_with_nan: ifg.open()
def setUp(self): # faked cell sizes self.xs = 0.75 self.ys = 0.8 self.ifg = Ifg(join(SML_TEST_TIF, 'geo_060619-061002_unw.tif')) self.ifg.open() self.ifg.nodata_value = 0 self.m = MockIfg(self.ifg, 3, 4) self.m.x_size = self.xs self.m.y_size = self.ys
def prepare_ifgs_without_phase(ifg_paths, params): ifgs = [Ifg(p) for p in ifg_paths] for i in ifgs: if not i.is_open: i.open(readonly=False) nan_conversion = params[cf.NAN_CONVERSION] if nan_conversion: # nan conversion happens here in networkx mst # if not ifg.nan_converted: i.nodata_value = params[cf.NO_DATA_VALUE] i.convert_to_nans() return ifgs
def pre_prepare_ifgs(ifg_paths, params): """ Open ifg for reading """ ifgs = [Ifg(p) for p in ifg_paths] for i in ifgs: if not i.is_open: i.open(readonly=False) nan_and_mm_convert(i, params) log.info('Opened ifg for reading') return ifgs
def small5_ifgs(): """Convenience func to return a subset of 5 linked Ifgs from the testdata""" BASE_DIR = tempfile.mkdtemp() data_paths = [os.path.join(SML_TEST_TIF, p) for p in IFMS5.split()] new_data_paths = [ os.path.join(BASE_DIR, os.path.basename(d)) for d in data_paths ] for d in data_paths: shutil.copy(d, os.path.join(BASE_DIR, os.path.basename(d))) return [Ifg(p) for p in new_data_paths]
def _inner(ifg_path): """ convenient inner loop """ ifg = Ifg(ifg_path) ifg.open(readonly=False) phase_data = ifg.phase_data ref_phase = rpe.est_ref_phs_method1(phase_data, comp) phase_data -= ref_phase md = ifg.meta_data md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase(data=phase_data) ifg.close() return ref_phase
def _inner(ifg_path): ifg = Ifg(ifg_path) ifg.open(readonly=False) phase_data = ifg.phase_data ref_ph = rpe.est_ref_phs_method2(phase_data, half_chip_size, refpx, refpy, thresh) phase_data -= ref_ph md = ifg.meta_data md[ifc.REF_PHASE] = ifc.REF_PHASE_REMOVED ifg.write_modified_phase(data=phase_data) ifg.close() return ref_ph
def test_open_ifg_from_dataset(self): """ Test showing open() can not be used for Ifg created with gdal.Dataset object as Dataset has already been read in """ paths = [self.ifg.data_path] mlooked_phase_data = prepifg.prepare_ifgs(paths, crop_opt=prepifg.ALREADY_SAME_SIZE, xlooks=2, ylooks=2, write_to_disc=False) mlooked = [Ifg(m[1]) for m in mlooked_phase_data] self.assertRaises(RasterException, mlooked[0].open)
def _save_aps_corrected_phase(ifg_path, phase): """ Save (update) interferogram metadata and phase data after spatio-temporal filter (APS) correction. """ ifg = Ifg(ifg_path) ifg.open(readonly=False) ifg.phase_data[~np.isnan(ifg.phase_data)] = phase[~np.isnan(ifg.phase_data )] # set aps tags after aps error correction ifg.dataset.SetMetadataItem(ifc.PYRATE_APS_ERROR, ifc.APS_REMOVED) ifg.write_modified_phase() ifg.close()
def test_nodata(self): """Verify NODATA value copied correctly (amplitude band not copied)""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) for ex in [self.exp_files[0], self.exp_files[4]]: ifg = Ifg(ex) ifg.open() # NB: amplitude band doesn't have a NODATA value self.assertTrue( isnan(ifg.dataset.GetRasterBand(1).GetNoDataValue())) ifg.close() for i in self.ifgs: i.close()
def dem_or_ifg(data_path): """ Whether tif is a dem or an interferogram :param: data_path: xxxxx :return xxxx """ ds = gdal.Open(data_path) md = ds.GetMetadata() if 'DATE' in md: # ifg return Ifg(data_path) else: return DEM(data_path)
def test_write(self): base = TEMPDIR src = self.ifg.data_path dest = join(base, basename(self.ifg.data_path)) # shutil.copy needs to copy writeable permission from src os.chmod(src, S_IRGRP | S_IWGRP | S_IWOTH | S_IROTH | S_IRUSR | S_IWUSR) shutil.copy(src, dest) os.chmod(src, S_IRGRP | S_IROTH | S_IRUSR) # revert i = Ifg(dest) i.open() i.phase_data[0, 1:] = nan i.write_modified_phase() del i # reopen to ensure data/nans can be read back out i = Ifg(dest) i.open(readonly=True) assert_array_equal(True, isnan(i.phase_data[0, 1:])) i.close() os.remove(dest)
def remove_orbital_error(ifgs, params, preread_ifgs=None): """ Wrapper for orbital error removal functionality. :param ifgs: List of interferograms or interferogram paths :param params: Dict corresponding to config parameters :param preread_ifgs: Dict containing information regarding MPI jobs (optional) :return xxxx """ log.info('Calculating orbital error correction') if not params[cf.ORBITAL_FIT]: log.info('Orbital correction not required') return if preread_ifgs: # don't check except for mpi tests # remove non ifg keys _ = [preread_ifgs.pop(k) for k in ['gt', 'epochlist', 'md', 'wkt']] # perform some general error/sanity checks if mpiops.rank == 0: _check_orbital_ifgs(preread_ifgs) ifg_paths = [i.data_path for i in ifgs] \ if isinstance(ifgs[0], Ifg) else ifgs mlooked = None # mlooking is not necessary for independent correction # can use multiple procesing if write_to_disc=True if params[cf.ORBITAL_FIT_METHOD] == 2: mlooked_dataset = prepifg.prepare_ifgs( ifg_paths, crop_opt=prepifg.ALREADY_SAME_SIZE, xlooks=params[cf.ORBITAL_FIT_LOOKS_X], ylooks=params[cf.ORBITAL_FIT_LOOKS_Y], thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD], write_to_disc=False) mlooked = [Ifg(m[1]) for m in mlooked_dataset] for m in mlooked: m.initialize() m.nodata_value = params[cf.NO_DATA_VALUE] m.convert_to_nans() m.convert_to_mm() orbital_correction(ifgs, params, mlooked=mlooked, preread_ifgs=preread_ifgs)
def get_tiles(ifg_path, rows, cols): """ Break up the interferograms into tiles based on user supplied rows and columns. :param ifg_path: List of destination tifs :param rows: Number of rows to break each interferogram into :param cols: Number of columns to break each interferogram into :return tiles: List of shared.Tile instances """ ifg = Ifg(ifg_path) ifg.open(readonly=True) tiles = create_tiles(ifg.shape, nrows=rows, ncols=cols) ifg.close() return tiles
def data_setup(datafiles): """ xxxxxxxxxxx. :param datafiles: xxxx :return Interferogram objects for the files in the small_test data directory. The input phase data are in radians (not converted to mm). """ datafiles.sort() ifgs = [Ifg(i) for i in datafiles] for i in ifgs: i.open() return ifgs
def dem_or_ifg(data_path): """ Returns an Ifg or DEM class object from input geotiff file. :param str data_path: file path name :return: Interferogram or DEM object from input file :rtype: Ifg or DEM class object """ ds = gdal.Open(data_path) md = ds.GetMetadata() if 'DATE' in md: # ifg return Ifg(data_path) else: return DEM(data_path)
def ref_pixel_setup(ifgs_or_paths, params): """ Sets up the grid for reference pixel computation and saves numpy files to disk for later use during ref pixel computation. :param list ifgs_or_paths: List of interferogram filenames or Ifg objects :param dict params: Dictionary of configuration parameters :return: half_patch_size: size of patch :rtype: float :return: thresh :rtype: float :return: list(product(ysteps, xsteps)) :rtype: list """ log.info('Setting up ref pixel computation') refnx, refny, chipsize, min_frac = params[cf.REFNX], \ params[cf.REFNY], \ params[cf.REF_CHIP_SIZE], \ params[cf.REF_MIN_FRAC] if len(ifgs_or_paths) < 1: msg = 'Reference pixel search requires 2+ interferograms' raise RefPixelError(msg) if isinstance(ifgs_or_paths[0], str): head = Ifg(ifgs_or_paths[0]) head.open(readonly=True) else: head = ifgs_or_paths[0] # sanity check inputs _validate_chipsize(chipsize, head) _validate_minimum_fraction(min_frac) _validate_search_win(refnx, refny, chipsize, head) # pre-calculate useful amounts half_patch_size = chipsize // 2 chipsize = half_patch_size * 2 + 1 thresh = min_frac * chipsize * chipsize # do window searches across dataset, central pixel of stack with smallest # mean is the reference pixel rows, cols = head.shape ysteps = _step(rows, refny, half_patch_size) xsteps = _step(cols, refnx, half_patch_size) log.info('Ref pixel setup finished') return half_patch_size, thresh, list(product(ysteps, xsteps))
def test_nans(self): """Verify NaNs replace 0 in the multilooked phase band""" xlooks = ylooks = 1 prepare_ifgs(self.ifg_paths, MINIMUM_CROP, xlooks, ylooks) for ex in [self.exp_files[0], self.exp_files[4]]: ifg = Ifg(ex) ifg.open() phase = ifg.phase_band.ReadAsArray() self.assertFalse((phase == 0).any()) self.assertTrue((isnan(phase)).any()) ifg.close() self.assertAlmostEqual(nanmax(phase), 4.247, 3) # copied from gdalinfo self.assertAlmostEqual(nanmin(phase), 0.009, 3) # copied from gdalinfo for i in self.ifgs: i.close()
def test_same_size_multilooking(self): ifgs = same_exts_ifgs() ifg_data_paths = [d.data_path for d in ifgs] xlooks = ylooks = 2 prepare_ifgs(ifg_data_paths, ALREADY_SAME_SIZE, xlooks, ylooks) looks_paths = [ mlooked_path(d, looks=xlooks, crop_out=ALREADY_SAME_SIZE) for d in ifg_data_paths ] mlooked = [Ifg(i) for i in looks_paths] for m in mlooked: m.open() self.assertEqual(len(mlooked), 2) for ifg in mlooked: self.assertAlmostEqual(ifg.x_step, xlooks * self.xs) self.assertAlmostEqual(ifg.x_step, ylooks * self.xs)
def parse(self, string): """ override of :py:meth:`DictParam.parse`. """ dct = super(RasterParam, self).parse(string) raster_type = dct['type'] path = dct['path'] if raster_type == 'DEM': return DEM(path) elif raster_type == 'Ifg': return Ifg(path) elif raster_type == 'Incidence': return Incidence(path) else: # pragma: no cover raise luigi.parameter.UnknownParameterException( 'rasterBase must be an inscance DEM, ' 'Ifg or Incidence is valid')
def small_data_setup(datafiles=None, is_dir=False): """Returns Ifg objs for the files in the small test dir input phase data is in radians; these ifgs are in radians - not converted to mm""" if is_dir: datafiles = glob.glob(join(datafiles, "*.tif")) else: if datafiles: for i, d in enumerate(datafiles): datafiles[i] = os.path.join(SML_TEST_TIF, d) else: datafiles = glob.glob(join(SML_TEST_TIF, "*.tif")) datafiles.sort() ifgs = [Ifg(i) for i in datafiles] for i in ifgs: i.open() i.nodata_value = 0 return ifgs
def remove_orbital_error(ifgs, params, preread_ifgs=None): """ Wrapper function for PyRate orbital error removal functionality. NB: the ifg data is modified in situ, rather than create intermediate files. The network method assumes the given ifgs have already been reduced to a minimum spanning tree network. :param list ifgs: List of interferograms class objects :param dict params: Dictionary containing configuration parameters :param dict preread_ifgs: Dictionary containing information specifically for MPI jobs (optional) :return: None - interferogram phase data is updated and saved to disk """ ifg_paths = [i.data_path for i in ifgs] \ if isinstance(ifgs[0], Ifg) else ifgs mlooked = None # mlooking is not necessary for independent correction # can use multiple procesing if write_to_disc=True if params[cf.ORBITAL_FIT_METHOD] == NETWORK_METHOD: mlooked_dataset = prepifg.prepare_ifgs( ifg_paths, crop_opt=prepifg.ALREADY_SAME_SIZE, xlooks=params[cf.ORBITAL_FIT_LOOKS_X], ylooks=params[cf.ORBITAL_FIT_LOOKS_Y], thresh=params[cf.NO_DATA_AVERAGING_THRESHOLD], write_to_disc=False) mlooked = [Ifg(m[1]) for m in mlooked_dataset] for m in mlooked: m.initialize() m.nodata_value = params[cf.NO_DATA_VALUE] m.convert_to_nans() m.convert_to_mm() _orbital_correction(ifgs, params, mlooked=mlooked, preread_ifgs=preread_ifgs)