def _prepifg_multiprocessing(path, xlooks, ylooks, exts, thresh, crop, params):
"""
Multiprocessing wrapper for prepifg
"""
processor = params[cf.PROCESSOR] # roipac or gamma
if processor == GAMMA:
header = gamma.gamma_header(path, params)
elif processor == ROIPAC:
header = roipac.roipac_header(path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
# If we're performing coherence masking, find the coherence file for this IFG.
# TODO: Refactor _is_interferogram to be unprotected (remove '_')
if params[cf.COH_MASK] and shared._is_interferogram(header):
coherence_path = cf.coherence_paths_for(path, params, tif=True)[0]
coherence_thresh = params[cf.COH_THRESH]
else:
coherence_path = None
coherence_thresh = None
prepifg_helper.prepare_ifg(path,
xlooks,
ylooks,
exts,
thresh,
crop,
out_path=params[cf.OUT_DIR],
header=header,
coherence_path=coherence_path,
coherence_thresh=coherence_thresh)
def _geotiff_multiprocessing(unw_path, params):
"""
Multiprocessing wrapper for full-res geotiff conversion
"""
# TODO: Need a more robust method for identifying coherence files.
if params[cf.COH_FILE_DIR] and unw_path.endswith('.cc'):
# If the user has provided a dir for coherence files, place
# converted coherence files in that directory.
dest = shared.output_tiff_filename(unw_path, params[cf.COH_FILE_DIR])
else:
dest = shared.output_tiff_filename(unw_path, params[cf.OBS_DIR])
processor = params[cf.PROCESSOR] # roipac or gamma
# Create full-res geotiff if not already on disk
if not os.path.exists(dest):
if processor == GAMMA:
header = gamma.gamma_header(unw_path, params)
elif processor == ROIPAC:
header = roipac.roipac_header(unw_path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
shared.write_fullres_geotiff(header, unw_path, dest,
nodata=params[cf.NO_DATA_VALUE])
return dest
else:
log.info("Full-res geotiff already exists")
return None
def _prepifg_multiprocessing(path, xlooks, ylooks, exts, thresh, crop, params):
"""
Multiprocessing wrapper for prepifg
"""
processor = params[cf.PROCESSOR] # roipac, gamma or geotif
if (processor == GAMMA) or (processor == GEOTIF):
header = gamma.gamma_header(path, params)
elif processor == ROIPAC:
log.info("Warning: ROI_PAC support will be deprecated in a future PyRate release")
header = roipac.roipac_header(path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
# If we're performing coherence masking, find the coherence file for this IFG.
if params[cf.COH_MASK] and shared._is_interferogram(header):
coherence_path = cf.coherence_paths_for(path, params, tif=True)
coherence_thresh = params[cf.COH_THRESH]
else:
coherence_path = None
coherence_thresh = None
if params[cf.LARGE_TIFS]:
op = output_tiff_filename(path, params[cf.OUT_DIR])
looks_path = cf.mlooked_path(op, ylooks, crop)
return path, coherence_path, looks_path
else:
prepifg_helper.prepare_ifg(path, xlooks, ylooks, exts, thresh, crop, out_path=params[cf.OUT_DIR],
header=header, coherence_path=coherence_path, coherence_thresh=coherence_thresh)
def setUp(self):
self.BASE_DIR = tempfile.mkdtemp()
# change to orbital error correction method 2
self.params = Configuration(common.TEST_CONF_ROIPAC).__dict__
self.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
self.params[cf.ORBITAL_FIT_LOOKS_X] = 1
self.params[cf.ORBITAL_FIT_LOOKS_Y] = 1
data_paths = [
os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()
]
self.new_data_paths = [
os.path.join(self.BASE_DIR, os.path.basename(d))
for d in data_paths
]
for d in data_paths:
d_copy = os.path.join(self.BASE_DIR, os.path.basename(d))
shutil.copy(d, d_copy)
os.chmod(d_copy, 0o660)
self.ifgs = small_data_setup(datafiles=self.new_data_paths)
self.headers = [
roipac.roipac_header(i.data_path, self.params) for i in self.ifgs
]
for i in self.ifgs:
if not i.is_open:
i.open()
if not i.nan_converted:
i.convert_to_nans()
if not i.mm_converted:
i.convert_to_mm()
i.write_modified_phase()
def _geotiff_multiprocessing(unw_path: MultiplePaths,
params: dict) -> Tuple[str, bool]:
"""
Multiprocessing wrapper for full-res geotiff conversion
"""
# TODO: Need a more robust method for identifying coherence files.
dest = unw_path.converted_path
processor = params[C.PROCESSOR] # roipac or gamma
# Create full-res geotiff if not already on disk
if not os.path.exists(dest):
if processor == GAMMA:
header = gamma.gamma_header(unw_path.unwrapped_path, params)
elif processor == ROIPAC:
log.info(
"Warning: ROI_PAC support will be deprecated in a future PyRate release"
)
header = roipac.roipac_header(unw_path.unwrapped_path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
header[ifc.INPUT_TYPE] = unw_path.input_type
shared.write_fullres_geotiff(header,
unw_path.unwrapped_path,
dest,
nodata=params[C.NO_DATA_VALUE])
Path(dest).chmod(0o444) # readonly output
return dest, True
else:
log.warning(
f"Full-res geotiff already exists in {dest}! Returning existing geotiff!"
)
return dest, False
def setup_class(cls):
cls.xs = 0.000833333
cls.ys = -cls.xs
cls.ifgs, cls.random_dir = diff_exts_ifgs()
cls.ifg_paths = [i.data_path for i in cls.ifgs]
cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.params[C.OUT_DIR] = cls.random_dir
cls.params[C.GEOMETRY_DIR] = Path(cls.random_dir).joinpath(
C.GEOMETRY_DIR)
cls.params[C.GEOMETRY_DIR].mkdir(exist_ok=True)
cls.params[C.INTERFEROGRAM_DIR] = Path(cls.random_dir).joinpath(
C.INTERFEROGRAM_DIR)
cls.params[C.INTERFEROGRAM_DIR].mkdir(exist_ok=True)
cls.headers = [
roipac.roipac_header(i.data_path, cls.params) for i in cls.ifgs
]
paths = [
"060619-061002_ifg.tif", "060619-061002_ifg.tif",
"060619-061002_ifg.tif", "060619-061002_ifg.tif",
"070326-070917_ifg.tif", "070326-070917_ifg.tif",
"070326-070917_ifg.tif", "070326-070917_ifg.tif"
]
cls.exp_files = [
join(cls.random_dir, C.INTERFEROGRAM_DIR, p) for p in paths
]
def repair_params_for_correct_tests(out_dir, params):
base_ifg_paths = [c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
params[cf.INTERFEROGRAM_FILES] = params[cf.INTERFEROGRAM_FILES][:-2]
dest_paths = [
Path(out_dir).joinpath(Path(c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES]
]
for p, d in zip(params[cf.INTERFEROGRAM_FILES], dest_paths): # hack
p.sampled_path = d
return dest_paths, headers
def find_header(path: MultiplePaths, params: dict):
processor = params[cf.PROCESSOR] # roipac, gamma or geotif
tif_path = path.converted_path
if (processor == GAMMA) or (processor == GEOTIF):
header = gamma.gamma_header(tif_path, params)
elif processor == ROIPAC:
log.info("Warning: ROI_PAC support will be deprecated in a future PyRate release")
header = roipac.roipac_header(tif_path, params)
else:
raise PreprocessError('Processor must be ROI_PAC (0) or GAMMA (1)')
header[ifc.INPUT_TYPE] = path.input_type
return header
def setUpClass(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
shutil.rmtree(params[cf.OUT_DIR])
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = cls.temp_out_dir
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 1
params[cf.PARALLEL] = True
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
# Estimate reference pixel location
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
for i in ifgs:
i.close()
ifgs = common.pre_prepare_ifgs(dest_paths, params)
for i in ifgs:
i.close()
cls.ref_phs, cls.ifgs = process._ref_phase_estimation(
dest_paths, params, refx, refy)
def setUpClass(cls):
params = Configuration(TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(cls.temp_out_dir, cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
params[cf.REF_EST_METHOD] = 2
conv2tif.main(params)
prepifg.main(params)
cls.params = params
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
dest_paths = [c.converted_path for c in params[cf.INTERFEROGRAM_FILES]]
dest_paths = dest_paths[:-2]
for i in dest_paths:
Path(i).chmod(
0o664
) # assign write permission as conv2tif output is readonly
ifgs = common.pre_prepare_ifgs(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
headers = [roipac.roipac_header(i, cls.params) for i in base_ifg_paths]
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, cls.ifgs = process._ref_phase_estimation(dest_paths, params, refx,
refy)
ifgs[0].open()
r_dist = RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
cls.maxvar = [
cvd(i,
params,
r_dist,
calc_alpha=True,
save_acg=True,
write_vals=True)[0] for i in dest_paths
]
cls.vcmt = get_vcmt(ifgs, cls.maxvar)
for ifg in ifgs:
ifg.close()
def setUp(self):
from tests.common import small_data_setup
self.ifgs = small_data_setup()
self.ifg_paths = [i.data_path for i in self.ifgs]
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
self.headers = [
roipac.roipac_header(i.data_path, params) for i in self.ifgs
]
prepare_ifgs(self.ifg_paths,
crop_opt=1,
xlooks=1,
ylooks=1,
headers=self.headers)
looks_paths = [
mlooked_path(d, looks=1, crop_out=1) for d in self.ifg_paths
]
self.ifgs_with_nan = [Ifg(i) for i in looks_paths]
for ifg in self.ifgs_with_nan:
ifg.open()
def setUp(self):
self.xs = 0.000833333
self.ys = -self.xs
self.ifgs, self.random_dir = diff_exts_ifgs()
self.ifg_paths = [i.data_path for i in self.ifgs]
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
self.headers = [
roipac.roipac_header(i.data_path, params) for i in self.ifgs
]
paths = [
"geo_060619-061002_unw_1rlks_1cr.tif",
"geo_060619-061002_unw_1rlks_2cr.tif",
"geo_060619-061002_unw_1rlks_3cr.tif",
"geo_060619-061002_unw_4rlks_3cr.tif",
"geo_070326-070917_unw_1rlks_1cr.tif",
"geo_070326-070917_unw_1rlks_2cr.tif",
"geo_070326-070917_unw_1rlks_3cr.tif",
"geo_070326-070917_unw_4rlks_3cr.tif"
]
self.exp_files = [join(self.random_dir, p) for p in paths]
def setup_class(cls):
from tests.common import small_data_setup
cls.ifgs = small_data_setup()
cls.ifg_paths = [i.data_path for i in cls.ifgs]
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.headers = [
roipac.roipac_header(i.data_path, params) for i in cls.ifgs
]
params[C.IFG_LKSX], params[C.IFG_LKSY], params[
C.IFG_CROP_OPT] = 1, 1, 1
prepare_ifgs(cls.ifg_paths,
crop_opt=1,
xlooks=1,
ylooks=1,
headers=cls.headers,
params=params)
looks_paths = [
mlooked_path(d, params, t)
for d, t in zip(cls.ifg_paths, [InputTypes.IFG] * len(cls.ifgs))
]
cls.ifgs_with_nan = [Ifg(i) for i in looks_paths]
for ifg in cls.ifgs_with_nan:
ifg.open()
def setup_class(cls):
# change to orbital error correction method 2
cls.params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.BASE_DIR = cls.params[cf.OUT_DIR]
cls.params[cf.ORBITAL_FIT_METHOD] = NETWORK_METHOD
cls.params[cf.ORBITAL_FIT_LOOKS_X] = 1
cls.params[cf.ORBITAL_FIT_LOOKS_Y] = 1
cls.params[cf.ORBFIT_OFFSET] = True
cls.params[cf.OUT_DIR] = cls.BASE_DIR
data_paths = [os.path.join(SML_TEST_TIF, p) for p in small_ifg_file_list()]
cls.new_data_paths = [os.path.join(cls.BASE_DIR, os.path.basename(d)) for d in data_paths]
cls.params[cf.INTERFEROGRAM_FILES] = [MultiplePaths(file_name=d, params=cls.params) for d in data_paths]
for p in cls.params[cf.INTERFEROGRAM_FILES]:
p.sampled_path = p.converted_path
# copy the files from the dir into temp dir
for d in data_paths:
d_copy = os.path.join(cls.BASE_DIR, os.path.basename(d))
shutil.copy(d, d_copy)
os.chmod(d_copy, 0o660)
cls.headers = [roipac.roipac_header(i, cls.params) for i in cls.new_data_paths]
cls.orb_error_dir = Path(cls.params[cf.OUT_DIR]).joinpath(ORB_ERROR_DIR)
cls.orb_error_dir.mkdir(parents=True, exist_ok=True)
def setUpClass(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.PARALLEL] = 0
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 2
params[cf.PARALLEL] = 1
# Calculate time series
cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs,
params,
vcmt,
mst=mst_grid)
params[cf.PARALLEL] = 0
# Calculate time series serailly by the pixel
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
# copy legacy data
SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series')
tsincr_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_incr_interp0_method2.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_cum_interp0_method2.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls):
params = Configuration(TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate stacked rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
# Calculate stacked rate map
params[cf.PARALLEL] = 0
cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate')
cls.rate_container = np.genfromtxt(os.path.join(
stackrate_dir, 'stackmap.csv'),
delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(
stackrate_dir, 'errormap.csv'),
delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(
stackrate_dir, 'coh_sta.csv'),
delimiter=',')
for ifg in ifgs:
ifg.close()