def setUpClass(cls):
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
# change the required params
cls.params = cf.get_config_params(cls.test_conf)
cls.params[cf.OBS_DIR] = common.SML_TEST_GAMMA
cls.params[cf.PROCESSOR] = 1 # gamma
file_list = list(
cf.parse_namelist(os.path.join(common.SML_TEST_GAMMA, 'ifms_17')))
fd, cls.params[cf.IFG_FILE_LIST] = tempfile.mkstemp(suffix='.conf',
dir=cls.tif_dir)
os.close(fd)
# write a short filelist with only 3 gamma unws
with open(cls.params[cf.IFG_FILE_LIST], 'w') as fp:
for f in file_list[:3]:
fp.write(os.path.join(common.SML_TEST_GAMMA, f) + '\n')
cls.params[cf.OUT_DIR] = cls.tif_dir
cls.params[cf.PARALLEL] = 0
cls.params[cf.REF_EST_METHOD] = 1
cls.params[cf.DEM_FILE] = common.SML_TEST_DEM_GAMMA
# base_unw_paths need to be geotiffed and multilooked by run_prepifg
cls.base_unw_paths = cf.original_ifg_paths(
cls.params[cf.IFG_FILE_LIST], cls.params[cf.OBS_DIR])
cls.base_unw_paths.append(common.SML_TEST_DEM_GAMMA)
xlks, ylks, crop = cf.transform_params(cls.params)
# dest_paths are tifs that have been geotif converted and multilooked
conv2tif.main(cls.params)
prepifg.main(cls.params)
# run_prepifg.gamma_prepifg(cls.base_unw_paths, cls.params)
cls.base_unw_paths.pop() # removed dem as we don't want it in ifgs
cls.dest_paths = cf.get_dest_paths(cls.base_unw_paths, crop,
cls.params, xlks)
cls.ifgs = common.small_data_setup(datafiles=cls.dest_paths)
def parallel_ifgs(gamma_conf):
tdir = Path(tempfile.mkdtemp())
params_p = manipulate_test_conf(gamma_conf, tdir)
params_p[cf.PARALLEL] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params_p, output_conf_file=output_conf)
params_p = Configuration(output_conf).__dict__
gtif_paths = conv2tif.main(params_p)
prepifg.main(params_p)
parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix))
p_ifgs = small_data_setup(datafiles=parallel_df)
yield p_ifgs
shutil.rmtree(params_p[cf.OBS_DIR])
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[C.COH_MASK] = coh_mask
params[C.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
pyrate.configuration.write_config_file(params=params,
output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
# reread params from config
params_s = Configuration(output_conf).__dict__
prepifg.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_unw.tif'))
assert len(ifg_files) == 17
mlooked_files = list(
Path(tdir.joinpath(params_s[C.INTERFEROGRAM_DIR])).glob('*_ifg.tif'))
assert len(mlooked_files) == 17
coh_files = list(
Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_cc.tif'))
mlooked_coh_files = list(
Path(tdir.joinpath(params_s[C.COHERENCE_DIR])).glob('*_coh.tif'))
if coh_mask:
assert len(coh_files) == 17
assert len(mlooked_coh_files) == 17
dem_file = list(
Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('*_dem.tif'))[0]
mlooked_dem_file = list(
Path(tdir.joinpath(params_s[C.GEOMETRY_DIR])).glob('dem.tif'))[0]
import itertools
# assert coherence and ifgs have correct metadata
for i in itertools.chain(
*[ifg_files, mlooked_files, coh_files, mlooked_coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_unw.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
assert ifc.IFG_LKSX not in md
assert ifc.IFG_LKSY not in md
assert ifc.IFG_CROP not in md
continue
if i.name.endswith('_cc.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
assert ifc.IFG_LKSX not in md
assert ifc.IFG_LKSY not in md
assert ifc.IFG_CROP not in md
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH
assert md[ifc.IFG_LKSX] == '1'
assert md[ifc.IFG_LKSY] == '1'
assert md[ifc.IFG_CROP] == '1'
continue
if i.name.endswith('_ifg.tif'):
if coh_mask:
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_COH_MASKED_IFG
assert md[ifc.IFG_LKSX] == '1'
assert md[ifc.IFG_LKSY] == '1'
assert md[ifc.IFG_CROP] == '1'
else:
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED
assert md[ifc.IFG_LKSX] == '1'
assert md[ifc.IFG_LKSY] == '1'
assert md[ifc.IFG_CROP] == '1'
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
assert ifc.IFG_LKSX not in md
assert ifc.IFG_LKSY not in md
assert ifc.IFG_CROP not in md
dem = DEM(mlooked_dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM
assert ifc.IFG_LKSX in md
assert ifc.IFG_LKSY in md
assert ifc.IFG_CROP in md
shutil.rmtree(tdir)
def test_num_gamma_tifs_equals_num_unws(gamma_params):
gtifs = conv2tif.main(gamma_params)
# 17 unws + dem
assert len(gtifs) == 18
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
params[C.TEMP_MLOOKED_DIR] = os.path.join(params[C.OUT_DIR],
C.TEMP_MLOOKED_DIR)
# force error maps to 1-sigma to match legacy
params[C.VELERROR_NSIG] = 1
conv2tif.main(params)
prepifg.main(params)
params[C.REF_EST_METHOD] = 2
xlks, _, crop = pyrate.core.prepifg_helper.transform_params(params)
dest_paths, headers = common.repair_params_for_correct_tests(
params[C.INTERFEROGRAM_DIR], params)
correct._copy_mlooked(params)
copied_dest_paths = [
os.path.join(params[C.TEMP_MLOOKED_DIR], os.path.basename(d))
for d in dest_paths
]
del dest_paths
# start run_pyrate copy
ifgs = pre_prepare_ifgs(copied_dest_paths, params)
mst_grid = tests.common.mst_calculation(copied_dest_paths, params)
refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
params[C.REFX] = refx
params[C.REFY] = refy
params[C.ORBFIT_OFFSET] = True
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(copied_dest_paths, params)
for ifg in ifgs:
ifg.close()
correct._update_params_with_tiles(params)
_, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [
vcm_module.cvd(i, params, r_dist)[0] for i in copied_dest_paths
]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate stacked rate map
params[C.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
# Calculate stacked rate map
params[C.PARALLEL] = 0
cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate')
cls.rate_container = np.genfromtxt(os.path.join(
stackrate_dir, 'stackmap.csv'),
delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(
stackrate_dir, 'errormap.csv'),
delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(
stackrate_dir, 'coh_sta.csv'),
delimiter=',')
for ifg in ifgs:
ifg.close()
cls.params = params
def setUpClass(cls):
rate_types = ['stack_rate', 'stack_error', 'stack_samples']
cls.tif_dir = tempfile.mkdtemp()
cls.test_conf = common.TEST_CONF_GAMMA
from pyrate.configuration import Configuration
# change the required params
params = Configuration(cls.test_conf).__dict__
params[cf.OBS_DIR] = common.SML_TEST_GAMMA
params[cf.PROCESSES] = 4
params[cf.PROCESSOR] = 1 # gamma
params[cf.IFG_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
params[cf.OUT_DIR] = cls.tif_dir
params[cf.PARALLEL] = 1
params[cf.APS_CORRECTION] = False
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
rows, cols = params["rows"], params["cols"]
# xlks, ylks, crop = cf.transform_params(params)
# base_unw_paths need to be geotiffed by converttogeotif
# and multilooked by run_prepifg
base_unw_paths = list(cf.parse_namelist(params[cf.IFG_FILE_LIST]))
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
# dest_paths are tifs that have been geotif converted and multilooked
cls.converted_paths = [b.converted_path for b in multi_paths]
cls.sampled_paths = [b.sampled_path for b in multi_paths]
from copy import copy
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
tiles = pyrate.core.shared.get_tiles(cls.sampled_paths[0], rows, cols)
ifgs = common.small_data_setup()
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel_p, cls.maxvar_p, cls.vcmt_p = process.process_ifgs(
cls.sampled_paths, params, rows, cols)
cls.mst_p = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate_p, cls.error_p, cls.samples_p = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
common.remove_tifs(params[cf.OBS_DIR])
# now create the non parallel version
cls.tif_dir_s = tempfile.mkdtemp()
params[cf.PARALLEL] = 0
params[cf.PROCESSES] = 1
params[cf.OUT_DIR] = cls.tif_dir_s
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
multi_paths = [
MultiplePaths(params[cf.OUT_DIR],
b,
ifglksx=params[cf.IFG_LKSX],
ifgcropopt=params[cf.IFG_CROP_OPT])
for b in base_unw_paths
]
cls.converted_paths_s = [b.converted_path for b in multi_paths]
cls.sampled_paths_s = [b.sampled_path for b in multi_paths]
orig_params = copy(params)
conv2tif.main(params)
prepifg.main(orig_params)
params[cf.INTERFEROGRAM_FILES] = multi_paths
cls.refpixel, cls.maxvar, cls.vcmt = process.process_ifgs(
cls.sampled_paths_s, params, rows, cols)
cls.mst = common.reconstruct_mst(ifgs[0].shape, tiles,
params[cf.TMPDIR])
cls.rate, cls.error, cls.samples = \
[common.reconstruct_stack_rate(ifgs[0].shape, tiles, params[cf.TMPDIR], t) for t in rate_types]
def setUpClass(cls):
from pyrate import conv2tif, prepifg
cls.params = config.get_config_params(TEST_CONF_GAMMA)
conv2tif.main(cls.params)
prepifg.main(cls.params)
def setup_class(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
params[cf.TEMP_MLOOKED_DIR] = os.path.join(params[cf.OUT_DIR],
cf.TEMP_MLOOKED_DIR)
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
dest_paths, headers = common.repair_params_for_correct_tests(
params[cf.OUT_DIR], params)
correct._copy_mlooked(params)
copied_dest_paths = [
os.path.join(params[cf.TEMP_MLOOKED_DIR], os.path.basename(d))
for d in dest_paths
]
del dest_paths
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(copied_dest_paths, params)
mst_grid = common.mst_calculation(copied_dest_paths, params)
refx, refy = pyrate.core.refpixel.ref_pixel_calc_wrapper(params)
params[cf.REFX] = refx
params[cf.REFY] = refy
params[cf.ORBFIT_OFFSET] = True
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(copied_dest_paths, params)
for ifg in ifgs:
ifg.close()
correct._update_params_with_tiles(params)
_, ifgs = pyrate.core.ref_phs_est.ref_phase_est_wrapper(params)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
maxvar = [
covariance.cvd(i, params, r_dist)[0] for i in copied_dest_paths
]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 2
params[cf.PARALLEL] = 1
# Calculate time series
cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs,
params,
vcmt,
mst=mst_grid)
params[cf.PARALLEL] = 0
# Calculate time series serailly by the pixel
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
# copy legacy data
SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series')
tsincr_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_incr_interp0_method2.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_cum_interp0_method2.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
cls.params = params
def setUpClass(cls):
params = cf.get_config_params(common.TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
sys.argv = ['prepifg.py', common.TEST_CONF_ROIPAC]
params[cf.OUT_DIR] = cls.temp_out_dir
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, ylks, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 2
params[cf.PARALLEL] = 1
# Calculate time series
cls.tsincr, cls.tscum, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 2
# Calculate time series
cls.tsincr_2, cls.tscum_2, _ = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
params[cf.PARALLEL] = 0
# Calculate time series serailly by the pixel
cls.tsincr_0, cls.tscum_0, _ = \
common.calculate_time_series(ifgs, params, vcmt, mst=mst_grid)
# copy legacy data
SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR,
'time_series')
tsincr_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_incr_interp0_method2.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_cum_interp0_method2.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape, order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls):
from pyrate import conv2tif, prepifg
cls.params = Configuration(TEST_CONF_GAMMA).__dict__
conv2tif.main(cls.params)
prepifg.main(cls.params)
def setUpClass(cls):
params = cf.get_config_params(TEST_CONF_ROIPAC)
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = cf.original_ifg_paths(params[cf.IFG_FILE_LIST],
params[cf.OBS_DIR])
dest_paths = cf.get_dest_paths(base_ifg_paths, crop, params, xlks)
print(f"base_ifg_paths={base_ifg_paths}")
print(f"dest_paths={dest_paths}")
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate linear rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_linear_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
params[cf.PARALLEL] = 2
cls.rate_2, cls.error_2, cls.samples_2 = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
params[cf.PARALLEL] = 0
# Calculate linear rate map
cls.rate_s, cls.error_s, cls.samples_s = \
tests.common.calculate_linear_rate(ifgs, params, vcmt,
mst_mat=mst_grid)
linrate_dir = os.path.join(SML_TEST_DIR, 'linrate')
cls.rate_container = np.genfromtxt(os.path.join(
linrate_dir, 'stackmap.csv'),
delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(
linrate_dir, 'errormap.csv'),
delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(
linrate_dir, 'coh_sta.csv'),
delimiter=',')
for ifg in ifgs:
ifg.close()
def test_tifs_succeeds(self):
conv2tif.main(self.params)
prepifg.main(self.params)
def test_num_roipac_tifs_equals_num_unws(self):
gtifs = conv2tif.main(self.roipac_params)
# 17 unws + dem
self.assertEqual(len(gtifs), 18)
def test_num_gamma_tifs_equals_num_unws(self):
gtifs = conv2tif.main(self.gamma_params)
# 17 unws + incidence + dem
self.assertEqual(len(gtifs), 19)
def test_tifs_succeeds(gamma_params):
conv2tif.main(gamma_params)
prepifg.main(gamma_params)
def test_num_roipac_tifs_equals_num_unws(roipac_params):
gtifs = conv2tif.main(roipac_params)
# 17 unws + dem
assert len(gtifs) == 18
def main():
start_time = time.time()
parser = argparse.ArgumentParser(prog='pyrate', description=CLI_DESCRIPTION, add_help=True,
formatter_class=RawTextHelpFormatter)
parser.add_argument('-v', '--verbosity', type=str, default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
help="Increase output verbosity")
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
parser_conv2tif = subparsers.add_parser('conv2tif', help='Convert interferograms to geotiff.', add_help=True)
parser_conv2tif.add_argument('-f', '--config_file', action="store", type=str, default=None,
help="Pass configuration file", required=True)
parser_prepifg = subparsers.add_parser('prepifg', help='Perform multilooking and cropping on geotiffs.',
add_help=True)
parser_prepifg.add_argument('-f', '--config_file', action="store", type=str, default=None,
help="Pass configuration file", required=True)
parser_process = subparsers.add_parser(
'process', help='Main processing workflow including corrections, time series and stacking computation.',
add_help=True)
parser_process.add_argument('-f', '--config_file', action="store", type=str, default=None,
help="Pass configuration file", required=True)
parser_merge = subparsers.add_parser('merge', help="Reassemble computed tiles and save as geotiffs.",
add_help=True)
parser_merge.add_argument('-f', '--config_file', action="store", type=str, default=None,
help="Pass configuration file", required=False)
parser_workflow = subparsers.add_parser('workflow', help="Run all the PyRate processes", add_help=True)
parser_workflow.add_argument('-f', '--config_file', action="store", type=str, default=None,
help="Pass configuration file", required=False)
args = parser.parse_args()
params = mpiops.run_once(_params_from_conf, args.config_file)
configure_stage_log(args.verbosity, args.command, Path(params[cf.OUT_DIR]).joinpath('pyrate.log.').as_posix())
log.debug("Starting PyRate")
log.debug("Arguments supplied at command line: ")
log.debug(args)
if args.verbosity:
log.setLevel(args.verbosity)
log.info("Verbosity set to " + str(args.verbosity) + ".")
if args.command == "conv2tif":
conv2tif.main(params)
if args.command == "prepifg":
prepifg.main(params)
if args.command == "process":
process.main(params)
if args.command == "merge":
merge.main(params)
if args.command == "workflow":
log.info("***********CONV2TIF**************")
conv2tif.main(params)
log.info("***********PREPIFG**************")
params = mpiops.run_once(_params_from_conf, args.config_file)
prepifg.main(params)
log.info("***********PROCESS**************")
# reset params as prepifg modifies params
params = mpiops.run_once(_params_from_conf, args.config_file)
process.main(params)
# process might modify params too
params = mpiops.run_once(_params_from_conf, args.config_file)
log.info("***********MERGE**************")
merge.main(params)
log.debug("--- %s seconds ---" % (time.time() - start_time))
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[cf.COH_MASK] = coh_mask
params[cf.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
# reread params from config
params_s = Configuration(output_conf).__dict__
prepifg.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif'))
assert len(ifg_files) == 17
mlooked_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg_1rlks_1cr.tif'))
assert len(mlooked_files) == 17
coh_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_cc_coh.tif'))
mlooked_coh_files = list(
Path(tdir.joinpath(
params_s[cf.OUT_DIR])).glob('*_cc_coh_1rlks_1cr.tif'))
if coh_mask:
assert len(coh_files) == 17
assert len(mlooked_coh_files) == 17
dem_file = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0]
mlooked_dem_file = list(
Path(tdir.joinpath(
params_s[cf.OUT_DIR])).glob('*_dem_1rlks_1cr.tif'))[0]
import itertools
# assert coherence and ifgs have correct metadata
for i in itertools.chain(
*[ifg_files, mlooked_files, coh_files, mlooked_coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_ifg.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
continue
if i.name.endswith('_cc_coh_1rlks_1cr.tif'):
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH
continue
if i.name.endswith('_ifg_1rlks_1cr.tif'):
if coh_mask:
assert md[ifc.DATA_TYPE] == ifc.COHERENCE
else:
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
dem = DEM(mlooked_dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM
shutil.rmtree(tdir)
def main():
start_time = time.time()
parser = argparse.ArgumentParser(prog='pyrate', description=CLI_DESCRIPTION, add_help=True,
formatter_class=RawTextHelpFormatter)
parser.add_argument('-v', '--verbosity', type=str, default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR'],
help="Increase output verbosity")
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
parser_conv2tif = subparsers.add_parser('conv2tif', help='<Optional> Convert interferograms to geotiff.',
add_help=True)
parser_prepifg = subparsers.add_parser(
'prepifg', help='Perform multilooking, cropping and coherence masking to interferogram geotiffs.',
add_help=True)
parser_correct = subparsers.add_parser(
'correct', help='Calculate and apply corrections to interferogram phase data.',
add_help=True)
parser_ts = subparsers.add_parser(
'timeseries', help='<Optional> Timeseries inversion of interferogram phase data.', add_help=True
)
parser_stack = subparsers.add_parser('stack', help='<Optional> Stacking of interferogram phase data.',
add_help=True)
parser_merge = subparsers.add_parser(
'merge', help="Reassemble computed tiles and save as geotiffs.",
add_help=True)
parser_workflow = subparsers.add_parser(
'workflow', help="<Optional> Sequentially run all the PyRate processing steps.",
add_help=True)
for p in [parser_conv2tif, parser_prepifg, parser_correct, parser_merge, parser_ts, parser_stack, parser_workflow]:
p.add_argument('-f', '--config_file', action="store", type=str, default=None,
help="Pass configuration file", required=False)
args = parser.parse_args()
params = mpiops.run_once(_params_from_conf, args.config_file)
configure_stage_log(args.verbosity, args.command, Path(params[C.OUT_DIR]).joinpath('pyrate.log.').as_posix())
log.debug("Starting PyRate")
log.debug("Arguments supplied at command line: ")
log.debug(args)
if args.verbosity:
log.setLevel(args.verbosity)
log.info("Verbosity set to " + str(args.verbosity) + ".")
if args.command == "conv2tif":
conv2tif.main(params)
if args.command == "prepifg":
prepifg.main(params)
if args.command == "correct":
config_file = os.path.abspath(args.config_file)
config = Configuration(config_file)
correct.main(config)
if args.command == "timeseries":
config_file = os.path.abspath(args.config_file)
config = Configuration(config_file)
timeseries(config)
if args.command == "stack":
config_file = os.path.abspath(args.config_file)
config = Configuration(config_file)
stack(config)
if args.command == "merge":
merge.main(params)
if args.command == "workflow":
log.info("***********CONV2TIF**************")
conv2tif.main(params)
log.info("***********PREPIFG**************")
params = mpiops.run_once(_params_from_conf, args.config_file)
prepifg.main(params)
log.info("***********CORRECT**************")
# reset params as prepifg modifies params
config_file = os.path.abspath(args.config_file)
config = Configuration(config_file)
correct.main(config)
log.info("***********TIMESERIES**************")
config = Configuration(config_file)
timeseries(config)
log.info("***********STACK**************")
config = Configuration(config_file)
stack(config)
log.info("***********MERGE**************")
params = mpiops.run_once(_params_from_conf, args.config_file)
merge.main(params)
log.info("--- Runtime = %s seconds ---" % (time.time() - start_time))
def setUpClass(cls):
params = Configuration(common.TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.PARALLEL] = 0
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = common.pre_prepare_ifgs(dest_paths, params)
mst_grid = common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
ifgs = common.prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = covariance.RDist(ifgs[0])()
ifgs[0].close()
# Calculate interferogram noise
maxvar = [covariance.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = covariance.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
ifg.nodata_value = 0.0
params[cf.TIME_SERIES_METHOD] = 2
params[cf.PARALLEL] = 1
# Calculate time series
cls.tsincr, cls.tscum, _ = common.calculate_time_series(ifgs,
params,
vcmt,
mst=mst_grid)
params[cf.PARALLEL] = 0
# Calculate time series serailly by the pixel
cls.tsincr_0, cls.tscum_0, _ = common.calculate_time_series(
ifgs, params, vcmt, mst=mst_grid)
# copy legacy data
SML_TIME_SERIES_DIR = os.path.join(common.SML_TEST_DIR, 'time_series')
tsincr_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_incr_interp0_method2.csv')
ts_incr = np.genfromtxt(tsincr_path)
tscum_path = os.path.join(SML_TIME_SERIES_DIR,
'ts_cum_interp0_method2.csv')
ts_cum = np.genfromtxt(tscum_path)
cls.ts_incr = np.reshape(ts_incr,
newshape=cls.tsincr_0.shape,
order='F')
cls.ts_cum = np.reshape(ts_cum, newshape=cls.tscum_0.shape, order='F')
def setUpClass(cls):
params = Configuration(TEST_CONF_ROIPAC).__dict__
cls.temp_out_dir = tempfile.mkdtemp()
params[cf.OUT_DIR] = cls.temp_out_dir
params[cf.TMPDIR] = os.path.join(params[cf.OUT_DIR], cf.TMPDIR)
shared.mkdir_p(params[cf.TMPDIR])
conv2tif.main(params)
prepifg.main(params)
params[cf.REF_EST_METHOD] = 2
xlks, _, crop = cf.transform_params(params)
base_ifg_paths = [
c.unwrapped_path for c in params[cf.INTERFEROGRAM_FILES]
]
headers = [roipac.roipac_header(i, params) for i in base_ifg_paths]
dest_paths = [
Path(cls.temp_out_dir).joinpath(Path(
c.sampled_path).name).as_posix()
for c in params[cf.INTERFEROGRAM_FILES][:-2]
]
# start run_pyrate copy
ifgs = pre_prepare_ifgs(dest_paths, params)
mst_grid = tests.common.mst_calculation(dest_paths, params)
refx, refy = process._ref_pixel_calc(dest_paths, params)
# Estimate and remove orbit errors
pyrate.core.orbital.remove_orbital_error(ifgs, params, headers)
ifgs = prepare_ifgs_without_phase(dest_paths, params)
for ifg in ifgs:
ifg.close()
_, ifgs = process._ref_phase_estimation(dest_paths, params, refx, refy)
ifgs[0].open()
r_dist = vcm_module.RDist(ifgs[0])()
ifgs[0].close()
maxvar = [vcm_module.cvd(i, params, r_dist)[0] for i in dest_paths]
for ifg in ifgs:
ifg.open()
vcmt = vcm_module.get_vcmt(ifgs, maxvar)
for ifg in ifgs:
ifg.close()
ifg.open()
# Calculate stacked rate map
params[cf.PARALLEL] = 1
cls.rate, cls.error, cls.samples = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
# Calculate stacked rate map
params[cf.PARALLEL] = 0
cls.rate_s, cls.error_s, cls.samples_s = tests.common.calculate_stack_rate(
ifgs, params, vcmt, mst_mat=mst_grid)
stackrate_dir = os.path.join(SML_TEST_DIR, 'stackrate')
cls.rate_container = np.genfromtxt(os.path.join(
stackrate_dir, 'stackmap.csv'),
delimiter=',')
cls.error_container = np.genfromtxt(os.path.join(
stackrate_dir, 'errormap.csv'),
delimiter=',')
cls.samples_container = np.genfromtxt(os.path.join(
stackrate_dir, 'coh_sta.csv'),
delimiter=',')
for ifg in ifgs:
ifg.close()
