def __workflow(params, tdir):
copytree(params[cf.OBS_DIR], tdir)
# manipulate params
params[cf.OBS_DIR] = tdir.as_posix()
outdir = tdir.joinpath('out')
outdir.mkdir(exist_ok=True)
params[cf.OUT_DIR] = outdir.as_posix()
params[cf.DEM_FILE] = tdir.joinpath(Path(
params[cf.DEM_FILE]).name).as_posix()
params[cf.DEM_HEADER_FILE] = tdir.joinpath(
Path(params[cf.DEM_HEADER_FILE]).name).as_posix()
params[cf.HDR_FILE_LIST] = tdir.joinpath(
Path(params[cf.HDR_FILE_LIST]).name).as_posix()
params[cf.SLC_DIR] = tdir.as_posix()
params[cf.IFG_FILE_LIST] = tdir.joinpath(
Path(params[cf.IFG_FILE_LIST]).name).as_posix()
params[cf.COH_FILE_DIR] = tdir.as_posix()
params[cf.APS_INCIDENCE_MAP] = tdir.joinpath(
Path(params[cf.APS_INCIDENCE_MAP]).name).as_posix()
params[cf.TMPDIR] = tdir.joinpath(Path(params[cf.TMPDIR]).name).as_posix()
output_conf = tdir.joinpath('roipac_temp.conf')
cf.write_config_file(params=params, output_conf_file=output_conf)
params = configuration.Configuration(output_conf).__dict__
conv2tif.main(params)
prepifg.main(params)
return params
def test_conv2tif_file_types(tempdir, gamma_conf):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[cf.COH_MASK] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
params_s = configuration.Configuration(output_conf).__dict__
conv2tif.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif'))
coh_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_coh.tif'))
dem_file = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0]
# assert coherence and ifgs have correct metadata
for i in itertools.chain(*[ifg_files, coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_ifg.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
shutil.rmtree(tdir)
def modify_params(conf_file, parallel, output_conf_file):
tdir = Path(tempdir())
params = manipulate_test_conf(conf_file, tdir)
params[cf.COH_MASK] = coh_mask
params[cf.LARGE_TIFS] = 1
params[cf.PARALLEL] = parallel
params[cf.PROCESSES] = 4
params[cf.APSEST] = 1
params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks
params[cf.REFNX], params[cf.REFNY] = 4, 4
params[cf.IFG_CROP_OPT] = local_crop
params[cf.ORBITAL_FIT_LOOKS_X], params[
cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks, orbfit_lks
params[cf.ORBITAL_FIT] = 1
params[cf.ORBITAL_FIT_METHOD] = orbfit_method
params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
params[cf.REF_EST_METHOD] = ref_est_method
params["rows"], params["cols"] = 3, 2
params["tiles"] = params["rows"] * params["cols"]
print(params)
# write new temp config
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
return output_conf, params
def modify_params(conf_file, parallel_vs_serial, output_conf_file):
tdir = Path(tempdir())
params = manipulate_test_conf(conf_file, tdir)
if params[cf.PROCESSOR] == 1: # turn on coherence for gamma
params[cf.COH_MASK] = 1
params[cf.PARALLEL] = parallel_vs_serial
params[cf.PROCESSES] = 4
params[cf.APSEST] = 1
params[cf.IFG_LKSX], params[cf.IFG_LKSY] = get_lks, get_lks
params[cf.REFNX], params[cf.REFNY] = 2, 2
params[cf.IFG_CROP_OPT] = get_crop
params[cf.ORBITAL_FIT_LOOKS_X], params[
cf.ORBITAL_FIT_LOOKS_Y] = orbfit_lks, orbfit_lks
params[cf.ORBITAL_FIT] = 1
params[cf.ORBITAL_FIT_METHOD] = orbfit_method
params[cf.ORBITAL_FIT_DEGREE] = orbfit_degrees
params[cf.REF_EST_METHOD] = ref_est_method
params["rows"], params["cols"] = 3, 2
params["savenpy"] = 1
params["tiles"] = params["rows"] * params["cols"]
print(params)
# write new temp config
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
return output_conf, params
def test_vcm_legacy_vs_mpi(mpisync, tempdir, roipac_or_gamma_conf):
params = configuration.Configuration(roipac_or_gamma_conf).__dict__
LEGACY_VCM_DIR = os.path.join(SML_TEST_DIR, 'vcm')
legacy_vcm = np.genfromtxt(os.path.join(LEGACY_VCM_DIR, 'vcmt.csv'), delimiter=',')
tmpdir = Path(mpiops.run_once(tempdir))
mpiops.run_once(common.copytree, params[cf.OBS_DIR], tmpdir)
params[cf.OUT_DIR] = tmpdir.joinpath('out')
params[cf.PARALLEL] = 0
output_conf = Path(tmpdir).joinpath('conf.cfg')
cf.write_config_file(params=params, output_conf_file=output_conf)
params = configuration.Configuration(output_conf).__dict__
dest_paths = [p.sampled_path for p in params[cf.INTERFEROGRAM_FILES]]
# run conv2tif and prepifg, create the dest_paths files
conv2tif.main(params)
params[cf.INTERFEROGRAM_FILES].pop()
prepifg.main(params)
params[cf.INTERFEROGRAM_FILES].pop()
preread_ifgs = process._create_ifg_dict(dest_paths, params=params)
refpx, refpy = process._ref_pixel_calc(dest_paths, params)
process._orb_fit_calc(params[cf.INTERFEROGRAM_FILES], params)
process._ref_phase_estimation(dest_paths, params, refpx, refpy)
maxvar, vcmt = process._maxvar_vcm_calc(dest_paths, params, preread_ifgs)
# phase data after ref pixel has changed due to commit bf2f7ebd
# Legacy tests won't match anymore
np.testing.assert_array_almost_equal(maxvar, legacy_maxvar, decimal=4)
np.testing.assert_array_almost_equal(legacy_vcm, vcmt, decimal=3)
mpiops.run_once(shutil.rmtree, tmpdir)
def series_ifgs(gamma_conf):
print('======================setup series==========================')
tdir = Path(tempfile.mkdtemp())
params_s = manipulate_test_conf(gamma_conf, tdir)
params_s[cf.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params_s, output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
gtif_paths = conv2tif.main(params_s)
prepifg.main(params_s)
parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix))
s_ifgs = small_data_setup(datafiles=parallel_df)
yield s_ifgs
print('======================teardown series==========================')
shutil.rmtree(params_s[cf.OBS_DIR])
def test_new_config_file_and_original_match(self):
params = config.get_config_params(TEST_CONF_GAMMA)
temp_config = tempfile.mktemp(suffix='.conf')
config.write_config_file(params, temp_config)
new_params = config.get_config_params(temp_config)
self.maxDiff = None
self.assertDictEqual(params, new_params)
os.remove(temp_config)
def test_coherence_files_exist(self):
self.params[COH_MASK] = 1
self.params[COH_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'ifms_17')
validate_coherence_files(self.params[IFG_FILE_LIST], self.params)
self.params[COH_FILE_LIST] = os.path.join(common.SML_TEST_GAMMA,
'bad_epochs_ifms_17')
tmp_config = tempfile.mktemp('.conf')
write_config_file(self.params, output_conf_file=tmp_config)
with pytest.raises(ConfigException):
Configuration(tmp_config).__dict__
def create_stack_output(tempdir, gamma_conf):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
output_conf_file = tdir.joinpath('conf.cfg')
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
check_call(f"pyrate conv2tif -f {output_conf}", shell=True)
check_call(f"pyrate prepifg -f {output_conf}", shell=True)
check_call(f"pyrate process -f {output_conf}", shell=True)
params = Configuration(output_conf).__dict__
return params, tdir
def _get_mlooked_files(gamma_conf, tdir, refx, refy):
params = manipulate_test_conf(gamma_conf, tdir)
params[cf.REFX] = refx
params[cf.REFY] = refy
output_conf_file = 'config.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
check_call(f"pyrate conv2tif -f {output_conf}", shell=True)
check_call(f"pyrate prepifg -f {output_conf}", shell=True)
stdout = run(f"pyrate process -f {output_conf}",
shell=True,
capture_output=True,
text=True)
print("============================================", stdout)
return params, stdout
def setUp(self):
self.params = cf.get_config_params(TEST_CONF_ROIPAC)
self.params[cf.PARALLEL] = 0
self.params[cf.OUT_DIR], self.ifg_paths = copy_small_ifg_file_list()
conf_file = Path(self.params[cf.OUT_DIR], 'conf_file.conf')
cf.write_config_file(params=self.params, output_conf_file=conf_file)
self.params = Configuration(conf_file).__dict__
self.params_alt_ref_frac = copy.copy(self.params)
self.params_alt_ref_frac[cf.REF_MIN_FRAC] = 0.5
self.params_all_2s = copy.copy(self.params)
self.params_all_2s[cf.REFNX] = 2
self.params_all_2s[cf.REFNY] = 2
self.params_chipsize_15 = copy.copy(self.params_all_2s)
self.params_chipsize_15[cf.REF_CHIP_SIZE] = 15
self.params_all_1s = copy.copy(self.params)
self.params_all_1s[cf.REFNX] = 1
self.params_all_1s[cf.REFNY] = 1
self.params_all_1s[cf.REF_MIN_FRAC] = 0.7
def test_prepifg_treat_inputs_read_only(gamma_conf, tempdir, coh_mask):
tdir = Path(tempdir())
params = common.manipulate_test_conf(gamma_conf, tdir)
params[cf.COH_MASK] = coh_mask
output_conf = tdir.joinpath('conf.cfg')
cf.write_config_file(params=params, output_conf_file=output_conf)
check_call(f"mpirun -n 3 pyrate conv2tif -f {output_conf}", shell=True)
tifs = list(Path(params[cf.OUT_DIR]).glob('*_unw_ifg.tif'))
assert len(tifs) == 17
check_call(f"mpirun -n 3 pyrate prepifg -f {output_conf}", shell=True)
cropped = list(Path(params[cf.OUT_DIR]).glob('*cr.tif'))
if coh_mask: # 17 + 1 dem + 17 coh files
assert len(cropped) == 35
else: # 17 + 1 dem
assert len(cropped) == 18
# check all tifs from conv2tif are still readonly
for t in tifs:
assert t.stat().st_mode == 33060
def parallel_ifgs(gamma_conf):
tdir = Path(tempfile.mkdtemp())
params_p = manipulate_test_conf(gamma_conf, tdir)
params_p[cf.PARALLEL] = 1
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params_p, output_conf_file=output_conf)
params_p = Configuration(output_conf).__dict__
gtif_paths = conv2tif.main(params_p)
prepifg.main(params_p)
parallel_df = list(Path(tdir).joinpath('out').glob(glob_prefix))
p_ifgs = small_data_setup(datafiles=parallel_df)
yield p_ifgs
shutil.rmtree(params_p[cf.OBS_DIR])
def test_prepifg_file_types(tempdir, gamma_conf, coh_mask):
tdir = Path(tempdir())
params = manipulate_test_conf(gamma_conf, tdir)
params[cf.COH_MASK] = coh_mask
params[cf.PARALLEL] = 0
output_conf_file = 'conf.conf'
output_conf = tdir.joinpath(output_conf_file)
cf.write_config_file(params=params, output_conf_file=output_conf)
params_s = Configuration(output_conf).__dict__
conv2tif.main(params_s)
# reread params from config
params_s = Configuration(output_conf).__dict__
prepifg.main(params_s)
ifg_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg.tif'))
assert len(ifg_files) == 17
mlooked_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_ifg_1rlks_1cr.tif'))
assert len(mlooked_files) == 17
coh_files = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_cc_coh.tif'))
mlooked_coh_files = list(
Path(tdir.joinpath(
params_s[cf.OUT_DIR])).glob('*_cc_coh_1rlks_1cr.tif'))
if coh_mask:
assert len(coh_files) == 17
assert len(mlooked_coh_files) == 17
dem_file = list(
Path(tdir.joinpath(params_s[cf.OUT_DIR])).glob('*_dem.tif'))[0]
mlooked_dem_file = list(
Path(tdir.joinpath(
params_s[cf.OUT_DIR])).glob('*_dem_1rlks_1cr.tif'))[0]
import itertools
# assert coherence and ifgs have correct metadata
for i in itertools.chain(
*[ifg_files, mlooked_files, coh_files, mlooked_coh_files]):
ifg = Ifg(i)
ifg.open()
md = ifg.meta_data
if i.name.endswith('_ifg.tif'):
assert md[ifc.DATA_TYPE] == ifc.ORIG
continue
if i.name.endswith('_coh.tif'):
assert md[ifc.DATA_TYPE] == ifc.COH
continue
if i.name.endswith('_cc_coh_1rlks_1cr.tif'):
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED_COH
continue
if i.name.endswith('_ifg_1rlks_1cr.tif'):
if coh_mask:
assert md[ifc.DATA_TYPE] == ifc.COHERENCE
else:
assert md[ifc.DATA_TYPE] == ifc.MULTILOOKED
continue
# assert dem has correct metadata
dem = DEM(dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.DEM
dem = DEM(mlooked_dem_file.as_posix())
dem.open()
md = dem.dataset.GetMetadata()
assert md[ifc.DATA_TYPE] == ifc.MLOOKED_DEM
shutil.rmtree(tdir)
def test_write_config_file(self):
params = config.get_config_params(TEST_CONF_GAMMA)
temp_config = tempfile.mktemp(suffix='.conf')
config.write_config_file(params, temp_config)
self.assertTrue(os.path.exists(temp_config))
os.remove(temp_config)
