def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf):
params_1, _ = _get_mlooked_files(gamma_conf,
Path(tempdir()),
refx=-1,
refy=-1)
params_2, _ = _get_mlooked_files(gamma_conf,
Path(tempdir()),
refx=150.941666654,
refy=-34.218333314)
assert_two_dirs_equal(params_1[cf.OUT_DIR], params_2[cf.OUT_DIR],
["*_ifg.tif", '*_coh.tif', 'dem.tif'], 35)
def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf):
params_1, _ = _get_mlooked_files(gamma_conf,
Path(tempdir()),
refx=-1,
refy=-1)
params_2, _ = _get_mlooked_files(gamma_conf,
Path(tempdir()),
refx=150.941666654,
refy=-34.218333314)
assert_two_dirs_equal(params_1[cf.OUT_DIR], params_2[cf.OUT_DIR],
f"*{params_1[cf.IFG_CROP_OPT]}cr.tif", 18)
def test_coh_stats_equality(mexico_cropa_params):
subprocess.run(
f"mpirun -n 3 pyrate prepifg -f {MEXICO_CROPA_CONF.as_posix()}",
shell=True)
params = mexico_cropa_params
mexico_cropa_coh_stats_dir = Path(BASE_TEST).joinpath(
"cropA", 'coherence_stats')
assert_two_dirs_equal(params[C.COHERENCE_DIR],
mexico_cropa_coh_stats_dir,
ext="coh*.tif",
num_files=3)
# assert metadata was written
from pyrate.prepifg import out_type_md_dict
for stat_tif in mexico_cropa_coh_stats_dir.glob("coh*.tif"):
ds = gdal.Open(stat_tif.as_posix())
md = ds.GetMetadata()
expected_md = out_type_md_dict[stat_tif.stem.upper()]
assert ifc.DATA_TYPE in md.keys()
assert expected_md in md.values()
def test_prepifg_largetifs_vs_python(modified_config_largetifs, gamma_conf,
create_mpi_files):
print("\n\n")
print("===x===" * 10)
if TRAVIS and np.random.randint(0,
1000) > 499: # skip 50% of tests randomly
pytest.skip("Randomly skipping as part of 50 percent")
params = create_mpi_files(gamma_conf)
sr_conf, params_p = modified_config_largetifs(gamma_conf, 1,
'parallel_conf.conf')
check_call(f"pyrate conv2tif -f {sr_conf}", shell=True)
check_call(f"pyrate prepifg -f {sr_conf}", shell=True)
# convert2tif tests, 17 interferograms
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR],
"*_unw.tif", 17)
# if coherence masking, compare coh files were converted
if params[cf.COH_MASK]:
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR],
"*_utm.tif", 17)
print("coherence files compared")
# prepifg
# 17 ifgs + 1 dem
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR],
f"*{params[cf.IFG_CROP_OPT]}cr.tif", 18)
print("==========================xxx===========================")
shutil.rmtree(params[cf.OBS_DIR])
shutil.rmtree(params_p[cf.OBS_DIR])
def test_gamma_ref_pixel_search_vs_lat_lon(tempdir, gamma_conf):
params_1, _ = _get_mlooked_files(gamma_conf,
Path(tempdir()),
refx=-1,
refy=-1)
params_2, _ = _get_mlooked_files(gamma_conf,
Path(tempdir()),
refx=150.941666654,
refy=-34.218333314)
assert_two_dirs_equal(params_1[C.COHERENCE_DIR], params_2[C.COHERENCE_DIR],
['*_coh.tif', '*_cc.tif'], 34)
assert_two_dirs_equal(params_1[C.INTERFEROGRAM_DIR],
params_2[C.INTERFEROGRAM_DIR],
["*_ifg.tif", '*_unw.tif'], 34)
assert_two_dirs_equal(params_1[C.GEOMETRY_DIR], params_2[C.GEOMETRY_DIR],
["*.tif"], 8)
def test_stack_and_ts_mpi_vs_parallel_vs_serial(modified_config_short, gamma_conf, create_mpi_files, parallel):
"""
Checks performed:
1. mpi vs single process pipeline
2. mpi vs parallel (python multiprocess) pipeline.
3. Doing 1 and 2 means we have checked single vs parallel python multiprocess pipelines
4. This also checks the entire pipeline using largetifs (new prepifg) vs old perpifg (python based)
"""
if np.random.randint(0, 1000) > 100: # skip 90% of tests randomly
pytest.skip("Randomly skipping as part of 60 percent")
print("\n\n")
print("===x==="*10)
params = create_mpi_files(modified_config_short, gamma_conf)
sr_conf, params_p = modified_config_short(gamma_conf, parallel, 'parallel_conf.conf', 0)
check_call(f"pyrate workflow -f {sr_conf}", shell=True)
# convert2tif tests, 17 interferograms
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_unw.tif", 17)
# if coherence masking, compare coh files were converted
if params[cf.COH_FILE_LIST] is not None:
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "*_cc.tif", 17)
print("coherence files compared")
# prepifg + correct steps that overwrite tifs test
# 17 mlooked ifgs + 1 dem + 17 mlooked coherence files
if params[cf.COH_FILE_LIST] is not None:
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], ["*_ifg.tif", "*_coh.tif", 'dem.tif'], 35)
else:
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], ["*_ifg.tif", 'dem.tif'], 18)
assert_two_dirs_equal(params[cf.TEMP_MLOOKED_DIR], params_p[cf.TEMP_MLOOKED_DIR], "*_ifg.tif", 17)
# ifg phase checking in the previous step checks the correct pipeline upto APS correction
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "tsincr_*.npy", params['notiles'] * 2)
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "tscuml_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "linear_rate_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "linear_error_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "linear_samples_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "linear_intercept_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "linear_rsquared_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "stack_rate_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "stack_error_*.npy", params['notiles'])
assert_two_dirs_equal(params[cf.TMPDIR], params_p[cf.TMPDIR], "stack_samples_*.npy", params['notiles'])
# compare merge step
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.tif", 3)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.kml", 2)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.png", 2)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "stack*.npy", 3)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "linear*.tif", 5)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "linear*.kml", 3)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "linear*.png", 3)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "linear*.npy", 5)
assert_two_dirs_equal(params[cf.OUT_DIR], params_p[cf.OUT_DIR], "tscuml*.tif")
print("==========================xxx===========================")
shutil.rmtree(params[cf.OBS_DIR])
shutil.rmtree(params_p[cf.OBS_DIR])
def test_stack_and_ts_mpi_vs_parallel_vs_serial(modified_config_short,
gamma_conf, create_mpi_files,
parallel):
"""
Checks performed:
1. mpi vs single process pipeline
2. mpi vs parallel (python multiprocess) pipeline.
3. Doing 1 and 2 means we have checked single vs parallel python multiprocess pipelines
4. This also checks the entire pipeline using largetifs (new prepifg) vs old perpifg (python based)
"""
if np.random.randint(0, 1000) > 300: # skip 90% of tests randomly
pytest.skip("Randomly skipping as part of 60 percent")
print("\n\n")
print("===x===" * 10)
params = create_mpi_files(modified_config_short, gamma_conf)
sr_conf, params_p = modified_config_short(gamma_conf, parallel,
'parallel_conf.conf', 0)
check_call(f"pyrate workflow -f {sr_conf}", shell=True)
# convert2tif tests, 17 interferograms
assert_two_dirs_equal(params[C.INTERFEROGRAM_DIR],
params_p[C.INTERFEROGRAM_DIR], "*_unw.tif", 17)
# if coherence masking, compare coh files were converted
if params[C.COH_FILE_LIST] is not None:
assert_two_dirs_equal(params[C.COHERENCE_DIR],
params_p[C.COHERENCE_DIR], "*_cc.tif", 17)
print("coherence files compared")
assert_two_dirs_equal(params[C.INTERFEROGRAM_DIR],
params_p[C.INTERFEROGRAM_DIR], ["*_ifg.tif"], 17)
# one original dem, another multilooked dem
assert_two_dirs_equal(params[C.GEOMETRY_DIR], params_p[C.GEOMETRY_DIR],
['*dem.tif'], 2)
assert_two_dirs_equal(params[C.GEOMETRY_DIR], params_p[C.GEOMETRY_DIR],
[t + "*.tif" for t in C.GEOMETRY_OUTPUT_TYPES],
6) # 2 dems, 6 geom
assert_two_dirs_equal(params[C.TEMP_MLOOKED_DIR],
params_p[C.TEMP_MLOOKED_DIR], "*_ifg.tif", 17)
# ifg phase checking in the previous step checks the correct pipeline upto APS correction
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR], "tsincr_*.npy",
params['notiles'] * 2)
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR], "tscuml_*.npy",
params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"linear_rate_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"linear_error_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"linear_samples_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"linear_intercept_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"linear_rsquared_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"stack_rate_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"stack_error_*.npy", params['notiles'])
assert_two_dirs_equal(params[C.TMPDIR], params_p[C.TMPDIR],
"stack_samples_*.npy", params['notiles'])
# compare merge step
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"stack*.tif", 3)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"stack*.kml", 2)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"stack*.png", 2)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"stack*.npy", 3)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"linear*.tif", 5)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"linear*.kml", 3)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"linear*.png", 3)
assert_two_dirs_equal(params[C.VELOCITY_DIR], params_p[C.VELOCITY_DIR],
"linear*.npy", 5)
assert_two_dirs_equal(params[C.TIMESERIES_DIR], params_p[C.TIMESERIES_DIR],
"tscuml*.tif")
assert_two_dirs_equal(params[C.TIMESERIES_DIR], params_p[C.TIMESERIES_DIR],
"tsincr*.tif")
assert_two_dirs_equal(params[C.TIMESERIES_DIR], params_p[C.TIMESERIES_DIR],
"tscuml*.npy")
assert_two_dirs_equal(params[C.TIMESERIES_DIR], params_p[C.TIMESERIES_DIR],
"tsincr*.npy")
print("==========================xxx===========================")
shutil.rmtree(params[WORKING_DIR])
shutil.rmtree(params_p[WORKING_DIR])
def test_prepifg_largetifs_vs_python(modified_config_largetifs, gamma_conf,
create_mpi_files):
print("\n\n")
print("===x===" * 10)
if GITHUB_ACTIONS and np.random.randint(
0, 1000) > 899: # skip 90% of tests randomly
pytest.skip("Randomly skipping as part of 50 percent")
params = create_mpi_files(gamma_conf)
sr_conf, params_p = modified_config_largetifs(gamma_conf, 1,
'parallel_conf.conf')
params_p = Configuration(sr_conf).__dict__
conv2tif.main(params_p)
params_p = Configuration(sr_conf).__dict__
prepifg.main(params_p)
params_p = Configuration(sr_conf).__dict__
# convert2tif tests, 17 interferograms
assert_two_dirs_equal(params[C.INTERFEROGRAM_DIR],
params_p[C.INTERFEROGRAM_DIR], "*_unw.tif", 17)
# if coherence masking, compare coh files were converted
if params[C.COH_FILE_LIST] is not None:
assert_two_dirs_equal(params[C.COHERENCE_DIR],
params_p[C.COHERENCE_DIR], "*_cc.tif", 17)
# 17 ifgs + 1 dem + 17 mlooked file
assert_two_dirs_equal(params[C.COHERENCE_DIR],
params_p[C.COHERENCE_DIR], "*_coh.tif", 17)
assert_two_dirs_equal(params[C.COHERENCE_DIR],
params_p[C.COHERENCE_DIR], ["coh_*.tif"], 3)
assert_two_dirs_equal(params[C.GEOMETRY_DIR], params_p[C.GEOMETRY_DIR],
"*_dem.tif", 1)
assert_two_dirs_equal(params[C.GEOMETRY_DIR], params_p[C.GEOMETRY_DIR],
[t + '.tif' for t in C.GEOMETRY_OUTPUT_TYPES], 6)
# prepifg
# 17 ifgs + 1 dem
assert_two_dirs_equal(params[C.INTERFEROGRAM_DIR],
params_p[C.INTERFEROGRAM_DIR], "*_ifg.tif", 17)
assert_two_dirs_equal(params[C.GEOMETRY_DIR], params_p[C.GEOMETRY_DIR],
"dem.tif", 1)
print("==========================xxx===========================")
shutil.rmtree(params[C.OUT_DIR])
shutil.rmtree(params_p[C.OUT_DIR])