def __save_ifgs_dict_with_headers_and_epochs(dest_tifs, ifgs_dict, params,
process_tifs):
tmpdir = params[cf.TMPDIR]
if not os.path.exists(tmpdir):
shared.mkdir_p(tmpdir)
preread_ifgs_file = Configuration.preread_ifgs(params)
nifgs = len(dest_tifs)
# add some extra information that's also useful later
gt, md, wkt = shared.get_geotiff_header_info(
process_tifs[0].tmp_sampled_path)
epochlist = algorithm.get_epochs(ifgs_dict)[0]
log.info('Found {} unique epochs in the {} interferogram network'.format(
len(epochlist.dates), nifgs))
ifgs_dict['epochlist'] = epochlist
ifgs_dict['gt'] = gt
ifgs_dict['md'] = md
ifgs_dict['wkt'] = wkt
# dump ifgs_dict file for later use
cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb'))
for k in ['gt', 'epochlist', 'md', 'wkt']:
ifgs_dict.pop(k)
return ifgs_dict
def test_get_epochs(self):
def str2date(s):
segs = s[:4], s[4:6], s[6:] # year, month, day
return date(*[int(sg) for sg in segs])
raw_date = [
'20060619', '20060828', '20061002', '20061106', '20061211',
'20070115', '20070219', '20070326', '20070430', '20070604',
'20070709', '20070813', '20070917'
]
exp_dates = [str2date(d) for d in raw_date]
exp_repeat = [1, 1, 3, 3, 4, 3, 3, 3, 3, 3, 3, 2, 2]
exp_spans = [
0, 0.1916, 0.2875, 0.3833, 0.4791, 0.5749, 0.6708, 0.7666, 0.8624,
0.9582, 1.0541, 1.1499, 1.2457
]
ifms = join(SML_TEST_TIF, "ifms_17")
ifgs = [Ifg(join(SML_TEST_TIF, p)) for p in parse_namelist(ifms)]
for i in ifgs:
i.open()
epochs = get_epochs(ifgs)[0]
self.assertTrue((exp_dates == epochs.dates).all())
self.assertTrue((exp_repeat == epochs.repeat).all())
assert_array_almost_equal(exp_spans, epochs.spans, decimal=4)
def test_mst_matrix_as_array(self):
# Verifies MST matrix func returns array with dict/trees in each cell
for i in self.ifgs[3:]:
i.phase_data[0, 1] = 0 # partial stack of NODATA to one cell
for i in self.ifgs:
i.convert_to_nans() # zeros to NaN/NODATA
epochs = algorithm.get_epochs(self.ifgs)[0]
res = mst._mst_matrix_as_array(self.ifgs)
ys, xs = res.shape
for y, x in product(range(ys), range(xs)):
r = res[y, x]
num_nodes = len(r)
self.assertTrue(num_nodes < len(epochs.dates))
stack = array([i.phase_data[y, x]
for i in self.ifgs]) # 17 ifg stack
self.assertTrue(
0 == nsum(stack == 0)) # all 0s should be converted
nc = nsum(isnan(stack))
exp_count = len(epochs.dates) - 1
if nc == 0:
self.assertEqual(num_nodes, exp_count)
elif nc > 5:
# rough test: too many nans must reduce the total tree size
self.assertTrue(num_nodes <= (17 - nc))
def _create_ifg_dict(dest_tifs, params):
"""
1. Convert ifg phase data into numpy binary files.
2. Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
ifgs_dict = {}
nifgs = len(dest_tifs)
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = shared._prep_ifg(d, params)
ifgs_dict[d] = PrereadIfg(path=d,
nan_fraction=ifg.nan_fraction,
master=ifg.master,
slave=ifg.slave,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data)
ifg.close()
ifgs_dict = _join_dicts(mpiops.comm.allgather(ifgs_dict))
preread_ifgs_file = join(params[cf.TMPDIR], 'preread_ifgs.pk')
if mpiops.rank == MASTER_PROCESS:
# add some extra information that's also useful later
gt, md, wkt = shared.get_geotiff_header_info(process_tifs[0])
epochlist = algorithm.get_epochs(ifgs_dict)[0]
log.info(
'Found {} unique epochs in the {} interferogram network'.format(
len(epochlist.dates), nifgs))
ifgs_dict['epochlist'] = epochlist
ifgs_dict['gt'] = gt
ifgs_dict['md'] = md
ifgs_dict['wkt'] = wkt
# dump ifgs_dict file for later use
cp.dump(ifgs_dict, open(preread_ifgs_file, 'wb'))
mpiops.comm.barrier()
preread_ifgs = OrderedDict(
sorted(cp.load(open(preread_ifgs_file, 'rb')).items()))
log.debug('Finished converting phase_data to numpy in process {}'.format(
mpiops.rank))
return preread_ifgs
def write_timeseries_geotiff(ifgs, params, tsincr, pr_type):
# setup metadata for writing into result files
gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path)
epochlist = algorithm.get_epochs(ifgs)[0]
for i in range(tsincr.shape[2]):
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
md['SEQUENCE_POSITION'] = i + 1 # sequence position
data = tsincr[:, :, i]
dest = join(params[cf.OUT_DIR],
pr_type + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = pr_type
write_output_geotiff(md, gt, wkt, data, dest, np.nan)
def linear_rate_array(tscuml, ifgs, params):
"""
This function loops over all pixels in a 3-dimensional cumulative
time series array and calculates the linear rate (line of best fit)
for each pixel using linear regression.
:param ndarray tscuml: 3-dimensional cumulative time series array
:param list ifgs: list of interferogram class objects.
:param dict params: Configuration parameters
:return: linrate: Linear rate map from linear regression
:rtype: ndarray
:return: intercept: Array of Y-axis intercepts from linear regression
:rtype: ndarray
:return: rsquared: Array of R-squared value of the linear regression
:rtype: ndarray
:return: error: Array of standard errors of the linear regression
:rtype: ndarray
:return: samples: Number of observations used in linear regression for each pixel
:rtype: ndarray
"""
b0_mat, interp, p_thresh, sm_factor, sm_order, ts_method, ifg_data, mst, \
ncols, nrows, nvelpar, span, tsvel_matrix = \
_time_series_setup(ifgs, params)
epochlist = get_epochs(ifgs)[0]
# get cumulative time per epoch
t = asarray(epochlist.spans)
# test for equal length of input vectors
if tscuml.shape[2] != len(t):
raise TimeSeriesError(
"linear_rate_array: tscuml and nepochs are not equal length")
# preallocate empty arrays for results
linrate = np.empty([nrows, ncols], dtype=float32)
rsquared = np.empty([nrows, ncols], dtype=float32)
error = np.empty([nrows, ncols], dtype=float32)
intercept = np.empty([nrows, ncols], dtype=np.float32)
samples = np.empty([nrows, ncols], dtype=np.float32)
# pixel-by-pixel calculation.
# nested loops to loop over the 2 image dimensions
for i in range(nrows):
for j in range(ncols):
linrate[i, j], intercept[i, j], rsquared[i, j], error[i, j], samples[i, j] = \
linear_rate_pixel(tscuml[i, j, :], t)
return linrate, intercept, rsquared, error, samples
def write_stackrate_tifs(ifgs, params, res):
rate, error, samples = res
gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path)
epochlist = algorithm.get_epochs(ifgs)[0]
dest = join(params[cf.OUT_DIR], "stack_rate.tif")
md[ifc.EPOCH_DATE] = epochlist.dates
md[ifc.DATA_TYPE] = ifc.STACKRATE
write_output_geotiff(md, gt, wkt, rate, dest, np.nan)
dest = join(params[cf.OUT_DIR], "stack_error.tif")
md[ifc.DATA_TYPE] = ifc.STACKERROR
write_output_geotiff(md, gt, wkt, error, dest, np.nan)
dest = join(params[cf.OUT_DIR], "stack_samples.tif")
md[ifc.DATA_TYPE] = ifc.STACKSAMP
write_output_geotiff(md, gt, wkt, samples, dest, np.nan)
write_stackrate_numpy_files(error, rate, samples, params)
def write_linrate_tifs(ifgs, params, res):
# log.info('Writing linrate results')
rate, error, samples = res
gt, md, wkt = get_geotiff_header_info(ifgs[0].data_path)
epochlist = algorithm.get_epochs(ifgs)[0]
dest = join(params[cf.OUT_DIR], "linrate.tif")
md[ifc.EPOCH_DATE] = epochlist.dates
md[ifc.DATA_TYPE] = ifc.LINRATE
write_output_geotiff(md, gt, wkt, rate, dest, np.nan)
dest = join(params[cf.OUT_DIR], "linerror.tif")
md[ifc.DATA_TYPE] = ifc.LINERROR
write_output_geotiff(md, gt, wkt, error, dest, np.nan)
dest = join(params[cf.OUT_DIR], "linsamples.tif")
md[ifc.DATA_TYPE] = ifc.LINSAMP
write_output_geotiff(md, gt, wkt, samples, dest, np.nan)
write_linrate_numpy_files(error, rate, samples, params)
def _time_series_setup(ifgs, mst, params):
"""
Convenience function for setting up time series computation parameters
"""
if len(ifgs) < 1:
msg = 'Time series requires 2+ interferograms'
raise TimeSeriesError(msg)
# if mst is not a single tree then do interpolation
interp = 0 if mst_module.mst_from_ifgs(ifgs)[1] else 1
# Parallel Processing parameters
parallel = params[cf.PARALLEL]
# Time Series parameters
tsmethod = params[cf.TIME_SERIES_METHOD]
pthresh, smfactor, smorder = _validate_params(params, tsmethod)
epochlist = get_epochs(ifgs)[0]
nrows = ifgs[0].nrows
ncols = ifgs[0].ncols
nifgs = len(ifgs)
span = diff(epochlist.spans)
nepoch = len(epochlist.dates) # epoch number
nvelpar = nepoch - 1 # velocity parameters number
# nlap = nvelpar - smorder # Laplacian observations number
mast_slave_ids = master_slave_ids(epochlist.dates)
imaster = [mast_slave_ids[ifg.master] for ifg in ifgs]
islave = [mast_slave_ids[ifg.slave] for ifg in ifgs]
imaster = min(imaster, islave)
islave = max(imaster, islave)
b0_mat = zeros((nifgs, nvelpar))
for i in range(nifgs):
b0_mat[i, imaster[i]:islave[i]] = span[imaster[i]:islave[i]]
# change the sign if slave is earlier than master
isign = where(np.atleast_1d(imaster) > np.atleast_1d(islave))
b0_mat[isign[0], :] = -b0_mat[isign[0], :]
tsvel_matrix = np.empty(shape=(nrows, ncols, nvelpar), dtype=float32)
ifg_data = np.zeros((nifgs, nrows, ncols), dtype=float32)
for ifg_num in range(nifgs):
ifg_data[ifg_num] = ifgs[ifg_num].phase_data
if mst is None:
mst = ~isnan(ifg_data)
return b0_mat, interp, pthresh, smfactor, smorder, tsmethod, ifg_data, \
mst, ncols, nrows, nvelpar, parallel, span, tsvel_matrix
def get_nml(ifg_list_instance, nodata_value, nan_conversion=False):
"""
:param xxx(eg str, tuple, int, float...) ifg_list_instance: xxxx
:param float nodata_value: No data value in image
:param bool nan_conversion: Convert NaNs
:return: ifg_list_instance: replaces in place
:rtype: list
:return: _epoch_list: list of epochs
:rtype: list
"""
_epoch_list, n = algorithm.get_epochs(ifg_list_instance.ifgs)
ifg_list_instance.reshape_n(n)
if nan_conversion:
ifg_list_instance.update_nan_frac(nodata_value)
# turn on for nan conversion
ifg_list_instance.convert_nans(nan_conversion=nan_conversion)
ifg_list_instance.make_data_stack()
return ifg_list_instance, _epoch_list
def _ts_to_ifgs(tsincr, preread_ifgs):
"""
Function that converts an incremental displacement time series into
interferometric phase observations. Used to re-construct an interferogram
network from a time series.
:param ndarray tsincr: incremental time series array of size
(ifg.shape, nepochs-1)
:param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances
:return: None, interferograms are saved to disk
"""
log.debug('Reconstructing interferometric observations from time series')
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
_, n = get_epochs(ifgs)
index_master, index_slave = n[:len(ifgs)], n[len(ifgs):]
for i, ifg in enumerate(ifgs):
phase = np.sum(tsincr[:, :, index_master[i]:index_slave[i]], axis=2)
_save_aps_corrected_phase(ifg.path, phase)
