def find_min_mean(mean_sds, grid):
"""
Determine the ref pixel block with minimum mean value
:param list mean_sds: List of mean standard deviations from each
reference pixel grid
:param list grid: List of ref pixel coordinates tuples
:return: Tuple of (refy, refx) with minimum mean
:rtype: tuple
"""
log.debug('Ranking ref pixel candidates based on mean values')
try:
refp_index = np.nanargmin(mean_sds)
return grid[refp_index]
except RefPixelError as v:
log.error(v)
return v
def __calc_time_series_for_tile(tile, params):
"""
Wrapper for time series calculation on a single tile
"""
preread_ifgs = params[cf.PREREAD_IFGS]
vcmt = params[cf.VCMT]
ifg_paths = [
ifg_path.tmp_sampled_path
for ifg_path in params[cf.INTERFEROGRAM_FILES]
]
output_dir = params[cf.TMPDIR]
log.debug(f"Calculating time series for tile {tile.index}")
ifg_parts = [
shared.IfgPart(p, tile, preread_ifgs, params) for p in ifg_paths
]
mst_tile = np.load(Configuration.mst_path(params, tile.index))
tsincr, tscuml, _ = time_series(ifg_parts, params, vcmt, mst_tile)
np.save(file=os.path.join(output_dir, 'tscuml_{}.npy'.format(tile.index)),
arr=tscuml)
# optional save of tsincr npy tiles
if params["savetsincr"] == 1:
np.save(file=os.path.join(output_dir,
'tsincr_{}.npy'.format(tile.index)),
arr=tsincr)
tscuml = np.insert(tscuml, 0, 0,
axis=2) # add zero epoch to tscuml 3D array
log.info('Calculating linear regression of cumulative time series')
linrate, intercept, r_squared, std_err, samples = linear_rate_array(
tscuml, ifg_parts, params)
np.save(file=os.path.join(output_dir,
'linear_rate_{}.npy'.format(tile.index)),
arr=linrate)
np.save(file=os.path.join(output_dir,
'linear_intercept_{}.npy'.format(tile.index)),
arr=intercept)
np.save(file=os.path.join(output_dir,
'linear_rsquared_{}.npy'.format(tile.index)),
arr=r_squared)
np.save(file=os.path.join(output_dir,
'linear_error_{}.npy'.format(tile.index)),
arr=std_err)
np.save(file=os.path.join(output_dir,
'linear_samples_{}.npy'.format(tile.index)),
arr=samples)
def _ts_to_ifgs(tsincr, preread_ifgs):
"""
Function that converts an incremental displacement time series into
interferometric phase observations. Used to re-construct an interferogram
network from a time series.
:param ndarray tsincr: incremental time series array of size
(ifg.shape, nepochs-1)
:param dict preread_ifgs: Dictionary of shared.PrereadIfg class instances
:return: None, interferograms are saved to disk
"""
log.debug('Reconstructing interferometric observations from time series')
ifgs = list(OrderedDict(sorted(preread_ifgs.items())).values())
_, n = get_epochs(ifgs)
index_master, index_slave = n[:len(ifgs)], n[len(ifgs):]
for i, ifg in enumerate(ifgs):
phase = np.sum(tsincr[:, :, index_master[i]:index_slave[i]], axis=2)
_save_aps_corrected_phase(ifg.path, phase)
def temporal_high_pass_filter(tsincr: np.ndarray, epochlist: EpochList,
params: dict) -> np.ndarray:
"""
Isolate high-frequency components of time series data by subtracting
low-pass components obtained using a Gaussian filter defined by a
cut-off time period (in days).
:param tsincr: Array of incremental time series data of shape (ifg.shape, n_epochs).
:param epochlist: A pyrate.core.shared.EpochList Class instance.
:param params: Dictionary of PyRate configuration parameters.
:return: ts_hp: Filtered high frequency time series data; shape (ifg.shape, nepochs).
"""
log.info('Applying temporal high-pass filter')
threshold = params[C.TLPF_PTHR]
cutoff_day = params[C.TLPF_CUTOFF]
if cutoff_day < 1 or type(cutoff_day) != int:
raise ValueError(f'tlpf_cutoff must be an integer greater than or '
f'equal to 1 day. Value provided = {cutoff_day}')
# convert cutoff in days to years
cutoff_yr = cutoff_day / ifc.DAYS_PER_YEAR
log.info(f'Gaussian temporal filter cutoff is {cutoff_day} days '
f'({cutoff_yr:.4f} years)')
intv = np.diff(epochlist.spans) # time interval for the neighboring epochs
span = epochlist.spans[:tsincr.shape[2]] + intv / 2 # accumulated time
rows, cols = tsincr.shape[:2]
tsfilt_row = {}
process_rows = mpiops.array_split(list(range(rows)))
for r in process_rows:
tsfilt_row[r] = np.empty(tsincr.shape[1:], dtype=np.float32) * np.nan
for j in range(cols):
# Result of gaussian filter is low frequency time series
tsfilt_row[r][j, :] = gaussian_temporal_filter(
tsincr[r, j, :], cutoff_yr, span, threshold)
tsfilt_combined = shared.join_dicts(mpiops.comm.allgather(tsfilt_row))
tsfilt = np.array([v[1] for v in tsfilt_combined.items()])
log.debug("Finished applying temporal high-pass filter")
# Return the high-pass time series by subtracting low-pass result from input
return tsincr - tsfilt
def maxvar_vcm_calc_wrapper(params):
"""
MPI wrapper for maxvar and vcmt computation
"""
preread_ifgs = params[C.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
log.info('Calculating the temporal variance-covariance matrix')
def _get_r_dist(ifg_path):
"""
Get RDIst class object
"""
ifg = Ifg(ifg_path)
ifg.open()
r_dist = RDist(ifg)()
ifg.close()
return r_dist
r_dist = mpiops.run_once(_get_r_dist, ifg_paths[0])
prcs_ifgs = mpiops.array_split(list(enumerate(ifg_paths)))
process_maxvar = {}
for n, i in prcs_ifgs:
log.debug(
f'Calculating maxvar for {n} of process ifgs {len(prcs_ifgs)} of total {len(ifg_paths)}'
)
process_maxvar[int(n)] = cvd(i,
params,
r_dist,
calc_alpha=True,
write_vals=True,
save_acg=True)[0]
maxvar_d = shared.join_dicts(mpiops.comm.allgather(process_maxvar))
maxvar = [v[1] for v in sorted(maxvar_d.items(), key=lambda s: s[0])]
vcmt = mpiops.run_once(get_vcmt, preread_ifgs, maxvar)
log.debug("Finished maxvar and vcm calc!")
params[C.MAXVAR], params[C.VCMT] = maxvar, vcmt
np.save(Configuration.vcmt_path(params), arr=vcmt)
return maxvar, vcmt
def _save_merged_files(ifgs_dict,
outdir,
array,
out_type,
index=None,
savenpy=None):
"""
Convenience function to save PyRate geotiff and numpy array files
"""
log.debug('Saving PyRate outputs {}'.format(out_type))
gt, md, wkt = ifgs_dict['gt'], ifgs_dict['md'], ifgs_dict['wkt']
epochlist = ifgs_dict['epochlist']
if out_type in ('tsincr', 'tscuml'):
epoch = epochlist.dates[index + 1]
dest = join(outdir, out_type + "_" + str(epoch) + ".tif")
npy_file = join(outdir, out_type + "_" + str(epoch) + ".npy")
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = index + 1
md[ifc.EPOCH_DATE] = epoch
else:
dest = join(outdir, out_type + ".tif")
npy_file = join(outdir, out_type + '.npy')
md[ifc.EPOCH_DATE] = epochlist.dates
if out_type == 'stack_rate':
md[ifc.DATA_TYPE] = ifc.STACKRATE
elif out_type == 'stack_error':
md[ifc.DATA_TYPE] = ifc.STACKERROR
elif out_type == 'stack_samples':
md[ifc.DATA_TYPE] = ifc.STACKSAMP
elif out_type == 'tsincr':
md[ifc.DATA_TYPE] = ifc.INCR
else: #tscuml
md[ifc.DATA_TYPE] = ifc.CUML
shared.write_output_geotiff(md, gt, wkt, array, dest, np.nan)
if savenpy:
np.save(file=npy_file, arr=array)
log.debug('Finished saving {}'.format(out_type))
def wrap_spatio_temporal_filter(params):
"""
A wrapper for the spatio-temporal filter so it can be tested.
See docstring for spatio_temporal_filter.
"""
if params[cf.APSEST]:
log.info('Doing APS spatio-temporal filtering')
else:
log.info('APS spatio-temporal filtering not required')
return
tiles = params[cf.TILES]
preread_ifgs = params[cf.PREREAD_IFGS]
ifg_paths = [
ifg_path.tmp_sampled_path
for ifg_path in params[cf.INTERFEROGRAM_FILES]
]
# perform some checks on existing ifgs
log.debug('Checking APS correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_APS_ERROR):
log.debug('Finished APS correction')
return # return if True condition returned
aps_error_files_on_disc = [
MultiplePaths.aps_error_path(i, params) for i in ifg_paths
]
if all(a.exists() for a in aps_error_files_on_disc):
log.warning("Reusing APS errors from previous run!!!")
for ifg_path, a in mpiops.array_split(
list(zip(ifg_paths, aps_error_files_on_disc))):
phase = np.load(a)
_save_aps_corrected_phase(ifg_path, phase)
else:
tsincr = _calc_svd_time_series(ifg_paths, params, preread_ifgs, tiles)
mpiops.comm.barrier()
spatio_temporal_filter(tsincr, ifg_paths, params, preread_ifgs)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
def parse_baseline_header(path: str) -> dict:
"""
Returns dictionary of Baseline metadata required for PyRate.
Will read the Precise baseline estimate, if available,
otherwise will read the Initial baseline estimate.
:param path: Full path to GAMMA base.par file
:return: bdict: Dictionary of baseline values
"""
lookup = _parse_header(path) # read file contents in to a dict
# split the initial and precise baselines
initial = lookup[GAMMA_INITIAL_BASELINE]
initial_rate = lookup[GAMMA_INITIAL_BASELINE_RATE]
precise = lookup[GAMMA_PRECISION_BASELINE]
precise_rate = lookup[GAMMA_PRECISION_BASELINE_RATE]
# read the initial baseline if all precise components are zero
# (indicates that the precise baseline estimation was not ran in GAMMA workflow)
if float(precise[0]) == 0.0 and float(precise[1]) == 0.0 and float(
precise[2]) == 0.0:
log.debug('Reading Initial GAMMA baseline values')
baseline, baseline_rate = initial, initial_rate
else:
log.debug('Reading Precise GAMMA baseline values')
baseline, baseline_rate = precise, precise_rate
# Extract and return a dict of baseline values
bdict = {}
# baseline vector (along Track, aCross track, Normal to the track)
bdict[ifc.PYRATE_BASELINE_T] = float(baseline[0])
bdict[ifc.PYRATE_BASELINE_C] = float(baseline[1])
bdict[ifc.PYRATE_BASELINE_N] = float(baseline[2])
bdict[ifc.PYRATE_BASELINE_RATE_T] = float(baseline_rate[0])
bdict[ifc.PYRATE_BASELINE_RATE_C] = float(baseline_rate[1])
bdict[ifc.PYRATE_BASELINE_RATE_N] = float(baseline_rate[2])
return bdict
def __find_closed_loops(edges: List[Edge],
max_loop_length: int) -> List[List[date]]:
g = nx.Graph()
edges = [(we.first, we.second) for we in edges]
g.add_edges_from(edges)
A = nx.adjacency_matrix(g)
graph = np.asarray(A.todense())
loops = []
for n in range(3, max_loop_length + 1):
log.debug(
f"Searching for loops of length {n} using Depth First Search")
_, all_loops = find_loops(graph=graph, loop_length=n)
loops_ = dedupe_loops(all_loops)
log.debug(
f"Selected number of loops of length {n} after deduplication is {len(loops_)}"
)
loops.extend(loops_)
node_list = g.nodes()
node_list_dict = {i: n for i, n in enumerate(node_list)}
loop_subset = []
for l in loops:
loop = []
for ll in l:
loop.append(node_list_dict[ll])
loop_subset.append(loop)
log.debug(f"Total number of loops is {len(loop_subset)}")
return loop_subset
def _create_ifg_dict(params):
"""
1. Convert ifg phase data into numpy binary files.
2. Save the preread_ifgs dict with information about the ifgs that are
later used for fast loading of Ifg files in IfgPart class
:param list dest_tifs: List of destination tifs
:param dict params: Config dictionary
:param list tiles: List of all Tile instances
:return: preread_ifgs: Dictionary containing information regarding
interferograms that are used later in workflow
:rtype: dict
"""
dest_tifs = [ifg_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
ifgs_dict = {}
process_tifs = mpiops.array_split(dest_tifs)
for d in process_tifs:
ifg = shared._prep_ifg(d.sampled_path, params)
ifgs_dict[d.tmp_sampled_path] = PrereadIfg(
path=d.sampled_path,
tmp_path=d.tmp_sampled_path,
nan_fraction=ifg.nan_fraction,
first=ifg.first,
second=ifg.second,
time_span=ifg.time_span,
nrows=ifg.nrows,
ncols=ifg.ncols,
metadata=ifg.meta_data)
ifg.close()
ifgs_dict = join_dicts(mpiops.comm.allgather(ifgs_dict))
ifgs_dict = mpiops.run_once(__save_ifgs_dict_with_headers_and_epochs,
dest_tifs, ifgs_dict, params, process_tifs)
params[cf.PREREAD_IFGS] = ifgs_dict
log.debug('Finished converting phase_data to numpy in process {}'.format(
mpiops.rank))
return ifgs_dict
def discard_loops_containing_max_ifg_count(loops: List[WeightedLoop],
params) -> List[WeightedLoop]:
"""
This function will discard loops when each ifg participating in a loop has met the max loop count criteria.
:param loops: list of loops
:param params: params dict
:return: selected loops satisfying MAX_LOOP_REDUNDANCY criteria
"""
selected_loops = []
ifg_counter = defaultdict(int)
for loop in loops:
edge_appearances = np.array([ifg_counter[e] for e in loop.edges])
if not np.all(edge_appearances > params[C.MAX_LOOP_REDUNDANCY]):
selected_loops.append(loop)
for e in loop.edges:
ifg_counter[e] += 1
else:
log.debug(
f"Loop {loop.loop} ignored: all constituent ifgs have been in a loop "
f"{params[C.MAX_LOOP_REDUNDANCY]} times or more")
return selected_loops
def _mst_calc(dest_tifs, params, tiles, preread_ifgs):
"""
MPI wrapper function for MST calculation
"""
process_tiles = mpiops.array_split(tiles)
log.info('Calculating minimum spanning tree matrix')
def _save_mst_tile(tile, i, preread_ifgs):
"""
Convenient inner loop for mst tile saving
"""
mst_tile = mst.mst_multiprocessing(tile, dest_tifs, preread_ifgs,
params)
# locally save the mst_mat
mst_file_process_n = join(params[cf.TMPDIR],
'mst_mat_{}.npy'.format(i))
np.save(file=mst_file_process_n, arr=mst_tile)
for t in process_tiles:
_save_mst_tile(t, t.index, preread_ifgs)
log.debug('Finished mst calculation for process {}'.format(mpiops.rank))
mpiops.comm.barrier()
def mst_matrix_networkx(ifgs):
"""
Generates MST network for a single pixel for the given ifgs using
NetworkX-package algorithms.
:param list ifgs: Sequence of interferogram objects
:return: y: pixel y coordinate
:rtype: int
:return: x: pixel x coordinate
:rtype: int
:return: mst: list of tuples for edges in the minimum spanning tree
:rtype: list
"""
# make default MST to optimise result when no Ifg cells in a stack are nans
edges_with_weights = [(i.master, i.slave, i.nan_fraction) for i in ifgs]
edges, g_nx = _minimum_spanning_edges_from_mst(edges_with_weights)
# TODO: memory efficiencies can be achieved here with tiling
list_of_phase_data = [i.phase_data for i in ifgs]
log.debug("list_of_phase_data length: " + str(len(list_of_phase_data)))
for row in list_of_phase_data:
log.debug("row length in list_of_phase_data: " + str(len(row)))
log.debug("row in list_of_phase_data: " + str(row))
data_stack = array(list_of_phase_data, dtype=float32)
# create MSTs for each pixel in the ifg data stack
nifgs = len(ifgs)
for y, x in product(range(ifgs[0].nrows), range(ifgs[0].ncols)):
values = data_stack[:, y,
x] # vertical stack of ifg values for a pixel
nan_count = nsum(isnan(values))
# optimisations: use pre-created results for all nans/no nans
if nan_count == 0:
yield y, x, edges
continue
elif nan_count == nifgs:
yield y, x, nan
continue
# dynamically modify graph to reuse a single graph: this should avoid
# repeatedly creating new graph objs & reduce RAM use
ebunch_add = []
ebunch_delete = []
for value, edge in zip(values, edges_with_weights):
if not isnan(value):
if not g_nx.has_edge(edge[0], edge[1]):
ebunch_add.append(edge)
else:
if g_nx.has_edge(edge[0], edge[1]):
ebunch_delete.append(edge)
if ebunch_add:
g_nx.add_weighted_edges_from(ebunch_add)
if ebunch_delete:
g_nx.remove_edges_from(ebunch_delete)
yield y, x, nx.minimum_spanning_tree(g_nx).edges()
def convert_to_radians(self):
"""
Convert phase_data units from millimetres to radians.
Note: converted phase_data held in memory and not written to disc
(see shared.write_modified_phase)
"""
if self.meta_data[ifc.DATA_UNITS] == MILLIMETRES:
self.phase_data = convert_mm_to_radians(self.phase_data,
wavelength=self.wavelength)
self.meta_data[ifc.DATA_UNITS] = RADIANS
self.mm_converted = False
msg = '{}: converted phase units to radians'.format(self.data_path)
log.debug(msg)
return
elif self.meta_data[ifc.DATA_UNITS] == RADIANS:
self.mm_converted = False
msg = '{}: ignored as phase units are already ' \
'radians'.format(self.data_path)
log.debug(msg)
return
else: # pragma: no cover
msg = 'Phase units are not millimetres or radians'
raise IfgException(msg)
def mask_pixels_with_unwrapping_errors(ifgs_breach_count: NDArray[(
Any, Any, Any), UInt16], num_occurrences_each_ifg: NDArray[(Any, ),
UInt16],
params: dict) -> None:
"""
Find pixels in the phase data that breach closure_thr, and mask
(assign NaNs) to those pixels in those ifgs.
:param ifgs_breach_count: unwrapping issues at pixels in all loops
:param num_occurrences_each_ifg: frequency of ifgs appearing in all loops
:param params: params dict
"""
log.debug("Masking phase data of retained ifgs")
for i, m_p in enumerate(params[C.INTERFEROGRAM_FILES]):
pix_index = ifgs_breach_count[:, :, i] == num_occurrences_each_ifg[i]
ifg = Ifg(m_p.tmp_sampled_path)
ifg.open()
nan_and_mm_convert(ifg, params)
ifg.phase_data[pix_index] = np.nan
ifg.write_modified_phase()
log.info(f"Masked phase data of {i + 1} retained ifgs after phase closure")
return None
def _make_aps_corrections(ts_aps: np.ndarray, ifgs: List[Ifg],
params: dict) -> None:
"""
Function to convert the time series APS filter output into interferometric
phase corrections and save them to disc.
:param ts_aps: Incremental APS time series array.
:param ifgs: List of Ifg class objects.
:param params: Dictionary of PyRate configuration parameters.
"""
log.debug('Reconstructing interferometric observations from time series')
# get first and second image indices
_, n = mpiops.run_once(get_epochs, ifgs)
index_first, index_second = n[:len(ifgs)], n[len(ifgs):]
num_ifgs_tuples = mpiops.array_split(list(enumerate(ifgs)))
for i, ifg in [(int(num), ifg) for num, ifg in num_ifgs_tuples]:
# sum time slice data from first to second epoch
ifg_aps = np.sum(ts_aps[:, :, index_first[i]:index_second[i]], axis=2)
aps_error_on_disc = MultiplePaths.aps_error_path(ifg.tmp_path, params)
np.save(file=aps_error_on_disc, arr=ifg_aps) # save APS as numpy array
mpiops.comm.barrier()
def mst_calc_wrapper(params):
"""
MPI wrapper function for MST calculation
"""
log.info('Calculating minimum spanning tree matrix')
def _save_mst_tile(tile: Tile, params: dict) -> None:
"""
Convenient inner loop for mst tile saving
"""
preread_ifgs = params[cf.PREREAD_IFGS]
dest_tifs = [ifg_path.tmp_sampled_path for ifg_path in params[cf.INTERFEROGRAM_FILES]]
mst_file_process_n = Configuration.mst_path(params, index=tile.index)
if mst_file_process_n.exists():
return
mst_tile = mst_multiprocessing(tile, dest_tifs, preread_ifgs, params)
# locally save the mst_mat
np.save(file=mst_file_process_n, arr=mst_tile)
tiles_split(_save_mst_tile, params)
log.debug('Finished minimum spanning tree calculation')
def _merge_timeseries(params: dict, tstype: str) -> None:
"""
Merge tiled time series outputs
"""
log.info('Merging {} time series outputs'.format(tstype))
shape, tiles, ifgs_dict = __merge_setup(params)
# load the first time series file to determine the number of time series tifs
ts_file = join(params[C.TMPDIR], tstype + '_0.npy')
ts = np.load(file=ts_file)
# pylint: disable=no-member
no_ts_tifs = ts.shape[2]
process_tifs = mpiops.array_split(range(no_ts_tifs))
# depending on nvelpar, this will not fit in memory
# e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory
# 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB
# the double for loop helps us overcome the memory limit
log.info('Process {} writing {} {} time series tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), tstype,
no_ts_tifs))
for i in process_tifs:
ts_arr = assemble_tiles(shape,
params[C.TMPDIR],
tiles,
out_type=tstype,
index=i)
__save_merged_files(ifgs_dict,
params,
ts_arr,
out_type=tstype,
index=i,
savenpy=params["savenpy"])
mpiops.comm.barrier()
log.debug('Process {} finished writing {} {} time series tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), tstype,
no_ts_tifs))
def convert_to_mm(self):
"""
Convert phase data units from radians to millimetres.
"""
self.mm_converted = True
if self.dataset.GetMetadataItem(ifc.DATA_UNITS) == MILLIMETRES:
msg = '{}: ignored as previous phase unit conversion ' \
'already applied'.format(self.data_path)
log.debug(msg)
self.phase_data = self.phase_data
return
elif self.dataset.GetMetadataItem(ifc.DATA_UNITS) == RADIANS:
self.phase_data = convert_radians_to_mm(self.phase_data,
self.wavelength)
self.meta_data[ifc.DATA_UNITS] = MILLIMETRES
# self.write_modified_phase()
# otherwise NaN's don't write to bytecode properly
# and numpy complains
# self.dataset.FlushCache()
msg = '{}: converted phase units to millimetres'.format(self.data_path)
log.debug(msg)
else: # pragma: no cover
msg = 'Phase units are not millimetres or radians'
raise IfgException(msg)
def convert_to_nans(self):
"""
Convert phase data of given value to NaN
"""
if (self._nodata_value is None) \
or (self.dataset is None): # pragma: no cover
msg = 'nodata value needs to be set for nan conversion.' \
'Use ifg.nodata_value = NoDataValue to set nodata_value'
log.warning(msg)
raise RasterException(msg)
if ((self.dataset.GetMetadataItem(ifc.NAN_STATUS) == ifc.NAN_CONVERTED)
or self.nan_converted):
self.phase_data = self.phase_data
self.nan_converted = True
msg = '{}: ignored as previous nan ' \
'conversion detected'.format(self.data_path)
log.debug(msg)
return
else:
self.phase_data = where(
isclose(self.phase_data, self._nodata_value, atol=1e-6), nan,
self.phase_data)
self.meta_data[ifc.NAN_STATUS] = ifc.NAN_CONVERTED
self.nan_converted = True
def _orb_fit_calc(multi_paths: List[MultiplePaths],
params,
preread_ifgs=None) -> None:
"""
MPI wrapper for orbital fit correction
"""
if not params[cf.ORBITAL_FIT]:
log.info('Orbital correction not required!')
print('Orbital correction not required!')
return
log.info('Calculating orbital correction')
ifg_paths = [p.sampled_path for p in multi_paths]
if preread_ifgs: # don't check except for mpi tests
# perform some general error/sanity checks
log.debug('Checking Orbital error correction status')
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_ORBITAL_ERROR):
log.debug(
'Orbital error correction not required as all ifgs are already corrected!'
)
return # return if True condition returned
if params[cf.ORBITAL_FIT_METHOD] == 1:
prcs_ifgs = mpiops.array_split(ifg_paths)
orbital.remove_orbital_error(prcs_ifgs, params, preread_ifgs)
else:
# Here we do all the multilooking in one process, but in memory
# can use multiple processes if we write data to disc during
# remove_orbital_error step
# A performance comparison should be made for saving multilooked
# files on disc vs in memory single process multilooking
if mpiops.rank == MASTER_PROCESS:
headers = [find_header(p, params) for p in multi_paths]
orbital.remove_orbital_error(ifg_paths,
params,
headers,
preread_ifgs=preread_ifgs)
mpiops.comm.barrier()
log.debug('Finished Orbital error correction')
def check_correction_status(ifgs, meta): # pragma: no cover
"""
Generic function for checking if a correction has already been performed
in a previous run by interrogating PyRate meta data entries
:param preread_ifgs: Dictionary of pre-read interferogram information
:param str meta: Meta data flag to check for
:return: True if correction has been performed, otherwise False
:rtype: bool
"""
def close_all(ifgs):
for ifg in ifgs:
ifg.close()
if not isinstance(ifgs[0], Ifg):
ifgs = [Ifg(ifg_path) for ifg_path in ifgs]
for ifg in ifgs:
if not ifg.is_open:
ifg.open()
flags = [meta in ifg.meta_data for ifg in ifgs]
if all(flags):
log.info('Skipped: interferograms already corrected')
return True
elif not all(flags) and any(flags):
log.debug('Detected mix of corrected and uncorrected interferograms')
for i, flag in zip(ifgs, flags):
if flag:
msg = '{}: correction detected'.format(i.data_path)
else:
msg = '{}: correction NOT detected'.format(i.data_path)
log.debug(msg)
close_all(ifgs)
raise CorrectionStatusError(msg)
else:
log.debug('Calculating corrections')
close_all(ifgs)
return False
def _merge_timeseries(rows, cols, params):
"""
Merge time series output
"""
xlks, _, crop = cf.transform_params(params)
base_unw_paths = []
for p in Path(params[OUT_DIR]).rglob("*rlks_*cr.tif"):
if "dem" not in str(p):
base_unw_paths.append(str(p))
if "tif" in base_unw_paths[0].split(".")[1]:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
for i, dest_tif in enumerate(dest_tifs):
dest_tifs[i] = dest_tif.replace("_tif", "")
else:
dest_tifs = base_unw_paths # cf.get_dest_paths(base_unw_paths, crop, params, xlks)
output_dir = params[cf.TMPDIR]
# load previously saved prepread_ifgs dict
preread_ifgs_file = join(output_dir, 'preread_ifgs.pk')
ifgs = cp.load(open(preread_ifgs_file, 'rb'))
# metadata and projections
gt, md, wkt = ifgs['gt'], ifgs['md'], ifgs['wkt']
epochlist = ifgs['epochlist']
ifgs = [v for v in ifgs.values() if isinstance(v, PrereadIfg)]
tiles = shared.get_tiles(dest_tifs[0], rows, cols)
# load the first tsincr file to determine the number of time series tifs
tsincr_file = os.path.join(output_dir, 'tsincr_0.npy')
tsincr = np.load(file=tsincr_file)
# pylint: disable=no-member
no_ts_tifs = tsincr.shape[2]
# we create 2 x no_ts_tifs as we are splitting tsincr and tscuml
# to all processes.
process_tifs = mpiops.array_split(range(2 * no_ts_tifs))
# depending on nvelpar, this will not fit in memory
# e.g. nvelpar=100, nrows=10000, ncols=10000, 32bit floats need 40GB memory
# 32 * 100 * 10000 * 10000 / 8 bytes = 4e10 bytes = 40 GB
# the double for loop helps us overcome the memory limit
log.info('Process {} writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs), no_ts_tifs * 2))
for i in process_tifs:
tscum_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
if i < no_ts_tifs:
for n, t in enumerate(tiles):
_assemble_tiles(i, n, t, tscum_g, output_dir, 'tscuml')
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
dest = os.path.join(
params[cf.OUT_DIR],
'tscuml' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.CUML
shared.write_output_geotiff(md, gt, wkt, tscum_g, dest, np.nan)
else:
tsincr_g = np.empty(shape=ifgs[0].shape, dtype=np.float32)
i %= no_ts_tifs
for n, t in enumerate(tiles):
_assemble_tiles(i, n, t, tsincr_g, output_dir, 'tsincr')
md[ifc.EPOCH_DATE] = epochlist.dates[i + 1]
# sequence position; first time slice is #0
md['SEQUENCE_POSITION'] = i + 1
dest = os.path.join(
params[cf.OUT_DIR],
'tsincr' + "_" + str(epochlist.dates[i + 1]) + ".tif")
md[ifc.DATA_TYPE] = ifc.INCR
shared.write_output_geotiff(md, gt, wkt, tsincr_g, dest, np.nan)
mpiops.comm.barrier()
log.debug('Process {} finished writing {} timeseries tifs of '
'total {}'.format(mpiops.rank, len(process_tifs),
no_ts_tifs * 2))
def _process_dem_error_per_tile(tile: Tile, params: dict) -> None:
"""
Convenience function for processing DEM error in tiles
:param tile: pyrate.core.shared.Tile Class object.
:param params: Dictionary of PyRate configuration parameters.
"""
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
ifg0_path = ifg_paths[0]
ifg0 = Ifg(ifg0_path)
ifg0.open(readonly=True)
# read lon and lat values of multi-looked ifg (first ifg only)
lon, lat = geometry.get_lonlat_coords(ifg0)
# read azimuth and range coords and DEM from tif files generated in prepifg
geom_files = Configuration.geometry_files(params)
rdc_az_file = geom_files['rdc_azimuth']
geom_az = Geometry(rdc_az_file)
rdc_rg_file = geom_files['rdc_range']
geom_rg = Geometry(rdc_rg_file)
dem_file = params[C.DEM_FILE_PATH].sampled_path
dem = DEM(dem_file)
preread_ifgs = params[C.PREREAD_IFGS]
threshold = params[C.DE_PTHR]
ifg_parts = [
shared.IfgPart(p, tile, preread_ifgs, params) for p in ifg_paths
]
lon_parts = lon(tile)
lat_parts = lat(tile)
az_parts = geom_az(tile)
rg_parts = geom_rg(tile)
dem_parts = dem(tile)
log.debug(
f"Calculating per-pixel baseline for tile {tile.index} during DEM error correction"
)
bperp, look_angle, range_dist = _calculate_bperp_wrapper(
ifg_paths, az_parts, rg_parts, lat_parts, lon_parts, dem_parts)
log.debug(
f"Calculating DEM error for tile {tile.index} during DEM error correction"
)
# mst_tile = np.load(Configuration.mst_path(params, tile.index))
# calculate the DEM error estimate and the correction values for each IFG
# current implementation uses the look angle and range distance matrix of the primary SLC in the last IFG
# todo: check the impact of using the same information from another SLC
dem_error, dem_error_correction, _ = calc_dem_errors(
ifg_parts, bperp, look_angle, range_dist, threshold)
# dem_error contains the estimated DEM error for each pixel (i.e. the topographic change relative to the DEM)
# size [row, col]
# dem_error_correction contains the correction value for each interferogram
# size [num_ifg, row, col]
# save tiled data in tmpdir
np.save(file=os.path.join(params[C.TMPDIR],
'dem_error_{}.npy'.format(tile.index)),
arr=dem_error)
# swap the axes of 3D array to fit the style used in function assemble_tiles
tmp_array = np.moveaxis(dem_error_correction, 0, -1)
# new dimension is [row, col, num_ifg]
# save tiled data into tmpdir
np.save(file=os.path.join(
params[C.TMPDIR], 'dem_error_correction_{}.npy'.format(tile.index)),
arr=tmp_array)
# Calculate and save the average perpendicular baseline for the tile
bperp_avg = np.nanmean(bperp, axis=(1, 2), dtype=np.float64)
np.save(file=os.path.join(params[C.TMPDIR],
'bperp_avg_{}.npy'.format(tile.index)),
arr=bperp_avg)
def _update_phase_metadata(ifg):
ifg.meta_data[ifc.PYRATE_REF_PHASE] = ifc.REF_PHASE_REMOVED
ifg.write_modified_phase()
log.debug(f"Reference phase corrected for {ifg.data_path}")
def create_png_and_kml_from_tif(output_folder_path: str,
output_type: str) -> None:
"""
Function to create a preview PNG format image from a geotiff, and a KML file
"""
log.info(f'Creating quicklook image for stack_{output_type}')
# open raster and choose band to find min, max
raster_path = join(output_folder_path, f"stack_{output_type}.tif")
if not isfile(raster_path):
raise Exception(f"stack_{output_type}.tif file not found at: " +
raster_path)
gtif = gdal.Open(raster_path)
# find bounds of image
west, north, east, south = "", "", "", ""
for line in gdal.Info(gtif).split('\n'):
if "Upper Left" in line:
west, north = line.split(")")[0].split("(")[1].split(",")
if "Lower Right" in line:
east, south = line.split(")")[0].split("(")[1].split(",")
# write KML file
kml_file_path = join(output_folder_path, f"stack_{output_type}.kml")
kml_file_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://earth.google.com/kml/2.1">
<Document>
<name>stack_{output_type}.kml</name>
<GroundOverlay>
<name>stack_{output_type}.png</name>
<Icon>
<href>stack_{output_type}.png</href>
</Icon>
<LatLonBox>
<north> """ + north + """ </north>
<south> """ + south + """ </south>
<east> """ + east + """ </east>
<west> """ + west + """ </west>
</LatLonBox>
</GroundOverlay>
</Document>
</kml>"""
with open(kml_file_path, "w") as f:
f.write(kml_file_content)
# Get raster statistics
srcband = gtif.GetRasterBand(1)
minimum, maximum, _, _ = srcband.GetStatistics(True, True)
del gtif # close geotiff (used to calculate statistics)
# steps used for the colourmap, must be even (currently hard-coded to 254 resulting in 255 values)
no_of_steps = 254
# slightly different code required for rate map and rate error map
if output_type == 'rate':
# minimum value might be negative
maximum = max(abs(minimum), abs(maximum))
minimum = -1 * maximum
# colours: blue -> white -> red (white==0)
# note that an extra value will be added for zero (i.e. white: 255 255 255)
# generate a colourmap for odd number of values (currently hard-coded to 255)
mid = int(no_of_steps * 0.5)
# allocate RGB values to three numpy arrays r, g, b
r = np.arange(0, mid) / mid
g = r
r = np.concatenate((r, np.ones(mid + 1)))
g = np.concatenate((g, np.array([1]), np.flipud(g)))
b = np.flipud(r)
# change direction of colours (blue: positve, red: negative)
r = np.flipud(r) * 255
g = np.flipud(g) * 255
b = np.flipud(b) * 255
if output_type == 'error':
# colours: white -> red (minimum error -> maximum error
# allocate RGB values to three numpy arrays r, g, b
r = np.ones(no_of_steps + 1) * 255
g = np.arange(0, no_of_steps + 1) / (no_of_steps)
g = np.flipud(g) * 255
b = g
# generate the colourmap file in the output folder
color_map_path = join(output_folder_path, f"colourmap_{output_type}.txt")
log.info(
'Saving colour map to file {}; min/max values: {:.2f}/{:.2f}'.format(
color_map_path, minimum, maximum))
with open(color_map_path, "w") as f:
f.write("nan 0 0 0 0\n")
for i, value in enumerate(
np.linspace(minimum, maximum, no_of_steps + 1)):
f.write("%f %f %f %f 255\n" % (value, r[i], g[i], b[i]))
input_tif_path = join(output_folder_path, f"stack_{output_type}.tif")
output_png_path = join(output_folder_path, f"stack_{output_type}.png")
subprocess.check_call([
"gdaldem", "color-relief", "-of", "PNG", input_tif_path, "-alpha",
color_map_path, output_png_path, "-nearest_color_entry"
])
log.debug(f'Finished creating quicklook image for stack_{output_type}')
def sum_phase_closures(ifg_files: List[str], loops: List[WeightedLoop], params: dict) -> \
Tuple[NDArray[(Any, Any, Any), Float32], NDArray[(Any, Any, Any), UInt16], NDArray[(Any,), UInt16]]:
"""
Compute the closure sum for each pixel in each loop, and count the number of times a pixel
contributes to a failed closure loop (where the summed closure is above/below the
CLOSURE_THR threshold).
:param ifg_files: list of ifg files
:param loops: list of loops
:param params: params dict
:return: Tuple of closure, ifgs_breach_count, num_occurrences_each_ifg
closure: summed closure for each loop.
ifgs_breach_count: shape=(ifg.shape, n_ifgs) number of times a pixel in an ifg fails the closure
check (i.e., has unwrapping error) in all loops under investigation.
num_occurrences_each_ifg: frequency of ifg appearance in all loops.
"""
edge_to_indexed_ifgs = __create_ifg_edge_dict(ifg_files, params)
ifgs = [v.IfgPhase for v in edge_to_indexed_ifgs.values()]
n_ifgs = len(ifgs)
if params[C.PARALLEL]:
# rets = Parallel(n_jobs=params[cf.PROCESSES], verbose=joblib_log_level(cf.LOG_LEVEL))(
# delayed(__compute_ifgs_breach_count)(ifg0, n_ifgs, weighted_loop, edge_to_indexed_ifgs, params)
# for weighted_loop in loops
# )
# for k, r in enumerate(rets):
# closure_dict[k], ifgs_breach_count_dict[k] = r
# TODO: enable multiprocessing - needs pickle error workaround
closure = np.zeros(shape=(* ifgs[0].phase_data.shape, len(loops)), dtype=np.float32)
ifgs_breach_count = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
for k, weighted_loop in enumerate(loops):
closure[:, :, k], ifgs_breach_count_l = __compute_ifgs_breach_count(weighted_loop, edge_to_indexed_ifgs,
params)
ifgs_breach_count += ifgs_breach_count_l
else:
process_loops = mpiops.array_split(loops)
closure_process = np.zeros(shape=(* ifgs[0].phase_data.shape, len(process_loops)), dtype=np.float32)
ifgs_breach_count_process = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
for k, weighted_loop in enumerate(process_loops):
closure_process[:, :, k], ifgs_breach_count_l = \
__compute_ifgs_breach_count(weighted_loop, edge_to_indexed_ifgs, params)
ifgs_breach_count_process += ifgs_breach_count_l # process
total_gb = mpiops.comm.allreduce(ifgs_breach_count_process.nbytes / 1e9, op=mpiops.MPI.SUM)
log.debug(f"Memory usage to compute ifgs_breach_count_process was {total_gb} GB")
log.debug(f"shape of ifgs_breach_count_process is {ifgs_breach_count_process.shape}")
log.debug(f"dtype of ifgs_breach_count_process is {ifgs_breach_count_process.dtype}")
total_gb = mpiops.comm.allreduce(closure_process.nbytes / 1e9, op=mpiops.MPI.SUM)
log.debug(f"Memory usage to compute closure_process was {total_gb} GB")
if mpiops.rank == 0:
ifgs_breach_count = np.zeros(shape=(ifgs[0].phase_data.shape + (n_ifgs,)), dtype=np.uint16)
# closure
closure = np.zeros(shape=(* ifgs[0].phase_data.shape, len(loops)), dtype=np.float32)
main_process_indices = mpiops.array_split(range(len(loops))).astype(np.uint16)
closure[:, :, main_process_indices] = closure_process
for rank in range(1, mpiops.size):
rank_indices = mpiops.array_split(range(len(loops)), rank).astype(np.uint16)
this_rank_closure = np.zeros(shape=(* ifgs[0].phase_data.shape, len(rank_indices)), dtype=np.float32)
mpiops.comm.Recv(this_rank_closure, source=rank, tag=rank)
closure[:, :, rank_indices] = this_rank_closure
else:
closure = None
ifgs_breach_count = None
mpiops.comm.Send(closure_process, dest=0, tag=mpiops.rank)
if mpiops.MPI_INSTALLED:
mpiops.comm.Reduce([ifgs_breach_count_process, mpiops.MPI.UINT16_T],
[ifgs_breach_count, mpiops.MPI.UINT16_T], op=mpiops.MPI.SUM, root=0) # global
else:
ifgs_breach_count = mpiops.comm.reduce(ifgs_breach_count_process, op=mpiops.sum0_op, root=0)
log.debug(f"successfully summed phase closure breach array")
num_occurrences_each_ifg = None
if mpiops.rank == 0:
num_occurrences_each_ifg = _find_num_occurrences_each_ifg(loops, edge_to_indexed_ifgs, n_ifgs)
return closure, ifgs_breach_count, num_occurrences_each_ifg
def ref_pixel_calc_wrapper(params: dict) -> Tuple[int, int]:
"""
Wrapper for reference pixel calculation
"""
__validate_supplied_lat_lon(params)
ifg_paths = [ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]]
lon = params[C.REFX]
lat = params[C.REFY]
ifg = Ifg(ifg_paths[0])
ifg.open(readonly=True)
# assume all interferograms have same projection and will share the same transform
transform = ifg.dataset.GetGeoTransform()
ref_pixel_file = Configuration.ref_pixel_path(params)
def __reuse_ref_pixel_file_if_exists():
if ref_pixel_file.exists():
refx, refy = np.load(ref_pixel_file)
log.info('Reusing pre-calculated ref-pixel values: ({}, {}) from file {}'.format(
refx, refy, ref_pixel_file.as_posix()))
log.warning("Reusing ref-pixel values from previous run!!!")
params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy)
return int(refx), int(refy)
else:
return None, None
# read and return
refx, refy = mpiops.run_once(__reuse_ref_pixel_file_if_exists)
if (refx is not None) and (refy is not None):
update_refpix_metadata(ifg_paths, int(refx), int(refy), transform, params)
return refx, refy
if lon == -1 or lat == -1:
log.info('Searching for best reference pixel location')
half_patch_size, thresh, grid = ref_pixel_setup(ifg_paths, params)
process_grid = mpiops.array_split(grid)
save_ref_pixel_blocks(process_grid, half_patch_size, ifg_paths, params)
mean_sds = _ref_pixel_mpi(process_grid, half_patch_size, ifg_paths, thresh, params)
mean_sds = mpiops.comm.gather(mean_sds, root=0)
if mpiops.rank == MAIN_PROCESS:
mean_sds = np.hstack(mean_sds)
refpixel_returned = mpiops.run_once(find_min_mean, mean_sds, grid)
if isinstance(refpixel_returned, ValueError):
raise RefPixelError(
"Reference pixel calculation returned an all nan slice!\n"
"Cannot continue downstream computation. Please change reference pixel algorithm used before "
"continuing.")
refy, refx = refpixel_returned # row first means first value is latitude
log.info('Selected reference pixel coordinate (x, y): ({}, {})'.format(refx, refy))
lon, lat = convert_pixel_value_to_geographic_coordinate(refx, refy, transform)
log.info('Selected reference pixel coordinate (lon, lat): ({}, {})'.format(lon, lat))
else:
log.info('Using reference pixel from config file (lon, lat): ({}, {})'.format(lon, lat))
log.warning("Ensure user supplied reference pixel values are in lon/lat")
refx, refy = convert_geographic_coordinate_to_pixel_value(lon, lat, transform)
log.info('Converted reference pixel coordinate (x, y): ({}, {})'.format(refx, refy))
np.save(file=ref_pixel_file, arr=[int(refx), int(refy)])
update_refpix_metadata(ifg_paths, refx, refy, transform, params)
log.debug("refpx, refpy: "+str(refx) + " " + str(refy))
ifg.close()
params[C.REFX_FOUND], params[C.REFY_FOUND] = int(refx), int(refy)
return int(refx), int(refy)
def ref_phase_est_wrapper(params):
"""
Wrapper for reference phase estimation.
"""
ifg_paths = [
ifg_path.tmp_sampled_path for ifg_path in params[C.INTERFEROGRAM_FILES]
]
refpx, refpy = params[C.REFX_FOUND], params[C.REFY_FOUND]
if len(ifg_paths) < 2:
raise ReferencePhaseError(
"At least two interferograms required for reference phase correction ({len_ifg_paths} "
"provided).".format(len_ifg_paths=len(ifg_paths)))
# this is not going to be true as we now start with fresh multilooked ifg copies - remove?
if mpiops.run_once(shared.check_correction_status, ifg_paths,
ifc.PYRATE_REF_PHASE):
log.warning(
'Reference phase correction already applied to ifgs; returning')
return
ifgs = [Ifg(ifg_path) for ifg_path in ifg_paths]
# Save reference phase numpy arrays to disk.
ref_phs_file = Configuration.ref_phs_file(params)
# If ref phase file exists on disk, then reuse - subtract ref_phase from ifgs and return
if ref_phs_file.exists():
ref_phs = np.load(ref_phs_file)
_update_phase_and_metadata(ifgs, ref_phs, params)
shared.save_numpy_phase(ifg_paths, params)
return ref_phs, ifgs
# determine the reference phase for each ifg
if params[C.REF_EST_METHOD] == 1:
log.info("Calculating reference phase as median of interferogram")
ref_phs = est_ref_phase_ifg_median(ifg_paths, params)
elif params[C.REF_EST_METHOD] == 2:
log.info(
'Calculating reference phase in a patch surrounding pixel (x, y): ({}, {})'
.format(refpx, refpy))
ref_phs = est_ref_phase_patch_median(ifg_paths, params, refpx, refpy)
else:
raise ReferencePhaseError(
"No such option, set parameter 'refest' to '1' or '2'.")
# gather all reference phases from distributed processes and save to disk
if mpiops.rank == MAIN_PROCESS:
collected_ref_phs = np.zeros(len(ifg_paths), dtype=np.float64)
process_indices = mpiops.array_split(range(len(ifg_paths))).astype(
np.uint16)
collected_ref_phs[process_indices] = ref_phs
for r in range(1, mpiops.size):
process_indices = mpiops.array_split(range(len(ifg_paths)),
r).astype(np.uint16)
this_process_ref_phs = np.zeros(shape=len(process_indices),
dtype=np.float64)
mpiops.comm.Recv(this_process_ref_phs, source=r, tag=r)
collected_ref_phs[process_indices] = this_process_ref_phs
np.save(file=ref_phs_file, arr=collected_ref_phs)
else:
collected_ref_phs = np.empty(len(ifg_paths), dtype=np.float64)
mpiops.comm.Send(ref_phs, dest=MAIN_PROCESS, tag=mpiops.rank)
mpiops.comm.Bcast(collected_ref_phs, root=0)
# subtract ref_phase from ifgs
_update_phase_and_metadata(ifgs, collected_ref_phs, params)
mpiops.comm.barrier()
shared.save_numpy_phase(ifg_paths, params)
log.debug("Finished reference phase correction")
# Preserve old return value so tests don't break.
return ref_phs, ifgs
def cvd_from_phase(phase, ifg, r_dist, calc_alpha, save_acg=False, params=None):
"""
A convenience function used to compute radial autocovariance from phase
data
:param ndarray phase: An array of interferogram phase data
:param Ifg class ifg: A pyrate.shared.Ifg class instance
:param ndarray r_dist: Array of distance values from the image centre
(See Rdist class for more details)
:param bool calc_alpha: If True calculate alpha
:param bool save_acg: If True write autocorrelation and radial distance
data to numpy array file on disk
:param dict params: [optional] Dictionary of configuration parameters;
Must be provided if save_acg=True
:return: maxvar: The maximum variance (at zero lag)
:rtype: float
:return: alpha: the exponential length-scale of decay factor
:rtype: float
"""
# pylint: disable=invalid-name
# pylint: disable=too-many-locals
autocorr_grid = _get_autogrid(phase)
acg = reshape(autocorr_grid, phase.size, order='F')
# Symmetry in image; keep only unique points
# tmp = _unique_points(zip(acg, r_dist))
# Sudipta: Unlikely, as unique_point is a search/comparison,
# whereas keeping 1st half is just numpy indexing.
# If it is not faster, why was this done differently here?
# r_dist = r_dist[:int(ceil(phase.size / 2.0)) + nrows]
acg = acg[:len(r_dist)]
# Alternative method to remove duplicate cells
# r_dist = r_dist[:ceil(len(r_dist)/2)+nlines]
# Reason for '+nlines' term unknown
# eg. array([x for x in set([(1,1), (2,2), (1,1)])])
# the above shortens r_dist by some number of cells
# pick the smallest axis to determine circle search radius
if (ifg.x_centre * ifg.x_size) < (ifg.y_centre * ifg.y_size):
maxdist = (ifg.x_centre+1) * ifg.x_size / DISTFACT
else:
maxdist = (ifg.y_centre+1) * ifg.y_size / DISTFACT
# filter out data where the of lag distance is greater than maxdist
# r_dist = array([e for e in rorig if e <= maxdist]) #
# MG: prefers to use all the data
# acg = array([e for e in rorig if e <= maxdist])
indices_to_keep = r_dist < maxdist
acg = acg[indices_to_keep]
# optionally save acg vs dist observations to disk
if save_acg:
_save_cvd_data(acg, r_dist[indices_to_keep],
ifg.data_path, params[cf.TMPDIR])
if calc_alpha:
# bin width for collecting data
bin_width = max(ifg.x_size, ifg.y_size) * 2 / DISTFACT # km
r_dist = r_dist[indices_to_keep] # km
# classify values of r_dist according to bin number
rbin = ceil(r_dist / bin_width).astype(int)
maxbin = max(rbin) - 1 # consistent with Legacy data
cvdav = zeros(shape=(2, maxbin + 1))
# the following stays in numpy land
# distance instead of bin number
cvdav[0, :] = np.multiply(range(maxbin + 1), bin_width)
# mean variance for the bins
cvdav[1, :] = [mean(acg[rbin == b]) for b in range(maxbin + 1)]
# calculate best fit function maxvar*exp(-alpha*r_dist)
alphaguess = 2 / (maxbin * bin_width)
alpha = fmin(_pendiffexp, x0=alphaguess, args=(cvdav,), disp=False,
xtol=1e-6, ftol=1e-6)
log.debug("1st guess alpha {}, converged "
"alpha: {}".format(alphaguess, alpha))
# maximum variance usually at the zero lag: max(acg[:len(r_dist)])
return np.max(acg), alpha[0] # alpha unit 1/km
else:
return np.max(acg), None
