def ifgram_inversion(inps=None):
"""Phase triangulatino of small baseline interferograms
Parameters: inps - namespace
Example: inps = cmd_line_parse()
ifgram_inversion(inps)
"""
if not inps:
inps = cmd_line_parse()
start_time = time.time()
## 1. input info
stack_obj = ifgramStack(inps.ifgramStackFile)
stack_obj.open(print_msg=False)
date12_list = stack_obj.get_date12_list(dropIfgram=True)
date_list = stack_obj.get_date_list(dropIfgram=True)
length, width = stack_obj.length, stack_obj.width
# 1.1 read values on the reference pixel
inps.refPhase = stack_obj.get_reference_phase(
unwDatasetName=inps.obsDatasetName,
skip_reference=inps.skip_ref,
dropIfgram=True)
# 1.2 design matrix
A = stack_obj.get_design_matrix4timeseries(date12_list)[0]
num_ifgram, num_date = A.shape[0], A.shape[1] + 1
inps.numIfgram = num_ifgram
# 1.3 print key setup info
msg = '-------------------------------------------------------------------------------\n'
if inps.minNormVelocity:
suffix = 'deformation velocity'
else:
suffix = 'deformation phase'
msg += 'least-squares solution with L2 min-norm on: {}\n'.format(suffix)
msg += 'minimum redundancy: {}\n'.format(inps.minRedundancy)
msg += 'weight function: {}\n'.format(inps.weightFunc)
if inps.maskDataset:
if inps.maskDataset in ['coherence', 'offsetSNR']:
suffix = '{} < {}'.format(inps.maskDataset, inps.maskThreshold)
else:
suffix = '{} == 0'.format(inps.maskDataset)
msg += 'mask out pixels with: {}\n'.format(suffix)
else:
msg += 'mask: no\n'
if np.linalg.matrix_rank(A) < A.shape[1]:
msg += '***WARNING: the network is NOT fully connected.\n'
msg += '\tInversion result can be biased!\n'
msg += '\tContinue to use SVD to resolve the offset between different subsets.\n'
msg += '-------------------------------------------------------------------------------'
print(msg)
print('number of interferograms: {}'.format(num_ifgram))
print('number of acquisitions : {}'.format(num_date))
print('number of lines : {}'.format(length))
print('number of columns : {}'.format(width))
## 2. prepare output
# 2.1 metadata
meta = dict(stack_obj.metadata)
for key in configKeys:
meta[key_prefix + key] = str(vars(inps)[key])
# 2.2 instantiate time-series
dsNameDict = {
"date": (np.dtype('S8'), (num_date, )),
"bperp": (np.float32, (num_date, )),
"timeseries": (np.float32, (num_date, length, width)),
}
meta['FILE_TYPE'] = 'timeseries'
meta['UNIT'] = 'm'
meta['REF_DATE'] = date_list[0]
ts_obj = timeseries(inps.tsFile)
ts_obj.layout_hdf5(dsNameDict, meta)
# write date time-series
date_list_utf8 = [dt.encode('utf-8') for dt in date_list]
writefile.write_hdf5_block(inps.tsFile, date_list_utf8, datasetName='date')
# write bperp time-series
pbase = stack_obj.get_perp_baseline_timeseries(dropIfgram=True)
writefile.write_hdf5_block(inps.tsFile, pbase, datasetName='bperp')
# 2.3 instantiate temporal coherence
dsNameDict = {"temporalCoherence": (np.float32, (length, width))}
meta['FILE_TYPE'] = 'temporalCoherence'
meta['UNIT'] = '1'
meta.pop('REF_DATE')
writefile.layout_hdf5(inps.tempCohFile, dsNameDict, metadata=meta)
# 2.4 instantiate number of inverted observations
dsNameDict = {"mask": (np.float32, (length, width))}
meta['FILE_TYPE'] = 'mask'
meta['UNIT'] = '1'
writefile.layout_hdf5(inps.numInvFile, dsNameDict, metadata=meta)
## 3. run the inversion / estimation and write to disk
# 3.1 split ifgram_file into blocks to save memory
box_list, num_box = split2boxes(inps.ifgramStackFile,
memory_size=inps.memorySize)
# 3.2 prepare the input arguments for *_patch()
data_kwargs = {
"ifgram_file": inps.ifgramStackFile,
"ref_phase": inps.refPhase,
"obs_ds_name": inps.obsDatasetName,
"weight_func": inps.weightFunc,
"min_norm_velocity": inps.minNormVelocity,
"water_mask_file": inps.waterMaskFile,
"mask_ds_name": inps.maskDataset,
"mask_threshold": inps.maskThreshold,
"min_redundancy": inps.minRedundancy
}
# 3.3 invert / write block-by-block
for i, box in enumerate(box_list):
box_width = box[2] - box[0]
box_length = box[3] - box[1]
if num_box > 1:
print('\n------- processing patch {} out of {} --------------'.
format(i + 1, num_box))
print('box width: {}'.format(box_width))
print('box length: {}'.format(box_length))
# update box argument in the input data
data_kwargs['box'] = box
if inps.cluster == 'no':
# non-parallel
ts, temp_coh, num_inv_ifg = ifgram_inversion_patch(
**data_kwargs)[:-1]
else:
# parallel
print('\n\n------- start parallel processing using Dask -------')
# initiate the output data
ts = np.zeros((num_date, box_length, box_width), np.float32)
temp_coh = np.zeros((box_length, box_width), np.float32)
num_inv_ifg = np.zeros((box_length, box_width), np.float32)
# initiate dask cluster and client
cluster_obj = cluster.DaskCluster(inps.cluster,
inps.numWorker,
config_name=inps.config)
cluster_obj.open()
# run dask
ts, temp_coh, num_inv_ifg = cluster_obj.run(
func=ifgram_inversion_patch,
func_data=data_kwargs,
results=[ts, temp_coh, num_inv_ifg])
# close dask cluster and client
cluster_obj.close()
print('------- finished parallel processing -------\n\n')
# write the block to disk
# with 3D block in [z0, z1, y0, y1, x0, x1]
# and 2D block in [y0, y1, x0, x1]
# time-series - 3D
block = [0, num_date, box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(inps.tsFile,
data=ts,
datasetName='timeseries',
block=block)
# temporal coherence - 2D
block = [box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(inps.tempCohFile,
data=temp_coh,
datasetName='temporalCoherence',
block=block)
# number of inverted obs - 2D
writefile.write_hdf5_block(inps.numInvFile,
data=num_inv_ifg,
datasetName='mask',
block=block)
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
# 3.4 update output data on the reference pixel
if not inps.skip_ref:
# grab ref_y/x
ref_y = int(stack_obj.metadata['REF_Y'])
ref_x = int(stack_obj.metadata['REF_X'])
print('-' * 50)
print('update values on the reference pixel: ({}, {})'.format(
ref_y, ref_x))
print('set temporal coherence on the reference pixel to 1.')
with h5py.File(inps.tempCohFile, 'r+') as f:
f['temporalCoherence'][ref_y, ref_x] = 1.
print('set # of observations on the reference pixel as {}'.format(
num_ifgram))
with h5py.File(inps.numInvFile, 'r+') as f:
f['mask'][ref_y, ref_x] = num_ifgram
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.\n'.format(m, s))
return
def correct_dem_error(inps):
"""Correct DEM error of input timeseries file"""
start_time = time.time()
# limit the number of threads to 1
# for slight speedup and big CPU usage save
num_threads_dict = cluster.set_num_threads("1")
## 1. input info
# 1.1 read date info
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open()
num_date = ts_obj.numDate
length, width = ts_obj.length, ts_obj.width
num_step = len(inps.stepFuncDate)
# exclude dates
date_flag = read_exclude_date(inps.excludeDate, ts_obj.dateList)[0]
if inps.polyOrder > np.sum(date_flag):
raise ValueError(
"input poly order {} > number of acquisition {}! Reduce it!".
format(inps.polyOrder, np.sum(date_flag)))
# 1.2 design matrix part 1 - time func for surface deformation
G_defo = get_design_matrix4defo(inps)
## 2. prepare output
# 2.1 metadata
meta = dict(ts_obj.metadata)
print(
'add/update the following configuration metadata to file:\n{}'.format(
configKeys))
for key in configKeys:
meta[key_prefix + key] = str(vars(inps)[key])
# 2.2 instantiate est. DEM error
dem_err_file = 'demErr.h5'
meta['FILE_TYPE'] = 'dem'
meta['UNIT'] = 'm'
ds_name_dict = {'dem': [np.float32, (length, width), None]}
writefile.layout_hdf5(dem_err_file, ds_name_dict, metadata=meta)
# 2.3 instantiate corrected time-series
ts_cor_file = inps.outfile
meta['FILE_TYPE'] = 'timeseries'
writefile.layout_hdf5(ts_cor_file,
metadata=meta,
ref_file=inps.timeseries_file)
# 2.4 instantiate residual phase time-series
ts_res_file = os.path.join(os.path.dirname(inps.outfile),
'timeseriesResidual.h5')
writefile.layout_hdf5(ts_res_file,
metadata=meta,
ref_file=inps.timeseries_file)
## 3. run the estimation and write to disk
# 3.1 split ts_file into blocks to save memory
# 1st dimension size: ts (obs / cor / res / step) + dem_err/inc_angle/rg_dist (+pbase)
num_epoch = num_date * 3 + num_step + 3
if inps.geom_file:
geom_obj = geometry(inps.geom_file)
geom_obj.open(print_msg=False)
if 'bperp' in geom_obj.datasetNames:
num_epoch += num_date
# split in row/line direction based on the input memory limit
num_box = int(
np.ceil((num_epoch * length * width * 4) * 2.5 /
(inps.maxMemory * 1024**3)))
box_list = cluster.split_box2sub_boxes(box=(0, 0, width, length),
num_split=num_box,
dimension='y')
# 3.2 prepare the input arguments for *_patch()
data_kwargs = {
'G_defo': G_defo,
'ts_file': inps.timeseries_file,
'geom_file': inps.geom_file,
'date_flag': date_flag,
'phase_velocity': inps.phaseVelocity,
}
# 3.3 invert / write block-by-block
for i, box in enumerate(box_list):
box_wid = box[2] - box[0]
box_len = box[3] - box[1]
if num_box > 1:
print('\n------- processing patch {} out of {} --------------'.
format(i + 1, num_box))
print('box width: {}'.format(box_wid))
print('box length: {}'.format(box_len))
# update box argument in the input data
data_kwargs['box'] = box
# invert
if not inps.cluster:
# non-parallel
delta_z, ts_cor, ts_res = correct_dem_error_patch(
**data_kwargs)[:-1]
else:
# parallel
print('\n\n------- start parallel processing using Dask -------')
# initiate the output data
delta_z = np.zeros((box_len, box_wid), dtype=np.float32)
ts_cor = np.zeros((num_date, box_len, box_wid), dtype=np.float32)
ts_res = np.zeros((num_date, box_len, box_wid), dtype=np.float32)
# initiate dask cluster and client
cluster_obj = cluster.DaskCluster(inps.cluster,
inps.numWorker,
config_name=inps.config)
cluster_obj.open()
# run dask
delta_z, ts_cor, ts_res = cluster_obj.run(
func=correct_dem_error_patch,
func_data=data_kwargs,
results=[delta_z, ts_cor, ts_res])
# close dask cluster and client
cluster_obj.close()
print('------- finished parallel processing -------\n\n')
# write the block to disk
# with 3D block in [z0, z1, y0, y1, x0, x1]
# and 2D block in [y0, y1, x0, x1]
# DEM error - 2D
block = [box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(dem_err_file,
data=delta_z,
datasetName='dem',
block=block)
# corrected time-series - 3D
block = [0, num_date, box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(ts_cor_file,
data=ts_cor,
datasetName='timeseries',
block=block)
# residual time-series - 3D
block = [0, num_date, box[1], box[3], box[0], box[2]]
writefile.write_hdf5_block(ts_res_file,
data=ts_res,
datasetName='timeseries',
block=block)
# roll back to the origial number of threads
cluster.roll_back_num_threads(num_threads_dict)
# time info
m, s = divmod(time.time() - start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s))
return dem_err_file, ts_cor_file, ts_res_file