def manual_select_pairs_to_remove(File):
'''Manually select interferograms to remove'''
print '----------------------------------------------------------------------------'
print 'Manually select interferograms to remove'
print 'Click two dates - points - in the figure to select one pair of interferogram'
print 'repeat until you select all pairs you would like to remove'
print 'then close the figure to continue the program ...'
print '----------------------------------------------------------------------------'
# Display the network
fig = plt.figure()
ax = fig.add_subplot(111)
date12_orig = pnet.get_date12_list(File)
bperp_list = ut.perp_baseline_ifgram2timeseries(File)[0].tolist()
date8_list = ptime.ifgram_date_list(File)
ax = pnet.plot_network(ax, date12_orig, date8_list, bperp_list)
print 'display the network of interferogram of file: ' + File
date6_list = ptime.yymmdd(date8_list)
dates_array = np.array(ptime.date_list2vector(date8_list)[0])
dateNum_array = mdates.date2num(dates_array)
bperp_array = np.array(bperp_list)
date_click = []
date12_click = []
def onclick(event):
xClick = event.xdata
yClick = event.ydata
idx = nearest_neighbor(xClick, yClick, dateNum_array, bperp_array)
date6 = date6_list[idx]
print 'click at ' + date6
date_click.append(date6)
if len(date_click) % 2 == 0 and date_click[-2] != date_click[-1]:
[m_date, s_date] = sorted(date_click[-2:])
m_idx = date6_list.index(m_date)
s_idx = date6_list.index(s_date)
date12 = m_date + '-' + s_date
if date12 in date12_orig:
print 'select date12: ' + date12
date12_click.append(date12)
ax.plot([dateNum_array[m_idx], dateNum_array[s_idx]],
[bperp_array[m_idx], bperp_array[s_idx]],
'r',
lw=4)
else:
print date12 + ' is not existed in input file'
plt.draw()
cid = fig.canvas.mpl_connect('button_press_event', onclick)
plt.show()
return date12_click
def main(argv):
inps = cmdLineParse()
if not inps.disp_fig:
plt.switch_backend('Agg')
if inps.template_file:
inps = read_template2inps(inps.template_file, inps)
##### 1. Read Info
# Read dateList and bperpList
ext = os.path.splitext(inps.file)[1]
if ext in ['.h5']:
atr = readfile.read_attribute(inps.file)
k = atr['FILE_TYPE']
print 'reading date and perpendicular baseline from ' + k + ' file: ' + os.path.basename(
inps.file)
if not k in multi_group_hdf5_file:
raise ValueError('only the following file type are supported:\n' +
str(multi_group_hdf5_file))
if not inps.coherence_file and k == 'coherence':
inps.coherence_file = inps.file
pbase_list = ut.perp_baseline_ifgram2timeseries(inps.file)[0]
date8_list = ptime.ifgram_date_list(inps.file)
else:
print 'reading date and perpendicular baseline from baseline list file: ' + inps.bl_list_file
date8_list, pbase_list = pnet.read_baseline_file(
inps.bl_list_file)[0:2]
print 'number of acquisitions : ' + str(len(date8_list))
# Read Pairs Info
print 'reading pairs info from file: ' + inps.file
date12_list = pnet.get_date12_list(inps.file)
print 'number of interferograms: ' + str(len(date12_list))
# Read drop_ifgram
date8_list_drop = []
date12_list_drop = []
if ext in ['.h5', '.he5']:
h5 = h5py.File(inps.file, 'r')
ifgram_list_all = sorted(h5[k].keys())
ifgram_list_keep = ut.check_drop_ifgram(h5)
date12_list_keep = ptime.list_ifgram2date12(ifgram_list_keep)
# Get date12_list_drop
date12_list_drop = sorted(
list(set(date12_list) - set(date12_list_keep)))
print 'number of interferograms marked as dropped: ' + str(
len(date12_list_drop))
print 'number of interferograms marked as kept : ' + str(
len(date12_list_keep))
# Get date_list_drop
m_dates = [i.split('-')[0] for i in date12_list_keep]
s_dates = [i.split('-')[1] for i in date12_list_keep]
date8_list_keep = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
date8_list_drop = sorted(list(set(date8_list) - set(date8_list_keep)))
print 'number of acquisitions marked as dropped: ' + str(
len(date8_list_drop))
# Read Coherence List
inps.coherence_list = None
if inps.coherence_file and os.path.isfile(inps.coherence_file):
if inps.mask_file and not os.path.isfile(inps.mask_file):
inps.mask_file = None
inps.coherence_list, inps.coh_date12_list = ut.spatial_average(inps.coherence_file, inps.mask_file, \
saveList=True, checkAoi=False)
if all(np.isnan(inps.coherence_list)):
print 'WARNING: all coherence value are nan! Do not use this and continue.'
inps.coherence_list = None
# Check subset of date12 info between input file and coherence file
if not set(inps.coh_date12_list) >= set(date12_list):
print 'WARNING: not every pair/date12 from input file is in coherence file'
print 'turn off the color plotting of interferograms based on coherence'
inps.coherence_list = None
elif set(inps.coh_date12_list) > set(date12_list):
print 'extract coherence value for all pair/date12 in input file'
inps.coherence_list = [
inps.coherence_list[inps.coh_date12_list.index(i)]
for i in date12_list
]
#inps.coh_thres = 0.7
##### 2. Plot
inps.cbar_label = 'Average spatial coherence'
# Fig 1 - Baseline History
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_perp_baseline_hist(ax, date8_list, pbase_list, vars(inps),
date8_list_drop)
figName = 'BperpHistory' + inps.fig_ext
if inps.save_fig:
fig.savefig(figName, bbox_inches='tight')
print 'save figure to ' + figName
# Fig 2 - Coherence Matrix
if inps.coherence_list:
figName = 'CoherenceMatrix' + inps.fig_ext
if inps.fig_size:
fig = plt.figure(figsize=inps.fig_size)
else:
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_coherence_matrix(ax, date12_list, inps.coherence_list,\
date12_list_drop, plot_dict=vars(inps))
if inps.save_fig:
fig.savefig(figName, bbox_inches='tight', dpi=150)
print 'save figure to ' + figName
# Fig 3 - Min/Max Coherence History
if inps.coherence_list:
figName = 'CoherenceHistory' + inps.fig_ext
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_coherence_history(ax, date12_list, inps.coherence_list)
if inps.save_fig:
fig.savefig(figName, bbox_inches='tight')
print 'save figure to ' + figName
# Fig 4 - Interferogram Network
if inps.fig_size:
fig = plt.figure(figsize=inps.fig_size)
else:
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_network(ax, date12_list, date8_list, pbase_list, vars(inps),
date12_list_drop)
figName = 'Network' + inps.fig_ext
if inps.save_fig:
fig.savefig(figName, bbox_inches='tight')
print 'save figure to ' + figName
if inps.save_list:
txtFile = os.path.splitext(inps.file)[0] + '_date12_list.txt'
np.savetxt(txtFile, date12_list, fmt='%s')
print 'save pairs/date12 info to file: ' + txtFile
if inps.disp_fig:
plt.show()
def main(argv):
inps = cmdLineParse()
if not inps.disp_fig:
plt.switch_backend('Agg')
#print '\n******************** Plot Network **********************'
##### 1. Read Info
# Read dateList and bperpList
ext = os.path.splitext(inps.file)[1]
if ext in ['.h5']:
atr = readfile.read_attribute(inps.file)
k = atr['FILE_TYPE']
print 'reading date and perpendicular baseline from '+k+' file: '+os.path.basename(inps.file)
if not k in multi_group_hdf5_file:
raise ValueError('only the following file type are supported:\n'+str(multi_group_hdf5_file))
pbase_list = ut.perp_baseline_ifgram2timeseries(inps.file)[0]
date8_list = ptime.ifgram_date_list(inps.file)
else:
print 'reading date and perpendicular baseline from baseline list file: '+inps.bl_list_file
date8_list, pbase_list = pnet.read_baseline_file(inps.bl_list_file)[0:2]
print 'number of acquisitions : '+str(len(date8_list))
# Read Pairs Info
print 'reading pairs info from file: '+inps.file
date12_list = pnet.get_date12_list(inps.file)
print 'number of interferograms: '+str(len(date12_list))
# Read drop_ifgram
date8_list_drop = []
date12_list_drop = []
if ext in ['.h5','.he5']:
h5 = h5py.File(inps.file, 'r')
ifgram_list_all = sorted(h5[k].keys())
ifgram_list_keep = ut.check_drop_ifgram(h5, atr, ifgram_list_all)
date12_list_keep = ptime.list_ifgram2date12(ifgram_list_keep)
# Get date12_list_drop
date12_list_drop = sorted(list(set(date12_list) - set(date12_list_keep)))
print 'number of interferograms marked as dropped: '+str(len(date12_list_drop))
# Get date_list_drop
m_dates = [i.split('-')[0] for i in date12_list_keep]
s_dates = [i.split('-')[1] for i in date12_list_keep]
date8_list_keep = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
date8_list_drop = sorted(list(set(date8_list) - set(date8_list_keep)))
print 'number of acquisitions marked as dropped: '+str(len(date8_list_drop))
# Read Coherence List
inps.coherence_list = None
if inps.coherence_file and os.path.isfile(inps.coherence_file):
ext = os.path.splitext(inps.coherence_file)[1]
if ext in ['.h5']:
listFile = os.path.splitext(inps.coherence_file)[0]+'_spatialAverage.txt'
if os.path.isfile(listFile):
print 'reading coherence value from existed '+listFile
fcoh = np.loadtxt(listFile, dtype=str)
inps.coherence_list = [float(i) for i in fcoh[:,1]]
inps.coh_date12_list = [i for i in fcoh[:,0]]
else:
print 'calculating average coherence value from '+inps.coherence_file
if inps.mask_file:
mask = readfile.read(inps.mask_file)[0]
else:
mask = None
inps.coherence_list = ut.spatial_average(inps.coherence_file, mask, saveList=True)
inps.coh_date12_list = pnet.get_date12_list(inps.coherence_file)
else:
print 'reading coherence value from '+inps.coherence_file
fcoh = np.loadtxt(inps.coherence_file, dtype=str)
inps.coherence_list = [float(i) for i in fcoh[:,1]]
inps.coh_date12_list = [i for i in fcoh[:,0]]
# Check length of coherence file and input file
if not set(inps.coh_date12_list) == set(date12_list):
print 'WARNING: input coherence list has different pairs/date12 from input file'
print 'turn off the color plotting of interferograms based on coherence'
inps.coherence_list = None
#inps.coh_thres = 0.7
##### 2. Plot
# Fig 1 - Baseline History
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_perp_baseline_hist(ax, date8_list, pbase_list, vars(inps), date8_list_drop)
figName = 'BperpHistory'+inps.fig_ext
if inps.save_fig:
fig.savefig(figName,bbox_inches='tight')
print 'save figure to '+figName
# Fig 2 - Coherence Matrix
if inps.coherence_list:
figName = 'CoherenceMatrix'+inps.fig_ext
if inps.fig_size:
fig = plt.figure(figsize=inps.fig_size)
else:
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_coherence_matrix(ax, date12_list, inps.coherence_list)
if inps.save_fig:
fig.savefig(figName, bbox_inches='tight')
print 'save figure to '+figName
# Fig 3 - Min/Max Coherence History
if inps.coherence_list:
figName = 'CoherenceHistory'+inps.fig_ext
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_coherence_history(ax, date12_list, inps.coherence_list)
if inps.save_fig:
fig.savefig(figName, bbox_inches='tight')
print 'save figure to '+figName
# Fig 4 - Interferogram Network
if inps.fig_size:
fig = plt.figure(figsize=inps.fig_size)
else:
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pnet.plot_network(ax, date12_list, date8_list, pbase_list, vars(inps), date12_list_drop)
figName = 'Network'+inps.fig_ext
if inps.save_fig:
fig.savefig(figName,bbox_inches='tight')
print 'save figure to '+figName
if inps.save_list:
txtFile = os.path.splitext(inps.file)[0]+'_date12_list.txt'
np.savetxt(txtFile, date12_list, fmt='%s')
print 'save pairs/date12 info to file: '+txtFile
if inps.disp_fig:
plt.show()
def main(argv):
inps = cmdLineParse()
suffix = '_demErr'
if not inps.outfile:
inps.outfile = os.path.splitext(
inps.timeseries_file)[0] + suffix + os.path.splitext(
inps.timeseries_file)[1]
# 1. template_file
if inps.template_file:
print 'read option from template file: ' + inps.template_file
inps = read_template2inps(inps.template_file, inps)
# Read Time Series
print "loading time series: " + inps.timeseries_file
atr = readfile.read_attribute(inps.timeseries_file)
length = int(atr['FILE_LENGTH'])
width = int(atr['WIDTH'])
h5 = h5py.File(inps.timeseries_file)
date_list = sorted(h5['timeseries'].keys())
date_num = len(date_list)
print 'number of acquisitions: ' + str(date_num)
# Exclude date info
#inps.ex_date = ['20070115','20100310']
if inps.ex_date:
inps = get_exclude_date(inps, date_list)
if inps.ex_date:
inps.ex_flag = np.array([i not in inps.ex_date for i in date_list])
timeseries = np.zeros((len(date_list), length * width), np.float32)
prog_bar = ptime.progress_bar(maxValue=date_num, prefix='loading: ')
for i in range(date_num):
date = date_list[i]
d = h5['timeseries'].get(date)[:]
timeseries[i][:] = d.flatten('F')
prog_bar.update(i + 1, suffix=date)
del d
h5.close()
prog_bar.close()
# Perpendicular Baseline
print 'read perpendicular baseline'
try:
inps.pbase = ut.perp_baseline_timeseries(atr, dimension=0)
if inps.pbase.shape[1] > 1:
print '\tconsider P_BASELINE variation in azimuth direction'
else:
pbase = inps.pbase
except:
print '\tCannot find P_BASELINE_TIMESERIES from timeseries file.'
print '\tTrying to calculate it from interferograms file'
if inps.ifgram_file:
inps.pbase = np.array(
ut.perp_baseline_ifgram2timeseries(
inps.ifgram_file)[0]).reshape(date_num, 1)
else:
message = 'No interferogram file input!\n'+\
'Can not correct for DEM residula without perpendicular base info!'
raise Exception(message)
# Temporal Baseline
print 'read temporal baseline'
inps.tbase = np.array(ptime.date_list2tbase(date_list)[0]).reshape(
date_num, 1)
# Incidence angle (look angle in the paper)
if inps.incidence_angle:
if os.path.isfile(inps.incidence_angle):
print 'reading incidence angle from file: ' + inps.incidence_angle
inps.incidence_angle = readfile.read(inps.incidence_angle)[0]
else:
try:
inps.incidence_angle = np.array(float(inps.incidence_angle))
print 'use input incidence angle : ' + str(
inps.incidence_angle)
except:
raise ValueError('Can not read input incidence angle: ' +
str(inps.incidence_angle))
else:
print 'calculate incidence angle using attributes of time series file'
if inps.pbase.shape[1] > 1:
inps.incidence_angle = ut.incidence_angle(atr, dimension=2)
else:
inps.incidence_angle = ut.incidence_angle(atr, dimension=1)
inps.incidence_angle *= np.pi / 180.0
# Range distance
if inps.range_dis:
if os.path.isfile(inps.range_dis):
print 'reading range distance from file: ' + inps.range_dis
inps.range_dis = readfile.read(inps.range_dis)[0]
else:
try:
inps.range_dis = np.array(float(inps.range_dis))
print 'use input range distance : ' + str(inps.range_dis)
except:
raise ValueError('Can not read input incidence angle: ' +
str(inps.range_dis))
else:
print 'calculate range distance using attributes from time series file'
if inps.pbase.shape[1] > 1:
inps.range_dis = ut.range_distance(atr, dimension=2)
else:
inps.range_dis = ut.range_distance(atr, dimension=1)
# Design matrix - temporal deformation model using tbase
print '-------------------------------------------------'
if inps.phase_velocity:
print 'using phase velocity history'
A1 = np.ones((date_num - 1, 1))
A2 = (inps.tbase[1:date_num] + inps.tbase[0:date_num - 1]) / 2.0
A3 = (inps.tbase[1:date_num]**3 - inps.tbase[0:date_num - 1]**
3) / np.diff(inps.tbase, axis=0) / 6.0
#A3 = (inps.tbase[1:date_num]**2 + inps.tbase[1:date_num]*inps.tbase[0:date_num-1] +\
# inps.tbase[0:date_num-1]**2) / 6.0
else:
print 'using phase history'
A1 = np.hstack((np.ones((date_num, 1)), inps.tbase))
A2 = inps.tbase**2 / 2.0
A3 = inps.tbase**3 / 6.0
# Polynomial order of model
print "temporal deformation model's polynomial order = " + str(
inps.poly_order)
if inps.poly_order == 1: A_def = A1
elif inps.poly_order == 2: A_def = np.hstack((A1, A2))
elif inps.poly_order == 3: A_def = np.hstack((A1, A2, A3))
# step function
if inps.step_date:
print "temporal deformation model's step function step at " + inps.step_date
step_yy = ptime.yyyymmdd2years(inps.step_date)
yy_list = ptime.yyyymmdd2years(date_list)
flag_array = np.array(yy_list) >= step_yy
A_step = np.zeros((date_num, 1))
A_step[flag_array] = 1.0
A_def = np.hstack((A_def, A_step))
# Heresh's original code for phase history approach
#A_def = np.hstack((A2,A1,np.ones((date_num,1))))
print '-------------------------------------------------'
##---------------------------------------- Loop for L2-norm inversion -----------------------------------##
delta_z_mat = np.zeros([length, width], dtype=np.float32)
resid_n = np.zeros([A_def.shape[0], length * width], dtype=np.float32)
constC = np.zeros([length, width], dtype=np.float32)
#delta_a_mat = np.zeros([length, width])
if inps.incidence_angle.ndim == 2 and inps.range_dis.ndim == 2:
print 'inversing using L2-norm minimization (unweighted least squares)'\
' pixel by pixel: %d loops in total' % (length*width)
prog_bar = ptime.progress_bar(maxValue=length * width,
prefix='calculating: ')
for i in range(length * width):
row = i % length
col = i / length
range_dis = inps.range_dis[row, col]
inc_angle = inps.incidence_angle[row, col]
# Consider P_BASELINE variation within one interferogram
if inps.pbase.shape[1] > 1:
pbase = inps.pbase[:, row].reshape(date_num, 1)
# Design matrix - DEM error using pbase, range distance and incidence angle
A_delta_z = pbase / (range_dis * np.sin(inc_angle))
if inps.phase_velocity:
pbase_v = np.diff(pbase, axis=0) / np.diff(inps.tbase, axis=0)
A_delta_z_v = pbase_v / (range_dis * np.sin(inc_angle))
A = np.hstack((A_delta_z_v, A_def))
else:
A = np.hstack((A_delta_z, A_def))
# L-2 norm inversion
if inps.ex_date:
A_inv = np.linalg.pinv(A[inps.ex_flag, :])
else:
A_inv = np.linalg.pinv(A)
# Get unknown parameters X = [delta_z, vel, acc, delta_acc, ...]
ts_dis = timeseries[:, i]
if inps.phase_velocity:
ts_dis = np.diff(ts_dis, axis=0) / np.diff(inps.tbase, axis=0)
if inps.ex_date:
X = np.dot(A_inv, ts_dis[inps.ex_flag])
else:
X = np.dot(A_inv, ts_dis)
# Residual vector n
resid_n[:, i] = ts_dis - np.dot(A, X)
# Update DEM error / timeseries matrix
delta_z = X[0]
delta_z_mat[row, col] = delta_z
if inps.update_timeseries:
timeseries[:, i] -= np.dot(A_delta_z, delta_z).flatten()
prog_bar.update(i + 1, every=length * width / 100)
prog_bar.close()
elif inps.incidence_angle.ndim == 1 and inps.range_dis.ndim == 1:
print 'inversing using L2-norm minimization (unweighted least squares)'\
' column by column: %d loops in total' % (width)
prog_bar = ptime.progress_bar(maxValue=width, prefix='calculating: ')
for i in range(width):
range_dis = inps.range_dis[i]
inc_angle = inps.incidence_angle[i]
# Design matrix - DEM error using pbase, range distance and incidence angle
A_delta_z = pbase / (range_dis * np.sin(inc_angle))
if inps.phase_velocity:
pbase_v = np.diff(pbase, axis=0) / np.diff(inps.tbase, axis=0)
A_delta_z_v = pbase_v / (range_dis * np.sin(inc_angle))
A = np.hstack((A_delta_z_v, A_def))
else:
A = np.hstack((A_delta_z, A_def))
# L-2 norm inversion
if inps.ex_date:
A_inv = np.linalg.pinv(A[inps.ex_flag, :])
else:
A_inv = np.linalg.pinv(A)
# Get unknown parameters X = [delta_z, vel, acc, delta_acc, ...]
ts_dis = timeseries[:, i * length:(i + 1) * length]
if inps.phase_velocity:
ts_dis = np.diff(ts_dis, axis=0) / np.diff(inps.tbase, axis=0)
if inps.ex_date:
X = np.dot(A_inv, ts_dis[inps.ex_flag, :])
else:
X = np.dot(A_inv, ts_dis)
# Residual vector n
resid_n[:, i * length:(i + 1) * length] = ts_dis - np.dot(A, X)
constC[:, i] = X[1].reshape((1, length))
# Update DEM error / timeseries matrix
delta_z = X[0].reshape((1, length))
delta_z_mat[:, i] = delta_z
if inps.update_timeseries:
timeseries[:, i * length:(i + 1) * length] -= np.dot(
A_delta_z, delta_z)
prog_bar.update(i + 1, every=width / 100)
prog_bar.close()
elif inps.incidence_angle.ndim == 0 and inps.range_dis.ndim == 0:
print 'inversing using L2-norm minimization (unweighted least squares) for the whole area'
# Design matrix - DEM error using pbase, range distance and incidence angle
A_delta_z = pbase / (inps.range_dis * np.sin(inps.incidence_angle))
if inps.phase_velocity:
pbase_v = np.diff(pbase, axis=0) / np.diff(inps.tbase, axis=0)
A_delta_z_v = pbase_v / (inps.range_dis *
np.sin(inps.incidence_angle))
A = np.hstack((A_delta_z_v, A_def))
else:
A = np.hstack((A_delta_z, A_def))
# L-2 norm inversion
if inps.ex_date:
A_inv = np.linalg.pinv(A[inps.ex_flag, :])
else:
A_inv = np.linalg.pinv(A)
# Get unknown parameters X = [delta_z, vel, acc, delta_acc, ...]
if inps.phase_velocity:
timeseries = np.diff(timeseries, axis=0) / np.diff(inps.tbase,
axis=0)
if inps.ex_date:
X = np.dot(A_inv, timeseries[inps.ex_flag, :])
else:
X = np.dot(A_inv, timeseries)
# Residual vector n
resid_n = ts_dis - np.dot(A, X)
# Update DEM error / timeseries matrix
delta_z_mat = X[0].reshape((1, length * width))
if inps.update_timeseries:
timeseries -= np.dot(A_delta_z, delta_z_mat)
delta_z_mat = np.reshape(delta_z_mat, [length, width], order='F')
else:
print 'ERROR: Script only support same dimension for both incidence angle and range distance matrix.'
print 'dimension of incidence angle: ' + str(inps.incidence_angle.ndim)
print 'dimension of range distance: ' + str(inps.range_dis.ndim)
sys.exit(1)
##------------------------------------------------ Output --------------------------------------------##
# DEM error file
if 'Y_FIRST' in atr.keys():
dem_error_file = 'demGeo_error.h5'
else:
dem_error_file = 'demRadar_error.h5'
#if inps.phase_velocity: suffix = '_pha_poly'+str(inps.poly_order)
#else: suffix = '_vel_poly'+str(inps.poly_order)
#dem_error_file = os.path.splitext(dem_error_file)[0]+suffix+os.path.splitext(dem_error_file)[1]
print 'writing >>> ' + dem_error_file
atr_dem_error = atr.copy()
atr_dem_error['FILE_TYPE'] = 'dem'
atr_dem_error['UNIT'] = 'm'
writefile.write(delta_z_mat, atr_dem_error, dem_error_file)
## Phase Constant C = resid_n[0,:]
#atrC = atr.copy()
#atrC['FILE_TYPE'] = 'mask'
#atrC['UNIT'] = 'm'
#writefile.write(constC, atrC, 'constD.h5')
## Corrected DEM file
#if inps.dem_file:
# inps.dem_outfile = os.path.splitext(inps.dem_file)[0]+suffix+os.path.splitext(inps.dem_file)[1]
# print '--------------------------------------'
# print 'writing >>> '+inps.dem_outfile
# dem, atr_dem = readfile.read(inps.dem_file)
# writefile.write(dem+delta_z_mat, atr_dem, inps.dem_outfile)
#outfile = 'delta_acc.h5'
#print 'writing >>> '+outfile
#atr_dem_error = atr.copy()
#atr_dem_error['FILE_TYPE'] = 'velocity'
#atr_dem_error['UNIT'] = 'm/s'
#writefile.write(delta_a_mat, atr_dem_error, outfile)
#print '**************************************'
# Corrected Time Series
if inps.update_timeseries:
print 'writing >>> ' + inps.outfile
print 'number of dates: ' + str(len(date_list))
h5out = h5py.File(inps.outfile, 'w')
group = h5out.create_group('timeseries')
prog_bar = ptime.progress_bar(maxValue=date_num, prefix='writing: ')
for i in range(date_num):
date = date_list[i]
d = np.reshape(timeseries[i][:], [length, width], order='F')
dset = group.create_dataset(date, data=d, compression='gzip')
prog_bar.update(i + 1, suffix=date)
prog_bar.close()
for key, value in atr.iteritems():
group.attrs[key] = value
h5out.close()
outFile = os.path.splitext(inps.outfile)[0] + 'InvResid.h5'
print 'writing >>> ' + outFile
print 'number of dates: ' + str(A_def.shape[0])
h5out = h5py.File(outFile, 'w')
group = h5out.create_group('timeseries')
prog_bar = ptime.progress_bar(maxValue=A_def.shape[0], prefix='writing: ')
for i in range(A_def.shape[0]):
date = date_list[i]
d = np.reshape(resid_n[i][:], [length, width], order='F')
dset = group.create_dataset(date, data=d, compression='gzip')
prog_bar.update(i + 1, suffix=date)
prog_bar.close()
# Attribute
for key, value in atr.iteritems():
group.attrs[key] = value
if A_def.shape[0] == date_num:
group.attrs['UNIT'] = 'm'
else:
group.attrs['UNIT'] = 'm/yr'
h5out.close()
return
def ifgram_inversion(ifgramFile='unwrapIfgram.h5', coherenceFile='coherence.h5', meta=None):
'''Implementation of the SBAS algorithm.
modified from sbas.py written by scott baker, 2012
Inputs:
ifgramFile - string, HDF5 file name of the interferograms
coherenceFile - string, HDF5 file name of the coherence
meta - dict, including the following options:
weight_function
chunk_size - float, max number of data (ifgram_num*row_num*col_num)
to read per loop; to control the memory
Output:
timeseriesFile - string, HDF5 file name of the output timeseries
tempCohFile - string, HDF5 file name of temporal coherence
Example:
meta = dict()
meta['weight_function'] = 'variance'
meta['chunk_size'] = 0.5e9
meta['timeseriesFile'] = 'timeseries_var.h5'
meta['tempCohFile'] = 'temporalCoherence_var.h5'
ifgram_inversion('unwrapIfgram.h5', 'coherence.h5', meta)
'''
if 'tempCohFile' not in meta.keys():
meta['tempCohFile'] = 'temporalCoherence.h5'
meta['timeseriesStdFile'] = 'timeseriesDecorStd.h5'
total = time.time()
if not meta:
meta = vars(cmdLineParse())
if meta['update_mode'] and not ut.update_file(meta['timeseriesFile'], ifgramFile):
return meta['timeseriesFile'], meta['tempCohFile']
##### Basic Info
# length/width
atr = readfile.read_attribute(ifgramFile)
length = int(atr['FILE_LENGTH'])
width = int(atr['WIDTH'])
meta['length'] = length
meta['width'] = width
# ifgram_list
h5ifgram = h5py.File(ifgramFile,'r')
ifgram_list = sorted(h5ifgram['interferograms'].keys())
#if meta['weight_function'] in ['no','uniform']:
# ifgram_list = ut.check_drop_ifgram(h5ifgram)
ifgram_list = ut.check_drop_ifgram(h5ifgram)
meta['ifgram_list'] = ifgram_list
ifgram_num = len(ifgram_list)
# date12_list/date8_list/tbase_diff
date12_list = ptime.list_ifgram2date12(ifgram_list)
m_dates = [i.split('-')[0] for i in date12_list]
s_dates = [i.split('-')[1] for i in date12_list]
date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
date_num = len(date8_list)
meta['date8_list'] = date8_list
meta['date12_list'] = date12_list
tbase_list = ptime.date_list2tbase(date8_list)[0]
tbase_diff = np.diff(tbase_list).reshape((-1,1))
meta['tbase_diff'] = tbase_diff
print 'number of interferograms: %d' % (ifgram_num)
print 'number of acquisitions : %d' % (date_num)
print 'number of columns: %d' % (width)
print 'number of lines : %d' % (length)
##### ref_y/x/value
try:
ref_x = int(atr['ref_x'])
ref_y = int(atr['ref_y'])
print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x)
ref_value = np.zeros((ifgram_num,1), np.float32)
for j in range(ifgram_num):
ifgram = ifgram_list[j]
dset = h5ifgram['interferograms'][ifgram].get(ifgram)
ref_value[j] = dset[ref_y,ref_x]
meta['ref_y'] = ref_y
meta['ref_x'] = ref_x
meta['ref_value'] = ref_value
except:
if meta['skip_ref']:
meta['ref_value'] = 0.0
print 'skip checking reference pixel info - This is for SIMULATION ONLY.'
else:
print 'ERROR: No ref_x/y found! Can not invert interferograms without reference in space.'
print 'run seed_data.py '+ifgramFile+' --mark-attribute for a quick referencing.'
sys.exit(1)
h5ifgram.close()
##### Rank of Design matrix for weighted inversion
A, B = ut.design_matrix(ifgramFile, date12_list)
print '-------------------------------------------------------------------------------'
if meta['weight_function'] in ['no','uniform']:
print 'generic least square inversion with min-norm phase velocity'
print ' based on Berardino et al. (2002, IEEE-TGRS)'
print ' OLS for pixels with fully connected network'
print ' SVD for pixels with partially connected network'
if np.linalg.matrix_rank(A) < date_num-1:
print 'WARNING: singular design matrix! Inversion result can be biased!'
print 'continue using its SVD solution on all pixels'
else:
print 'weighted least square (WLS) inversion with min-norm phase, pixelwise'
if np.linalg.matrix_rank(A) < date_num-1:
print 'ERROR: singular design matrix!'
print ' Input network of interferograms is not fully connected!'
print ' Can not invert the weighted least square solution.'
print 'You could try:'
print ' 1) Add more interferograms to make the network fully connected:'
print ' a.k.a., no multiple subsets nor network islands'
print " 2) Use '-w no' option for non-weighted SVD solution."
sys.exit(-1)
print '-------------------------------------------------------------------------------'
##### Invert time-series phase
##Check parallel environment
if meta['weight_function'] in ['no','uniform']:
meta['parallel'] = False
if meta['parallel']:
num_cores, meta['parallel'], Parallel, delayed = ut.check_parallel(1000, print_msg=False)
##Split into chunks to reduce memory usage
r_step = meta['chunk_size']/ifgram_num/width #split in lines
if meta['weight_function'] not in ['no','uniform']: #more memory usage (coherence) for WLS
r_step /= 2.0
if meta['parallel']:
r_step /= num_cores
r_step = int(ceil_to_1(r_step))
meta['row_step'] = r_step
chunk_num = int((length-1)/r_step)+1
if chunk_num > 1:
print 'maximum chunk size: %.1E' % (meta['chunk_size'])
print 'split %d lines into %d patches for processing' % (length, chunk_num)
print ' with each patch up to %d lines' % (r_step)
if meta['parallel']:
print 'parallel processing using %d cores ...' % (min([num_cores,chunk_num]))
##Computing the inversion
box_list = []
for i in range(chunk_num):
r0 = i*r_step
r1 = min([length, r0+r_step])
box = (0,r0,width,r1)
box_list.append(box)
box_num = len(box_list)
if not meta['parallel']:
timeseries = np.zeros((date_num, length, width), np.float32)
timeseriesStd = np.zeros((date_num, length, width), np.float32)
tempCoh = np.zeros((length, width), np.float32)
for i in range(box_num):
if box_num > 1:
print '\n------- Processing Patch %d out of %d --------------' % (i+1, box_num)
box = box_list[i]
ts, tcoh, tsStd = ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box)
tempCoh[box[1]:box[3],box[0]:box[2]] = tcoh
timeseries[:,box[1]:box[3],box[0]:box[2]] = ts
timeseriesStd[:,box[1]:box[3],box[0]:box[2]] = tsStd
else:
##Temp file list
meta['ftemp_base'] = 'timeseries_temp_'
temp_file_list = [meta['ftemp_base']+str(i)+'.h5' for i in range(chunk_num)]
##Computation
Parallel(n_jobs=num_cores)(delayed(ifgram_inversion_patch)\
(ifgramFile, coherenceFile, meta, box) for box in box_list)
##Concatenate temp files
print 'concatenating temporary timeseries files ...'
timeseries = np.zeros((date_num, length, width), np.float32)
tempCoh = np.zeros((length, width), np.float32)
rmCmd = 'rm'
for i in range(chunk_num):
fname = temp_file_list[i]
box = box_list[i]
print 'reading '+fname
h5temp = h5py.File(fname, 'r')
dset = h5temp['timeseries'].get('timeseries')
timeseries[:,box[1]:box[3],box[0]:box[2]] = dset[0:-1,:,:]
tempCoh[box[1]:box[3],box[0]:box[2]] = dset[-1,:,:]
h5temp.close()
rmCmd += ' '+fname
print rmCmd
os.system(rmCmd)
print 'converting phase to range'
phase2range = -1*float(atr['WAVELENGTH'])/(4.*np.pi)
timeseries *= phase2range
timeseriesStd *= abs(phase2range)
##### Calculate time-series attributes
print 'calculating perpendicular baseline timeseries'
pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries(ifgramFile, ifgram_list)
pbase = str(pbase.tolist()).translate(None,'[],') # convert np.array into string separated by white space
pbase_top = str(pbase_top.tolist()).translate(None,'[],')
pbase_bottom = str(pbase_bottom.tolist()).translate(None,'[],')
atr['P_BASELINE_TIMESERIES'] = pbase
atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top
atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom
atr['ref_date'] = date8_list[0]
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
##### Output
## 1. Write time-series file
meta['timeseriesFile'] = write_timeseries_hdf5_file(timeseries, date8_list, atr,\
timeseriesFile=meta['timeseriesFile'])
if not np.all(timeseriesStd == 0.):
meta['timeseriesStdFile'] = write_timeseries_hdf5_file(timeseriesStd, date8_list, atr,\
timeseriesFile=meta['timeseriesStdFile'])
## 2. Write Temporal Coherence File
print 'writing >>> '+meta['tempCohFile']
atr['FILE_TYPE'] = 'temporal_coherence'
atr['UNIT'] = '1'
meta['tempCohFile'] = writefile.write(tempCoh, atr, meta['tempCohFile'])
print 'Time series inversion took ' + str(time.time()-total) +' secs\nDone.'
return meta['timeseriesFile'], meta['tempCohFile']
def timeseries_inversion(ifgramFile='unwrapIfgram.h5',
coherenceFile='coherence.h5',
inps_dict=None):
'''Implementation of the SBAS algorithm.
modified from sbas.py written by scott baker, 2012
Inputs:
ifgramFile - string, HDF5 file name of the interferograms
coherenceFile - string, HDF5 file name of the coherence
inps_dict - dict, including the following options:
weight_function
min_coherence
max_coherence
Output:
timeseriesFile - string, HDF5 file name of the output timeseries
tempCohFile - string, HDF5 file name of temporal coherence
'''
total = time.time()
if not inps_dict:
inps_dict = vars(cmdLineParse())
weight_func = inps_dict['weight_function']
min_coh = inps_dict['min_coherence']
max_coh = inps_dict['max_coherence']
# Basic Info
atr = readfile.read_attribute(ifgramFile)
length = int(atr['FILE_LENGTH'])
width = int(atr['WIDTH'])
pixel_num = length * width
h5ifgram = h5py.File(ifgramFile, 'r')
ifgram_list = sorted(h5ifgram['interferograms'].keys())
if inps_dict['weight_function'] == 'no':
ifgram_list = ut.check_drop_ifgram(h5ifgram, atr, ifgram_list)
ifgram_num = len(ifgram_list)
# Convert ifgram_list to date12/8_list
date12_list = ptime.list_ifgram2date12(ifgram_list)
m_dates = [i.split('-')[0] for i in date12_list]
s_dates = [i.split('-')[1] for i in date12_list]
date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
date_num = len(date8_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
tbase_diff = np.diff(tbase_list).reshape((date_num - 1, 1))
print 'number of interferograms: ' + str(ifgram_num)
print 'number of acquisitions : ' + str(date_num)
print 'number of pixels: ' + str(pixel_num)
# Reference pixel in space
try:
ref_x = int(atr['ref_x'])
ref_y = int(atr['ref_y'])
print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x)
except:
print 'ERROR: No ref_x/y found! Can not inverse interferograms without reference in space.'
print 'run seed_data.py ' + ifgramFile + ' --mark-attribute for a quick referencing.'
sys.exit(1)
##### Read Interferograms
print 'reading interferograms ...'
ifgram_data = np.zeros((ifgram_num, pixel_num), np.float32)
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for j in range(ifgram_num):
ifgram = ifgram_list[j]
d = h5ifgram['interferograms'][ifgram].get(ifgram)[:]
#d[d != 0.] -= d[ref_y, ref_x]
d -= d[ref_y, ref_x]
ifgram_data[j] = d.flatten()
prog_bar.update(j + 1, suffix=date12_list[j])
h5ifgram.close()
prog_bar.close()
#####---------------------- Inversion ----------------------#####
# Design matrix
A, B = ut.design_matrix(ifgramFile, date12_list)
if weight_func == 'no':
print 'generalized inversion using SVD (Berardino et al., 2002, IEEE-TGRS)'
print 'inversing time series ...'
B_inv = np.array(np.linalg.pinv(B), np.float32)
ts_rate = np.dot(B_inv, ifgram_data)
ts1 = ts_rate * np.tile(tbase_diff, (1, pixel_num))
ts0 = np.array([0.] * pixel_num, np.float32)
ts_data = np.vstack((ts0, np.cumsum(ts1, axis=0)))
del ts_rate, ts0, ts1
# Temporal coherence
print 'calculating temporal coherence (Tizzani et al., 2007, RSE)'
temp_coh = np.zeros((1, pixel_num), np.float32) + 0j
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for i in range(ifgram_num):
ifgram_est = np.dot(A[i, :], ts_data[1:, :])
ifgram_diff = ifgram_data[i, :] - ifgram_est
temp_coh += np.exp(1j * ifgram_diff)
prog_bar.update(i + 1, suffix=date12_list[i])
prog_bar.close()
del ifgram_data, ifgram_est, ifgram_diff
temp_coh = np.array((np.absolute(temp_coh) / ifgram_num).reshape(
(length, width)),
dtype=np.float32)
else:
print 'weighted least square (WLS) inversion using coherence pixel by pixel'
if np.linalg.matrix_rank(A) < date_num - 1:
print 'ERROR: singular design matrix!'
print ' Input network of interferograms is not fully connected!'
print ' Can not inverse the weighted least square solution.'
print 'You could try:'
print ' 1) Add more interferograms to make the network fully connected:'
print ' a.k.a., no multiple subsets nor network islands'
print " 2) Use '-w no' option for non-weighted SVD solution."
sys.exit(-1)
pixel_mask = np.ones(pixel_num, np.bool_)
print 'reading coherence: ' + os.path.basename(coherenceFile)
h5coh = h5py.File(coherenceFile, 'r')
coh_list = sorted(h5coh['coherence'].keys())
coh_data = np.zeros((ifgram_num, pixel_num), np.float32)
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for j in range(ifgram_num):
ifgram = coh_list[j]
d = h5coh['coherence'][ifgram].get(ifgram)[:].flatten()
d[np.isnan(d)] = 0.
pixel_mask[d == 0.] = 0
coh_data[j] = d
prog_bar.update(j + 1, suffix=date12_list[j])
h5coh.close()
prog_bar.close()
# Get mask of valid pixels to inverse
print 'skip pixels with zero coherence in at least one interferogram'
print 'skip pixels with zero phase in all interferograms'
ifgram_stack = ut.get_file_stack(ifgramFile).flatten()
pixel_mask[ifgram_stack == 0.] = 0
pixel_num2inv = np.sum(pixel_mask)
pixel_idx2inv = np.where(pixel_mask)[0]
ifgram_data = ifgram_data[:, pixel_mask]
coh_data = coh_data[:, pixel_mask]
print 'number of pixels to inverse: %d' % (pixel_num2inv)
##### Calculate Weight matrix
weight = coh_data
if weight_func.startswith('var'):
print 'convert coherence to weight using inverse of variance: x**2/(1-x**2) from Hanssen (2001, for 4.2.32)'
weight[weight > 0.999] = 0.999
if weight_func == 'variance-max-coherence':
print 'constrain the max coherence to %f' % max_coh
weight[weight > max_coh] = max_coh
weight = np.square(weight)
weight *= 1. / (1. - weight)
if weight_func == 'variance-log':
print 'use log(1/variance)+1 as weight'
weight = np.log(weight + 1)
elif weight_func.startswith('lin'):
print 'use coherence as weight directly (Tong et al., 2016, RSE)'
elif weight_func.startswith('norm'):
print 'convert coherence to weight using CDF of normal distribution: N(%f, %f)' % (
mu, std)
mu = (min_coh + max_coh) / 2.0
std = (max_coh - min_coh) / 6.0
chunk_size = 1000
chunk_num = int(pixel_num2inv / chunk_size) + 1
prog_bar = ptime.progress_bar(maxValue=chunk_num)
for i in range(chunk_num):
i0 = (i - 1) * chunk_size
i1 = min([pixel_num2inv, i0 + chunk_size])
weight[:, i0:i1] = norm.cdf(weight[:, i0:i1], mu, std)
prog_bar.update(i + 1, every=10)
prog_bar.close()
#weight = norm.cdf(weight, mu, std)
else:
print 'Un-recognized weight function: %s' % weight_func
sys.exit(-1)
##### Weighted Inversion pixel by pixel
print 'inversing time series ...'
ts_data = np.zeros((date_num, pixel_num), np.float32)
temp_coh = np.zeros(pixel_num, np.float32)
prog_bar = ptime.progress_bar(maxValue=pixel_num2inv)
for i in range(pixel_num2inv):
# Inverse timeseries
ifgram_pixel = ifgram_data[:, i]
weight_pixel = weight[:, i]
W = np.diag(weight_pixel)
ts = np.linalg.inv(A.T.dot(W).dot(A)).dot(
A.T).dot(W).dot(ifgram_pixel)
ts_data[1:, pixel_idx2inv[i]] = ts
# Calculate weighted temporal coherence
ifgram_diff = ifgram_pixel - np.dot(A, ts)
temp_coh_pixel = np.abs(
np.sum(np.multiply(weight_pixel, np.exp(1j * ifgram_diff)),
axis=0)) / np.sum(weight_pixel)
temp_coh[pixel_idx2inv[i]] = temp_coh_pixel
prog_bar.update(i + 1, every=2000, suffix=str(i + 1) + ' pixels')
prog_bar.close()
del ifgram_data, weight
#####---------------------- Outputs ----------------------#####
## 1.1 Convert time-series phase to displacement
print 'converting phase to range'
phase2range = -1 * float(atr['WAVELENGTH']) / (4. * np.pi)
ts_data *= phase2range
## 1.2 Write time-series data matrix
timeseriesFile = 'timeseries.h5'
print 'writing >>> ' + timeseriesFile
print 'number of acquisitions: ' + str(date_num)
h5timeseries = h5py.File(timeseriesFile, 'w')
group = h5timeseries.create_group('timeseries')
prog_bar = ptime.progress_bar(maxValue=date_num)
for i in range(date_num):
date = date8_list[i]
dset = group.create_dataset(date,
data=ts_data[i].reshape(length, width),
compression='gzip')
prog_bar.update(i + 1, suffix=date)
prog_bar.close()
## 1.3 Write time-series attributes
print 'calculating perpendicular baseline timeseries'
pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries(
ifgramFile, ifgram_list)
pbase = str(pbase.tolist()).translate(
None, '[],') # convert np.array into string separated by white space
pbase_top = str(pbase_top.tolist()).translate(None, '[],')
pbase_bottom = str(pbase_bottom.tolist()).translate(None, '[],')
atr['P_BASELINE_TIMESERIES'] = pbase
atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top
atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom
atr['ref_date'] = date8_list[0]
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
for key, value in atr.iteritems():
group.attrs[key] = value
h5timeseries.close()
del ts_data
## 2. Write Temporal Coherence File
tempCohFile = 'temporalCoherence.h5'
print 'writing >>> ' + tempCohFile
atr['FILE_TYPE'] = 'temporal_coherence'
atr['UNIT'] = '1'
writefile.write(temp_coh.reshape(length, width), atr, tempCohFile)
print 'Time series inversion took ' + str(time.time() -
total) + ' secs\nDone.'
return timeseriesFile, tempCohFile