def reconstruct_igrams_from_timeseries(h5timeseries,h5igrams):
dateList = h5timeseries['timeseries'].keys()
dateIndex={}
for ni in range(len(dateList)):
dateIndex[dateList[ni]]=ni
dset = h5timeseries['timeseries'].get(h5timeseries['timeseries'].keys()[0])
nrows,ncols=shape(dset)
timeseries = zeros((len(h5timeseries['timeseries'].keys()),shape(dset)[0]*shape(dset)[1]),float32)
for date in dateList:
dset = h5timeseries['timeseries'].get(date)
d = dset[0:dset.shape[0],0:dset.shape[1]]
timeseries[dateIndex[date]][:]=d.flatten(0)
del d
lt,numpixels=shape(timeseries)
A,B = ut.design_matrix(h5igrams)
lam = float(h5timeseries['timeseries'].attrs['WAVELENGTH'])
range2phase=-4*pi/lam
timeseries=range2phase*timeseries
p=-1*ones([A.shape[0],1])
Ap=hstack((p,A))
estData=dot(Ap,timeseries)
return estData,nrows,ncols
def reconstruct_igrams_from_timeseries(h5timeseries,h5igrams):
dateList = h5timeseries['timeseries'].keys()
dateIndex={}
for ni in range(len(dateList)):
dateIndex[dateList[ni]]=ni
dset = h5timeseries['timeseries'].get(h5timeseries['timeseries'].keys()[0])
nrows,ncols=shape(dset)
timeseries = zeros((len(h5timeseries['timeseries'].keys()),shape(dset)[0]*shape(dset)[1]),float32)
for date in dateList:
dset = h5timeseries['timeseries'].get(date)
d = dset[0:dset.shape[0],0:dset.shape[1]]
timeseries[dateIndex[date]][:]=d.flatten(0)
del d
lt,numpixels=shape(timeseries)
A,B = ut.design_matrix(h5igrams)
lam = float(h5timeseries['timeseries'].attrs['WAVELENGTH'])
range2phase=-4*pi/lam
timeseries=range2phase*timeseries
p=-1*ones([A.shape[0],1])
Ap=hstack((p,A))
estData=dot(Ap,timeseries)
return estData,nrows,ncols
def temporal_coherence(timeseriesFile, ifgramFile):
'''Calculate temporal coherence based on input timeseries file and interferograms file
Inputs:
timeseriesFile - string, path of time series file
ifgramFile - string, path of interferograms file
Output:
temp_coh - 2D np.array, temporal coherence in float32
'''
# Basic Info
atr_ts = readfile.read_attribute(timeseriesFile)
length = int(atr_ts['FILE_LENGTH'])
width = int(atr_ts['WIDTH'])
pixel_num = length * width
# Read time series data
h5timeseries = h5py.File(timeseriesFile, 'r')
date_list = sorted(h5timeseries['timeseries'].keys())
date_num = len(date_list)
print "load time series: " + timeseriesFile
print 'number of acquisitions: ' + str(date_num)
timeseries = np.zeros((date_num, pixel_num), np.float32)
prog_bar = ptime.progress_bar(maxValue=date_num, prefix='loading: ')
for i in range(date_num):
date = date_list[i]
d = h5timeseries['timeseries'].get(date)[:]
timeseries[i][:] = d.flatten(0)
prog_bar.update(i + 1, suffix=date)
prog_bar.close()
h5timeseries.close()
# Convert displacement from meter to radian
range2phase = -4 * np.pi / float(atr_ts['WAVELENGTH'])
timeseries *= range2phase
# interferograms data
print "interferograms file: " + ifgramFile
atr_ifgram = readfile.read_attribute(ifgramFile)
h5ifgram = h5py.File(ifgramFile, 'r')
ifgram_list = sorted(h5ifgram['interferograms'].keys())
ifgram_list = ut.check_drop_ifgram(h5ifgram, atr_ifgram, ifgram_list)
ifgram_num = len(ifgram_list)
# Design matrix
date12_list = ptime.list_ifgram2date12(ifgram_list)
A1, B = ut.design_matrix(ifgramFile, date12_list)
A0 = -1 * np.ones([ifgram_num, 1])
A = np.hstack((A0, A1))
# Get reference pixel
try:
ref_x = int(atr_ts['ref_x'])
ref_y = int(atr_ts['ref_y'])
print 'find reference pixel in y/x: [%d, %d]' % (ref_y, ref_x)
except ValueError:
print 'No ref_x/y found! Can not calculate temporal coherence without it.'
print 'calculating temporal coherence interferogram by interferogram ...'
print 'number of interferograms: ' + str(ifgram_num)
temp_coh = np.zeros(pixel_num, dtype=np.float32) + 0j
prog_bar = ptime.progress_bar(maxValue=ifgram_num, prefix='calculating: ')
for i in range(ifgram_num):
ifgram = ifgram_list[i]
# read interferogram
data = h5ifgram['interferograms'][ifgram].get(ifgram)[:]
data -= data[ref_y, ref_x]
data = data.flatten(0)
# calculate difference between observed and estimated data
dataEst = np.dot(A[i, :], timeseries)
dataDiff = data - dataEst
temp_coh += np.exp(1j * dataDiff)
prog_bar.update(i + 1, suffix=date12_list[i])
prog_bar.close()
del timeseries, data, dataEst, dataDiff
h5ifgram.close()
temp_coh = np.array((np.absolute(temp_coh) / ifgram_num).reshape(
(length, width)),
dtype=np.float32)
return temp_coh
def main(argv):
method = 'triangular_consistency' ## or 'bonding_point'
ramp_type = 'plane'
save_rampCor = 'yes'
plot_bonding_points = 'yes'
##### Check Inputs
if len(sys.argv) > 2:
try:
opts, args = getopt.getopt(argv, 'h:f:m:x:y:o:t:',
['ramp=', 'no-ramp-save'])
except getopt.GetoptError:
print 'Error while getting args'
usage()
sys.exit(1)
for opt, arg in opts:
if opt in ['-h', '--help']:
usage()
sys.exit()
elif opt in '-f':
File = arg
elif opt in '-m':
maskFile = arg
elif opt in '-o':
outName = arg
elif opt in '-x':
x = [int(i) for i in arg.split(',')]
method = 'bonding_point'
elif opt in '-y':
y = [int(i) for i in arg.split(',')]
method = 'bonding_point'
elif opt in '-t':
templateFile = arg
elif opt in '--ramp':
ramp_type = arg.lower()
elif opt in '--no-ramp-save':
save_rampCor = 'no'
elif len(sys.argv) == 2:
if argv[0] in ['-h', '--help']:
usage()
sys.exit()
elif os.path.isfile(argv[0]):
File = argv[0]
maskFile = argv[1]
else:
print 'Input file does not existed: ' + argv[0]
sys.exit(1)
else:
usage()
sys.exit(1)
##### Check template file
try:
templateFile
templateContents = readfile.read_template(templateFile)
except:
pass
try:
yx = [
int(i) for i in templateContents['pysar.unwrapError.yx'].split(',')
]
x = yx[1::2]
y = yx[0::2]
method = 'bonding_point'
except:
pass
##### Read Mask File
## Priority:
## Input mask file > pysar.mask.file > existed Modified_Mask.h5 > existed Mask.h5
try:
maskFile
except:
try:
maskFile = templateContents['pysar.mask.file']
except:
if os.path.isfile('Modified_Mask.h5'):
maskFile = 'Modified_Mask.h5'
elif os.path.isfile('Mask.h5'):
maskFile = 'Mask.h5'
else:
print 'No mask found!'
sys.exit(1)
try:
Mask, Matr = readfile.read(maskFile)
print 'mask: ' + maskFile
except:
print 'Can not open mask file: ' + maskFile
sys.exit(1)
##### Output file name
ext = os.path.splitext(File)[1]
try:
outName
except:
outName = File.split('.')[0] + '_unwCor' + ext
print '\n**************** Unwrapping Error Correction ******************'
#################### Triangular Consistency (Phase Closure) ####################
if method == 'triangular_consistency':
print 'Phase unwrapping error correction using Triangular Consistency / Phase Closure'
h5file = h5py.File(File)
ifgramList = h5file['interferograms'].keys()
sx = int(h5file['interferograms'][ifgramList[0]].attrs['WIDTH'])
sy = int(h5file['interferograms'][ifgramList[0]].attrs['FILE_LENGTH'])
curls, Triangles, C = ut.get_triangles(h5file)
A, B = ut.design_matrix(h5file)
ligram, lv = np.shape(B)
lcurls = np.shape(curls)[0]
print 'Number of all triangles: ' + str(lcurls)
print 'Number of interferograms: ' + str(ligram)
#print curls
curlfile = 'curls.h5'
if not os.path.isfile(curlfile):
ut.generate_curls(curlfile, h5file, Triangles, curls)
thr = 0.50
curls = np.array(curls)
n1 = curls[:, 0]
n2 = curls[:, 1]
n3 = curls[:, 2]
numPixels = sy * sx
print 'reading interferograms...'
data = np.zeros((ligram, numPixels), np.float32)
for ni in range(ligram):
dset = h5file['interferograms'][ifgramList[ni]].get(ifgramList[ni])
d = dset[0:dset.shape[0], 0:dset.shape[1]]
data[ni] = d.flatten(1)
print np.shape(data)
print 'reading curls ...'
h5curl = h5py.File(curlfile)
curlList = h5curl['interferograms'].keys()
curlData = np.zeros((lcurls, numPixels), np.float32)
for ni in range(lcurls):
dset = h5curl['interferograms'][curlList[ni]].get(curlList[ni])
d = dset[0:dset.shape[0], 0:dset.shape[1]]
curlData[ni] = d.flatten(1)
pi = np.pi
EstUnwrap = np.zeros((ligram, numPixels), np.float32)
#try:
# maskFile=argv[1]
# h5Mask=h5py.File(maskFile)
# dset = h5Mask['mask'].get('mask')
# Mask=dset[0:dset.shape[0],0:dset.shape[1]]
#except:
# dset = h5file['mask'].get('mask')
# Mask=dset[0:dset.shape[0],0:dset.shape[1]]
Mask = Mask.flatten(1)
for ni in range(numPixels):
#dU = np.zeros([ligram,1])
#print np.shape(dU)
#print np.shape(data[:,ni])
if Mask[ni] == 1:
dU = data[:, ni]
#nan_ndx = dataPoint == 0.
unwCurl = np.array(curlData[:, ni])
#print unwCurl
ind = np.abs(unwCurl) >= thr
N1 = n1[ind]
N2 = n2[ind]
N3 = n3[ind]
indC = np.abs(unwCurl) < thr
Nc1 = n1[indC]
Nc2 = n2[indC]
Nc3 = n3[indC]
N = np.hstack([N1, N2, N3])
UniN = np.unique(N)
Nc = np.hstack([Nc1, Nc2, Nc3])
UniNc = np.unique(Nc)
inter = list(set(UniNc) & set(UniN)) # intersetion
UniNc = list(UniNc)
for x in inter:
UniNc.remove(x)
D = np.zeros([len(UniNc), ligram])
for i in range(len(UniNc)):
D[i, UniNc[i]] = 1
AAA = np.vstack([-2 * pi * C, D])
#AAA1=np.hstack([AAA,np.zeros([AAA.shape[0],lv])])
#AAA2=np.hstack([-2*pi*np.eye(ligram),B])
#AAAA=np.vstack([AAA1,AAA2])
AAAA = np.vstack([AAA, 0.25 * np.eye(ligram)])
#print '************************'
#print np.linalg.matrix_rank(C)
#print np.linalg.matrix_rank(AAA)
#print np.linalg.matrix_rank(AAAA)
#print '************************'
#LLL=list(np.dot(C,dU)) + list(np.zeros(np.shape(UniNc)[0]))# + list(dU)
#ind=np.isnan(AAA)
#M1=pinv(AAA)
#M=np.dot(M1,LLL)
#EstUnwrap[:,ni]=np.round(M[0:ligram])*2.0*np.pi
##########
# with Tikhonov regularization:
AAAA = np.vstack([AAA, 0.25 * np.eye(ligram)])
LLL = list(np.dot(C, dU)) + list(np.zeros(
np.shape(UniNc)[0])) + list(np.zeros(ligram))
ind = np.isnan(AAAA)
M1 = pinv(AAAA)
M = np.dot(M1, LLL)
EstUnwrap[:, ni] = np.round(M[0:ligram]) * 2.0 * np.pi
#print M[0:ligram]
#print np.round(M[0:ligram])
else:
EstUnwrap[:, ni] = np.zeros([ligram])
if not np.remainder(ni, 10000):
print 'Processing point: %7d of %7d ' % (ni, numPixels)
##### Output
dataCor = data + EstUnwrap
unwCorFile = File.replace('.h5', '') + '_unwCor.h5'
print 'writing >>> ' + unwCorFile
h5unwCor = h5py.File(unwCorFile, 'w')
gg = h5unwCor.create_group('interferograms')
for i in range(ligram):
group = gg.create_group(ifgramList[i])
dset = group.create_dataset(ifgramList[i],
data=np.reshape(
dataCor[i, :], [sx, sy]).T,
compression='gzip')
for key, value in h5file['interferograms'][
ifgramList[i]].attrs.iteritems():
group.attrs[key] = value
try:
MASK = h5file['mask'].get('mask')
gm = h5unwCor.create_group('mask')
dset = gm.create_dataset('mask', data=MASK, compression='gzip')
except:
pass
h5unwCor.close()
h5file.close()
h5curl.close()
#################### Bonding Points (Spatial Continuity) ####################
elif method == 'bonding_point':
print 'Phase unwrapping error correction using Bonding Points / Spatial Continuity'
##### Read Bridge Points Info
try:
x
y
if len(x) != len(y) or np.mod(len(x), 2) != 0:
print 'Wrong number of bridge points input: ' + str(
len(x)) + ' for x, ' + str(len(y)) + ' for y'
usage()
sys.exit(1)
except:
print 'Error in reading bridge points info!'
usage()
sys.exit(1)
for i in range(0, len(x)):
if Mask[y[i], x[i]] == 0:
print '\nERROR: Connecting point (' + str(y[i]) + ',' + str(
x[i]) + ') is out of masked area! Select them again!\n'
sys.exit(1)
print 'Number of bonding point pairs: ' + str(len(x) / 2)
print 'Bonding points coordinates:\nx: ' + str(x) + '\ny: ' + str(y)
## Plot Connecting Pair of Points
if plot_bonding_points == 'yes':
point_yx = ''
line_yx = ''
n_bridge = len(x) / 2
for i in range(n_bridge):
pair_yx = str(y[2 * i]) + ',' + str(x[2 * i]) + ',' + str(
y[2 * i + 1]) + ',' + str(x[2 * i + 1])
if not i == n_bridge - 1:
point_yx += pair_yx + ','
line_yx += pair_yx + ';'
else:
point_yx += pair_yx
line_yx += pair_yx
try:
plot_cmd = 'view.py --point-yx="'+point_yx+'" --line-yx="'+line_yx+\
'" --nodisplay -o bonding_points.png -f '+maskFile
print plot_cmd
os.system(plot_cmd)
except:
pass
##### Ramp Info
ramp_mask = Mask == 1
print 'estimate phase ramp during the correction'
print 'ramp type: ' + ramp_type
if save_rampCor == 'yes':
outName_ramp = os.path.basename(outName).split(
ext)[0] + '_' + ramp_type + ext
########## PySAR ##########
if ext == '.h5':
##### Read
try:
h5file = h5py.File(File, 'r')
except:
print 'ERROR: Cannot open input file: ' + File
sys.exit(1)
k = h5file.keys()
if 'interferograms' in k:
k[0] = 'interferograms'
print 'Input file is ' + k[0]
else:
print 'Input file - ' + File + ' - is not interferograms.'
usage()
sys.exit(1)
igramList = sorted(h5file[k[0]].keys())
#### Write
h5out = h5py.File(outName, 'w')
gg = h5out.create_group(k[0])
print 'writing >>> ' + outName
if save_rampCor == 'yes':
h5out_ramp = h5py.File(outName_ramp, 'w')
gg_ramp = h5out_ramp.create_group(k[0])
print 'writing >>> ' + outName_ramp
##### Loop
print 'Number of interferograms: ' + str(len(igramList))
for igram in igramList:
print igram
data = h5file[k[0]][igram].get(igram)[:]
data_ramp, ramp = rm.remove_data_surface(
data, ramp_mask, ramp_type)
#ramp = data_ramp - data
data_rampCor = phase_bonding(data_ramp, Mask, x, y)
dataCor = data_rampCor - ramp
group = gg.create_group(igram)
dset = group.create_dataset(igram,
data=dataCor,
compression='gzip')
for key, value in h5file[k[0]][igram].attrs.iteritems():
group.attrs[key] = value
if save_rampCor == 'yes':
group_ramp = gg_ramp.create_group(igram)
dset = group_ramp.create_dataset(igram,
data=data_rampCor,
compression='gzip')
for key, value in h5file[k[0]][igram].attrs.iteritems():
group_ramp.attrs[key] = value
try:
mask = h5file['mask'].get('mask')
gm = h5out.create_group('mask')
dset = gm.create_dataset('mask',
data=mask[0:mask.shape[0],
0:mask.shape[1]],
compression='gzip')
except:
print 'no mask group found.'
h5file.close()
h5out.close()
if save_rampCor == 'yes':
h5out_ramp.close()
########## ROI_PAC ##########
elif ext == '.unw':
print 'Input file is ' + ext
a, data, atr = readfile.read_float32(File)
data_ramp, ramp = rm.remove_data_surface(data, ramp_mask,
ramp_type)
#ramp = data_ramp - data
data_rampCor = phase_bonding(data_ramp, Mask, x, y)
dataCor = data_rampCor - ramp
writefile.write(dataCor, atr, outName)
if save_rampCor == 'yes':
writefile.write(data_rampCor, atr, outName_ramp)
else:
print 'Un-supported file type: ' + ext
usage()
sys.exit(1)
def ifgram_inversion(ifgramFile='unwrapIfgram.h5', coherenceFile='coherence.h5', meta=None):
'''Implementation of the SBAS algorithm.
modified from sbas.py written by scott baker, 2012
Inputs:
ifgramFile - string, HDF5 file name of the interferograms
coherenceFile - string, HDF5 file name of the coherence
meta - dict, including the following options:
weight_function
chunk_size - float, max number of data (ifgram_num*row_num*col_num)
to read per loop; to control the memory
Output:
timeseriesFile - string, HDF5 file name of the output timeseries
tempCohFile - string, HDF5 file name of temporal coherence
Example:
meta = dict()
meta['weight_function'] = 'variance'
meta['chunk_size'] = 0.5e9
meta['timeseriesFile'] = 'timeseries_var.h5'
meta['tempCohFile'] = 'temporalCoherence_var.h5'
ifgram_inversion('unwrapIfgram.h5', 'coherence.h5', meta)
'''
if 'tempCohFile' not in meta.keys():
meta['tempCohFile'] = 'temporalCoherence.h5'
meta['timeseriesStdFile'] = 'timeseriesDecorStd.h5'
total = time.time()
if not meta:
meta = vars(cmdLineParse())
if meta['update_mode'] and not ut.update_file(meta['timeseriesFile'], ifgramFile):
return meta['timeseriesFile'], meta['tempCohFile']
##### Basic Info
# length/width
atr = readfile.read_attribute(ifgramFile)
length = int(atr['FILE_LENGTH'])
width = int(atr['WIDTH'])
meta['length'] = length
meta['width'] = width
# ifgram_list
h5ifgram = h5py.File(ifgramFile,'r')
ifgram_list = sorted(h5ifgram['interferograms'].keys())
#if meta['weight_function'] in ['no','uniform']:
# ifgram_list = ut.check_drop_ifgram(h5ifgram)
ifgram_list = ut.check_drop_ifgram(h5ifgram)
meta['ifgram_list'] = ifgram_list
ifgram_num = len(ifgram_list)
# date12_list/date8_list/tbase_diff
date12_list = ptime.list_ifgram2date12(ifgram_list)
m_dates = [i.split('-')[0] for i in date12_list]
s_dates = [i.split('-')[1] for i in date12_list]
date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
date_num = len(date8_list)
meta['date8_list'] = date8_list
meta['date12_list'] = date12_list
tbase_list = ptime.date_list2tbase(date8_list)[0]
tbase_diff = np.diff(tbase_list).reshape((-1,1))
meta['tbase_diff'] = tbase_diff
print 'number of interferograms: %d' % (ifgram_num)
print 'number of acquisitions : %d' % (date_num)
print 'number of columns: %d' % (width)
print 'number of lines : %d' % (length)
##### ref_y/x/value
try:
ref_x = int(atr['ref_x'])
ref_y = int(atr['ref_y'])
print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x)
ref_value = np.zeros((ifgram_num,1), np.float32)
for j in range(ifgram_num):
ifgram = ifgram_list[j]
dset = h5ifgram['interferograms'][ifgram].get(ifgram)
ref_value[j] = dset[ref_y,ref_x]
meta['ref_y'] = ref_y
meta['ref_x'] = ref_x
meta['ref_value'] = ref_value
except:
if meta['skip_ref']:
meta['ref_value'] = 0.0
print 'skip checking reference pixel info - This is for SIMULATION ONLY.'
else:
print 'ERROR: No ref_x/y found! Can not invert interferograms without reference in space.'
print 'run seed_data.py '+ifgramFile+' --mark-attribute for a quick referencing.'
sys.exit(1)
h5ifgram.close()
##### Rank of Design matrix for weighted inversion
A, B = ut.design_matrix(ifgramFile, date12_list)
print '-------------------------------------------------------------------------------'
if meta['weight_function'] in ['no','uniform']:
print 'generic least square inversion with min-norm phase velocity'
print ' based on Berardino et al. (2002, IEEE-TGRS)'
print ' OLS for pixels with fully connected network'
print ' SVD for pixels with partially connected network'
if np.linalg.matrix_rank(A) < date_num-1:
print 'WARNING: singular design matrix! Inversion result can be biased!'
print 'continue using its SVD solution on all pixels'
else:
print 'weighted least square (WLS) inversion with min-norm phase, pixelwise'
if np.linalg.matrix_rank(A) < date_num-1:
print 'ERROR: singular design matrix!'
print ' Input network of interferograms is not fully connected!'
print ' Can not invert the weighted least square solution.'
print 'You could try:'
print ' 1) Add more interferograms to make the network fully connected:'
print ' a.k.a., no multiple subsets nor network islands'
print " 2) Use '-w no' option for non-weighted SVD solution."
sys.exit(-1)
print '-------------------------------------------------------------------------------'
##### Invert time-series phase
##Check parallel environment
if meta['weight_function'] in ['no','uniform']:
meta['parallel'] = False
if meta['parallel']:
num_cores, meta['parallel'], Parallel, delayed = ut.check_parallel(1000, print_msg=False)
##Split into chunks to reduce memory usage
r_step = meta['chunk_size']/ifgram_num/width #split in lines
if meta['weight_function'] not in ['no','uniform']: #more memory usage (coherence) for WLS
r_step /= 2.0
if meta['parallel']:
r_step /= num_cores
r_step = int(ceil_to_1(r_step))
meta['row_step'] = r_step
chunk_num = int((length-1)/r_step)+1
if chunk_num > 1:
print 'maximum chunk size: %.1E' % (meta['chunk_size'])
print 'split %d lines into %d patches for processing' % (length, chunk_num)
print ' with each patch up to %d lines' % (r_step)
if meta['parallel']:
print 'parallel processing using %d cores ...' % (min([num_cores,chunk_num]))
##Computing the inversion
box_list = []
for i in range(chunk_num):
r0 = i*r_step
r1 = min([length, r0+r_step])
box = (0,r0,width,r1)
box_list.append(box)
box_num = len(box_list)
if not meta['parallel']:
timeseries = np.zeros((date_num, length, width), np.float32)
timeseriesStd = np.zeros((date_num, length, width), np.float32)
tempCoh = np.zeros((length, width), np.float32)
for i in range(box_num):
if box_num > 1:
print '\n------- Processing Patch %d out of %d --------------' % (i+1, box_num)
box = box_list[i]
ts, tcoh, tsStd = ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box)
tempCoh[box[1]:box[3],box[0]:box[2]] = tcoh
timeseries[:,box[1]:box[3],box[0]:box[2]] = ts
timeseriesStd[:,box[1]:box[3],box[0]:box[2]] = tsStd
else:
##Temp file list
meta['ftemp_base'] = 'timeseries_temp_'
temp_file_list = [meta['ftemp_base']+str(i)+'.h5' for i in range(chunk_num)]
##Computation
Parallel(n_jobs=num_cores)(delayed(ifgram_inversion_patch)\
(ifgramFile, coherenceFile, meta, box) for box in box_list)
##Concatenate temp files
print 'concatenating temporary timeseries files ...'
timeseries = np.zeros((date_num, length, width), np.float32)
tempCoh = np.zeros((length, width), np.float32)
rmCmd = 'rm'
for i in range(chunk_num):
fname = temp_file_list[i]
box = box_list[i]
print 'reading '+fname
h5temp = h5py.File(fname, 'r')
dset = h5temp['timeseries'].get('timeseries')
timeseries[:,box[1]:box[3],box[0]:box[2]] = dset[0:-1,:,:]
tempCoh[box[1]:box[3],box[0]:box[2]] = dset[-1,:,:]
h5temp.close()
rmCmd += ' '+fname
print rmCmd
os.system(rmCmd)
print 'converting phase to range'
phase2range = -1*float(atr['WAVELENGTH'])/(4.*np.pi)
timeseries *= phase2range
timeseriesStd *= abs(phase2range)
##### Calculate time-series attributes
print 'calculating perpendicular baseline timeseries'
pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries(ifgramFile, ifgram_list)
pbase = str(pbase.tolist()).translate(None,'[],') # convert np.array into string separated by white space
pbase_top = str(pbase_top.tolist()).translate(None,'[],')
pbase_bottom = str(pbase_bottom.tolist()).translate(None,'[],')
atr['P_BASELINE_TIMESERIES'] = pbase
atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top
atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom
atr['ref_date'] = date8_list[0]
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
##### Output
## 1. Write time-series file
meta['timeseriesFile'] = write_timeseries_hdf5_file(timeseries, date8_list, atr,\
timeseriesFile=meta['timeseriesFile'])
if not np.all(timeseriesStd == 0.):
meta['timeseriesStdFile'] = write_timeseries_hdf5_file(timeseriesStd, date8_list, atr,\
timeseriesFile=meta['timeseriesStdFile'])
## 2. Write Temporal Coherence File
print 'writing >>> '+meta['tempCohFile']
atr['FILE_TYPE'] = 'temporal_coherence'
atr['UNIT'] = '1'
meta['tempCohFile'] = writefile.write(tempCoh, atr, meta['tempCohFile'])
print 'Time series inversion took ' + str(time.time()-total) +' secs\nDone.'
return meta['timeseriesFile'], meta['tempCohFile']
def ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box=None):
'''
Inputs:
ifgramFile - string, interferograms hdf5 file
coherenceFile - string, coherence hdf5 file
box - 4-tuple, left, upper, right, and lower pixel coordinate of area of interest
meta - dict, including the following attributes:
#Interferograms
length/width - int, file size for each interferogram
ifgram_list - list of string, interferogram dataset name
date12_list - list of string, YYMMDD-YYMMDD
ref_value - np.array in size of (ifgram_num, 1)
reference pixel coordinate in row/column number
ref_y/x - int, reference pixel coordinate in row/column number
#Time-series
date8_list - list of string in YYYYMMDD
tbase_diff - np.array in size of (date_num-1, 1), differential temporal baseline
#Inversion
weight_function - no, fim, var, coh
Outputs:
ts - 3D np.array in size of (date_num, row_num, col_num)
temp_coh - 2D np.array in size of (row_num, col_num)
tsStd - 3D np.array in size of (date_num, row_num, col_num)
'''
##### Get patch size/index
if not box:
box = (0,0,meta['width'],meta['length'])
c0,r0,c1,r1 = box
print 'processing %8d/%d lines ...' % (r1, meta['length'])
## Initiate output data matrixs
row_num = r1-r0
col_num = c1-c0
pixel_num = row_num * col_num
date_num = len(meta['date8_list'])
ts = np.zeros((date_num, pixel_num), np.float32)
tsStd = np.zeros((date_num, pixel_num), np.float32)
temp_coh = np.zeros(pixel_num, np.float32)
##### Mask for pixels to invert
mask = np.ones(pixel_num, np.bool_)
## 1 - Water Mask
if meta['water_mask_file']:
print 'skip pixels on water with mask from file: %s' % (os.path.basename(meta['water_mask_file']))
try: waterMask = readfile.read(meta['water_mask_file'], epoch='waterMask')[0][r0:r1,c0:c1].flatten()
except: waterMask = readfile.read(meta['water_mask_file'], epoch='mask')[0][r0:r1,c0:c1].flatten()
mask *= np.array(waterMask, np.bool_)
## 2 - Mask for Zero Phase in ALL ifgrams
print 'skip pixels with zero/nan value in all interferograms'
ifgram_stack = ut.get_file_stack(ifgramFile)[r0:r1,c0:c1].flatten()
mask *= ~np.isnan(ifgram_stack)
mask *= ifgram_stack != 0.
## Invert pixels on mask 1+2
pixel_num2inv = np.sum(mask)
pixel_idx2inv = np.where(mask)[0]
print 'number of pixels to invert: %s out of %s' % (pixel_num2inv, pixel_num)
if pixel_num2inv < 1:
ts = ts.reshape(date_num, row_num, col_num)
temp_coh = temp_coh.reshape(row_num, col_num)
tsStd = tsStd.reshape(date_num, row_num, col_num)
return ts, temp_coh, tsStd
##### Read interferograms
ifgram_num = len(meta['ifgram_list'])
ifgram_data = np.zeros((ifgram_num, pixel_num), np.float32)
date12_list = meta['date12_list']
if meta['skip_zero_phase']:
print 'skip zero phase value (masked out and filled during phase unwrapping)'
atr = readfile.read_attribute(ifgramFile)
h5ifgram = h5py.File(ifgramFile,'r')
for j in range(ifgram_num):
ifgram = meta['ifgram_list'][j]
d = h5ifgram['interferograms'][ifgram].get(ifgram)[r0:r1,c0:c1].flatten()
if meta['skip_zero_phase']:
d[d != 0.] -= meta['ref_value'][j]
else:
d -= meta['ref_value'][j]
ifgram_data[j] = d
sys.stdout.write('\rreading interferograms %s/%s ...' % (j+1, ifgram_num))
sys.stdout.flush()
print ' '
h5ifgram.close()
#ifgram_data -= meta['ref_value']
## 3 - Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion)
maskAllNet = np.all(ifgram_data, axis=0)
maskAllNet *= mask
maskPartNet = mask ^ maskAllNet
##### Design matrix
A,B = ut.design_matrix(ifgramFile, date12_list)
try: ref_date = str(np.loadtxt('reference_date.txt', dtype=str))
except: ref_date = meta['date8_list'][0]
#print 'calculate decorrelation noise covariance with reference date = %s' % (ref_date)
refIdx = meta['date8_list'].index(ref_date)
timeIdx = [i for i in range(date_num)]
timeIdx.remove(refIdx)
Astd = ut.design_matrix(ifgramFile, date12_list, referenceDate=ref_date)[0]
##### Inversion
if meta['weight_function'] in ['no','uniform']:
if np.sum(maskAllNet) > 0:
print 'inverting pixels with valid phase in all ifgrams with OLS (%.0f pixels) ...' % (np.sum(maskAllNet))
ts1, tempCoh1 = network_inversion_sbas(B, ifgram_data[:,maskAllNet], meta['tbase_diff'], skipZeroPhase=False)
ts[1:,maskAllNet] = ts1
temp_coh[maskAllNet] = tempCoh1
if np.sum(maskPartNet) > 0:
print 'inverting pixels with valid phase in part of ifgrams with SVD ...'
pixel_num2inv = np.sum(maskPartNet)
pixel_idx2inv = np.where(maskPartNet)[0]
prog_bar = ptime.progress_bar(maxValue=pixel_num2inv)
for i in range(pixel_num2inv):
idx = pixel_idx2inv[i]
ts1, tempCoh1 = network_inversion_sbas(B, ifgram_data[:,idx], meta['tbase_diff'], meta['skip_zero_phase'])
ts[1:, idx] = ts1.flatten()
temp_coh[idx] = tempCoh1
prog_bar.update(i+1, every=100, suffix=str(i+1)+'/'+str(pixel_num2inv)+' pixels')
prog_bar.close()
else:
##### Read coherence
coh_data = np.zeros((ifgram_num, pixel_num), np.float32)
h5coh = h5py.File(coherenceFile,'r')
coh_list = sorted(h5coh['coherence'].keys())
coh_list = ut.check_drop_ifgram(h5coh)
for j in range(ifgram_num):
ifgram = coh_list[j]
d = h5coh['coherence'][ifgram].get(ifgram)[r0:r1,c0:c1]
d[np.isnan(d)] = 0.
coh_data[j] = d.flatten()
sys.stdout.write('\rreading coherence %s/%s ...' % (j+1, ifgram_num))
sys.stdout.flush()
print ' '
h5coh.close()
##### Calculate Weight matrix
weight = np.array(coh_data, np.float64)
L = int(atr['ALOOKS']) * int(atr['RLOOKS'])
epsilon = 1e-4
if meta['weight_function'].startswith('var'):
print 'convert coherence to weight using inverse of phase variance'
print ' with phase PDF for distributed scatterers from Tough et al. (1995)'
weight = 1.0 / coherence2phase_variance_ds(weight, L, print_msg=True)
elif meta['weight_function'].startswith(('lin','coh','cor')):
print 'use coherence as weight directly (Perissin & Wang, 2012; Tong et al., 2016)'
weight[weight < epsilon] = epsilon
elif meta['weight_function'].startswith(('fim','fisher')):
print 'convert coherence to weight using Fisher Information Index (Seymour & Cumming, 1994)'
weight = coherence2fisher_info_index(weight, L)
else:
print 'Un-recognized weight function: %s' % meta['weight_function']
sys.exit(-1)
##### Weighted Inversion pixel by pixel
print 'inverting time series ...'
prog_bar = ptime.progress_bar(maxValue=pixel_num2inv)
for i in range(pixel_num2inv):
idx = pixel_idx2inv[i]
ts1, tempCoh1, tsStd1 = network_inversion_wls(A, ifgram_data[:,idx], weight[:,idx], Astd=Astd,\
skipZeroPhase=meta['skip_zero_phase'])
ts[1:, idx] = ts1.flatten()
temp_coh[idx] = tempCoh1
tsStd[timeIdx, idx] = tsStd1.flatten()
prog_bar.update(i+1, every=100, suffix=str(i+1)+'/'+str(pixel_num2inv)+' pixels')
prog_bar.close()
ts = ts.reshape(date_num, row_num, col_num)
temp_coh = temp_coh.reshape(row_num, col_num)
tsStd = tsStd.reshape(date_num, row_num, col_num)
##Write to temp hdf5 files for parallel processing
if meta['parallel']:
fname = meta['ftemp_base']+str(int(r0/meta['row_step']))+'.h5'
print 'writing >>> '+fname
h5temp = h5py.File(fname, 'w')
group = h5temp.create_group('timeseries')
dset = group.create_dataset('timeseries', shape=(date_num+1, row_num, col_num), dtype=np.float32)
dset[0:-1,:,:] = ts
dset[1,:,:] = temp_coh
h5temp.close()
return
else:
return ts, temp_coh, tsStd
def main(argv):
method = 'triangular_consistency' ## or 'bonding_point'
ramp_type = 'plane'
save_rampCor = 'yes'
plot_bonding_points = 'yes'
##### Check Inputs
if len(sys.argv)>2:
try: opts, args = getopt.getopt(argv,'h:f:m:x:y:o:t:',['ramp=','no-ramp-save'])
except getopt.GetoptError: print 'Error while getting args'; Usage(); sys.exit(1)
for opt,arg in opts:
if opt in ['-h','--help']: Usage(); sys.exit()
elif opt in '-f': File = arg
elif opt in '-m': maskFile = arg
elif opt in '-o': outName = arg
elif opt in '-x': x = [int(i) for i in arg.split(',')]; method = 'bonding_point'
elif opt in '-y': y = [int(i) for i in arg.split(',')]; method = 'bonding_point'
elif opt in '-t': templateFile = arg
elif opt in '--ramp' : ramp_type = arg.lower()
elif opt in '--no-ramp-save' : save_rampCor = 'no'
elif len(sys.argv)==2:
if argv[0] in ['-h','--help']: Usage(); sys.exit()
elif os.path.isfile(argv[0]): File = argv[0]; maskFile = argv[1]
else: print 'Input file does not existed: '+argv[0]; sys.exit(1)
else: Usage(); sys.exit(1)
##### Check template file
try:
templateFile
templateContents = readfile.read_template(templateFile)
except: pass
try:
yx = [int(i) for i in templateContents['pysar.unwrapError.yx'].split(',')]
x = yx[1::2]
y = yx[0::2]
method = 'bonding_point'
except: pass
##### Read Mask File
## Priority:
## Input mask file > pysar.mask.file > existed Modified_Mask.h5 > existed Mask.h5
try: maskFile
except:
try: maskFile = templateContents['pysar.mask.file']
except:
if os.path.isfile('Modified_Mask.h5'): maskFile = 'Modified_Mask.h5'
elif os.path.isfile('Mask.h5'): maskFile = 'Mask.h5'
else: print 'No mask found!'; sys.exit(1)
try: Mask,Matr = readfile.read(maskFile); print 'mask: '+maskFile
except: print 'Can not open mask file: '+maskFile; sys.exit(1)
##### Output file name
ext = os.path.splitext(File)[1]
try: outName
except: outName = File.split('.')[0]+'_unwCor'+ext
print '\n**************** Unwrapping Error Correction ******************'
#################### Triangular Consistency (Phase Closure) ####################
if method == 'triangular_consistency':
print 'Phase unwrapping error correction using Triangular Consistency / Phase Closure'
h5file=h5py.File(File)
ifgramList = h5file['interferograms'].keys()
sx = int(h5file['interferograms'][ifgramList[0]].attrs['WIDTH'])
sy = int(h5file['interferograms'][ifgramList[0]].attrs['FILE_LENGTH'])
curls,Triangles,C=ut.get_triangles(h5file)
A,B = ut.design_matrix(h5file)
ligram,lv=np.shape(B)
lcurls=np.shape(curls)[0]
print 'Number of all triangles: '+ str(lcurls)
print 'Number of interferograms: '+ str(ligram)
#print curls
curlfile='curls.h5'
if not os.path.isfile(curlfile):
ut.generate_curls(curlfile,h5file,Triangles,curls)
thr=0.50
curls=np.array(curls); n1=curls[:,0]; n2=curls[:,1]; n3=curls[:,2]
numPixels=sy*sx
print 'reading interferograms...'
data = np.zeros((ligram,numPixels),np.float32)
for ni in range(ligram):
dset=h5file['interferograms'][ifgramList[ni]].get(ifgramList[ni])
d = dset[0:dset.shape[0],0:dset.shape[1]]
data[ni] = d.flatten(1)
print np.shape(data)
print 'reading curls ...'
h5curl=h5py.File(curlfile)
curlList=h5curl['interferograms'].keys()
curlData = np.zeros((lcurls,numPixels),np.float32)
for ni in range(lcurls):
dset=h5curl['interferograms'][curlList[ni]].get(curlList[ni])
d = dset[0:dset.shape[0],0:dset.shape[1]]
curlData[ni] = d.flatten(1)
pi=np.pi
EstUnwrap=np.zeros((ligram,numPixels),np.float32)
#try:
# maskFile=argv[1]
# h5Mask=h5py.File(maskFile)
# dset = h5Mask['mask'].get('mask')
# Mask=dset[0:dset.shape[0],0:dset.shape[1]]
#except:
# dset = h5file['mask'].get('mask')
# Mask=dset[0:dset.shape[0],0:dset.shape[1]]
Mask=Mask.flatten(1)
from scipy.linalg import pinv as pinv
for ni in range(numPixels):
#dU = np.zeros([ligram,1])
#print np.shape(dU)
#print np.shape(data[:,ni])
if Mask[ni]==1:
dU = data[:,ni]
#nan_ndx = dataPoint == 0.
unwCurl = np.array(curlData[:,ni])
#print unwCurl
ind = np.abs(unwCurl)>=thr; N1 =n1[ind]; N2 =n2[ind]; N3 =n3[ind]
indC = np.abs(unwCurl)< thr; Nc1=n1[indC]; Nc2=n2[indC]; Nc3=n3[indC]
N =np.hstack([N1, N2, N3]); UniN =np.unique(N)
Nc=np.hstack([Nc1,Nc2,Nc3]); UniNc=np.unique(Nc)
inter=list(set(UniNc) & set(UniN)) # intersetion
UniNc= list(UniNc)
for x in inter:
UniNc.remove(x)
D=np.zeros([len(UniNc),ligram])
for i in range(len(UniNc)):
D[i,UniNc[i]]=1
AAA=np.vstack([-2*pi*C,D])
#AAA1=np.hstack([AAA,np.zeros([AAA.shape[0],lv])])
#AAA2=np.hstack([-2*pi*np.eye(ligram),B])
#AAAA=np.vstack([AAA1,AAA2])
AAAA=np.vstack([AAA,0.25*np.eye(ligram)])
#print '************************'
#print np.linalg.matrix_rank(C)
#print np.linalg.matrix_rank(AAA)
#print np.linalg.matrix_rank(AAAA)
#print '************************'
#LLL=list(np.dot(C,dU)) + list(np.zeros(np.shape(UniNc)[0]))# + list(dU)
#ind=np.isnan(AAA)
#M1=pinv(AAA)
#M=np.dot(M1,LLL)
#EstUnwrap[:,ni]=np.round(M[0:ligram])*2.0*np.pi
##########
# with Tikhonov regularization:
AAAA=np.vstack([AAA,0.25*np.eye(ligram)])
LLL=list(np.dot(C,dU)) + list(np.zeros(np.shape(UniNc)[0])) + list(np.zeros(ligram))
ind=np.isnan(AAAA)
M1=pinv(AAAA)
M=np.dot(M1,LLL)
EstUnwrap[:,ni]=np.round(M[0:ligram])*2.0*np.pi
#print M[0:ligram]
#print np.round(M[0:ligram])
else:
EstUnwrap[:,ni]=np.zeros([ligram])
if not np.remainder(ni,10000): print 'Processing point: %7d of %7d ' % (ni,numPixels)
##### Output
dataCor = data+EstUnwrap
unwCorFile=File.replace('.h5','')+'_unwCor.h5'; print 'writing >>> '+unwCorFile
h5unwCor=h5py.File(unwCorFile,'w')
gg = h5unwCor.create_group('interferograms')
for i in range(ligram):
group = gg.create_group(ifgramList[i])
dset = group.create_dataset(ifgramList[i], data=np.reshape(dataCor[i,:],[sx,sy]).T, compression='gzip')
for key, value in h5file['interferograms'][ifgramList[i]].attrs.iteritems():
group.attrs[key] = value
try:
MASK=h5file['mask'].get('mask')
gm = h5unwCor.create_group('mask')
dset = gm.create_dataset('mask', data=MASK, compression='gzip')
except: pass
h5unwCor.close()
h5file.close()
h5curl.close()
#################### Bonding Points (Spatial Continuity) ####################
elif method == 'bonding_point':
print 'Phase unwrapping error correction using Bonding Points / Spatial Continuity'
##### Read Bridge Points Info
try:
x
y
if len(x) != len(y) or np.mod(len(x),2) != 0:
print 'Wrong number of bridge points input: '+str(len(x))+' for x, '+str(len(y))+' for y'
Usage(); sys.exit(1)
except: print 'Error in reading bridge points info!'; Usage(); sys.exit(1)
for i in range(0,len(x)):
if Mask[y[i],x[i]] == 0:
print '\nERROR: Connecting point ('+str(y[i])+','+str(x[i])+') is out of masked area! Select them again!\n'
sys.exit(1)
print 'Number of bonding point pairs: '+str(len(x)/2)
print 'Bonding points coordinates:\nx: '+str(x)+'\ny: '+str(y)
## Plot Connecting Pair of Points
if plot_bonding_points == 'yes':
point_yx = ''
line_yx = ''
n_bridge = len(x)/2
for i in range(n_bridge):
pair_yx = str(y[2*i])+','+str(x[2*i])+','+str(y[2*i+1])+','+str(x[2*i+1])
if not i == n_bridge-1:
point_yx += pair_yx+','
line_yx += pair_yx+';'
else:
point_yx += pair_yx
line_yx += pair_yx
try:
plot_cmd = 'view.py --point-yx="'+point_yx+'" --line-yx="'+line_yx+\
'" --nodisplay -o bonding_points.png -f '+maskFile
print plot_cmd
os.system(plot_cmd)
except: pass
##### Ramp Info
ramp_mask = Mask==1
print 'estimate phase ramp during the correction'
print 'ramp type: '+ramp_type
if save_rampCor == 'yes':
outName_ramp = os.path.basename(outName).split(ext)[0]+'_'+ramp_type+ext
########## PySAR ##########
if ext == '.h5':
##### Read
try: h5file=h5py.File(File,'r')
except: print 'ERROR: Cannot open input file: '+File; sys.exit(1)
k=h5file.keys()
if 'interferograms' in k: k[0] = 'interferograms'; print 'Input file is '+k[0]
else: print 'Input file - '+File+' - is not interferograms.'; Usage(); sys.exit(1)
igramList = h5file[k[0]].keys()
igramList = sorted(igramList)
#### Write
h5out = h5py.File(outName,'w')
gg = h5out.create_group(k[0])
print 'writing >>> '+outName
if save_rampCor == 'yes':
h5out_ramp = h5py.File(outName_ramp,'w')
gg_ramp = h5out_ramp.create_group(k[0])
print 'writing >>> '+outName_ramp
##### Loop
print 'Number of interferograms: '+str(len(igramList))
for igram in igramList:
print igram
data = h5file[k[0]][igram].get(igram)[:]
data_ramp,ramp = rm.remove_data_surface(data,ramp_mask,ramp_type)
#ramp = data_ramp - data
data_rampCor = phase_bonding(data_ramp,Mask,x,y)
dataCor = data_rampCor - ramp
group = gg.create_group(igram)
dset = group.create_dataset(igram, data=dataCor, compression='gzip')
for key, value in h5file[k[0]][igram].attrs.iteritems():
group.attrs[key]=value
if save_rampCor == 'yes':
group_ramp = gg_ramp.create_group(igram)
dset = group_ramp.create_dataset(igram, data=data_rampCor, compression='gzip')
for key, value in h5file[k[0]][igram].attrs.iteritems():
group_ramp.attrs[key]=value
try:
mask = h5file['mask'].get('mask');
gm = h5out.create_group('mask')
dset = gm.create_dataset('mask', data=mask[0:mask.shape[0],0:mask.shape[1]], compression='gzip')
except: print 'no mask group found.'
h5file.close()
h5out.close()
if save_rampCor == 'yes':
h5out_ramp.close()
########## ROI_PAC ##########
elif ext == '.unw':
print 'Input file is '+ext
a,data,atr = readfile.read_float32(File);
data_ramp,ramp = rm.remove_data_surface(data,ramp_mask,ramp_type)
#ramp = data_ramp - data
data_rampCor = phase_bonding(data_ramp,Mask,x,y)
dataCor = data_rampCor - ramp
writefile.write(dataCor, atr, outName)
if save_rampCor == 'yes':
writefile.write(data_rampCor,atr,outName_ramp)
else: print 'Un-supported file type: '+ext; Usage(); sys.exit(1)
def main(argv):
##### Inputs
try:
ifgram_file = argv[0]
timeseries_file = argv[1]
except:
usage(); sys.exit(1)
try: outfile = argv[2]
except: outfile = 'reconstructed_'+ifgram_file
atr = readfile.read_attribute(timeseries_file)
length = int(atr['FILE_LENGTH'])
width = int(atr['WIDTH'])
##### Read time-series file
print 'loading timeseries ...'
h5ts = h5py.File(timeseries_file, 'r')
date_list = sorted(h5ts['timeseries'].keys())
date_num = len(date_list)
timeseries = np.zeros((date_num, length*width))
print 'number of acquisitions: '+str(date_num)
prog_bar = ptime.progress_bar(maxValue=date_num)
for i in range(date_num):
date = date_list[i]
d = h5ts['timeseries'].get(date)[:]
timeseries[i,:] = d.flatten(0)
prog_bar.update(i+1, suffix=date)
prog_bar.close()
h5ts.close()
del d
range2phase = -4*np.pi/float(atr['WAVELENGTH'])
timeseries = range2phase*timeseries
##### Estimate interferograms from timeseries
print 'estimating interferograms from timeseries using design matrix from input interferograms'
A,B = ut.design_matrix(ifgram_file)
p = -1*np.ones([A.shape[0],1])
Ap = np.hstack((p,A))
estData = np.dot(Ap, timeseries)
del timeseries
##### Write interferograms file
print 'writing >>> '+outfile
h5 = h5py.File(ifgram_file,'r')
ifgram_list = sorted(h5['interferograms'].keys())
ifgram_num = len(ifgram_list)
date12_list = ptime.list_ifgram2date12(ifgram_list)
h5out = h5py.File(outfile,'w')
group = h5out.create_group('interferograms')
print 'number of interferograms: '+str(ifgram_num)
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for i in range(ifgram_num):
ifgram = ifgram_list[i]
data = np.reshape(estData[i,:],(length, width))
gg = group.create_group(ifgram)
dset = gg.create_dataset(ifgram, data=data, compression='gzip')
for key, value in h5['interferograms'][ifgram].attrs.iteritems():
gg.attrs[key] = value
prog_bar.update(i+1, suffix=date12_list[i])
prog_bar.close()
h5.close()
h5out.close()
print 'Done.'
return outfile
def timeseries_inversion(ifgramFile='unwrapIfgram.h5',
coherenceFile='coherence.h5',
inps_dict=None):
'''Implementation of the SBAS algorithm.
modified from sbas.py written by scott baker, 2012
Inputs:
ifgramFile - string, HDF5 file name of the interferograms
coherenceFile - string, HDF5 file name of the coherence
inps_dict - dict, including the following options:
weight_function
min_coherence
max_coherence
Output:
timeseriesFile - string, HDF5 file name of the output timeseries
tempCohFile - string, HDF5 file name of temporal coherence
'''
total = time.time()
if not inps_dict:
inps_dict = vars(cmdLineParse())
weight_func = inps_dict['weight_function']
min_coh = inps_dict['min_coherence']
max_coh = inps_dict['max_coherence']
# Basic Info
atr = readfile.read_attribute(ifgramFile)
length = int(atr['FILE_LENGTH'])
width = int(atr['WIDTH'])
pixel_num = length * width
h5ifgram = h5py.File(ifgramFile, 'r')
ifgram_list = sorted(h5ifgram['interferograms'].keys())
if inps_dict['weight_function'] == 'no':
ifgram_list = ut.check_drop_ifgram(h5ifgram, atr, ifgram_list)
ifgram_num = len(ifgram_list)
# Convert ifgram_list to date12/8_list
date12_list = ptime.list_ifgram2date12(ifgram_list)
m_dates = [i.split('-')[0] for i in date12_list]
s_dates = [i.split('-')[1] for i in date12_list]
date8_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
date_num = len(date8_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
tbase_diff = np.diff(tbase_list).reshape((date_num - 1, 1))
print 'number of interferograms: ' + str(ifgram_num)
print 'number of acquisitions : ' + str(date_num)
print 'number of pixels: ' + str(pixel_num)
# Reference pixel in space
try:
ref_x = int(atr['ref_x'])
ref_y = int(atr['ref_y'])
print 'reference pixel in y/x: [%d, %d]' % (ref_y, ref_x)
except:
print 'ERROR: No ref_x/y found! Can not inverse interferograms without reference in space.'
print 'run seed_data.py ' + ifgramFile + ' --mark-attribute for a quick referencing.'
sys.exit(1)
##### Read Interferograms
print 'reading interferograms ...'
ifgram_data = np.zeros((ifgram_num, pixel_num), np.float32)
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for j in range(ifgram_num):
ifgram = ifgram_list[j]
d = h5ifgram['interferograms'][ifgram].get(ifgram)[:]
#d[d != 0.] -= d[ref_y, ref_x]
d -= d[ref_y, ref_x]
ifgram_data[j] = d.flatten()
prog_bar.update(j + 1, suffix=date12_list[j])
h5ifgram.close()
prog_bar.close()
#####---------------------- Inversion ----------------------#####
# Design matrix
A, B = ut.design_matrix(ifgramFile, date12_list)
if weight_func == 'no':
print 'generalized inversion using SVD (Berardino et al., 2002, IEEE-TGRS)'
print 'inversing time series ...'
B_inv = np.array(np.linalg.pinv(B), np.float32)
ts_rate = np.dot(B_inv, ifgram_data)
ts1 = ts_rate * np.tile(tbase_diff, (1, pixel_num))
ts0 = np.array([0.] * pixel_num, np.float32)
ts_data = np.vstack((ts0, np.cumsum(ts1, axis=0)))
del ts_rate, ts0, ts1
# Temporal coherence
print 'calculating temporal coherence (Tizzani et al., 2007, RSE)'
temp_coh = np.zeros((1, pixel_num), np.float32) + 0j
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for i in range(ifgram_num):
ifgram_est = np.dot(A[i, :], ts_data[1:, :])
ifgram_diff = ifgram_data[i, :] - ifgram_est
temp_coh += np.exp(1j * ifgram_diff)
prog_bar.update(i + 1, suffix=date12_list[i])
prog_bar.close()
del ifgram_data, ifgram_est, ifgram_diff
temp_coh = np.array((np.absolute(temp_coh) / ifgram_num).reshape(
(length, width)),
dtype=np.float32)
else:
print 'weighted least square (WLS) inversion using coherence pixel by pixel'
if np.linalg.matrix_rank(A) < date_num - 1:
print 'ERROR: singular design matrix!'
print ' Input network of interferograms is not fully connected!'
print ' Can not inverse the weighted least square solution.'
print 'You could try:'
print ' 1) Add more interferograms to make the network fully connected:'
print ' a.k.a., no multiple subsets nor network islands'
print " 2) Use '-w no' option for non-weighted SVD solution."
sys.exit(-1)
pixel_mask = np.ones(pixel_num, np.bool_)
print 'reading coherence: ' + os.path.basename(coherenceFile)
h5coh = h5py.File(coherenceFile, 'r')
coh_list = sorted(h5coh['coherence'].keys())
coh_data = np.zeros((ifgram_num, pixel_num), np.float32)
prog_bar = ptime.progress_bar(maxValue=ifgram_num)
for j in range(ifgram_num):
ifgram = coh_list[j]
d = h5coh['coherence'][ifgram].get(ifgram)[:].flatten()
d[np.isnan(d)] = 0.
pixel_mask[d == 0.] = 0
coh_data[j] = d
prog_bar.update(j + 1, suffix=date12_list[j])
h5coh.close()
prog_bar.close()
# Get mask of valid pixels to inverse
print 'skip pixels with zero coherence in at least one interferogram'
print 'skip pixels with zero phase in all interferograms'
ifgram_stack = ut.get_file_stack(ifgramFile).flatten()
pixel_mask[ifgram_stack == 0.] = 0
pixel_num2inv = np.sum(pixel_mask)
pixel_idx2inv = np.where(pixel_mask)[0]
ifgram_data = ifgram_data[:, pixel_mask]
coh_data = coh_data[:, pixel_mask]
print 'number of pixels to inverse: %d' % (pixel_num2inv)
##### Calculate Weight matrix
weight = coh_data
if weight_func.startswith('var'):
print 'convert coherence to weight using inverse of variance: x**2/(1-x**2) from Hanssen (2001, for 4.2.32)'
weight[weight > 0.999] = 0.999
if weight_func == 'variance-max-coherence':
print 'constrain the max coherence to %f' % max_coh
weight[weight > max_coh] = max_coh
weight = np.square(weight)
weight *= 1. / (1. - weight)
if weight_func == 'variance-log':
print 'use log(1/variance)+1 as weight'
weight = np.log(weight + 1)
elif weight_func.startswith('lin'):
print 'use coherence as weight directly (Tong et al., 2016, RSE)'
elif weight_func.startswith('norm'):
print 'convert coherence to weight using CDF of normal distribution: N(%f, %f)' % (
mu, std)
mu = (min_coh + max_coh) / 2.0
std = (max_coh - min_coh) / 6.0
chunk_size = 1000
chunk_num = int(pixel_num2inv / chunk_size) + 1
prog_bar = ptime.progress_bar(maxValue=chunk_num)
for i in range(chunk_num):
i0 = (i - 1) * chunk_size
i1 = min([pixel_num2inv, i0 + chunk_size])
weight[:, i0:i1] = norm.cdf(weight[:, i0:i1], mu, std)
prog_bar.update(i + 1, every=10)
prog_bar.close()
#weight = norm.cdf(weight, mu, std)
else:
print 'Un-recognized weight function: %s' % weight_func
sys.exit(-1)
##### Weighted Inversion pixel by pixel
print 'inversing time series ...'
ts_data = np.zeros((date_num, pixel_num), np.float32)
temp_coh = np.zeros(pixel_num, np.float32)
prog_bar = ptime.progress_bar(maxValue=pixel_num2inv)
for i in range(pixel_num2inv):
# Inverse timeseries
ifgram_pixel = ifgram_data[:, i]
weight_pixel = weight[:, i]
W = np.diag(weight_pixel)
ts = np.linalg.inv(A.T.dot(W).dot(A)).dot(
A.T).dot(W).dot(ifgram_pixel)
ts_data[1:, pixel_idx2inv[i]] = ts
# Calculate weighted temporal coherence
ifgram_diff = ifgram_pixel - np.dot(A, ts)
temp_coh_pixel = np.abs(
np.sum(np.multiply(weight_pixel, np.exp(1j * ifgram_diff)),
axis=0)) / np.sum(weight_pixel)
temp_coh[pixel_idx2inv[i]] = temp_coh_pixel
prog_bar.update(i + 1, every=2000, suffix=str(i + 1) + ' pixels')
prog_bar.close()
del ifgram_data, weight
#####---------------------- Outputs ----------------------#####
## 1.1 Convert time-series phase to displacement
print 'converting phase to range'
phase2range = -1 * float(atr['WAVELENGTH']) / (4. * np.pi)
ts_data *= phase2range
## 1.2 Write time-series data matrix
timeseriesFile = 'timeseries.h5'
print 'writing >>> ' + timeseriesFile
print 'number of acquisitions: ' + str(date_num)
h5timeseries = h5py.File(timeseriesFile, 'w')
group = h5timeseries.create_group('timeseries')
prog_bar = ptime.progress_bar(maxValue=date_num)
for i in range(date_num):
date = date8_list[i]
dset = group.create_dataset(date,
data=ts_data[i].reshape(length, width),
compression='gzip')
prog_bar.update(i + 1, suffix=date)
prog_bar.close()
## 1.3 Write time-series attributes
print 'calculating perpendicular baseline timeseries'
pbase, pbase_top, pbase_bottom = ut.perp_baseline_ifgram2timeseries(
ifgramFile, ifgram_list)
pbase = str(pbase.tolist()).translate(
None, '[],') # convert np.array into string separated by white space
pbase_top = str(pbase_top.tolist()).translate(None, '[],')
pbase_bottom = str(pbase_bottom.tolist()).translate(None, '[],')
atr['P_BASELINE_TIMESERIES'] = pbase
atr['P_BASELINE_TOP_TIMESERIES'] = pbase_top
atr['P_BASELINE_BOTTOM_TIMESERIES'] = pbase_bottom
atr['ref_date'] = date8_list[0]
atr['FILE_TYPE'] = 'timeseries'
atr['UNIT'] = 'm'
for key, value in atr.iteritems():
group.attrs[key] = value
h5timeseries.close()
del ts_data
## 2. Write Temporal Coherence File
tempCohFile = 'temporalCoherence.h5'
print 'writing >>> ' + tempCohFile
atr['FILE_TYPE'] = 'temporal_coherence'
atr['UNIT'] = '1'
writefile.write(temp_coh.reshape(length, width), atr, tempCohFile)
print 'Time series inversion took ' + str(time.time() -
total) + ' secs\nDone.'
return timeseriesFile, tempCohFile
def ifgram_inversion_patch(ifgramFile, coherenceFile, meta, box=None):
'''
Inputs:
ifgramFile - string, interferograms hdf5 file
coherenceFile - string, coherence hdf5 file
box - 4-tuple, left, upper, right, and lower pixel coordinate of area of interest
meta - dict, including the following attributes:
#Interferograms
length/width - int, file size for each interferogram
ifgram_list - list of string, interferogram dataset name
date12_list - list of string, YYMMDD-YYMMDD
ref_value - np.array in size of (ifgram_num, 1)
reference pixel coordinate in row/column number
ref_y/x - int, reference pixel coordinate in row/column number
#Time-series
date8_list - list of string in YYYYMMDD
tbase_diff - np.array in size of (date_num-1, 1), differential temporal baseline
#Inversion
weight_function -
'''
##### Get patch size/index
if not box:
box = (0, 0, meta['width'], meta['length'])
c0, r0, c1, r1 = box
print 'processing %8d/%d lines ...' % (r1, meta['length'])
## Initiate output data matrixs
row_num = r1 - r0
col_num = c1 - c0
pixel_num = row_num * col_num
date_num = len(meta['date8_list'])
ts = np.zeros((date_num, pixel_num), np.float32)
temp_coh = np.zeros(pixel_num, np.float32)
##### Get mask of non-zero pixels
print 'skip pixels with zero/nan value in all interferograms'
ifgram_stack = ut.get_file_stack(ifgramFile)[r0:r1, c0:c1].flatten()
mask = ~np.isnan(ifgram_stack)
mask[ifgram_stack == 0.] = 0
pixel_num2inv = np.sum(mask)
pixel_idx2inv = np.where(mask)[0]
print 'number of pixels to inverse: %d' % (pixel_num2inv)
if pixel_num2inv < 1:
ts = ts.reshape(date_num, row_num, col_num)
temp_coh = temp_coh.reshape(row_num, col_num)
return ts, temp_coh
##### Read interferograms
ifgram_num = len(meta['ifgram_list'])
ifgram_data = np.zeros((ifgram_num, pixel_num2inv), np.float32)
date12_list = meta['date12_list']
atr = readfile.read_attribute(ifgramFile)
h5ifgram = h5py.File(ifgramFile, 'r')
for j in range(ifgram_num):
ifgram = meta['ifgram_list'][j]
d = h5ifgram['interferograms'][ifgram].get(ifgram)[r0:r1, c0:c1]
ifgram_data[j] = d.flatten()[mask]
sys.stdout.write('\rreading interferograms %s/%s ...' %
(j + 1, ifgram_num))
sys.stdout.flush()
print ' '
h5ifgram.close()
ifgram_data -= meta['ref_value']
##### Design matrix
A, B = ut.design_matrix(ifgramFile, date12_list)
B_inv = np.array(np.linalg.pinv(B), np.float32)
##### Inverse
if meta['weight_function'] in ['no', 'uniform']:
print 'inversing time-series ...'
ts[1:, mask] = network_inversion_sbas(B,
ifgram_data,
meta['tbase_diff'],
B_inv=B_inv)
print 'calculating temporal coherence ...'
temp_coh[mask] = temporal_coherence(A, ts[1:, mask], ifgram_data)
else:
##### Read coherence
coh_data = np.zeros((ifgram_num, pixel_num2inv), np.float32)
h5coh = h5py.File(coherenceFile, 'r')
coh_list = sorted(h5coh['coherence'].keys())
coh_list = ut.check_drop_ifgram(h5coh)
for j in range(ifgram_num):
ifgram = coh_list[j]
d = h5coh['coherence'][ifgram].get(ifgram)[r0:r1, c0:c1]
d[np.isnan(d)] = 0.
coh_data[j] = d.flatten()[mask]
sys.stdout.write('\rreading coherence %s/%s ...' %
(j + 1, ifgram_num))
sys.stdout.flush()
print ' '
h5coh.close()
##### Calculate Weight matrix
weight = coh_data
if meta['weight_function'].startswith('var'):
print 'convert coherence to weight using inverse of phase variance'
print ' with phase PDF for distributed scatterers from Tough et al. (1995)'
L = int(atr['ALOOKS']) * int(atr['RLOOKS'])
lineStr = ' number of multilooks L=%d' % L
if L > 80:
L = 80
lineStr += ', use L=80 to avoid dividing by 0 in calculation with Negligible effect'
print lineStr
weight = 1.0 / coherence2phase_variance_ds(weight, L)
elif meta['weight_function'].startswith(('lin', 'coh', 'cor')):
print 'use coherence as weight directly (Perissin & Wang, 2012; Tong et al., 2016)'
epsilon = 1e-4
weight[weight < epsilon] = epsilon
else:
print 'Un-recognized weight function: %s' % meta['weight_function']
sys.exit(-1)
##### Weighted Inversion pixel by pixel
print 'inversing time series ...'
prog_bar = ptime.progress_bar(maxValue=pixel_num2inv)
for i in range(pixel_num2inv):
ts[1:, pixel_idx2inv[i]] = network_inversion_wls(
A, ifgram_data[:, i], weight[:, i])[0].flatten()
prog_bar.update(i + 1,
every=100,
suffix=str(i + 1) + '/' + str(pixel_num2inv) +
' pixels')
prog_bar.close()
print 'calculating temporal coherence ...'
#temp_coh[mask] = temporal_coherence(A, ts[1:,mask], ifgram_data, weight)
temp_coh[mask] = temporal_coherence(A, ts[1:, mask], ifgram_data)
ts = ts.reshape(date_num, row_num, col_num)
temp_coh = temp_coh.reshape(row_num, col_num)
##Write to temp hdf5 files for parallel processing
if meta['parallel']:
fname = meta['ftemp_base'] + str(int(r0 / meta['row_step'])) + '.h5'
print 'writing >>> ' + fname
h5temp = h5py.File(fname, 'w')
group = h5temp.create_group('timeseries')
dset = group.create_dataset('timeseries',
shape=(date_num + 1, row_num, col_num),
dtype=np.float32)
dset[0:-1, :, :] = ts
dset[1, :, :] = temp_coh
h5temp.close()
return
else:
return ts, temp_coh
