def get_giant_ifg_list(fnames):
m_date_list = []
s_date_list = []
pbase_list = []
ext = os.path.splitext(fnames[0])[1]
if ext == '.h5':
obj = ifgramStack(fnames[0])
obj.open()
m_date_list = obj.mDates[obj.dropIfgram].tolist()
s_date_list = obj.sDates[obj.dropIfgram].tolist()
pbase_list = obj.pbaseIfgram[obj.dropIfgram].tolist()
else:
ifgramNum = len(fnames)
print('Number of interferograms: %d' % (ifgramNum))
for fname in fnames:
atr = readfile.read_attribute(fname)
m_date, s_date = ptime.yymmdd(atr['DATE12'].split('-'))
pbase = (float(atr['P_BASELINE_TOP_HDR']) +
float(atr['P_BASELINE_BOTTOM_HDR'])) / 2.
m_date_list.append(m_date)
s_date_list.append(s_date)
pbase_list.append(pbase)
return m_date_list, s_date_list, pbase_list
def split_into_boxes(ifgram_file, chunk_size=100e6, print_msg=True):
"""Split into chunks in rows to reduce memory usage
Parameters:
"""
shape = ifgramStack(ifgram_file).get_size()
# Get r_step / chunk_num
r_step = chunk_size / (shape[0] * shape[2]) # split in lines
#if inps.weightFunc != 'sbas': # more memory usage (coherence) for WLS
# r_step /= 2.0
r_step = int(ceil_to_1(r_step))
chunk_num = int((shape[1] - 1) / r_step) + 1
if print_msg and chunk_num > 1:
print('maximum chunk size: %.1E' % chunk_size)
print('split %d lines into %d patches for processing' %
(shape[1], chunk_num))
print(' with each patch up to %d lines' % r_step)
# Computing the inversion
box_list = []
for i in range(chunk_num):
r0 = i * r_step
r1 = min([shape[1], r0 + r_step])
box = (0, r0, shape[2], r1)
box_list.append(box)
return box_list
def print_date_list(fname, disp_num=False, drop_ifgram=False, print_msg=False):
"""Print time/date info of file"""
atr = readfile.read_attribute(fname)
k = atr['FILE_TYPE']
dateList = None
if k in ['timeseries']:
dateList = timeseries(fname).get_date_list()
elif k == 'HDFEOS':
obj = HDFEOS(fname)
obj.open(print_msg=False)
dateList = obj.dateList
elif k == 'giantTimeseries':
obj = giantTimeseries(fname)
obj.open(print_msg=False)
dateList = obj.dateList
elif k in ['ifgramStack']:
dateList = ifgramStack(fname).get_date12_list(dropIfgram=drop_ifgram)
elif k in ['giantIfgramStack']:
obj = giantIfgramStack(fname)
obj.open(print_msg=False)
dateList = obj.date12List
else:
print('--date option can not be applied to {} file, ignore it.'.format(k))
if print_msg and dateList is not None:
for i in range(len(dateList)):
if disp_num:
print('{}\t{}'.format(dateList[i], i))
else:
print(dateList[i])
return dateList
def read_network_info(inps):
k = readfile.read_attribute(inps.ifgram_file)['FILE_TYPE']
if k != 'ifgramStack':
raise ValueError('input file {} is not ifgramStack: {}'.format(
inps.ifgram_file, k))
obj = ifgramStack(inps.ifgram_file)
obj.open()
inps.date12_list = obj.get_date12_list(dropIfgram=False)
date12_kept = obj.get_date12_list(dropIfgram=True)
inps.ex_date12_list = sorted(
list(set(inps.date12_list) - set(date12_kept)))
inps.date_list = obj.get_date_list(dropIfgram=False)
print('number of all interferograms: {}'.format(len(inps.date12_list)))
print('number of dropped interferograms: {}'.format(
len(inps.ex_date12_list)))
print('number of kept interferograms: {}'.format(
len(inps.date12_list) - len(inps.ex_date12_list)))
print('number of acquisitions: {}'.format(len(inps.date_list)))
if not inps.yx:
inps.yx = (obj.refY, obj.refX)
print('plot initial coherence matrix at reference pixel: {}'.format(
inps.yx))
return inps
def update_object(outFile, inObj, box, updateMode=True):
"""Do not write h5 file if: 1) h5 exists and readable,
2) it contains all date12 from ifgramStackDict,
or all datasets from geometryDict"""
write_flag = True
if updateMode and ut.run_or_skip(outFile, check_readable=True) == 'skip':
if inObj.name == 'ifgramStack':
in_size = inObj.get_size(box=box)[1:]
in_date12_list = inObj.get_date12_list()
outObj = ifgramStack(outFile)
out_size = outObj.get_size()[1:]
out_date12_list = outObj.get_date12_list(dropIfgram=False)
if out_size == in_size and set(in_date12_list).issubset(
set(out_date12_list)):
print((
'All date12 exists in file {} with same size as required,'
' no need to re-load.'.format(outFile)))
write_flag = False
elif inObj.name == 'geometry':
outObj = geometry(outFile)
outObj.open(print_msg=False)
if (outObj.get_size() == inObj.get_size(box=box)
and all(i in outObj.datasetNames
for i in inObj.get_dataset_list())):
print((
'All datasets exists in file {} with same size as required,'
' no need to re-load.'.format(outFile)))
write_flag = False
return write_flag
def get_date12_list(File, dropIfgram=False):
"""Read Date12 info from input file: Pairs.list or multi-group hdf5 file
Inputs:
File - string, path/name of input multi-group hdf5 file or text file
check_drop_ifgram - bool, check the "DROP_IFGRAM" attribute or not for multi-group hdf5 file
Output:
date12_list - list of string in YYMMDD-YYMMDD format
Example:
date12List = get_date12_list('unwrapIfgram.h5')
date12List = get_date12_list('unwrapIfgram.h5', check_drop_ifgram=True)
date12List = get_date12_list('Pairs.list')
"""
date12_list = []
ext = os.path.splitext(File)[1].lower()
if ext == '.h5':
k = readfile.read_attribute(File)['FILE_TYPE']
if k == 'ifgramStack':
date12_list = ifgramStack(File).get_date12_list(
dropIfgram=dropIfgram)
else:
return None
else:
txtContent = np.loadtxt(File, dtype=bytes).astype(str)
if len(txtContent.shape) == 1:
txtContent = txtContent.reshape(-1, 1)
date12_list = [i for i in txtContent[:, 0]]
date12_list = sorted(date12_list)
return date12_list
def get_nonzero_phase_closure(ifgram_file,
out_file=None,
thres=0.1,
unwDatasetName='unwrapPhase'):
"""Calculate/Read number of non-zero phase closure
Parameters: ifgram_file : string, path of ifgram stack file
out_file : string, path of num non-zero phase closure file
Returns: num_nonzero_closure : 2D np.array in size of (length, width)
"""
if not out_file:
out_file = 'numNonzeroPhaseClosure_{}.h5'.format(unwDatasetName)
if os.path.isfile(out_file) and readfile.read_attribute(out_file):
print('1. read number of nonzero phase closure from file: {}'.format(
out_file))
num_nonzero_closure = readfile.read(out_file)[0]
else:
obj = ifgramStack(ifgram_file)
obj.open(print_msg=False)
length, width = obj.length, obj.width
ref_phase = obj.get_reference_phase(unwDatasetName=unwDatasetName,
dropIfgram=False)
C = obj.get_design_matrix4triplet(
obj.get_date12_list(dropIfgram=False))
# calculate phase closure line by line to save memory usage
num_nonzero_closure = np.zeros((length, width), np.float32)
print(
'1. calculating phase closure of all pixels from dataset - {} ...'.
format(unwDatasetName))
line_step = 10
num_loop = int(np.ceil(length / line_step))
prog_bar = ptime.progressBar(maxValue=num_loop)
for i in range(num_loop):
# read phase
i0, i1 = i * line_step, min(length, (i + 1) * line_step)
box = (0, i0, width, i1)
pha_data = ifginv.read_unwrap_phase(obj,
box,
ref_phase,
unwDatasetName=unwDatasetName,
dropIfgram=False,
print_msg=False)
# calculate phase closure
pha_closure = np.dot(C, pha_data)
pha_closure = np.abs(pha_closure - ut.wrap(pha_closure))
# get number of non-zero phase closure
num_nonzero = np.sum(pha_closure >= thres, axis=0)
num_nonzero_closure[i0:i1, :] = num_nonzero.reshape(i1 - i0, width)
prog_bar.update(i + 1,
every=1,
suffix='{}/{} lines'.format((i + 1) * line_step,
length))
prog_bar.close()
atr = dict(obj.metadata)
atr['FILE_TYPE'] = 'mask'
atr['UNIT'] = 1
writefile.write(num_nonzero_closure, out_file=out_file, metadata=atr)
return num_nonzero_closure
def read_network_info(inps):
k = readfile.read_attribute(inps.ifgram_file)['FILE_TYPE']
if k != 'ifgramStack':
raise ValueError('input file {} is not ifgramStack: {}'.format(inps.ifgram_file, k))
obj = ifgramStack(inps.ifgram_file)
obj.open(print_msg=inps.print_msg)
inps.date12_list = obj.get_date12_list(dropIfgram=False)
date12_kept = obj.get_date12_list(dropIfgram=True)
inps.ex_date12_list = sorted(list(set(inps.date12_list) - set(date12_kept)))
inps.date_list = obj.get_date_list(dropIfgram=False)
vprint('number of all interferograms: {}'.format(len(inps.date12_list)))
vprint('number of dropped interferograms: {}'.format(len(inps.ex_date12_list)))
vprint('number of kept interferograms: {}'.format(len(inps.date12_list) - len(inps.ex_date12_list)))
vprint('number of acquisitions: {}'.format(len(inps.date_list)))
if inps.lalo:
if not inps.lookup_file:
lookup_file = os.path.join(os.path.dirname(inps.ifgram_file), 'geometry*.h5')
inps.lookup_file = ut.get_lookup_file(filePattern=lookup_file)
coord = ut.coordinate(obj.metadata, lookup_file=inps.lookup_file)
inps.yx = coord.geo2radar(inps.lalo[0], inps.lalo[1])[0:2]
if not inps.yx:
inps.yx = (obj.refY, obj.refX)
vprint('plot initial coherence matrix at reference pixel: {}'.format(inps.yx))
return inps
def timeseries2ifgram(ts_file, ifgram_file, out_file='reconUnwrapIfgram.h5'):
# read time-series
atr = readfile.read_attribute(ts_file)
range2phase = -4. * np.pi / float(atr['WAVELENGTH'])
print('reading timeseries data from file {} ...'.format(ts_file))
ts_data = readfile.read(ts_file)[0] * range2phase
num_date, length, width = ts_data.shape
ts_data = ts_data.reshape(num_date, -1)
# reconstruct unwrapPhase
print('reconstructing the interferograms from timeseries')
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
A1 = stack_obj.get_design_matrix4timeseries_estimation(dropIfgram=False)[0]
num_ifgram = A1.shape[0]
A0 = -1. * np.ones((num_ifgram, 1))
A = np.hstack((A0, A1))
ifgram_est = np.dot(A, ts_data).reshape(num_ifgram, length, width)
ifgram_est = np.array(ifgram_est, dtype=ts_data.dtype)
del ts_data
# write to ifgram file
dsDict = {}
dsDict['unwrapPhase'] = ifgram_est
writefile.write(dsDict, out_file=out_file, ref_file=ifgram_file)
return ifgram_file
def calculate_temporal_coherence(ifgram_file,
timeseries_file,
ifg_num_file=None,
chunk_size=100e6):
"""Calculate temporal coherence based on input timeseries file and interferograms file
Parameters: ifgram_file : str, path of interferograms file
timeseries_file : str, path of time series file
ifg_num_file : str, path of file for number of interferograms used in inversion.
chunk_size : float
Returns: temp_coh : 2D np.array, temporal coherence in float32
"""
# get box list and size info
box_list = ifginv.split_into_boxes(ifgram_file, chunk_size=chunk_size)
num_box = len(box_list)
stack_shape = ifgramStack(ifgram_file).get_size()
temp_coh = np.zeros(stack_shape[1:3], np.float32)
for i in range(num_box):
if num_box > 1:
print('\n------- Processing Patch %d out of %d --------------' %
(i + 1, num_box))
box = box_list[i]
temp_cohi = calculate_temporal_coherence_patch(
ifgram_file, timeseries_file, box=box, ifg_num_file=ifg_num_file)
temp_coh[box[1]:box[3], box[0]:box[2]] = temp_cohi
return temp_coh
def get_ifgram_reference_phase(ifgram_file, skip_reference=False):
"""Read refPhase"""
stack_obj = ifgramStack(ifgram_file)
stack_obj.get_size()
stack_obj.get_metadata()
try:
ref_y = int(stack_obj.metadata['REF_Y'])
ref_x = int(stack_obj.metadata['REF_X'])
ref_phase = np.squeeze(
stack_obj.read(datasetName='unwrapPhase',
box=(ref_x, ref_y, ref_x + 1, ref_y + 1),
dropIfgram=False,
print_msg=False))
print('reference pixel in y/x: {}'.format((ref_y, ref_x)))
except:
if skip_reference:
ref_phase = np.zeros((stack_obj.numIfgram, ), np.float32)
print(
'skip checking reference pixel info - This is for SIMULATION ONLY.'
)
else:
msg = 'ERROR: No REF_X/Y found! Can not invert interferograms without reference in space.'
msg += '\nrun reference_point.py {} for a quick referencing.'.format(
stack_obj.file)
raise Exception(msg)
return ref_phase
def run_unwrap_error_closure(inps, dsNameIn='unwrapPhase', dsNameOut='unwrapPhase_phaseClosure', fast_mode=False):
"""Run unwrapping error correction in network of interferograms using phase closure.
Parameters: inps : Namespace of input arguments including the following:
ifgram_file : string, path of ifgram stack file
maskFile : string, path of mask file mark the pixels to be corrected
dsNameIn : string, dataset name to read in
dsnameOut : string, dataset name to write out
fast_mode : bool, enable fast processing mode or not
Returns: inps.ifgram_file : string, path of corrected ifgram stack file
"""
print('-'*50)
print('Unwrapping Error Coorrection based on Phase Closure Consistency (Fattahi, 2015) ...')
if fast_mode:
print('fast mode: ON, the following asuumption is ignored for fast processing')
print('\tzero phase jump constraint on ifgrams without unwrap error')
else:
print(('this step can be very slow'
'you could terminate this and try --fast option for quick correction process.'))
print('-'*50)
stack_obj = ifgramStack(inps.ifgram_file)
stack_obj.open()
ref_phase = stack_obj.get_reference_phase(unwDatasetName=dsNameIn, dropIfgram=False)
num_nonzero_closure = np.zeros((stack_obj.length, stack_obj.width), dtype=np.int16)
# split ifgram_file into blocks to save memory
num_tri = stack_obj.get_design_matrix4ifgram_triangle(dropIfgram=True).shape[0]
length, width = stack_obj.get_size()[1:3]
box_list = ifginv.split_into_boxes(dataset_shape=(num_tri, length, width), chunk_size=200e6)
num_box = len(box_list)
for i in range(num_box):
box = box_list[i]
if num_box > 1:
print('\n------- Processing Patch {} out of {} --------------'.format(i+1, num_box))
# estimate/correct ifgram
unw_cor, num_closure = run_unwrap_error_patch(inps.ifgram_file,
box=box,
mask_file=inps.maskFile,
ref_phase=ref_phase,
fast_mode=fast_mode,
dsNameIn=dsNameIn)
num_nonzero_closure[box[1]:box[3], box[0]:box[2]] = num_closure
# write ifgram
write_hdf5_file_patch(inps.ifgram_file,
data=unw_cor,
box=box,
dsName=dsNameOut)
# write number of nonzero phase closure into file
num_file = 'numNonzeroPhaseClosure.h5'
atr = dict(stack_obj.metadata)
atr['FILE_TYPE'] = 'mask'
atr['UNIT'] = '1'
writefile.write(num_nonzero_closure, out_file=num_file, metadata=atr)
print('writing >>> {}'.format(num_file))
return inps.ifgram_file
def main(iargs=None):
inps = cmd_line_parse(iargs)
if inps.template_file:
inps = read_template2inps(inps.template_file, inps)
elif all(not i for i in [
inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax,
inps.connNumMax, inps.excludeIfgIndex, inps.excludeDate,
inps.coherenceBased, inps.startDate, inps.endDate, inps.reset,
inps.manual
]):
msg = 'No input option found to remove interferogram, exit.\n'
msg += 'To manually modify network, please use --manual option '
raise Exception(msg)
if inps.reset:
print('--------------------------------------------------')
reset_network(inps.file)
return inps.file
inps.date12_to_drop = get_date12_to_drop(inps)
if inps.date12_to_drop is not None:
ifgramStack(inps.file).update_drop_ifgram(
date12List_to_drop=inps.date12_to_drop)
print('--------------------------------------------------')
ut.nonzero_mask(inps.file)
print('--------------------------------------------------')
ut.temporal_average(inps.file,
datasetName='coherence',
updateMode=True)
# Touch spatial average txt file of coherence if it's existed
ut.touch(
os.path.splitext(os.path.basename(inps.file))[0] +
'_coherence_spatialAvg.txt')
# Plot result
if inps.plot:
print('\nplot modified network and save to file.')
plotCmd = 'plot_network.py {} --nodisplay'.format(inps.file)
if inps.template_file:
plotCmd += ' --template {}'.format(inps.template_file)
print(plotCmd)
os.system(plotCmd)
print('Done.')
return
def calculate_temporal_coherence_patch(ifgram_file,
timeseries_file,
box=None,
ifg_num_file=None):
atr = readfile.read_attribute(timeseries_file)
if not box:
box = (0, 0, int(atr['WIDTH']), int(atr['LENGTH']))
# Read timeseries data
ts_obj = timeseries(timeseries_file)
ts_obj.open(print_msg=False)
print('reading timeseries data from file: {}'.format(timeseries_file))
ts_data = ts_obj.read(box=box, print_msg=False).reshape(ts_obj.numDate, -1)
ts_data = ts_data[1:, :]
ts_data *= -4 * np.pi / float(atr['WAVELENGTH'])
# Read ifgram data
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
A = stack_obj.get_design_matrix4timeseries_estimation(dropIfgram=True)[0]
print('reading unwrapPhase data from file: {}'.format(ifgram_file))
ifgram_data = stack_obj.read(datasetName='unwrapPhase',
box=box).reshape(A.shape[0], -1)
ref_value = stack_obj.get_reference_phase(dropIfgram=True).reshape((-1, 1))
ifgram_data -= np.tile(ref_value, (1, ifgram_data.shape[1]))
ifgram_diff = ifgram_data - np.dot(A, ts_data)
del ts_data
pixel_num = ifgram_data.shape[1]
temp_coh = np.zeros((pixel_num), np.float32)
# (fast) nasty solution, which used all phase value including invalid zero phase
if not ifg_num_file:
temp_coh = np.abs(np.sum(np.exp(1j * ifgram_diff),
axis=0)) / ifgram_diff.shape[0]
# (slow) same solution as ifgram_inversion.py, considering:
# 1) invalid zero phase in ifgram
# 2) design matrix rank deficiency.
else:
print(
'considering different number of interferograms used in network inversion for each pixel'
)
ifg_num_map = readfile.read(ifg_num_file, box=box)[0].flatten()
prog_bar = ptime.progressBar(maxValue=pixel_num)
for i in range(pixel_num):
if ifg_num_map[i] > 0:
idx = ifgram_data[:, i] != 0.
temp_diff = ifgram_diff[idx, i]
temp_coh[i] = np.abs(np.sum(np.exp(1j * temp_diff),
axis=0)) / temp_diff.shape[0]
prog_bar.update(i + 1,
every=1000,
suffix='{}/{}'.format(i + 1, pixel_num))
prog_bar.close()
temp_coh = np.reshape(temp_coh, (box[3] - box[1], box[2] - box[0]))
return temp_coh
def manual_select_pairs_to_remove(stackFile):
"""Manually select interferograms to remove"""
print('\n-------------------------------------------------------------')
print('Manually select interferograms to remove')
print('1) click two dates/points to select one pair of interferogram')
print('2) repeat until you select all pairs you would like to remove')
print('3) close the figure to continue the program ...')
print('-------------------------------------------------------------\n')
obj = ifgramStack(stackFile)
obj.open()
date12ListAll = obj.date12List
pbase = obj.get_perp_baseline_timeseries(dropIfgram=False)
dateList = obj.dateList
datesNum = mdates.date2num(np.array(ptime.date_list2vector(dateList)[0]))
date12ListKept = obj.get_date12_list(dropIfgram=True)
date12ListDropped = sorted(list(set(date12ListAll) - set(date12ListKept)))
# Display the network
fig = plt.figure()
ax = fig.add_subplot(111)
ax = pp.plot_network(ax,
date12ListAll,
dateList,
list(pbase),
date12List_drop=date12ListDropped)
print('display the network of interferogram of file: ' + stackFile)
date_click = []
date12_click = []
def onclick(event):
idx = nearest_neighbor(event.xdata, event.ydata, datesNum, pbase)
print('click at ' + dateList[idx])
date_click.append(dateList[idx])
if len(date_click) % 2 == 0 and date_click[-2] != date_click[-1]:
[mDate, sDate] = sorted(date_click[-2:])
mIdx = dateList.index(mDate)
sIdx = dateList.index(sDate)
date12 = mDate + '_' + sDate
if date12 in date12ListAll:
print('select date12: ' + date12)
date12_click.append(date12)
ax.plot([datesNum[mIdx], datesNum[sIdx]],
[pbase[mIdx], pbase[sIdx]],
'r',
lw=4)
else:
print(date12 + ' is not existed in input file')
plt.draw()
cid = fig.canvas.mpl_connect('button_press_event', onclick)
plt.show()
if not ut.yes_or_no('Proceed to drop the ifgrams/date12?'):
date12_click = None
return date12_click
def write2hdf5_file(ifgram_file,
metadata,
ts,
temp_coh,
ts_std=None,
num_inv_ifg=None,
suffix='',
inps=None):
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
date_list = stack_obj.get_date_list(dropIfgram=True)
# File 1 - timeseries.h5
ts_file = '{}{}.h5'.format(suffix, os.path.splitext(inps.outfile[0])[0])
metadata['REF_DATE'] = date_list[0]
metadata['FILE_TYPE'] = 'timeseries'
metadata['UNIT'] = 'm'
print('-' * 50)
print('converting phase to range')
phase2range = -1 * float(stack_obj.metadata['WAVELENGTH']) / (4. * np.pi)
ts *= phase2range
print('calculating perpendicular baseline timeseries')
pbase = stack_obj.get_perp_baseline_timeseries(dropIfgram=True)
ts_obj = timeseries(ts_file)
ts_obj.write2hdf5(data=ts, dates=date_list, bperp=pbase, metadata=metadata)
# File 2 - temporalCoherence.h5
out_file = '{}{}.h5'.format(suffix, os.path.splitext(inps.outfile[1])[0])
metadata['FILE_TYPE'] = 'temporalCoherence'
metadata['UNIT'] = '1'
print('-' * 50)
writefile.write(temp_coh, out_file=out_file, metadata=metadata)
# File 3 - timeseriesDecorStd.h5
if not np.all(ts_std == 0.):
out_file = 'timeseriesDecorStd{}.h5'.format(suffix)
metadata['FILE_TYPE'] = 'timeseries'
metadata['UNIT'] = 'm'
ts_std *= abs(phase2range)
print('-' * 50)
writefile.write(ts_std,
out_file=out_file,
metadata=metadata,
ref_file=ts_file)
# File 4 - numInvIfgram.h5
out_file = 'numInvIfgram{}.h5'.format(suffix)
metadata['FILE_TYPE'] = 'mask'
metadata['UNIT'] = '1'
print('-' * 50)
writefile.write(num_inv_ifg, out_file=out_file, metadata=metadata)
return
def reset_network(stackFile):
"""Reset/restore all pairs within the input file by set all DROP_IFGRAM=no"""
print("reset dataset 'dropIfgram' to True for all interferograms for file: "+stackFile)
obj = ifgramStack(stackFile)
obj.open(print_msg=False)
if np.all(obj.dropIfgram):
print('All dropIfgram are already True, no need to reset.')
else:
with h5py.File(stackFile, 'r+') as f:
f['dropIfgram'][:] = True
ut.touch(os.path.splitext(os.path.basename(stackFile))[0]+'_coherence_spatialAvg.txt')
return stackFile
def main(iargs=None):
inps = cmd_line_parse(iargs)
if inps.templateFile:
inps = read_template2inps(inps.templateFile, inps)
inps.timeseriesFile, inps.tempCohFile = inps.outfile
# --fast option
if inps.fast:
print("Enable fast network inversion.")
if inps.weightFunc != 'no':
inps.weightFunc = 'no'
print("\tforcing weightFunc = 'no'")
if inps.maskDataset is not None:
inps.maskDataset = None
print("\tforcing maskDataset = None")
# --dset option
if not inps.unwDatasetName:
stack_obj = ifgramStack(inps.ifgramStackFile)
stack_obj.open(print_msg=False)
inps.unwDatasetName = [
i for i in [
'unwrapPhase_bridging_phaseClosure', 'unwrapPhase_bridging',
'unwrapPhase_phaseClosure', 'unwrapPhase'
] if i in stack_obj.datasetNames
][0]
# --update option
inps.numIfgram = len(
ifgramStack(inps.ifgramStackFile).get_date12_list(dropIfgram=True))
if inps.update_mode and run_or_skip(inps) == 'skip':
return inps.outfile
# Network Inversion
if inps.residualNorm == 'L2':
ifgram_inversion(inps.ifgramStackFile, inps)
else:
raise NotImplementedError('L1 norm minimization is not fully tested.')
#ut.timeseries_inversion_L1(inps.ifgramStackFile, inps.timeseriesFile)
return inps.outfile
def nonzero_mask(File, out_file='maskConnComp.h5', datasetName=None):
"""Generate mask file for non-zero value of input multi-group hdf5 file"""
atr = readfile.read_attribute(File)
k = atr['FILE_TYPE']
if k == 'ifgramStack':
mask = ifgramStack(File).nonzero_mask(datasetName=datasetName)
else:
print('Only ifgramStack file is supported for now, input is '+k)
return None
atr['FILE_TYPE'] = 'mask'
writefile.write(mask, out_file=out_file, metadata=atr)
return out_file
def write_hdf5_file_patch(ifgram_file,
data,
box=None,
dsName='unwrapPhase_phaseClosure'):
"""Write a patch of 3D dataset into an existing h5 file.
Parameters: ifgram_file : string, name/path of output hdf5 file
data : 3D np.array to be written
box : tuple of 4 int, indicating of (x0, y0, x1, y1) of data in file
dsName : output dataset name
Returns: ifgram_file
"""
num_ifgram, length, width = ifgramStack(ifgram_file).get_size(
dropIfgram=False)
if not box:
box = (0, 0, width, length)
num_row = box[3] - box[1]
num_col = box[2] - box[0]
# write to existing HDF5 file
print('open {} with r+ mode'.format(ifgram_file))
f = h5py.File(ifgram_file, 'r+')
# get h5py.Dataset
msg = 'dataset /{d} of {t:<10} in size of {s}'.format(d=dsName,
t=str(data.dtype),
s=(num_ifgram,
box[3], box[2]))
if dsName in f.keys():
print('update ' + msg)
ds = f[dsName]
else:
print('create ' + msg)
ds = f.create_dataset(dsName, (num_ifgram, num_row, num_col),
maxshape=(None, None, None),
chunks=True,
compression=None)
# resize h5py.Dataset if current size is not enough
if ds.shape != (num_ifgram, length, width):
ds.resize((num_ifgram, box[3], box[2]))
# write data to file
ds[:, box[1]:box[3], box[0]:box[2]] = data
ds.attrs['MODIFICATION_TIME'] = str(time.time())
f.close()
print('close {}'.format(ifgram_file))
return ifgram_file
def print_date_list(fname, disp_ifgram='all', disp_num=False, print_msg=False):
"""Print time/date info of file"""
k = readfile.read_attribute(fname)['FILE_TYPE']
dateList = None
if k in ['timeseries']:
dateList = timeseries(fname).get_date_list()
elif k == 'HDFEOS':
dateList = HDFEOS(fname).get_date_list()
elif k == 'giantTimeseries':
dateList = giantTimeseries(fname).get_date_list()
elif k in ['giantIfgramStack']:
dateList = giantIfgramStack(fname).get_date12_list()
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
dateListAll = obj.get_date12_list(dropIfgram=False)
dateListKept = obj.get_date12_list(dropIfgram=True)
# show dropped ifgram or not
if disp_ifgram == 'all':
dateList = list(dateListAll)
elif disp_ifgram == 'kept':
dateList = list(dateListKept)
else:
dateList = sorted(list(set(dateListAll) - set(dateListKept)))
else:
print('--date option can not be applied to {} file, ignore it.'.format(
k))
# print list info
if print_msg and dateList is not None:
for d in dateList:
if disp_num:
if k in ['ifgramStack']:
num = dateListAll.index(d)
else:
num = dateList.index(d)
msg = '{}\t{}'.format(d, num)
else:
msg = d
print(msg)
return dateList
def read_input_index_list(idxList, stackFile=None):
"""Read ['2','3:5','10'] into ['2','3','4','5','10']"""
idxListOut = []
for idx in idxList:
c = sorted([int(i) for i in idx.split(':')])
if len(c) == 2:
idxListOut += list(range(c[0], c[1]+1))
elif len(c) == 1:
idxListOut.append(c[0])
else:
print('Unrecoganized input: '+idx)
idxListOut = sorted(set(idxListOut))
if stackFile:
obj = ifgramStack(stackFile)
obj.open(print_msg=False)
idxListOut = [i for i in idxListOut if i < obj.numIfgram]
obj.close(print_msg=False)
return idxListOut
def main(iargs=None):
inps = cmd_line_parse(iargs)
if not inps.datasetNameOut:
inps.datasetNameOut = '{}_bridging'.format(inps.datasetNameIn)
# update mode checking
atr = readfile.read_attribute(inps.ifgram_file)
if inps.update and atr['FILE_TYPE'] == 'ifgramStack':
obj = ifgramStack(inps.ifgram_file)
obj.open(print_msg=False)
if inps.datasetNameOut in obj.datasetNames:
print(("update mode is enabled AND {} already exists"
" skip this step.").format(inps.datasetNameOut))
return inps.ifgram_file
start_time = time.time()
# get mask_cc from phase closure
mask_cc_file = detect_unwrap_error(ifgram_file=inps.ifgram_file,
mask_file=inps.maskFile,
mask_cc_file='maskConnComp.h5',
unwDatasetName=inps.datasetNameIn,
cutoff=inps.cutoff)
# run bridging
bridges = search_bridge(mask_cc_file, radius=inps.bridgePtsRadius)
run_unwrap_error_bridge(inps.ifgram_file,
mask_cc_file,
bridges,
dsNameIn=inps.datasetNameIn,
dsNameOut=inps.datasetNameOut,
ramp_type=inps.ramp)
if inps.run_closure:
print('')
inps.datasetNameIn = inps.datasetNameOut
inps.datasetNameOut = '{}_phaseClosure'.format(inps.datasetNameIn)
run_unwrap_error_closure(inps,
dsNameIn=inps.datasetNameIn,
dsNameOut=inps.datasetNameOut,
fast_mode=True)
m, s = divmod(time.time() - start_time, 60)
print('\ntime used: {:02.0f} mins {:02.1f} secs\nDone.'.format(m, s))
return inps.ifgram_file
def split_ifgram_file(ifgram_file, chunk_size=100e6):
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
metadata = dict(stack_obj.metadata)
# get reference phase
ref_phase = get_ifgram_reference_phase(ifgram_file)
# get list of boxes
box_list = split_into_boxes(ifgram_file,
chunk_size=chunk_size,
print_msg=True)
num_box = len(box_list)
# read/write each patch file
outfile_list = []
for i in range(num_box):
box = box_list[i]
outfile = '{}_{:03d}{}'.format(
os.path.splitext(ifgram_file)[0], i + 1,
os.path.splitext(ifgram_file)[1])
# datasets
print('-' * 50)
print('reading all datasets in {} from file: {} ...'.format(
box, ifgram_file))
dsNames = readfile.get_dataset_list(ifgram_file)
dsDict = {}
dsDict['refPhase'] = ref_phase
for dsName in dsNames:
data = stack_obj.read(datasetName=dsName, box=box, print_msg=False)
dsDict[dsName] = data
# metadata
metadata['LENGTH'] = box[3] - box[1]
metadata['WIDTH'] = box[2] - box[0]
writefile.write(dsDict,
out_file=outfile,
metadata=metadata,
ref_file=ifgram_file)
outfile_list.append(outfile)
return outfile_list
def select_network_candidate(inps):
date_list, tbase_list, pbase_list, dop_list = read_baseline_info(
baseline_file=inps.baseline_file, reference_file=inps.referenceFile)
# Pair selection from reference
if inps.referenceFile:
log('select initial network from reference file: {}'.format(
inps.referenceFile))
stack_obj = ifgramStack(inps.referenceFile)
date12_list = stack_obj.get_date12_list(dropIfgram=True)
date12_list = ptime.yymmdd_date12(date12_list)
# Pais selection from method
elif inps.baseline_file:
log('select initial network with method: {}'.format(inps.method))
if inps.method == 'all':
date12_list = pnet.select_pairs_all(date_list)
elif inps.method == 'delaunay':
date12_list = pnet.select_pairs_delaunay(date_list, pbase_list,
inps.norm)
elif inps.method == 'star':
date12_list = pnet.select_pairs_star(date_list)
elif inps.method == 'sequential':
date12_list = pnet.select_pairs_sequential(date_list, inps.connNum)
elif inps.method == 'hierarchical':
date12_list = pnet.select_pairs_hierarchical(
date_list, pbase_list, inps.tempPerpList)
elif inps.method == 'mst':
date12_list = pnet.select_pairs_mst(date_list, pbase_list)
else:
raise Exception('Unrecoganized select method: ' + inps.method)
log('initial number of interferograms: {}'.format(len(date12_list)))
inps.date12_list = date12_list
inps.date_list = date_list
inps.tbase_list = tbase_list
inps.pbase_list = pbase_list
inps.dop_list = dop_list
return inps
def check_design_matrix(ifgram_file, weight_func='fim'):
"""Check Rank of Design matrix for weighted inversion"""
A = ifgramStack(ifgram_file).get_design_matrix(dropIfgram=True)[0]
print(
'-------------------------------------------------------------------------------'
)
if weight_func == 'sbas':
print('generic least square inversion with min-norm phase velocity')
print(' based on Berardino et al. (2002, IEEE-TGRS)')
print(' OLS for pixels with full rank network')
print(' SVD for pixels with rank deficient network')
if np.linalg.matrix_rank(A) < A.shape[1]:
print(
'WARNING: singular design matrix! Inversion result can be biased!'
)
print('continue using its SVD solution on all pixels')
else:
print(
'weighted least square (WLS) inversion with min-norm phase, pixelwise'
)
if np.linalg.matrix_rank(A) < A.shape[1]:
print('ERROR: singular design matrix!')
print(
' Input network of interferograms is not fully connected!')
print(' Can not invert the weighted least square solution.')
print('You could try:')
print(
' 1) Add more interferograms to make the network fully connected:'
)
print(' a.k.a., no multiple subsets nor network islands')
print(" 2) Use '-w no' option for non-weighted SVD solution.")
raise Exception()
print(
'-------------------------------------------------------------------------------'
)
return A
def check_design_matrix(ifgram_file, weight_func='var'):
"""Check Rank of Design matrix for weighted inversion"""
A = ifgramStack(ifgram_file).get_design_matrix4timeseries_estimation(
dropIfgram=True)[0]
if weight_func == 'no':
if np.linalg.matrix_rank(A) < A.shape[1]:
print(
'WARNING: singular design matrix! Inversion result can be biased!'
)
print('continue using its SVD solution on all pixels')
else:
if np.linalg.matrix_rank(A) < A.shape[1]:
print('ERROR: singular design matrix!')
print(
' Input network of interferograms is not fully connected!')
print(' Can not invert the weighted least square solution.')
print('You could try:')
print(
' 1) Add more interferograms to make the network fully connected:'
)
print(' a.k.a., no multiple subsets nor network islands')
print(" 2) Use '-w no' option for non-weighted SVD solution.")
raise Exception()
return A
def get_date_list(fname, print_msg=False):
atr = readfile.read_attribute(fname)
k = atr['FILE_TYPE']
dateList = None
if k in ['timeseries']:
obj = timeseries(fname)
obj.open(print_msg=False)
dateList = obj.dateList
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
dateList = obj.date12List
else:
print('--date option can not be applied to {} file, ignore it.'.format(
k))
try:
obj.close(print_msg=False)
except:
pass
if print_msg and dateList is not None:
for i in dateList:
print(i)
return dateList
def check_loaded_dataset(work_dir='./', print_msg=True):
"""Check the result of loading data for the following two rules:
1. file existance
2. file attribute readability
Parameters: work_dir : string, PySAR working directory
print_msg : bool, print out message
Returns: True, if all required files and dataset exist; otherwise, ERROR
If True, PROCESS, SLC folder could be removed.
stack_file :
geom_file :
lookup_file :
Example: work_dir = os.path.expandvars('$SCRATCHDIR/SinabungT495F50AlosA/PYSAR')
ut.check_loaded_dataset(work_dir)
"""
load_complete = True
if not work_dir:
work_dir = os.getcwd()
work_dir = os.path.abspath(work_dir)
# 1. interferograms stack file: unwrapPhase, coherence
flist = [os.path.join(work_dir, 'INPUTS/ifgramStack.h5')]
stack_file = is_file_exist(flist, abspath=True)
if stack_file is not None:
obj = ifgramStack(stack_file)
obj.open(print_msg=False)
for dname in ['unwrapPhase', 'coherence']:
if dname not in obj.datasetNames:
raise ValueError(
'required dataset "{}" is missing in file {}'.format(
dname, stack_file))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'./INPUTS/ifgramStack.h5')
atr = readfile.read_attribute(stack_file)
# 2. geom_file: height
if 'X_FIRST' in atr.keys():
flist = [os.path.join(work_dir, 'INPUTS/geometryGeo.h5')]
else:
flist = [os.path.join(work_dir, 'INPUTS/geometryRadar.h5')]
geom_file = is_file_exist(flist, abspath=True)
if geom_file is not None:
obj = geometry(geom_file)
obj.open(print_msg=False)
dname = geometryDatasetNames[0]
if dname not in obj.datasetNames:
raise ValueError(
'required dataset "{}" is missing in file {}'.format(
dname, geom_file))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'./INPUTS/geometry*.h5')
# 3. lookup_file: latitude,longitude or rangeCoord,azimuthCoord
# could be different than geometry file in case of roipac and gamma
flist = [os.path.join(work_dir, 'INPUTS/geometry*.h5')]
lookup_file = get_lookup_file(flist, abspath=True, print_msg=print_msg)
if lookup_file is not None:
obj = geometry(lookup_file)
obj.open(print_msg=False)
if atr['PROCESSOR'] in ['isce', 'doris']:
dnames = [geometryDatasetNames[1], geometryDatasetNames[2]]
elif atr['PROCESSOR'] in ['gamma', 'roipac']:
dnames = [geometryDatasetNames[3], geometryDatasetNames[4]]
else:
raise AttributeError('InSAR processor: {}'.format(
atr['PROCESSOR']))
for dname in dnames:
if dname not in obj.datasetNames:
load_complete = False
raise Exception(
'required dataset "{}" is missing in file {}'.format(
dname, lookup_file))
else:
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT),
'./INPUTS/geometry*.h5')
# print message
if print_msg:
print(('Loaded dataset are processed by '
'InSAR software: {}'.format(atr['PROCESSOR'])))
if 'X_FIRST' in atr.keys():
print('Loaded dataset is in GEO coordinates')
else:
print('Loaded dataset is in RADAR coordinates')
print('Interferograms Stack: {}'.format(stack_file))
print('Geometry File : {}'.format(geom_file))
print('Lookup Table File : {}'.format(lookup_file))
if load_complete:
print('-' * 50)
print(
'All data needed found/loaded/copied. Processed 2-pass InSAR data can be removed.'
)
print('-' * 50)
return load_complete, stack_file, geom_file, lookup_file
def get_slice_list(fname):
"""Get list of 2D slice existed in file (for display)"""
fbase, fext = os.path.splitext(os.path.basename(fname))
fext = fext.lower()
atr = read_attribute(fname)
k = atr['FILE_TYPE']
global slice_list
# HDF5 Files
if fext in ['.h5', '.he5']:
with h5py.File(fname, 'r') as f:
d1_list = [i for i in f.keys() if isinstance(f[i], h5py.Dataset)]
if k == 'timeseries' and k in d1_list:
obj = timeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['geometry'] and k not in d1_list:
obj = geometry(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['ifgramStack']:
obj = ifgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['HDFEOS']:
obj = HDFEOS(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantTimeseries']:
obj = giantTimeseries(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
elif k in ['giantIfgramStack']:
obj = giantIfgramStack(fname)
obj.open(print_msg=False)
slice_list = obj.sliceList
else:
## Find slice by walking through the file structure
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
def get_hdf5_2d_dataset(name, obj):
global slice_list
if isinstance(obj, h5py.Dataset) and obj.shape[-2:] == (length,
width):
if obj.ndim == 2:
slice_list.append(name)
else:
warnings.warn(
'file has un-defined {}D dataset: {}'.format(
obj.ndim, name))
slice_list = []
with h5py.File(fname, 'r') as f:
f.visititems(get_hdf5_2d_dataset)
# Binary Files
else:
if fext.lower() in ['.trans', '.utm_to_rdc']:
slice_list = ['rangeCoord', 'azimuthCoord']
elif fbase.startswith('los'):
slice_list = ['incidenceAngle', 'azimuthAngle']
elif atr.get('number_bands', '1') == '2' and 'unw' not in k:
slice_list = ['band1', 'band2']
else:
slice_list = ['']
return slice_list