def prepare_stack(inputDir, filePattern, metadata=dict(), baseline_dict=dict(), update_mode=True):
print('prepare .rsc file for ', filePattern)
isce_files = sorted(glob.glob(os.path.join(os.path.abspath(inputDir), '*', filePattern)))
if len(isce_files) == 0:
raise FileNotFoundError('no file found in pattern: {}'.format(filePattern))
# write .rsc file for each interferogram file
num_file = len(isce_files)
prog_bar = ptime.progressBar(maxValue=num_file)
for i in range(num_file):
isce_file = isce_files[i]
# prepare metadata for current file
ifg_metadata = readfile.read_attribute(isce_file, metafile_ext='.xml')
ifg_metadata.update(metadata)
dates = os.path.basename(os.path.dirname(isce_file)).split('_')
ifg_metadata = add_ifgram_metadata(ifg_metadata, dates, baseline_dict)
# write .rsc file
rsc_file = isce_file+'.rsc'
writefile.write_roipac_rsc(ifg_metadata, rsc_file,
update_mode=update_mode,
print_msg=False)
prog_bar.update(i+1, suffix='{}_{}'.format(dates[0], dates[1]))
prog_bar.close()
return
def run_2to3_timeseries(py2_file, py3_file):
"""Convert timeseries file from py2 pysar format to py3 pysar format"""
# read data from py2_file
atr = readfile.read_attribute(py2_file)
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
with h5py.File(py2_file, 'r') as f:
date_list = list(f['timeseries'].keys())
num_date = len(date_list)
ts_data = np.zeros((num_date, length, width), np.float32)
print('reading time-series ...')
prog_bar = ptime.progressBar(maxValue=num_date)
for i in range(num_date):
ts_data[i, :, :] = f['timeseries/{}'.format(date_list[i])][:]
prog_bar.update(i + 1, suffix=date_list[i])
prog_bar.close()
# prepare metadata
bperp = np.array([float(i) for i in atr['P_BASELINE_TIMESERIES'].split()],
dtype=np.float32)
dates = np.array(date_list, np.string_)
atr['REF_DATE'] = date_list[0]
for key in [
'P_BASELINE_TIMESERIES', 'P_BASELINE_TOP_TIMESERIES',
'P_BASELINE_BOTTOM_TIMESERIES'
]:
try:
atr.pop(key)
except:
pass
# write to py3_file
ts_obj = timeseries(py3_file)
ts_obj.write2hdf5(data=ts_data, dates=dates, bperp=bperp, metadata=atr)
return py3_file
def bin_variance(distance,
variance,
step=5e3,
min_pair_num=100e3,
print_msg=True):
x_steps = np.arange(0, np.max(distance), step)
num_step = len(x_steps)
std = np.zeros(x_steps.shape)
stdStd = np.zeros(std.shape)
p_num = np.zeros(x_steps.shape)
if print_msg:
prog_bar = ptime.progressBar(maxValue=num_step)
for i in range(num_step):
x = x_steps[i]
idx = (distance > max(0, x - step / 2.)) * (distance < x + step / 2.)
p_num[i] = np.sum(idx)
std[i] = np.mean(np.sqrt(variance[idx]))
stdStd[i] = np.std(np.sqrt(variance[idx]))
if print_msg:
prog_bar.update(i + 1, every=10)
if print_msg:
prog_bar.close()
max_step_idx = int(max(np.argwhere(p_num > min_pair_num)))
return x_steps[0:max_step_idx], std[0:max_step_idx], stdStd[0:max_step_idx]
def get_nonzero_phase_closure(ifgram_file,
out_file=None,
thres=0.1,
unwDatasetName='unwrapPhase'):
"""Calculate/Read number of non-zero phase closure
Parameters: ifgram_file : string, path of ifgram stack file
out_file : string, path of num non-zero phase closure file
Returns: num_nonzero_closure : 2D np.array in size of (length, width)
"""
if not out_file:
out_file = 'numNonzeroPhaseClosure_{}.h5'.format(unwDatasetName)
if os.path.isfile(out_file) and readfile.read_attribute(out_file):
print('1. read number of nonzero phase closure from file: {}'.format(
out_file))
num_nonzero_closure = readfile.read(out_file)[0]
else:
obj = ifgramStack(ifgram_file)
obj.open(print_msg=False)
length, width = obj.length, obj.width
ref_phase = obj.get_reference_phase(unwDatasetName=unwDatasetName,
dropIfgram=False)
C = obj.get_design_matrix4triplet(
obj.get_date12_list(dropIfgram=False))
# calculate phase closure line by line to save memory usage
num_nonzero_closure = np.zeros((length, width), np.float32)
print(
'1. calculating phase closure of all pixels from dataset - {} ...'.
format(unwDatasetName))
line_step = 10
num_loop = int(np.ceil(length / line_step))
prog_bar = ptime.progressBar(maxValue=num_loop)
for i in range(num_loop):
# read phase
i0, i1 = i * line_step, min(length, (i + 1) * line_step)
box = (0, i0, width, i1)
pha_data = ifginv.read_unwrap_phase(obj,
box,
ref_phase,
unwDatasetName=unwDatasetName,
dropIfgram=False,
print_msg=False)
# calculate phase closure
pha_closure = np.dot(C, pha_data)
pha_closure = np.abs(pha_closure - ut.wrap(pha_closure))
# get number of non-zero phase closure
num_nonzero = np.sum(pha_closure >= thres, axis=0)
num_nonzero_closure[i0:i1, :] = num_nonzero.reshape(i1 - i0, width)
prog_bar.update(i + 1,
every=1,
suffix='{}/{} lines'.format((i + 1) * line_step,
length))
prog_bar.close()
atr = dict(obj.metadata)
atr['FILE_TYPE'] = 'mask'
atr['UNIT'] = 1
writefile.write(num_nonzero_closure, out_file=out_file, metadata=atr)
return num_nonzero_closure
def run_resample(self,
src_data,
interp_method='nearest',
fill_value=np.nan,
nprocs=1,
print_msg=True):
"""Run interpolation operation for input 2D/3D data
Parameters: src_data : 2D/3D np.array, source data to be geocoded
interp_method : string, nearest | linear
fill_value : NaN or number
nprocs : int, number of processes to be used
print_msg : bool
Returns: geo_data : 2D/3D np.array
"""
# use pyresample
if self.processor == 'pyresample':
if len(src_data.shape) == 3:
src_data = np.moveaxis(src_data, 0, -1)
if src_data.dtype == np.bool_:
fill_value = False
print('restrict fill value to False for bool type source data')
# resample source data into target data
geo_data = self.run_pyresample(src_data=src_data,
interp_method=interp_method,
fill_value=fill_value,
nprocs=nprocs,
radius=None,
print_msg=True)
if len(geo_data.shape) == 3:
geo_data = np.moveaxis(geo_data, -1, 0)
# use scipy.interpolater.RegularGridInterpolator
else:
if print_msg:
print(
'resampling using scipy.interpolate.RegularGridInterpolator ...'
)
if len(src_data.shape) == 3:
geo_data = np.empty(
(src_data.shape[0], self.length, self.width),
src_data.dtype)
prog_bar = ptime.progressBar(maxValue=src_data.shape[0])
for i in range(src_data.shape[0]):
geo_data[i, :, :] = self.run_regular_grid_interpolator(
src_data=src_data[i, :, :],
interp_method=interp_method,
fill_value=fill_value,
print_msg=True)
prog_bar.update(i + 1)
prog_bar.close()
else:
geo_data = self.run_regular_grid_interpolator(
src_data=src_data,
interp_method=interp_method,
fill_value=fill_value,
print_msg=True)
return geo_data
def structure_function(data,
lat,
lon,
step=5e3,
min_pair_num=100e3,
print_msg=True):
num_sample = len(data)
distance = np.zeros((num_sample**2))
variance = np.zeros((num_sample**2))
if print_msg:
prog_bar = ptime.progressBar(maxValue=num_sample)
for i in range(num_sample):
distance[i * num_sample:(i + 1) * num_sample] = get_distance(
lat, lon, i)
variance[i * num_sample:(i + 1) * num_sample] = np.square(data -
data[i])
if print_msg:
prog_bar.update(i + 1, every=10)
if print_msg:
prog_bar.close()
dist, std, stdStd = bin_variance(distance,
variance,
step=step,
min_pair_num=min_pair_num,
print_msg=print_msg)
return dist, std, stdStd
def main(argv):
try:
file = argv[0]
except:
usage()
sys.exit(1)
g_name = 'unwrapPhase'
g_name_out = 'unwrapPhase_laplace'
print('calculate Laplace filter of {} based on approximate second derivatives.'.format(g_name))
f = h5py.File(file, 'a')
ds = f[g_name]
if g_name_out in f.keys():
ds_out = f[g_name_out]
else:
ds_out = f.create_dataset(g_name_out, shape=ds.shape, dtype=np.float32, chunks=True, compression=None)
print('write to dataset /{}'.format(g_name_out))
num_ifgram = ds.shape[0]
prog_bar = ptime.progressBar(maxValue=num_ifgram)
for i in range(num_ifgram):
unw = ds[i, :, :]
ds_out[i, :, :] = laplace(unw)
prog_bar.update(i+1, suffix='{}/{}'.format(i+1, num_ifgram))
prog_bar.close()
f.close()
print('finished writing to {}'.format(file))
return
def calculate_temporal_coherence_patch(ifgram_file,
timeseries_file,
box=None,
ifg_num_file=None):
atr = readfile.read_attribute(timeseries_file)
if not box:
box = (0, 0, int(atr['WIDTH']), int(atr['LENGTH']))
# Read timeseries data
ts_obj = timeseries(timeseries_file)
ts_obj.open(print_msg=False)
print('reading timeseries data from file: {}'.format(timeseries_file))
ts_data = ts_obj.read(box=box, print_msg=False).reshape(ts_obj.numDate, -1)
ts_data = ts_data[1:, :]
ts_data *= -4 * np.pi / float(atr['WAVELENGTH'])
# Read ifgram data
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
A = stack_obj.get_design_matrix4timeseries_estimation(dropIfgram=True)[0]
print('reading unwrapPhase data from file: {}'.format(ifgram_file))
ifgram_data = stack_obj.read(datasetName='unwrapPhase',
box=box).reshape(A.shape[0], -1)
ref_value = stack_obj.get_reference_phase(dropIfgram=True).reshape((-1, 1))
ifgram_data -= np.tile(ref_value, (1, ifgram_data.shape[1]))
ifgram_diff = ifgram_data - np.dot(A, ts_data)
del ts_data
pixel_num = ifgram_data.shape[1]
temp_coh = np.zeros((pixel_num), np.float32)
# (fast) nasty solution, which used all phase value including invalid zero phase
if not ifg_num_file:
temp_coh = np.abs(np.sum(np.exp(1j * ifgram_diff),
axis=0)) / ifgram_diff.shape[0]
# (slow) same solution as ifgram_inversion.py, considering:
# 1) invalid zero phase in ifgram
# 2) design matrix rank deficiency.
else:
print(
'considering different number of interferograms used in network inversion for each pixel'
)
ifg_num_map = readfile.read(ifg_num_file, box=box)[0].flatten()
prog_bar = ptime.progressBar(maxValue=pixel_num)
for i in range(pixel_num):
if ifg_num_map[i] > 0:
idx = ifgram_data[:, i] != 0.
temp_diff = ifgram_diff[idx, i]
temp_coh[i] = np.abs(np.sum(np.exp(1j * temp_diff),
axis=0)) / temp_diff.shape[0]
prog_bar.update(i + 1,
every=1000,
suffix='{}/{}'.format(i + 1, pixel_num))
prog_bar.close()
temp_coh = np.reshape(temp_coh, (box[3] - box[1], box[2] - box[0]))
return temp_coh
def main(argv):
try:
file = argv[0]
except:
usage()
sys.exit(1)
outfile = os.path.splitext(file)[0] + '_wrap' + os.path.splitext(file)[1]
one_cycle = 2 * np.pi
one_cycle = 0.05
atr = readfile.read_attribute(file)
k = atr['FILE_TYPE']
if k in ['interferograms', 'coherence', 'wrapped', 'timeseries']:
h5 = h5py.File(file, 'r')
epochList = sorted(h5[k].keys())
epoch_num = len(epochList)
prog_bar = ptime.progressBar(maxValue=epoch_num)
print('writing >>> ' + outfile)
h5out = h5py.File(outfile, 'w')
group = h5out.create_group(k)
if k in ['interferograms', 'coherence', 'wrapped']:
date12_list = ptime.list_ifgram2date12(epochList)
print('number of interferograms: ' + str(len(epochList)))
for i in range(epoch_num):
epoch = epochList[i]
data = h5[k][epoch].get(epoch)[:]
data_wrap = rewrap(data)
gg = group.create_group(epoch)
dset = gg.create_dataset(epoch, data=data_wrap)
for key, value in h5[k][epoch].attrs.items():
gg.attrs[key] = value
prog_bar.update(i + 1, suffix=date12_list[i])
elif k == 'timeseries':
print('number of acquisitions: ' + str(len(epochList)))
for i in range(epoch_num):
epoch = epochList[i]
data = h5[k].get(epoch)[:]
data_wrap = rewrap(data, one_cycle)
dset = group.create_dataset(epoch, data=data_wrap)
prog_bar.update(i + 1, suffix=epoch)
for key, value in h5[k].attrs.items():
group.attrs[key] = value
h5.close()
h5out.close()
prog_bar.close()
print('Done.')
return outfile
def main(iargs=None):
inps = cmd_line_parse(iargs)
# read timeseries info / data
obj = timeseries(inps.timeseries_file)
obj.open()
tbase = np.array(obj.yearList, np.float32).reshape(-1, 1)
tbase -= tbase[obj.refIndex]
ts_data = obj.read().reshape(obj.numDate, -1)
# Smooth acquisitions / moving window in time one by one
print('-' * 50)
print('filtering in time Gaussian window with size of {:.1f} years'.format(
inps.time_win))
ts_data_filt = np.zeros(ts_data.shape, np.float32)
prog_bar = ptime.progressBar(maxValue=obj.numDate)
for i in range(obj.numDate):
# Weight from Gaussian (normal) distribution in time
tbase_diff = tbase[i] - tbase
weight = np.exp(-0.5 * (tbase_diff**2) / (inps.time_win**2))
weight /= np.sum(weight)
# Smooth the current acquisition
ts_data_filt[i, :] = np.sum(ts_data * weight, axis=0)
prog_bar.update(i + 1, suffix=obj.dateList[i])
prog_bar.close()
del ts_data
ts_data_filt -= ts_data_filt[obj.refIndex, :]
ts_data_filt = np.reshape(ts_data_filt,
(obj.numDate, obj.length, obj.width))
# write filtered timeseries file
if not inps.outfile:
inps.outfile = '{}_tempGaussian.h5'.format(
os.path.splitext(inps.timeseries_file)[0])
obj_out = timeseries(inps.outfile)
obj_out.write2hdf5(ts_data_filt, refFile=inps.timeseries_file)
return inps.outfile
def run_unwrap_error_bridge(ifgram_file,
water_mask_file,
ramp_type=None,
radius=50,
ccName='connectComponent',
dsNameIn='unwrapPhase',
dsNameOut='unwrapPhase_bridging'):
"""Run unwrapping error correction with bridging
Parameters: ifgram_file : str, path of ifgram stack file
water_mask_file : str, path of water mask file
ramp_type : str, name of phase ramp to be removed during the phase jump estimation
ccName : str, dataset name of connected components
dsNameIn : str, dataset name of unwrap phase to be corrected
dsNameOut : str, dataset name of unwrap phase to be saved after correction
Returns: ifgram_file : str, path of ifgram stack file
"""
print('-' * 50)
print(
'correct unwrapping error in {} with bridging ...'.format(ifgram_file))
if ramp_type is not None:
print('estimate and remove a {} ramp while calculating phase offset'.
format(ramp_type))
# read water mask
if water_mask_file and os.path.isfile(water_mask_file):
print('read water mask from file:', water_mask_file)
water_mask = readfile.read(water_mask_file)[0]
else:
water_mask = None
# file info
atr = readfile.read_attribute(ifgram_file)
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
k = atr['FILE_TYPE']
# correct unwrap error ifgram by ifgram
if k == 'ifgramStack':
date12_list = ifgramStack(ifgram_file).get_date12_list(
dropIfgram=False)
num_ifgram = len(date12_list)
shape_out = (num_ifgram, length, width)
# prepare output data writing
print('open {} with r+ mode'.format(ifgram_file))
f = h5py.File(ifgram_file, 'r+')
print('input dataset:', dsNameIn)
print('output dataset:', dsNameOut)
if dsNameOut in f.keys():
ds = f[dsNameOut]
print('access /{d} of np.float32 in size of {s}'.format(
d=dsNameOut, s=shape_out))
else:
ds = f.create_dataset(dsNameOut,
shape_out,
maxshape=(None, None, None),
chunks=True,
compression=None)
print('create /{d} of np.float32 in size of {s}'.format(
d=dsNameOut, s=shape_out))
# correct unwrap error ifgram by ifgram
prog_bar = ptime.progressBar(maxValue=num_ifgram)
for i in range(num_ifgram):
# read unwrapPhase and connectComponent
date12 = date12_list[i]
unw = np.squeeze(f[dsNameIn][i, :, :])
cc = np.squeeze(f[ccName][i, :, :])
if water_mask is not None:
cc[water_mask == 0] = 0
# bridging
cc_obj = connectComponent(conncomp=cc, metadata=atr)
cc_obj.label()
cc_obj.find_mst_bridge()
unw_cor = cc_obj.unwrap_conn_comp(unw, ramp_type=ramp_type)
# write to hdf5 file
ds[i, :, :] = unw_cor
prog_bar.update(i + 1, suffix=date12)
prog_bar.close()
ds.attrs['MODIFICATION_TIME'] = str(time.time())
f.close()
print('close {} file.'.format(ifgram_file))
if k == '.unw':
# read unwrap phase
unw = readfile.read(ifgram_file)[0]
# read connected components
cc_files0 = [
ifgram_file + '.conncomp',
os.path.splitext(ifgram_file)[0] + '_snap_connect.byt'
]
cc_files = [i for i in cc_files0 if os.path.isfile(i)]
if len(cc_files) == 0:
raise FileNotFoundError(cc_files0)
cc = readfile.read(cc_files[0])[0]
if water_mask is not None:
cc[water_mask == 0] = 0
# bridging
cc_obj = connectComponent(conncomp=cc, metadata=atr)
cc_obj.label()
cc_obj.find_mst_bridge()
unw_cor = cc_obj.unwrap_conn_comp(unw, ramp_type=ramp_type)
# write to hdf5 file
out_file = '{}_unwCor{}'.format(
os.path.splitext(ifgram_file)[0],
os.path.splitext(ifgram_file)[1])
print('writing >>> {}'.format(out_file))
writefile.write(unw_cor, out_file=out_file, ref_file=ifgram_file)
return ifgram_file
def get_delay_timeseries(inps, atr):
"""Calculate delay time-series and write it to HDF5 file.
Parameters: inps : namespace, all input parameters
atr : dict, metadata to be saved in trop_file
Returns: trop_file : str, file name of ECMWF.h5
"""
def get_dataset_size(fname):
atr = readfile.read_attribute(fname)
return (atr['LENGTH'], atr['WIDTH'])
# check 1 - existing tropo delay file
if (ut.run_or_skip(out_file=inps.trop_file,
in_file=inps.grib_file_list,
print_msg=False) == 'skip'
and get_dataset_size(inps.trop_file) == get_dataset_size(
inps.geom_file)):
print(
'{} file exists and is newer than all GRIB files, skip updating.'.
format(inps.trop_file))
return
# check 2 - geometry file
if any(i is None for i in [inps.geom_file, inps.ref_yx]):
print(
'No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.'
)
if not os.path.isfile(inps.trop_file):
inps.trop_file = None
return
# prepare geometry data
geom_obj = geometry(inps.geom_file)
geom_obj.open()
inps.dem = geom_obj.read(datasetName='height')
inps.inc = geom_obj.read(datasetName='incidenceAngle')
if 'latitude' in geom_obj.datasetNames:
inps.lat = geom_obj.read(datasetName='latitude')
inps.lon = geom_obj.read(datasetName='longitude')
else:
inps.lat, inps.lon = get_lat_lon(geom_obj.metadata)
# calculate phase delay
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
num_date = len(inps.grib_file_list)
date_list = [str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list]
trop_data = np.zeros((num_date, length, width), np.float32)
print(
'calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...'
)
print('number of grib files used: {}'.format(num_date))
if verbose:
prog_bar = ptime.progressBar(maxValue=num_date)
for i in range(num_date):
grib_file = inps.grib_file_list[i]
trop_data[i] = get_delay(grib_file, inps)
if verbose:
prog_bar.update(i + 1, suffix=os.path.basename(grib_file))
if verbose:
prog_bar.close()
# Convert relative phase delay on reference date
inps.ref_date = atr.get('REF_DATE', date_list[0])
print('convert to relative phase delay with reference date: ' +
inps.ref_date)
inps.ref_idx = date_list.index(inps.ref_date)
trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1))
# Write tropospheric delay to HDF5
atr['REF_Y'] = inps.ref_yx[0]
atr['REF_X'] = inps.ref_yx[1]
ts_obj = timeseries(inps.trop_file)
ts_obj.write2hdf5(data=trop_data,
dates=date_list,
metadata=atr,
refFile=inps.timeseries_file)
return
def run_unwrap_error_bridge(ifgram_file, mask_cc_file, bridges, dsNameIn='unwrapPhase',
dsNameOut='unwrapPhase_bridging', ramp_type=None):
"""Run unwrapping error correction with bridging
Parameters: ifgram_file : str, path of ifgram stack file
mask_cc_file : str, path of conn comp mask file
bridges : list of dicts, check bridge_unwrap_error() for details
dsNameIn : str, dataset name of unwrap phase to be corrected
dsNameOut : str, dataset name of unwrap phase to be saved after correction
ramp_type : str, name of phase ramp to be removed during the phase jump estimation
Returns: ifgram_file : str, path of ifgram stack file
"""
print('-'*50)
print('correct unwrapping error in {} with bridging ...'.format(ifgram_file))
# file info
atr = readfile.read_attribute(ifgram_file)
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
ref_y, ref_x = int(atr['REF_Y']), int(atr['REF_X'])
k = atr['FILE_TYPE']
# read mask
print('read mask from file: {}'.format(mask_cc_file))
mask_cc = readfile.read(mask_cc_file, datasetName='mask')[0]
if ramp_type is not None:
print('estimate and remove phase ramp of {} during the correction'.format(ramp_type))
mask4ramp = (mask_cc == mask_cc[ref_y, ref_x])
# correct unwrap error ifgram by ifgram
if k == 'ifgramStack':
date12_list = ifgramStack(ifgram_file).get_date12_list(dropIfgram=False)
num_ifgram = len(date12_list)
shape_out = (num_ifgram, length, width)
# prepare output data writing
print('open {} with r+ mode'.format(ifgram_file))
f = h5py.File(ifgram_file, 'r+')
print('input dataset:', dsNameIn)
print('output dataset:', dsNameOut)
if dsNameOut in f.keys():
ds = f[dsNameOut]
print('access /{d} of np.float32 in size of {s}'.format(d=dsNameOut, s=shape_out))
else:
ds = f.create_dataset(dsNameOut, shape_out, maxshape=(None, None, None),
chunks=True, compression=None)
print('create /{d} of np.float32 in size of {s}'.format(d=dsNameOut, s=shape_out))
# correct unwrap error ifgram by ifgram
prog_bar = ptime.progressBar(maxValue=num_ifgram)
for i in range(num_ifgram):
# read unwrapPhase
date12 = date12_list[i]
unw = np.squeeze(f[dsNameIn][i, :, :])
unw[unw != 0.] -= unw[ref_y, ref_x]
# remove phase ramp before phase jump estimation
if ramp_type is not None:
unw, unw_ramp = deramp(unw, mask4ramp, ramp_type, metadata=atr)
# estimate/correct phase jump
unw_cor = bridge_unwrap_error(unw, mask_cc, bridges)
if ramp_type is not None:
unw_cor += unw_ramp
# write to hdf5 file
ds[i, :, :] = unw_cor
prog_bar.update(i+1, suffix=date12)
prog_bar.close()
ds.attrs['MODIFICATION_TIME'] = str(time.time())
f.close()
print('close {} file.'.format(ifgram_file))
if k == '.unw':
# read data
unw = readfile.read(ifgram_file)[0]
unw[unw != 0.] -= unw[ref_y, ref_x]
# remove phase ramp before phase jump estimation
if ramp_type is not None:
unw, unw_ramp = deramp(unw, mask4ramp, ramp_type, metadata=atr)
# estimate/correct phase jump
unw_cor = bridge_unwrap_error(unw, mask_cc, bridges)
if ramp_type is not None:
unw_cor += unw_ramp
# write to hdf5 file
out_file = '{}_unwCor{}'.format(os.path.splitext(ifgram_file)[0],
os.path.splitext(ifgram_file)[1])
print('writing >>> {}'.format(out_file))
writefile.write(unw_cor, out_file=out_file, ref_file=ifgram_file)
return ifgram_file
def run_unwrap_error_patch(ifgram_file, box=None, mask_file=None, ref_phase=None, fast_mode=False,
thres=0.1, dsNameIn='unwrapPhase'):
"""Estimate/Correct unwrapping error in ifgram stack on area defined by box.
Parameters: ifgram_file : string, ifgramStack file
box : tuple of 4 int, indicating areas to be read and analyzed
mask_file : string, file name of mask file for pixels to be analyzed
ref_pahse : 1D np.array in size of (num_ifgram,) phase value on reference pixel, because:
1) phase value stored in pysar is not reference yet
2) reference point may be out of box definition
fast_mode : bool, apply zero jump constraint on ifgrams without unwrapping error.
thres : float, threshold of non-zero phase closure to be identified as unwrapping error.
Returns: pha_data : 3D np.array in size of (num_ifgram_all, box[3]-box[2], box[2]-box[0]),
unwrapped phase value after error correction
"""
# Basic info
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
num_ifgram = stack_obj.numIfgram
# Size Info - Patch
if box:
num_row = box[3] - box[1]
num_col = box[2] - box[0]
else:
num_row = stack_obj.length
num_col = stack_obj.width
num_pixel = num_row * num_col
C = stack_obj.get_design_matrix4ifgram_triangle(dropIfgram=True)
print('number of interferograms: {}'.format(C.shape[1]))
print('number of triangles: {}'.format(C.shape[0]))
# read unwrapPhase
pha_data_all = ifginv.read_unwrap_phase(stack_obj, box, ref_phase,
unwDatasetName=dsNameIn,
dropIfgram=False)
pha_data = np.array(pha_data_all[stack_obj.dropIfgram, :])
# mask of pixels to analyze
mask = np.ones((num_pixel), np.bool_)
print('number of pixels read: {}'.format(num_pixel))
# mask 1. mask of water or area of interest
if mask_file:
dsNames = readfile.get_dataset_list(mask_file)
dsName = [i for i in dsNames if i in ['waterMask', 'mask']][0]
waterMask = readfile.read(mask_file, datasetName=dsName, box=box)[0].flatten()
mask *= np.array(waterMask, np.bool_)
del waterMask
print('number of pixels left after mask: {}'.format(np.sum(mask)))
# mask 2. mask of pixels without unwrap error: : zero phase closure on all triangles
print('calculating phase closure of all possible triangles ...')
pha_closure = np.dot(C, pha_data)
pha_closure = np.abs(pha_closure - ut.wrap(pha_closure)) # Eq 4.2 (Fattahi, 2015)
num_nonzero_closure = np.sum(pha_closure >= thres, axis=0)
mask *= (num_nonzero_closure != 0.)
del pha_closure
print('number of pixels left after checking phase closure: {}'.format(np.sum(mask)))
# mask summary
num_pixel2proc = int(np.sum(mask))
if num_pixel2proc > 0:
ifgram = pha_data[:, mask]
ifgram_cor = np.array(ifgram, np.float32)
print('number of pixels to process: {} out of {} ({:.2f}%)'.format(num_pixel2proc, num_pixel,
num_pixel2proc/num_pixel*100))
# correcting unwrap error based on phase closure
print('correcting unwrapping error ...')
if fast_mode:
ifgram_cor = correct_unwrap_error(ifgram, C, Dconstraint=False)[0]
else:
prog_bar = ptime.progressBar(maxValue=num_pixel2proc)
for i in range(num_pixel2proc):
ifgram_cor[:, i] = correct_unwrap_error(ifgram[:, i], C, Dconstraint=True)[0].flatten()
prog_bar.update(i+1, every=10, suffix='{}/{}'.format(i+1, num_pixel2proc))
prog_bar.close()
pha_data[:, mask] = ifgram_cor
pha_data_all[stack_obj.dropIfgram, :] = pha_data
pha_data_all = pha_data_all.reshape(num_ifgram, num_row, num_col)
num_nonzero_closure = num_nonzero_closure.reshape(num_row, num_col)
return pha_data_all, num_nonzero_closure
def get_common_region_int_ambiguity(ifgram_file,
cc_mask_file,
water_mask_file=None,
num_sample=100,
dsNameIn='unwrapPhase'):
"""Solve the phase unwrapping integer ambiguity for the common regions among all interferograms
Parameters: ifgram_file : str, path of interferogram stack file
cc_mask_file : str, path of common connected components file
water_mask_file : str, path of water mask file
num_sample : int, number of pixel sampled for each region
dsNameIn : str, dataset name of the unwrap phase to be corrected
Returns: common_regions : list of skimage.measure._regionprops._RegionProperties object
modified by adding two more variables:
sample_coords : 2D np.ndarray in size of (num_sample, 2) in int64 format
int_ambiguity : 1D np.ndarray in size of (num_ifgram,) in int format
"""
print('-' * 50)
print(
'calculating the integer ambiguity for the common regions defined in',
cc_mask_file)
# stack info
stack_obj = ifgramStack(ifgram_file)
stack_obj.open()
date12_list = stack_obj.get_date12_list(dropIfgram=True)
num_ifgram = len(date12_list)
C = matrix(
ifgramStack.get_design_matrix4triplet(date12_list).astype(float))
ref_phase = stack_obj.get_reference_phase(unwDatasetName=dsNameIn,
dropIfgram=True).reshape(
num_ifgram, -1)
# prepare common label
print('read common mask from', cc_mask_file)
cc_mask = readfile.read(cc_mask_file)[0]
if water_mask_file is not None and os.path.isfile(water_mask_file):
water_mask = readfile.read(water_mask_file)[0]
print('refine common mask based on water mask file', water_mask_file)
cc_mask[water_mask == 0] = 0
label_img, num_label = connectComponent.get_large_label(cc_mask,
min_area=2.5e3,
print_msg=True)
common_regions = measure.regionprops(label_img)
print('number of common regions:', num_label)
# add sample_coords / int_ambiguity
print('number of samples per region:', num_sample)
print('solving the phase-unwrapping integer ambiguity for {}'.format(
dsNameIn))
print(
'\tbased on the closure phase of interferograms triplets (Yunjun et al., 2019)'
)
print(
'\tusing the L1-norm regularzed least squares approximation (LASSO) ...'
)
for i in range(num_label):
common_reg = common_regions[i]
# sample_coords
idx = sorted(
np.random.choice(common_reg.area, num_sample, replace=False))
common_reg.sample_coords = common_reg.coords[idx, :].astype(int)
# solve for int_ambiguity
U = np.zeros((num_ifgram, num_sample))
if common_reg.label == label_img[stack_obj.refY, stack_obj.refX]:
print('{}/{} skip calculation for the reference region'.format(
i + 1, num_label))
else:
prog_bar = ptime.progressBar(maxValue=num_sample,
prefix='{}/{}'.format(
i + 1, num_label))
for j in range(num_sample):
# read unwrap phase
y, x = common_reg.sample_coords[j, :]
unw = ifginv.read_unwrap_phase(stack_obj,
box=(x, y, x + 1, y + 1),
ref_phase=ref_phase,
unwDatasetName=dsNameIn,
dropIfgram=True,
print_msg=False).reshape(
num_ifgram, -1)
# calculate closure_int
closure_pha = np.dot(C, unw)
closure_int = matrix(
np.round(
(closure_pha - ut.wrap(closure_pha)) / (2. * np.pi)))
# solve for U
U[:, j] = np.round(
l1regls(-C, closure_int, alpha=1e-2,
show_progress=0)).flatten()
prog_bar.update(j + 1, every=5)
prog_bar.close()
# add int_ambiguity
common_reg.int_ambiguity = np.median(U, axis=1)
common_reg.date12_list = date12_list
#sort regions by size to facilitate the region matching later
common_regions.sort(key=lambda x: x.area, reverse=True)
# plot sample result
fig_size = pp.auto_figure_size(label_img.shape, disp_cbar=False)
fig, ax = plt.subplots(figsize=fig_size)
ax.imshow(label_img, cmap='jet')
for common_reg in common_regions:
ax.plot(common_reg.sample_coords[:, 1],
common_reg.sample_coords[:, 0],
'k.',
ms=2)
pp.auto_flip_direction(stack_obj.metadata, ax, print_msg=False)
out_img = 'common_region_sample.png'
fig.savefig(out_img, bbox_inches='tight', transparent=True, dpi=300)
print('saved common regions and sample pixels to file', out_img)
return common_regions
def write2hdf5(self,
outputFile='geometryRadar.h5',
access_mode='w',
box=None,
compression='gzip',
extra_metadata=None):
'''
/ Root level
Attributes Dictionary for metadata. 'X/Y_FIRST/STEP' attribute for geocoded.
/height 2D array of float32 in size of (l, w ) in meter.
/latitude (azimuthCoord) 2D array of float32 in size of (l, w ) in degree.
/longitude (rangeCoord) 2D array of float32 in size of (l, w ) in degree.
/incidenceAngle 2D array of float32 in size of (l, w ) in degree.
/slantRangeDistance 2D array of float32 in size of (l, w ) in meter.
/azimuthAngle 2D array of float32 in size of (l, w ) in degree. (optional)
/shadowMask 2D array of bool in size of (l, w ). (optional)
/waterMask 2D array of bool in size of (l, w ). (optional)
/bperp 3D array of float32 in size of (n, l, w) in meter (optional)
/date 1D array of string in size of (n, ) in YYYYMMDD(optional)
...
'''
if len(self.datasetDict) == 0:
print(
'No dataset file path in the object, skip HDF5 file writing.')
return None
self.outputFile = outputFile
f = h5py.File(self.outputFile, access_mode)
print('create HDF5 file {} with {} mode'.format(
self.outputFile, access_mode))
#groupName = self.name
#group = f.create_group(groupName)
#print('create group /{}'.format(groupName))
maxDigit = max([len(i) for i in geometryDatasetNames])
length, width = self.get_size(box=box)
self.length, self.width = self.get_size()
###############################
for dsName in self.dsNames:
# 3D datasets containing bperp
if dsName == 'bperp':
self.dateList = list(self.datasetDict[dsName].keys())
dsDataType = dataType
self.numDate = len(self.dateList)
dsShape = (self.numDate, length, width)
ds = f.create_dataset(dsName,
shape=dsShape,
maxshape=(None, dsShape[1], dsShape[2]),
dtype=dsDataType,
chunks=True,
compression=compression)
print(('create dataset /{d:<{w}} of {t:<25} in size of {s}'
' with compression = {c}').format(d=dsName,
w=maxDigit,
t=str(dsDataType),
s=dsShape,
c=str(compression)))
print(
'read coarse grid baseline files and linear interpolate into full resolution ...'
)
prog_bar = ptime.progressBar(maxValue=self.numDate)
for i in range(self.numDate):
fname = self.datasetDict[dsName][self.dateList[i]]
data = read_isce_bperp_file(fname=fname,
out_shape=(self.length,
self.width),
box=box)
ds[i, :, :] = data
prog_bar.update(i + 1, suffix=self.dateList[i])
prog_bar.close()
# Write 1D dataset date
dsName = 'date'
dsShape = (self.numDate, )
dsDataType = np.string_
print(('create dataset /{d:<{w}} of {t:<25}'
' in size of {s}').format(d=dsName,
w=maxDigit,
t=str(dsDataType),
s=dsShape))
data = np.array(self.dateList, dtype=dsDataType)
ds = f.create_dataset(dsName, data=data)
# 2D datasets containing height, latitude, incidenceAngle, shadowMask, etc.
else:
dsDataType = dataType
if dsName.lower().endswith('mask'):
dsDataType = np.bool_
dsShape = (length, width)
print(('create dataset /{d:<{w}} of {t:<25} in size of {s}'
' with compression = {c}').format(d=dsName,
w=maxDigit,
t=str(dsDataType),
s=dsShape,
c=str(compression)))
data = np.array(self.read(family=dsName, box=box)[0],
dtype=dsDataType)
ds = f.create_dataset(dsName,
data=data,
chunks=(100, 300),
compression=compression)
###############################
# Generate Dataset if not existed in binary file: incidenceAngle, slantRangeDistance
for dsName in [
i for i in ['incidenceAngle', 'slantRangeDistance']
if i not in self.dsNames
]:
# Calculate data
data = None
if dsName == 'incidenceAngle':
data = self.get_incidence_angle(box=box)
elif dsName == 'slantRangeDistance':
data = self.get_slant_range_distance(box=box)
# Write dataset
if data is not None:
dsShape = data.shape
dsDataType = dataType
print(('create dataset /{d:<{w}} of {t:<25} in size of {s}'
' with compression = {c}').format(d=dsName,
w=maxDigit,
t=str(dsDataType),
s=dsShape,
c=str(compression)))
ds = f.create_dataset(dsName,
data=data,
dtype=dataType,
chunks=(100, 300),
compression=compression)
###############################
# Attributes
self.get_metadata()
if extra_metadata:
self.metadata.update(extra_metadata)
print('add extra metadata: {}'.format(extra_metadata))
self.metadata = ut.subset_attribute(self.metadata, box)
self.metadata['FILE_TYPE'] = self.name
for key, value in self.metadata.items():
f.attrs[key] = value
f.close()
print('Finished writing to {}'.format(self.outputFile))
return self.outputFile
def get_delay_timeseries(inps, atr):
"""Calculate delay time-series and write it to HDF5 file.
Parameters: inps : namespace, all input parameters
atr : dict, metadata to be saved in trop_file
Returns: trop_file : str, file name of ECMWF.h5
"""
def get_dataset_size(fname):
atr = readfile.read_attribute(fname)
return (atr['LENGTH'], atr['WIDTH'])
if (ut.run_or_skip(out_file=inps.trop_file,
in_file=inps.grib_file_list,
print_msg=False) == 'skip'
and get_dataset_size(inps.trop_file) == get_dataset_size(
inps.geom_file)):
print(
'{} file exists and is newer than all GRIB files, skip updating.'.
format(inps.trop_file))
else:
if any(i is None for i in [inps.geom_file, inps.ref_yx]):
print(
'No DEM / incidenceAngle / ref_yx found, skip calculating tropospheric delays.'
)
if not os.path.isfile(inps.trop_file):
inps.trop_file = None
return
# calculate phase delay
length, width = int(atr['LENGTH']), int(atr['WIDTH'])
num_date = len(inps.grib_file_list)
date_list = [
str(re.findall('\d{8}', i)[0]) for i in inps.grib_file_list
]
trop_data = np.zeros((num_date, length, width), np.float32)
print(
'calcualting delay for each date using PyAPS (Jolivet et al., 2011; 2014) ...'
)
print('number of grib files used: {}'.format(num_date))
prog_bar = ptime.progressBar(maxValue=num_date)
for i in range(num_date):
grib_file = inps.grib_file_list[i]
trop_data[i] = get_delay(grib_file, inps)
prog_bar.update(i + 1, suffix=os.path.basename(grib_file))
prog_bar.close()
# Convert relative phase delay on reference date
try:
inps.ref_date = atr['REF_DATE']
except:
inps.ref_date = date_list[0]
print('convert to relative phase delay with reference date: ' +
inps.ref_date)
inps.ref_idx = date_list.index(inps.ref_date)
trop_data -= np.tile(trop_data[inps.ref_idx, :, :], (num_date, 1, 1))
# Write tropospheric delay to HDF5
atr['REF_Y'] = inps.ref_yx[0]
atr['REF_X'] = inps.ref_yx[1]
ts_obj = timeseries(inps.trop_file)
ts_obj.write2hdf5(data=trop_data,
dates=date_list,
metadata=atr,
refFile=inps.timeseries_file)
# Delete temporary DEM file in ROI_PAC format
if inps.geom_file:
temp_files = [
fname for fname in
[inps.dem_file, inps.inc_angle_file, inps.lat_file, inps.lon_file]
if (fname is not None and 'pyaps' in fname)
]
if temp_files:
print('delete temporary geometry files')
rmCmd = 'rm '
for fname in temp_files:
rmCmd += ' {f} {f}.rsc '.format(f=fname)
print(rmCmd)
os.system(rmCmd)
return
def ifgram_inversion_patch(ifgram_file,
box=None,
ref_phase=None,
weight_func='fim',
mask_dataset_name=None,
mask_threshold=0.4,
water_mask_file=None,
skip_zero_phase=True):
"""Invert one patch of an ifgram stack into timeseries.
Parameters: ifgram_file : str, interferograms stack HDF5 file, e.g. ./INPUTS/ifgramStack.h5
box : tuple of 4 int, indicating (x0, y0, x1, y1) pixel coordinate of area of interest
or None, to process the whole file and write output file
ref_phase : 1D array in size of (num_ifgram)
or None
weight_func : str, weight function, choose in ['sbas', 'fim', 'var', 'coh']
mask_dataset_name : str, dataset name in ifgram_file used to mask unwrapPhase pixelwisely
mask_threshold : float, min coherence of pixels if mask_dataset_name='coherence'
water_mask_file : str, water mask filename if available,
skip inversion on water to speed up the process
skip_zero_phase : bool, skip zero value of unwrapped phase or not, default yes, for comparison
Returns: ts : 3D array in size of (num_date, num_row, num_col)
temp_coh : 2D array in size of (num_row, num_col)
ts_std : 3D array in size of (num_date, num_row, num_col)
num_inv_ifgram : 2D array in size of (num_row, num_col)
Example: ifgram_inversion_patch('ifgramStack.h5', box=(0,200,1316,400), ref_phase=np.array(),
weight_func='fim', mask_dataset_name='coherence')
ifgram_inversion_patch('ifgramStack_001.h5', box=None, ref_phase=None,
weight_func='fim', mask_dataset_name='coherence')
"""
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
# Size Info - Patch
if box:
print('processing %8d/%d lines ...' % (box[3], stack_obj.length))
num_row = box[3] - box[1]
num_col = box[2] - box[0]
else:
num_row = stack_obj.length
num_col = stack_obj.width
num_pixel = num_row * num_col
# Design matrix
date12_list = stack_obj.get_date12_list(dropIfgram=True)
A, B = stack_obj.get_design_matrix(date12_list=date12_list)
num_ifgram = len(date12_list)
num_date = A.shape[1] + 1
try:
ref_date = str(
np.loadtxt('reference_date.txt', dtype=bytes).astype(str))
except:
ref_date = stack_obj.dateList[0]
ref_idx = stack_obj.dateList.index(ref_date)
time_idx = [i for i in range(num_date)]
time_idx.remove(ref_idx)
Astd = stack_obj.get_design_matrix(refDate=ref_date, dropIfgram=True)[0]
# Initialization of output matrix
print('number of interferograms: {}'.format(num_ifgram))
print('number of acquisitions : {}'.format(num_date))
print('number of lines : {}'.format(stack_obj.length))
print('number of columns: {}'.format(stack_obj.width))
ts = np.zeros((num_date, num_pixel), np.float32)
ts_std = np.zeros((num_date, num_pixel), np.float32)
temp_coh = np.zeros(num_pixel, np.float32)
num_inv_ifgram = np.zeros(num_pixel, np.int16)
# Read/Mask unwrapPhase
pha_data = read_unwrap_phase(stack_obj,
box,
ref_phase,
skip_zero_phase=skip_zero_phase)
pha_data = mask_unwrap_phase(pha_data,
stack_obj,
box,
mask_ds_name=mask_dataset_name,
mask_threshold=mask_threshold)
# Mask for pixels to invert
mask = np.ones(num_pixel, np.bool_)
# 1 - Water Mask
if water_mask_file:
print(('skip pixels on water with mask from'
' file: {}').format(os.path.basename(water_mask_file)))
dsNames = readfile.get_dataset_list(water_mask_file)
dsName = [i for i in dsNames if i in ['waterMask', 'mask']][0]
waterMask = readfile.read(water_mask_file, datasetName=dsName,
box=box)[0].flatten()
mask *= np.array(waterMask, np.bool_)
del waterMask
# 2 - Mask for Zero Phase in ALL ifgrams
print('skip pixels with zero/nan value in all interferograms')
phase_stack = np.nanmean(pha_data, axis=0)
mask *= np.multiply(~np.isnan(phase_stack), phase_stack != 0.)
del phase_stack
# Invert pixels on mask 1+2
num_pixel2inv = int(np.sum(mask))
idx_pixel2inv = np.where(mask)[0]
print(('number of pixels to invert: {} out of {}'
' ({:.1f}%)').format(num_pixel2inv, num_pixel,
num_pixel2inv / num_pixel * 100))
if num_pixel2inv < 1:
ts = ts.reshape(num_date, num_row, num_col)
temp_coh = temp_coh.reshape(num_row, num_col)
ts_std = ts_std.reshape(num_date, num_row, num_col)
num_inv_ifgram = num_inv_ifgram.reshape(num_row, num_col)
return ts, temp_coh, ts_std, num_inv_ifgram
# Inversion - SBAS
if weight_func == 'sbas':
# get tbase_diff (for SBAS approach)
date_list = stack_obj.get_date_list(dropIfgram=True)
tbase = np.array(ptime.date_list2tbase(date_list)[0],
np.float32) / 365.25
tbase_diff = np.diff(tbase).reshape(-1, 1)
# Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion)
mask_all_net = np.all(pha_data, axis=0)
mask_all_net *= mask
mask_part_net = mask ^ mask_all_net
if np.sum(mask_all_net) > 0:
print(('inverting pixels with valid phase in all ifgrams'
' ({:.0f} pixels) ...').format(np.sum(mask_all_net)))
# num_all_net = int(np.sum(mask_all_net))
# pha_data_temp = pha_data[:, mask_all_net]
# ts1 = np.zeros((num_date-1, num_all_net))
# temp_coh1 = np.zeros(num_all_net)
# step = 1000
# loop_num = int(np.floor(num_all_net/step))
# prog_bar = ptime.progressBar(maxValue=loop_num)
# for i in range(loop_num):
# [i0, i1] = [i * step, min((i + 1) * step, num_all_net)]
# ts1[:, i0:i1], temp_coh1[i0:i1] = network_inversion_sbas(B,
# ifgram=pha_data_temp[:, i0:i1],
# tbase_diff=tbase_diff,
# skip_zero_phase=False)
# prog_bar.update(i+1, suffix=i0)
# prog_bar.close()
ts1, temp_coh1, ifg_num1 = network_inversion_sbas(
B,
ifgram=pha_data[:, mask_all_net],
tbase_diff=tbase_diff,
skip_zero_phase=False)
ts[1:, mask_all_net] = ts1
temp_coh[mask_all_net] = temp_coh1
num_inv_ifgram[mask_all_net] = ifg_num1
if np.sum(mask_part_net) > 0:
print(('inverting pixels with valid phase in some ifgrams'
' ({:.0f} pixels) ...').format(np.sum(mask_part_net)))
num_pixel2inv = int(np.sum(mask_part_net))
idx_pixel2inv = np.where(mask_part_net)[0]
prog_bar = ptime.progressBar(maxValue=num_pixel2inv)
for i in range(num_pixel2inv):
idx = idx_pixel2inv[i]
ts1, temp_coh1, ifg_num1 = network_inversion_sbas(
B,
ifgram=pha_data[:, idx],
tbase_diff=tbase_diff,
skip_zero_phase=skip_zero_phase)
ts[1:, idx] = ts1.flatten()
temp_coh[idx] = temp_coh1
num_inv_ifgram[idx] = ifg_num1
prog_bar.update(i + 1,
every=100,
suffix='{}/{} pixels'.format(
i + 1, num_pixel2inv))
prog_bar.close()
# Inversion - WLS
else:
weight = read_coherence2weight(stack_obj,
box=box,
weight_func=weight_func)
# Converting to 32 bit floats leads to 2X speedup
# (comment it out as we now convert it beforehand)
# A = np.array(A, np.float32)
# pha_data = np.array(pha_data, np.float32)
# weight = np.array(weight, np.float32)
# Astd = np.array(Astd, np.float32)
# Weighted Inversion pixel by pixel
print('inverting network of interferograms into time series ...')
prog_bar = ptime.progressBar(maxValue=num_pixel2inv)
for i in range(num_pixel2inv):
idx = idx_pixel2inv[i]
ts1, temp_coh1, ts_std1, ifg_numi = network_inversion_wls(
A,
ifgram=pha_data[:, idx],
weight=weight[:, idx],
skip_zero_phase=skip_zero_phase,
Astd=Astd)
ts[1:, idx] = ts1.flatten()
temp_coh[idx] = temp_coh1
ts_std[time_idx, idx] = ts_std1.flatten()
num_inv_ifgram[idx] = ifg_numi
prog_bar.update(i + 1,
every=100,
suffix='{}/{} pixels'.format(i + 1, num_pixel2inv))
prog_bar.close()
ts = ts.reshape(num_date, num_row, num_col)
ts_std = ts_std.reshape(num_date, num_row, num_col)
temp_coh = temp_coh.reshape(num_row, num_col)
num_inv_ifgram = num_inv_ifgram.reshape(num_row, num_col)
# write output files if input file is splitted (box == None)
if box is None:
# metadata
metadata = dict(stack_obj.metadata)
metadata[key_prefix + 'weightFunc'] = weight_func
suffix = re.findall('_\d{3}', ifgram_file)[0]
write2hdf5_file(ifgram_file, metadata, ts, temp_coh, ts_std,
num_inv_ifgram, suffix)
return
else:
return ts, temp_coh, ts_std, num_inv_ifgram
def correct_dem_error(inps, A_def):
"""Correct DEM error of input timeseries file"""
# Read Date Info
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open()
num_date = ts_obj.numDate
num_pixel = ts_obj.numPixel
tbase = np.array(ts_obj.tbase, np.float32) / 365.25
num_step = len(inps.stepFuncDate)
drop_date, inps.excludeDate = read_exclude_date(inps.excludeDate,
ts_obj.dateList)
if inps.polyOrder > np.sum(drop_date):
raise ValueError(("input poly order {} > number of acquisition {}!"
" Reduce it!").format(inps.polyOrder,
np.sum(drop_date)))
# Read time-series Data
print('reading time-series data ...')
ts_data = ts_obj.read().reshape((num_date, -1))
##-------------------------------- Loop for L2-norm inversion --------------------------------##
print('inverting DEM error ...')
delta_z = np.zeros(num_pixel, dtype=np.float32)
ts_cor = np.zeros((num_date, num_pixel), dtype=np.float32)
ts_res = np.zeros((num_date, num_pixel), dtype=np.float32)
if num_step > 0:
step_model = np.zeros((num_step, num_pixel), dtype=np.float32)
print('skip pixels with zero/NaN value in all acquisitions')
ts_mean = np.nanmean(ts_data, axis=0)
mask = np.multiply(~np.isnan(ts_mean), ts_mean != 0.)
del ts_mean
if inps.rangeDist.size == 1:
A_geom = inps.pbase / (inps.rangeDist * inps.sinIncAngle)
A = np.hstack((A_geom, A_def))
ts_data = ts_data[:, mask]
(delta_z_i, ts_cor_i, ts_res_i,
step_model_i) = estimate_dem_error(ts_data,
A,
tbase=tbase,
drop_date=drop_date,
phaseVelocity=inps.phaseVelocity,
num_step=num_step)
delta_z[mask] = delta_z_i
ts_cor[:, mask] = ts_cor_i
ts_res[:, mask] = ts_res_i
if num_step > 0:
step_model[:, mask] = step_model_i
else:
# mask
print(
'skip pixels with zero/nan value in geometry: incidence angle or range distance'
)
mask *= np.multiply(inps.sinIncAngle != 0., inps.rangeDist != 0.)
num_pixel2inv = np.sum(mask)
idx_pixel2inv = np.where(mask)[0]
print(('number of pixels to invert: {} out of {}'
' ({:.1f}%)').format(num_pixel2inv, num_pixel,
num_pixel2inv / num_pixel * 100))
# update data matrix to save memory and IO
ts_data = ts_data[:, mask]
inps.rangeDist = inps.rangeDist[mask]
inps.sinIncAngle = inps.sinIncAngle[mask]
if inps.pbase.shape[1] != 1:
inps.pbase = inps.pbase[:, mask]
# loop pixel by pixel
prog_bar = ptime.progressBar(maxValue=num_pixel2inv)
for i in range(num_pixel2inv):
prog_bar.update(i + 1,
every=1000,
suffix='{}/{}'.format(i + 1, num_pixel2inv))
idx = idx_pixel2inv[i]
# design matrix
if inps.pbase.shape[1] == 1:
pbase = inps.pbase
else:
pbase = inps.pbase[:, i].reshape(-1, 1)
A_geom = pbase / (inps.rangeDist[i] * inps.sinIncAngle[i])
A = np.hstack((A_geom, A_def))
(delta_z_i, ts_cor_i, ts_res_i, step_model_i) = estimate_dem_error(
ts_data[:, i],
A,
tbase=tbase,
drop_date=drop_date,
phaseVelocity=inps.phaseVelocity,
num_step=num_step)
delta_z[idx:idx + 1] = delta_z_i
ts_cor[:, idx:idx + 1] = ts_cor_i
ts_res[:, idx:idx + 1] = ts_res_i
if num_step > 0:
step_model[:, idx:idx + 1] = step_model_i
prog_bar.close()
del ts_data
##---------------------------------------- Output -----------------------------------------##
# prepare for output
ts_cor = ts_cor.reshape((num_date, ts_obj.length, ts_obj.width))
ts_res = ts_res.reshape((num_date, ts_obj.length, ts_obj.width))
delta_z = delta_z.reshape((ts_obj.length, ts_obj.width))
if num_step > 0:
step_model = step_model.reshape(
(num_step, ts_obj.length, ts_obj.width))
atr = dict(ts_obj.metadata)
# config parameter
print(
'add/update the following configuration metadata to file:\n{}'.format(
configKeys))
for key in configKeys:
atr[key_prefix + key] = str(vars(inps)[key])
# 1. Estimated DEM error
outfile = 'demErr.h5'
atr['FILE_TYPE'] = 'dem'
atr['UNIT'] = 'm'
writefile.write(delta_z, out_file=outfile, metadata=atr)
# 2. Time-series corrected for DEM error
atr['FILE_TYPE'] = 'timeseries'
writefile.write(ts_cor,
out_file=inps.outfile,
metadata=atr,
ref_file=ts_obj.file)
# 3. Time-series of inversion residual
ts_ref_file = os.path.join(os.path.dirname(inps.outfile),
'timeseriesResidual.h5')
writefile.write(ts_res,
out_file=ts_ref_file,
metadata=atr,
ref_file=ts_obj.file)
# 4. Time-series of estimated Step Model
if num_step > 0:
atr.pop('REF_DATE')
step_obj = timeseries(
os.path.join(os.path.dirname(inps.outfile),
'timeseriesStepModel.h5'))
step_obj.write2hdf5(data=step_model,
metadata=atr,
dates=inps.stepFuncDate)
## 5. Time-series of estimated Deformation Model = poly model + step model
#ts_def_obj = timeseries(os.path.join(os.path.dirname(inps.outfile), 'timeseriesDefModel.h5'))
#ts_def_obj.write2hdf5(data=ts_cor - ts_res, refFile=ts_obj.file)
#del ts_cor, ts_res
return inps
def unwrap_error_correction_phase_closure(ifgram_file, mask_file, ifgram_cor_file=None):
"""Correct unwrapping errors in network of interferograms using phase closure.
Inputs:
ifgram_file - string, name/path of interferograms file
mask_file - string, name/path of mask file to mask the pixels to be corrected
ifgram_cor_file - string, optional, name/path of corrected interferograms file
Output:
ifgram_cor_file
Example:
'unwrapIfgram_unwCor.h5' = unwrap_error_correction_phase_closure('Seeded_unwrapIfgram.h5','mask.h5')
"""
print('read mask from file: '+mask_file)
mask = readfile.read(mask_file, datasetName='mask')[0].flatten(1)
atr = readfile.read_attribute(ifgram_file)
length = int(atr['LENGTH'])
width = int(atr['WIDTH'])
k = atr['FILE_TYPE']
pixel_num = length*width
# Check reference pixel
try:
ref_y = int(atr['REF_Y'])
ref_x = int(atr['REF_X'])
print('reference pixel in y/x: %d/%d' % (ref_y, ref_x))
except:
sys.exit('ERROR: Can not find ref_y/x value, input file is not referenced in space!')
h5 = h5py.File(ifgram_file, 'r')
ifgram_list = sorted(h5[k].keys())
ifgram_num = len(ifgram_list)
# Prepare curls
curls, Triangles, C = ut.get_triangles(h5)
curl_num = np.shape(curls)[0]
print('Number of triangles: ' + str(curl_num))
curl_file = 'curls.h5'
if not os.path.isfile(curl_file):
print('writing >>> '+curl_file)
ut.generate_curls(curl_file, h5, Triangles, curls)
thr = 0.50
curls = np.array(curls)
n1 = curls[:, 0]
n2 = curls[:, 1]
n3 = curls[:, 2]
print('reading interferograms...')
print('Number of interferograms: ' + str(ifgram_num))
data = np.zeros((ifgram_num, pixel_num), np.float32)
prog_bar = ptime.progressBar(maxValue=ifgram_num)
for ni in range(ifgram_num):
ifgram = ifgram_list[ni]
d = h5[k][ifgram].get(ifgram)[:].flatten(1)
data[ni, :] = d
prog_bar.update(ni+1)
prog_bar.close()
print('reading curls ...')
print('number of culrs: '+str(curl_num))
h5curl = h5py.File(curl_file, 'r')
curl_list = sorted(h5curl[k].keys())
curl_data = np.zeros((curl_num, pixel_num), np.float32)
prog_bar = ptime.progressBar(maxValue=curl_num)
for ni in range(curl_num):
d = h5curl[k][curl_list[ni]].get(curl_list[ni])[:].flatten(1)
curl_data[ni, :] = d.flatten(1)
prog_bar.update(ni+1)
prog_bar.close()
h5curl.close()
print('estimating unwrapping error pixel by pixel ...')
EstUnwrap = np.zeros((ifgram_num, pixel_num), np.float32)
prog_bar = ptime.progressBar(maxValue=pixel_num)
for ni in range(pixel_num):
if mask[ni] == 1:
dU = data[:, ni]
unwCurl = np.array(curl_data[:, ni])
ind = np.abs(unwCurl) >= thr
N1 = n1[ind]
N2 = n2[ind]
N3 = n3[ind]
indC = np.abs(unwCurl) < thr
Nc1 = n1[indC]
Nc2 = n2[indC]
Nc3 = n3[indC]
N = np.hstack([N1, N2, N3])
UniN = np.unique(N)
Nc = np.hstack([Nc1, Nc2, Nc3])
UniNc = np.unique(Nc)
inter = list(set(UniNc) & set(UniN)) # intersetion
UniNc = list(UniNc)
for x in inter:
UniNc.remove(x)
D = np.zeros([len(UniNc), ifgram_num])
for i in range(len(UniNc)):
D[i, UniNc[i]] = 1
AAA = np.vstack([-2*np.pi*C, D])
AAAA = np.vstack([AAA, 0.25*np.eye(ifgram_num)])
##########
# with Tikhonov regularization:
LLL = list(np.dot(C, dU)) + list(np.zeros(np.shape(UniNc)[0])) + list(np.zeros(ifgram_num))
ind = np.isnan(AAAA)
M1 = pinv(AAAA)
M = np.dot(M1, LLL)
EstUnwrap[:, ni] = np.round(M[0:ifgram_num])*2.0*np.pi
prog_bar.update(ni+1, suffix='%s/%d' % (ni, pixel_num))
prog_bar.close()
dataCor = data + EstUnwrap
# Output
if not ifgram_cor_file:
ifgram_cor_file = os.path.splitext(ifgram_file)[0]+'_unwCor.h5'
print('writing >>> '+ifgram_cor_file)
h5unwCor = h5py.File(ifgram_cor_file, 'w')
gg = h5unwCor.create_group(k)
prog_bar = ptime.progressBar(maxValue=ifgram_num)
for i in range(ifgram_num):
ifgram = ifgram_list[i]
group = gg.create_group(ifgram)
dset = group.create_dataset(ifgram, data=np.reshape(dataCor[i, :], [width, length]).T)
for key, value in h5[k][ifgram].attrs.items():
group.attrs[key] = value
prog_bar.update(i+1)
prog_bar.close()
h5unwCor.close()
h5.close()
return ifgram_cor_file
def unwrap_error_correction_bridging(ifgram_file, mask_file, y_list, x_list, ramp_type='plane',
ifgram_cor_file=None, save_cor_deramp_file=False):
"""Unwrapping error correction with bridging.
Inputs:
ifgram_file : string, name/path of interferogram(s) to be corrected
mask_file : string, name/path of mask file to mark different patches
y/x_list : list of int, bonding points in y/x
ifgram_cor_file : string, optional, output file name
save_cor_deramp_file : bool, optional
Output:
ifgram_cor_file
Example:
y_list = [235, 270, 350, 390]
x_list = [880, 890, 1200, 1270]
unwrap_error_correction_bridging('unwrapIfgram.h5', 'mask_all.h5', y_list, x_list, 'quadratic')
"""
##### Mask and Ramp
mask = readfile.read(mask_file, datasetName='mask')[0]
ramp_mask = mask == 1
print('estimate phase ramp during the correction')
print('ramp type: '+ramp_type)
# Bridge Info
# Check
for i in range(len(x_list)):
if mask[y_list[i], x_list[i]] == 0:
print('\nERROR: Connecting point (%d,%d) is out of masked area! Select them again!\n' % (y_list[i],
x_list[i]))
sys.exit(1)
print('Number of bridges: '+str(len(x_list)/2))
print('Bonding points coordinates:\nx: '+str(x_list)+'\ny: '+str(y_list))
# Plot Connecting Pair of Points
plot_bonding_points = False
if plot_bonding_points:
point_yx = ''
line_yx = ''
n_bridge = len(x)/2
for i in range(n_bridge):
pairs_yx = '{},{},{},{}'.format(y[2*i],
x[2*i],
y[2*i+1],
x[2*i+1])
if not i == n_bridge-1:
point_yx += pair_yx+','
line_yx += pair_yx+';'
else:
point_yx += pair_yx
line_yx += pair_yx
try:
plot_cmd = 'view.py --point-yx="'+point_yx+'" --line-yx="'+line_yx +\
'" --nodisplay -o bonding_points.png -f '+maskFile
print(plot_cmd)
os.system(plot_cmd)
except:
pass
# Basic info
ext = os.path.splitext(ifgram_file)[1]
atr = readfile.read_attribute(ifgram_file)
k = atr['FILE_TYPE']
try:
ref_y = int(atr['REF_Y'])
ref_x = int(atr['REF_X'])
print('reference pixel in y/x: %d/%d' % (ref_y, ref_x))
except:
sys.exit('ERROR: Can not find ref_y/x value, input file is not referenced in space!')
# output file name
if not ifgram_cor_file:
ifgram_cor_file = os.path.splitext(ifgram_file)[0]+'_unwCor'+ext
ifgram_cor_deramp_file = os.path.splitext(ifgram_cor_file)[0]+'_'+ramp_type+ext
# HDF5 file
if ext == '.h5':
# Read
h5 = h5py.File(ifgram_file, 'r')
ifgram_list = sorted(h5[k].keys())
ifgram_num = len(ifgram_list)
h5out = h5py.File(ifgram_cor_file, 'w')
group = h5out.create_group(k)
print('writing >>> '+ifgram_cor_file)
if save_cor_deramp_file:
h5out_deramp = h5py.File(ifgram_cor_deramp_file, 'w')
group_deramp = h5out_deramp.create_group(k)
print('writing >>> '+ifgram_cor_deramp_file)
# Loop
print('Number of interferograms: '+str(ifgram_num))
prog_bar = ptime.progressBar(maxValue=ifgram_num)
date12_list = ptime.list_ifgram2date12(ifgram_list)
for i in range(ifgram_num):
ifgram = ifgram_list[i]
data = h5[k][ifgram].get(ifgram)[:]
data -= data[ref_y, ref_x]
data_deramp, ramp = deramp.remove_data_surface(data, ramp_mask, ramp_type)
data_derampCor = bridging_data(data_deramp, mask, x_list, y_list)
ramp[data == 0.] = 0.
gg = group.create_group(ifgram)
dset = gg.create_dataset(ifgram, data=data_derampCor+ramp)
for key, value in h5[k][ifgram].attrs.items():
gg.attrs[key] = value
if save_cor_deramp_file:
gg_deramp = group_deramp.create_group(ifgram)
dset = gg_deramp.create_dataset(ifgram, data=data_derampCor)
for key, value in h5[k][ifgram].attrs.items():
gg_deramp.attrs[key] = value
prog_bar.update(i+1, suffix=date12_list[i])
prog_bar.close()
h5.close()
h5out.close()
try:
h5out_deramp.close()
except:
pass
# .unw file
elif ext == '.unw':
print('read '+ifgram_file)
data = readfile.read(ifgram_file)[0]
data -= data[ref_y, ref_x]
data_deramp, ramp = deramp.remove_data_surface(data, ramp_mask, ramp_type)
data_derampCor = bridging_data(data_deramp, mask, x_list, y_list)
print('writing >>> '+ifgram_cor_file)
ramp[data == 0.] = 0.
ifgram_cor_file = writefile.write(data_derampCor+ramp,
out_file=ifgram_cor_file,
metadata=atr)
if save_cor_deramp_file:
print('writing >>> '+ifgram_cor_deramp_file)
ifgram_cor_deramp_file = writefile.write(data_derampCor,
out_file=ifgram_cor_deramp_file,
metadata=atr)
else:
sys.exit('Un-supported file type: '+ext)
return ifgram_cor_file, ifgram_cor_deramp_file
def main(argv):
# Inputs
try:
ifgram_file = argv[0]
timeseries_file = argv[1]
except:
usage()
sys.exit(1)
try:
outfile = argv[2]
except:
outfile = 'recon_'+ifgram_file
atr = readfile.read_attribute(timeseries_file)
length = int(atr['LENGTH'])
width = int(atr['WIDTH'])
# Read time-series file
print('loading timeseries ...')
h5ts = h5py.File(timeseries_file, 'r')
date_list = sorted(h5ts['timeseries'].keys())
date_num = len(date_list)
timeseries = np.zeros((date_num, length*width))
print('number of acquisitions: '+str(date_num))
prog_bar = ptime.progressBar(maxValue=date_num)
for i in range(date_num):
date = date_list[i]
d = h5ts['timeseries'].get(date)[:]
timeseries[i, :] = d.flatten(0)
prog_bar.update(i+1, suffix=date)
prog_bar.close()
h5ts.close()
del d
range2phase = -4*np.pi/float(atr['WAVELENGTH'])
timeseries = range2phase*timeseries
# Estimate interferograms from timeseries
print('estimating interferograms from timeseries using design matrix from input interferograms')
A, B = ut.design_matrix(ifgram_file)
p = -1*np.ones([A.shape[0], 1])
Ap = np.hstack((p, A))
estData = np.dot(Ap, timeseries)
del timeseries
# Write interferograms file
print('writing >>> '+outfile)
h5 = h5py.File(ifgram_file, 'r')
ifgram_list = sorted(h5['interferograms'].keys())
ifgram_num = len(ifgram_list)
date12_list = ptime.list_ifgram2date12(ifgram_list)
h5out = h5py.File(outfile, 'w')
group = h5out.create_group('interferograms')
print('number of interferograms: '+str(ifgram_num))
prog_bar = ptime.progressBar(maxValue=ifgram_num)
for i in range(ifgram_num):
ifgram = ifgram_list[i]
data = np.reshape(estData[i, :], (length, width))
gg = group.create_group(ifgram)
dset = gg.create_dataset(ifgram, data=data)
for key, value in h5['interferograms'][ifgram].attrs.items():
gg.attrs[key] = value
prog_bar.update(i+1, suffix=date12_list[i])
prog_bar.close()
h5.close()
h5out.close()
print('Done.')
return outfile
def run_unwrap_error_phase_closure(ifgram_file,
common_regions,
water_mask_file=None,
ccName='connectComponent',
dsNameIn='unwrapPhase',
dsNameOut='unwrapPhase_phaseClosure'):
print('-' * 50)
print('correct unwrapping error in {} with phase closure ...'.format(
ifgram_file))
stack_obj = ifgramStack(ifgram_file)
stack_obj.open()
length, width = stack_obj.length, stack_obj.width
ref_y, ref_x = stack_obj.refY, stack_obj.refX
date12_list = stack_obj.get_date12_list(dropIfgram=False)
num_ifgram = len(date12_list)
shape_out = (num_ifgram, length, width)
# read water mask
if water_mask_file and os.path.isfile(water_mask_file):
print('read water mask from file:', water_mask_file)
water_mask = readfile.read(water_mask_file)[0]
else:
water_mask = None
# prepare output data writing
print('open {} with r+ mode'.format(ifgram_file))
f = h5py.File(ifgram_file, 'r+')
print('input dataset:', dsNameIn)
print('output dataset:', dsNameOut)
if dsNameOut in f.keys():
ds = f[dsNameOut]
print('access /{d} of np.float32 in size of {s}'.format(d=dsNameOut,
s=shape_out))
else:
ds = f.create_dataset(dsNameOut,
shape_out,
maxshape=(None, None, None),
chunks=True,
compression=None)
print('create /{d} of np.float32 in size of {s}'.format(d=dsNameOut,
s=shape_out))
# correct unwrap error ifgram by ifgram
prog_bar = ptime.progressBar(maxValue=num_ifgram)
for i in range(num_ifgram):
date12 = date12_list[i]
# read unwrap phase to be updated
unw_cor = np.squeeze(f[dsNameIn][i, :, :]).astype(np.float32)
unw_cor -= unw_cor[ref_y, ref_x]
# update kept interferograms only
if stack_obj.dropIfgram[i]:
# get local region info from connectComponent
cc = np.squeeze(f[ccName][i, :, :])
if water_mask is not None:
cc[water_mask == 0] = 0
cc_obj = connectComponent(conncomp=cc, metadata=stack_obj.metadata)
cc_obj.label()
local_regions = measure.regionprops(cc_obj.labelImg)
# matching regions and correct unwrap error
idx_common = common_regions[0].date12_list.index(date12)
for local_reg in local_regions:
local_mask = cc_obj.labelImg == local_reg.label
U = 0
for common_reg in common_regions:
y = common_reg.sample_coords[:, 0]
x = common_reg.sample_coords[:, 1]
if all(local_mask[y, x]):
U = common_reg.int_ambiguity[idx_common]
break
unw_cor[local_mask] += 2. * np.pi * U
# write to hdf5 file
ds[i, :, :] = unw_cor
prog_bar.update(i + 1, suffix=date12)
prog_bar.close()
ds.attrs['MODIFICATION_TIME'] = str(time.time())
f.close()
print('close {} file.'.format(ifgram_file))
return ifgram_file
def run_deramp(fname, ramp_type, mask_file=None, out_file=None, datasetName=None):
""" Remove ramp from each 2D matrix of input file
Parameters: fname : str, data file to be derampped
ramp_type : str, name of ramp to be estimated.
mask_file : str, file of mask of pixels used for ramp estimation
out_file : str, output file name
datasetName : str, output dataset name, for ifgramStack file type only
Returns: out_file : str, output file name
"""
print('remove {} ramp from file: {}'.format(ramp_type, fname))
if not out_file:
fbase, fext = os.path.splitext(fname)
out_file = '{}_ramp{}'.format(fbase, fext)
start_time = time.time()
atr = readfile.read_attribute(fname)
# mask
if os.path.isfile(mask_file):
mask = readfile.read(mask_file, datasetName='mask')[0]
print('read mask file: '+mask_file)
else:
mask = np.ones((int(atr['LENGTH']), int(atr['WIDTH'])))
print('use mask of the whole area')
# deramping
k = atr['FILE_TYPE']
if k == 'timeseries':
print('reading data ...')
data = readfile.read(fname)[0]
print('estimating phase ramp ...')
data = deramp(data, mask, ramp_type=ramp_type, metadata=atr)[0]
writefile.write(data, out_file, ref_file=fname)
elif k == 'ifgramStack':
obj = ifgramStack(fname)
obj.open(print_msg=False)
if not datasetName:
datasetName = 'unwrapPhase'
with h5py.File(fname, 'a') as f:
ds = f[datasetName]
dsNameOut = '{}_ramp'.format(datasetName)
if dsNameOut in f.keys():
dsOut = f[dsNameOut]
print('access HDF5 dataset /{}'.format(dsNameOut))
else:
dsOut = f.create_dataset(dsNameOut, shape=(obj.numIfgram, obj.length, obj.width),
dtype=np.float32, chunks=True, compression=None)
print('create HDF5 dataset /{}'.format(dsNameOut))
prog_bar = ptime.progressBar(maxValue=obj.numIfgram)
for i in range(obj.numIfgram):
data = ds[i, :, :]
data = deramp(data, mask, ramp_type=ramp_type, metadata=atr)[0]
dsOut[i, :, :] = data
prog_bar.update(i+1, suffix='{}/{}'.format(i+1, obj.numIfgram))
prog_bar.close()
print('finished writing to file: '.format(fname))
# Single Dataset File
else:
data = readfile.read(fname)[0]
data = deramp(data, mask, ramp_type, metadata=atr)[0]
print('writing >>> {}'.format(out_file))
writefile.write(data, out_file=out_file, ref_file=fname)
m, s = divmod(time.time()-start_time, 60)
print('time used: {:02.0f} mins {:02.1f} secs.'.format(m, s))
return out_file
def write2hdf5(self,
outputFile='ifgramStack.h5',
access_mode='w',
box=None,
compression=None,
extra_metadata=None):
'''Save/write an ifgramStackDict object into an HDF5 file with the structure below:
/ Root level
Attributes Dictionary for metadata
/date 2D array of string in size of (m, 2 ) in YYYYMMDD format for master and slave date
/bperp 1D array of float32 in size of (m, ) in meter.
/dropIfgram 1D array of bool in size of (m, ).
/unwrapPhase 3D array of float32 in size of (m, l, w) in radian.
/coherence 3D array of float32 in size of (m, l, w).
/connectComponent 3D array of int16 in size of (m, l, w). (optional)
/wrapPhase 3D array of float32 in size of (m, l, w) in radian. (optional)
/rangeOffset 3D array of float32 in size of (m, l, w). (optional)
/azimuthOffset 3D array of float32 in size of (m, l, w). (optional)
Parameters: outputFile : str, Name of the HDF5 file for the InSAR stack
access_mode : str, access mode of output File, e.g. w, r+
box : tuple, subset range in (x0, y0, x1, y1)
extra_metadata : dict, extra metadata to be added into output file
Returns: outputFile
'''
self.outputFile = outputFile
f = h5py.File(self.outputFile, access_mode)
print('create HDF5 file {} with {} mode'.format(
self.outputFile, access_mode))
self.pairs = sorted([pair for pair in self.pairsDict.keys()])
self.dsNames = list(self.pairsDict[self.pairs[0]].datasetDict.keys())
self.dsNames = [i for i in ifgramDatasetNames if i in self.dsNames]
maxDigit = max([len(i) for i in self.dsNames])
self.get_size(box)
self.bperp = np.zeros(self.numIfgram)
###############################
# 3D datasets containing unwrapPhase, coherence, connectComponent, wrapPhase, etc.
for dsName in self.dsNames:
dsShape = (self.numIfgram, self.length, self.width)
dsDataType = dataType
if dsName in ['connectComponent']:
dsDataType = np.bool_
print(('create dataset /{d:<{w}} of {t:<25} in size of {s}'
' with compression = {c}').format(d=dsName,
w=maxDigit,
t=str(dsDataType),
s=dsShape,
c=str(compression)))
ds = f.create_dataset(
dsName,
shape=dsShape,
maxshape=(None, dsShape[1], dsShape[2]),
dtype=dsDataType,
chunks=(5, 100, 300), #True
compression=compression)
prog_bar = ptime.progressBar(maxValue=self.numIfgram)
for i in range(self.numIfgram):
ifgramObj = self.pairsDict[self.pairs[i]]
data = ifgramObj.read(dsName, box=box)[0]
ds[i, :, :] = data
self.bperp[i] = ifgramObj.get_perp_baseline()
prog_bar.update(i + 1,
suffix='{}_{}'.format(self.pairs[i][0],
self.pairs[i][1]))
prog_bar.close()
ds.attrs['MODIFICATION_TIME'] = str(time.time())
###############################
# 2D dataset containing master and slave dates of all pairs
dsName = 'date'
dsDataType = np.string_
dsShape = (self.numIfgram, 2)
print('create dataset /{d:<{w}} of {t:<25} in size of {s}'.format(
d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape))
data = np.array(self.pairs, dtype=dsDataType)
ds = f.create_dataset(dsName, data=data)
###############################
# 1D dataset containing perpendicular baseline of all pairs
dsName = 'bperp'
dsDataType = dataType
dsShape = (self.numIfgram, )
print('create dataset /{d:<{w}} of {t:<25} in size of {s}'.format(
d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape))
data = np.array(self.bperp, dtype=dsDataType)
ds = f.create_dataset(dsName, data=data)
###############################
# 1D dataset containing bool value of dropping the interferograms or not
dsName = 'dropIfgram'
dsDataType = np.bool_
dsShape = (self.numIfgram, )
print('create dataset /{d:<{w}} of {t:<25} in size of {s}'.format(
d=dsName, w=maxDigit, t=str(dsDataType), s=dsShape))
data = np.ones(dsShape, dtype=dsDataType)
dsDate = f.create_dataset(dsName, data=data)
###############################
# Attributes
self.get_metadata()
if extra_metadata:
self.metadata.update(extra_metadata)
print('add extra metadata: {}'.format(extra_metadata))
self.metadata = ut.subset_attribute(self.metadata, box)
self.metadata['FILE_TYPE'] = self.name
for key, value in self.metadata.items():
f.attrs[key] = value
f.close()
print('Finished writing to {}'.format(self.outputFile))
return self.outputFile
def ifgram_inversion_patch(ifgram_file,
box=None,
ref_phase=None,
unwDatasetName='unwrapPhase',
weight_func='var',
min_norm_velocity=True,
mask_dataset_name=None,
mask_threshold=0.4,
min_redundancy=1.0,
water_mask_file=None,
skip_zero_phase=True):
"""Invert one patch of an ifgram stack into timeseries.
Parameters: ifgram_file : str, interferograms stack HDF5 file, e.g. ./INPUTS/ifgramStack.h5
box : tuple of 4 int, indicating (x0, y0, x1, y1) pixel coordinate of area of interest
or None, to process the whole file and write output file
ref_phase : 1D array in size of (num_ifgram)
or None
weight_func : str, weight function, choose in ['no', 'fim', 'var', 'coh']
mask_dataset_name : str, dataset name in ifgram_file used to mask unwrapPhase pixelwisely
mask_threshold : float, min coherence of pixels if mask_dataset_name='coherence'
water_mask_file : str, water mask filename if available,
skip inversion on water to speed up the process
skip_zero_phase : bool, skip zero value of unwrapped phase or not, default yes, for comparison
Returns: ts : 3D array in size of (num_date, num_row, num_col)
temp_coh : 2D array in size of (num_row, num_col)
ts_std : 3D array in size of (num_date, num_row, num_col)
num_inv_ifg : 2D array in size of (num_row, num_col)
Example: ifgram_inversion_patch('ifgramStack.h5', box=(0,200,1316,400), ref_phase=np.array(),
weight_func='var', min_norm_velocity=True, mask_dataset_name='coherence')
ifgram_inversion_patch('ifgramStack_001.h5', box=None, ref_phase=None,
weight_func='var', min_norm_velocity=True, mask_dataset_name='coherence')
"""
stack_obj = ifgramStack(ifgram_file)
stack_obj.open(print_msg=False)
## debug
#y, x = 258, 454
#box = (x, y, x+1, y+1)
# Size Info - Patch
if box:
#print('processing \t %d-%d / %d lines ...' % (box[1], box[3], stack_obj.length))
num_row = box[3] - box[1]
num_col = box[2] - box[0]
else:
num_row = stack_obj.length
num_col = stack_obj.width
num_pixel = num_row * num_col
# get tbase_diff
date_list = stack_obj.get_date_list(dropIfgram=True)
num_date = len(date_list)
tbase = np.array(ptime.date_list2tbase(date_list)[0], np.float32) / 365.25
tbase_diff = np.diff(tbase).reshape(-1, 1)
# Design matrix
date12_list = stack_obj.get_date12_list(dropIfgram=True)
A, B = stack_obj.get_design_matrix4timeseries_estimation(
date12_list=date12_list)[0:2]
num_ifgram = len(date12_list)
# prep for decor std time-series
try:
ref_date = str(
np.loadtxt('reference_date.txt', dtype=bytes).astype(str))
except:
ref_date = date_list[0]
Astd = stack_obj.get_design_matrix4timeseries_estimation(
refDate=ref_date, dropIfgram=True)[0]
#ref_idx = date_list.index(ref_date)
#time_idx = [i for i in range(num_date)]
#time_idx.remove(ref_idx)
# Initialization of output matrix
ts = np.zeros((num_date, num_pixel), np.float32)
ts_std = np.zeros((num_date, num_pixel), np.float32)
temp_coh = np.zeros(num_pixel, np.float32)
num_inv_ifg = np.zeros(num_pixel, np.int16)
# Read/Mask unwrapPhase
pha_data = read_unwrap_phase(stack_obj,
box,
ref_phase,
unwDatasetName=unwDatasetName,
dropIfgram=True,
skip_zero_phase=skip_zero_phase)
pha_data = mask_unwrap_phase(pha_data,
stack_obj,
box,
dropIfgram=True,
mask_ds_name=mask_dataset_name,
mask_threshold=mask_threshold)
# Mask for pixels to invert
mask = np.ones(num_pixel, np.bool_)
# 1 - Water Mask
if water_mask_file:
print(('skip pixels on water with mask from'
' file: {}').format(os.path.basename(water_mask_file)))
atr_msk = readfile.read_attribute(water_mask_file)
if (int(atr_msk['LENGTH']), int(
atr_msk['WIDTH'])) != (stack_obj.length, stack_obj.width):
raise ValueError(
'Input water mask file has different size from ifgramStack file.'
)
del atr_msk
dsName = [
i for i in readfile.get_dataset_list(water_mask_file)
if i in ['waterMask', 'mask']
][0]
waterMask = readfile.read(water_mask_file, datasetName=dsName,
box=box)[0].flatten()
mask *= np.array(waterMask, np.bool_)
del waterMask
# 2 - Mask for Zero Phase in ALL ifgrams
print('skip pixels with zero/nan value in all interferograms')
phase_stack = np.nanmean(pha_data, axis=0)
mask *= np.multiply(~np.isnan(phase_stack), phase_stack != 0.)
del phase_stack
# Invert pixels on mask 1+2
num_pixel2inv = int(np.sum(mask))
idx_pixel2inv = np.where(mask)[0]
print(('number of pixels to invert: {} out of {}'
' ({:.1f}%)').format(num_pixel2inv, num_pixel,
num_pixel2inv / num_pixel * 100))
if num_pixel2inv < 1:
ts = ts.reshape(num_date, num_row, num_col)
ts_std = ts_std.reshape(num_date, num_row, num_col)
temp_coh = temp_coh.reshape(num_row, num_col)
num_inv_ifg = num_inv_ifg.reshape(num_row, num_col)
return ts, temp_coh, ts_std, num_inv_ifg
# Inversion - SBAS
if weight_func in ['no', 'sbas']:
# Mask for Non-Zero Phase in ALL ifgrams (share one B in sbas inversion)
mask_all_net = np.all(pha_data, axis=0)
mask_all_net *= mask
mask_part_net = mask ^ mask_all_net
if np.sum(mask_all_net) > 0:
print(('inverting pixels with valid phase in all ifgrams'
' ({:.0f} pixels) ...').format(np.sum(mask_all_net)))
tsi, tcohi, num_ifgi = estimate_timeseries(
A,
B,
tbase_diff,
ifgram=pha_data[:, mask_all_net],
weight_sqrt=None,
min_norm_velocity=min_norm_velocity,
skip_zero_phase=skip_zero_phase,
min_redundancy=min_redundancy)
ts[:, mask_all_net] = tsi
temp_coh[mask_all_net] = tcohi
num_inv_ifg[mask_all_net] = num_ifgi
if np.sum(mask_part_net) > 0:
print(('inverting pixels with valid phase in some ifgrams'
' ({:.0f} pixels) ...').format(np.sum(mask_part_net)))
num_pixel2inv = int(np.sum(mask_part_net))
idx_pixel2inv = np.where(mask_part_net)[0]
prog_bar = ptime.progressBar(maxValue=num_pixel2inv)
for i in range(num_pixel2inv):
idx = idx_pixel2inv[i]
tsi, tcohi, num_ifgi = estimate_timeseries(
A,
B,
tbase_diff,
ifgram=pha_data[:, idx],
weight_sqrt=None,
min_norm_velocity=min_norm_velocity,
skip_zero_phase=skip_zero_phase,
min_redundancy=min_redundancy)
ts[:, idx] = tsi.flatten()
temp_coh[idx] = tcohi
num_inv_ifg[idx] = num_ifgi
prog_bar.update(i + 1,
every=1000,
suffix='{}/{} pixels'.format(
i + 1, num_pixel2inv))
prog_bar.close()
# Inversion - WLS
else:
L = int(stack_obj.metadata['ALOOKS']) * int(
stack_obj.metadata['RLOOKS'])
weight = read_coherence(stack_obj, box=box, dropIfgram=True)
weight = coherence2weight(weight,
weight_func=weight_func,
L=L,
epsilon=5e-2)
weight = np.sqrt(weight)
# Weighted Inversion pixel by pixel
print('inverting network of interferograms into time-series ...')
prog_bar = ptime.progressBar(maxValue=num_pixel2inv)
for i in range(num_pixel2inv):
idx = idx_pixel2inv[i]
tsi, tcohi, num_ifgi = estimate_timeseries(
A,
B,
tbase_diff,
ifgram=pha_data[:, idx],
weight_sqrt=weight[:, idx],
min_norm_velocity=min_norm_velocity,
skip_zero_phase=skip_zero_phase,
min_redundancy=min_redundancy)
ts[:, idx] = tsi.flatten()
temp_coh[idx] = tcohi
num_inv_ifg[idx] = num_ifgi
prog_bar.update(i + 1,
every=1000,
suffix='{}/{} pixels'.format(i + 1, num_pixel2inv))
prog_bar.close()
ts = ts.reshape(num_date, num_row, num_col)
ts_std = ts_std.reshape(num_date, num_row, num_col)
temp_coh = temp_coh.reshape(num_row, num_col)
num_inv_ifg = num_inv_ifg.reshape(num_row, num_col)
# write output files if input file is splitted (box == None)
if box is None:
# metadata
metadata = dict(stack_obj.metadata)
metadata[key_prefix + 'weightFunc'] = weight_func
suffix = re.findall('_\d{3}', ifgram_file)[0]
write2hdf5_file(ifgram_file,
metadata,
ts,
temp_coh,
ts_std,
num_inv_ifg,
suffix,
inps=inps)
return
else:
return ts, temp_coh, ts_std, num_inv_ifg
def remove_surface(fname, surf_type, mask_file=None, out_file=None, ysub=None):
start_time = time.time()
atr = readfile.read_attribute(fname)
k = atr['FILE_TYPE']
print('remove {} ramp from file: '.format(surf_type, fname))
if not out_file:
out_file = '{base}_{ramp}{ext}'.format(base=os.path.splitext(fname)[0],
ramp=surf_type,
ext=os.path.splitext(fname)[1])
if os.path.isfile(mask_file):
mask = readfile.read(mask_file, datasetName='mask')[0]
print('read mask file: ' + mask_file)
else:
mask = np.ones((int(atr['LENGTH']), int(atr['WIDTH'])))
print('use mask of the whole area')
if k == 'timeseries':
obj = timeseries(fname)
data = obj.read()
numDate = data.shape[0]
print('estimating phase ramp ...')
prog_bar = ptime.progressBar(maxValue=numDate)
for i in range(numDate):
if not ysub:
data[i, :, :] = remove_data_surface(np.squeeze(data[i, :, :]),
mask, surf_type)[0]
else:
data[i, :, :] = remove_data_multiple_surface(
np.squeeze(data), mask, surf_type, ysub)
prog_bar.update(i + 1, suffix=obj.dateList[i])
prog_bar.close()
objOut = timeseries(out_file)
objOut.write2hdf5(data=data, refFile=fname)
elif k == 'ifgramStack':
obj = ifgramStack(fname)
obj.open(print_msg=False)
with h5py.File(fname, 'a') as f:
ds = f['unwrapPhase']
dsName = 'unwrapPhase_{}'.format(surf_type)
if dsName in f.keys():
dsOut = f[dsName]
print('access HDF5 dataset /{}'.format(dsName))
else:
dsOut = f.create_dataset(dsName,
shape=(obj.numIfgram, obj.length,
obj.width),
dtype=np.float32,
chunks=True,
compression=None)
print('create HDF5 dataset /{}'.format(dsName))
prog_bar = ptime.progressBar(maxValue=obj.numIfgram)
for i in range(obj.numIfgram):
data = ds[i, :, :]
mask_n = np.array(mask, np.bool_)
mask_n[data == 0.] = 0
if not ysub:
data_n = remove_data_surface(data, mask, surf_type)[0]
else:
data_n = remove_data_multiple_surface(
data, mask, surf_type, ysub)
dsOut[i, :, :] = data_n
prog_bar.update(i + 1,
suffix='{}/{}'.format(i + 1, obj.numIfgram))
prog_bar.close()
# Single Dataset File
else:
data, atr = readfile.read(fname)
data_n, ramp = remove_data_surface(data, mask, surf_type)
writefile.write(data_n, out_file=out_file, metadata=atr)
m, s = divmod(time.time() - start_time, 60)
print('\ntime used: {:02.0f} mins {:02.1f} secs'.format(m, s))
return out_file
