def read_template2inps(template_file, inps=None):
"""Read input template file into inps.excludeDate"""
if not inps:
inps = cmd_line_parse()
iDict = vars(inps)
print('read options from template file: ' +
os.path.basename(template_file))
template = readfile.read_template(inps.template_file)
template = ut.check_template_auto_value(template)
# Read template option
prefix = 'mintpy.velocity.'
keyList = [i for i in list(iDict.keys()) if prefix + i in template.keys()]
for key in keyList:
value = template[prefix + key]
if key in ['bootstrap']:
iDict[key] = value
if value:
if key in ['startDate', 'endDate']:
iDict[key] = ptime.yyyymmdd(value)
elif key in ['excludeDate']:
value = value.replace('[', '').replace(']',
'').replace(',', ' ')
iDict[key] = ptime.yyyymmdd(value.split())
elif key in ['bootstrapCount']:
iDict[key] = int(value)
key = 'mintpy.compute.maxMemory'
if key in template.keys() and template[key]:
inps.maxMemory = float(template[key])
return inps
def select_pairs_star(date_list, m_date=None, pbase_list=[], date12_format='YYMMDD-YYMMDD'):
"""Select Star-like network/interferograms/pairs, it's a single master network, similar to PS approach.
Usage:
m_date : master date, choose it based on the following cretiria:
1) near the center in temporal and spatial baseline
2) prefer winter season than summer season for less temporal decorrelation
Reference:
Ferretti, A., C. Prati, and F. Rocca (2001), Permanent scatterers in SAR interferometry, IEEE TGRS, 39(1), 8-20.
"""
date8_list = sorted(ptime.yyyymmdd(date_list))
date6_list = ptime.yymmdd(date8_list)
# Select master date if not existed
if not m_date:
m_date = select_master_date(date8_list, pbase_list)
print(('auto select master date: '+m_date))
# Check input master date
m_date8 = ptime.yyyymmdd(m_date)
if m_date8 not in date8_list:
print('Input master date is not existed in date list!')
print(('Input master date: '+str(m_date8)))
print(('Input date list: '+str(date8_list)))
m_date8 = None
# Generate star/ps network
m_idx = date8_list.index(m_date8)
date12_idx_list = [sorted([m_idx, s_idx]) for s_idx in range(len(date8_list))
if s_idx is not m_idx]
date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]]
for idx in date12_idx_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def select_pairs_star(date_list, m_date=None, pbase_list=[], date12_format='YYMMDD-YYMMDD'):
"""Select Star-like network/interferograms/pairs, it's a single master network, similar to PS approach.
Usage:
m_date : master date, choose it based on the following cretiria:
1) near the center in temporal and spatial baseline
2) prefer winter season than summer season for less temporal decorrelation
Reference:
Ferretti, A., C. Prati, and F. Rocca (2001), Permanent scatterers in SAR interferometry, IEEE TGRS, 39(1), 8-20.
"""
date8_list = sorted(ptime.yyyymmdd(date_list))
date6_list = ptime.yymmdd(date8_list)
# Select master date if not existed
if not m_date:
m_date = select_master_date(date8_list, pbase_list)
print(('auto select master date: '+m_date))
# Check input master date
m_date8 = ptime.yyyymmdd(m_date)
if m_date8 not in date8_list:
print('Input master date is not existed in date list!')
print(('Input master date: '+str(m_date8)))
print(('Input date list: '+str(date8_list)))
m_date8 = None
# Generate star/ps network
m_idx = date8_list.index(m_date8)
date12_idx_list = [sorted([m_idx, s_idx]) for s_idx in range(len(date8_list))
if s_idx is not m_idx]
date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]]
for idx in date12_idx_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def read_exclude_date(inps, dateListAll):
# Merge ex_date/startDate/endDate into ex_date
yy_list_all = ptime.yyyymmdd2years(dateListAll)
exDateList = []
# ex_date
exDateList += ptime.read_date_list(list(inps.excludeDate),
date_list_all=dateListAll)
if exDateList:
print('exclude date:' + str(exDateList))
# startDate
if inps.startDate:
print('start date: ' + inps.startDate)
yy_min = ptime.yyyymmdd2years(ptime.yyyymmdd(inps.startDate))
for i in range(len(dateListAll)):
date = dateListAll[i]
if yy_list_all[i] < yy_min and date not in exDateList:
print(' remove date: ' + date)
exDateList.append(date)
# endDate
if inps.endDate:
print('end date: ' + inps.endDate)
yy_max = ptime.yyyymmdd2years(ptime.yyyymmdd(inps.endDate))
for i in range(len(dateListAll)):
date = dateListAll[i]
if yy_list_all[i] > yy_max and date not in exDateList:
print(' remove date: ' + date)
exDateList.append(date)
exDateList = sorted(list(set(exDateList)))
return exDateList
def read_exclude_date(inps, date_list_all):
inps.excludeDate = ptime.yyyymmdd(inps.excludeDate)
if not inps.excludeDate:
inps.excludeDate = []
else:
log('input exclude dates: {}'.format(inps.excludeDate))
if inps.startDate:
log('input start date: ' + inps.startDate)
inps.excludeDate += [
i for i in date_list_all
if float(i) < float(ptime.yyyymmdd(inps.startDate))
]
inps.excludeDate = sorted(inps.excludeDate)
if inps.endDate:
log('input end date: ' + inps.endDate)
inps.excludeDate += [
i for i in date_list_all
if float(i) > float(ptime.yyyymmdd(inps.endDate))
]
inps.excludeDate = sorted(inps.excludeDate)
if inps.excludeDate:
log('exclude dates: ({})\n{}'.format(len(inps.excludeDate),
inps.excludeDate))
return inps.excludeDate
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from template file: '+os.path.basename(template_file))
template = readfile.read_template(inps.template_file)
template = ut.check_template_auto_value(template)
# Update inps if key existed in template file
prefix = 'mintpy.network.'
keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()]
for key in keyList:
value = template[prefix+key]
if key in ['coherenceBased', 'areaRatioBased', 'keepMinSpanTree']:
inpsDict[key] = value
elif value:
if key in ['minCoherence', 'minAreaRatio', 'tempBaseMax', 'perpBaseMax']:
inpsDict[key] = float(value)
elif key in ['connNumMax']:
inpsDict[key] = int(value)
elif key in ['maskFile', 'referenceFile']:
inpsDict[key] = value
elif key == 'aoiYX':
tmp = [i.replace('[','').replace(']','').strip() for i in value.split(',')]
sub_y = sorted([int(i.strip()) for i in tmp[0].split(':')])
sub_x = sorted([int(i.strip()) for i in tmp[1].split(':')])
inps.aoi_pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1])
elif key == 'aoiLALO':
tmp = [i.replace('[','').replace(']','').strip() for i in value.split(',')]
sub_lat = sorted([float(i.strip()) for i in tmp[0].split(':')])
sub_lon = sorted([float(i.strip()) for i in tmp[1].split(':')])
inps.aoi_geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0])
# Check lookup file
if not inps.lookupFile:
print('Warning: no lookup table file found! Can not use '+key+' option without it.')
print('skip this option.')
inps.aoi_pix_box = None
elif key in ['startDate', 'endDate']:
inpsDict[key] = ptime.yyyymmdd(value)
elif key == 'excludeDate':
value = value.replace('[','').replace(']','').replace(',', ' ')
inpsDict[key] = ptime.yyyymmdd(value.split())
elif key == 'excludeIfgIndex':
value = value.replace('[','').replace(']','').replace(',', ' ')
inpsDict[key] += value.split()
inpsDict[key] = read_input_index_list(inpsDict[key], stackFile=inps.file)
# Turn reset on if 1) no input options found to drop ifgram AND 2) there is template input
if all(not i for i in [inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax, inps.connNumMax,
inps.excludeIfgIndex, inps.excludeDate, inps.coherenceBased, inps.areaRatioBased,
inps.startDate, inps.endDate, inps.reset, inps.manual]):
print('No input option found to remove interferogram')
print('Keep all interferograms by enable --reset option')
inps.reset = True
return inps
def read_inps2date_time(inps):
"""Read dates and time info from input arguments.
Related options: --file OR --date-list, --hour
Parameters: inps - Namespace for the input arguments
Returns: date_list - list of str, dates in YYYYMMDD format
hour - str, hour in 2-digit with zero padding
"""
# if --file is specified
if inps.dis_file:
# 1) ignore --date-list and --hour
for key in ['date_list', 'hour']:
if vars(inps)[key] is not None:
vars(inps)[key] = None
msg = 'input "{:<10}" is ignored'.format(key)
msg += ', use info from file {} instead'.format(inps.dis_file)
print(msg)
# 2) read dates/time from time-series file
print('read dates/time info from file: {}'.format(inps.dis_file))
atr = readfile.read_attribute(inps.dis_file)
if atr['FILE_TYPE'] == 'timeseries':
ts_obj = timeseries(inps.dis_file)
ts_obj.open(print_msg=False)
inps.date_list = ts_obj.dateList
else:
inps.date_list = ptime.yyyymmdd(atr['DATE12'].split('-'))
inps.hour = closest_weather_model_hour(atr['CENTER_LINE_UTC'],
grib_source=inps.tropo_model)
# read dates if --date-list is text file
if len(inps.date_list) == 1 and os.path.isfile(inps.date_list[0]):
date_file = inps.date_list[0]
if date_file.startswith('SAFE_'):
print(
'read date list and hour info from Sentinel-1 SAFE filenames: {}'
.format(date_file))
inps.date_list, inps.hour = safe2date_time(date_file,
inps.tropo_model)
else:
print('read date list from text file: {}'.format(date_file))
inps.date_list = np.loadtxt(date_file, dtype=bytes,
usecols=(0, )).astype(str).tolist()
inps.date_list = ptime.yyyymmdd(inps.date_list)
# at east 2 dates are required (for meaningful calculation)
if len(inps.date_list) < 2:
raise AttributeError('input number of dates < 2!')
# print time info
if inps.hour is None:
raise AttributeError('time info (--hour) not found!')
print('time of cloest available product: {}:00 UTC'.format(inps.hour))
return inps.date_list, inps.hour
def read_template2inps(template_file, inps=None):
"""Read input template options into Namespace inps"""
if not inps:
inps = cmd_line_parse()
inpsDict = vars(inps)
print('read options from template file: '+os.path.basename(template_file))
template = readfile.read_template(inps.template_file)
template = ut.check_template_auto_value(template)
# Update inps if key existed in template file
prefix = 'mintpy.network.'
keyList = [i for i in list(inpsDict.keys()) if prefix+i in template.keys()]
for key in keyList:
value = template[prefix+key]
if key in ['coherenceBased', 'keepMinSpanTree']:
inpsDict[key] = value
elif value:
if key in ['minCoherence', 'tempBaseMax', 'perpBaseMax']:
inpsDict[key] = float(value)
elif key in ['connNumMax']:
inpsDict[key] = int(value)
elif key in ['maskFile', 'referenceFile']:
inpsDict[key] = value
elif key == 'aoiYX':
tmp = [i.strip() for i in value.split(',')]
sub_y = sorted([int(i.strip()) for i in tmp[0].split(':')])
sub_x = sorted([int(i.strip()) for i in tmp[1].split(':')])
inps.aoi_pix_box = (sub_x[0], sub_y[0], sub_x[1], sub_y[1])
elif key == 'aoiLALO':
tmp = [i.strip() for i in value.split(',')]
sub_lat = sorted([float(i.strip()) for i in tmp[0].split(':')])
sub_lon = sorted([float(i.strip()) for i in tmp[1].split(':')])
inps.aoi_geo_box = (sub_lon[0], sub_lat[1], sub_lon[1], sub_lat[0])
# Check lookup file
if not inps.lookupFile:
print('Warning: no lookup table file found! Can not use '+key+' option without it.')
print('skip this option.')
inps.aoi_pix_box = None
elif key in ['startDate', 'endDate']:
inpsDict[key] = ptime.yyyymmdd(value)
elif key == 'excludeDate':
inpsDict[key] = ptime.yyyymmdd(value.replace(',', ' ').split())
elif key == 'excludeIfgIndex':
inpsDict[key] += value.replace(',', ' ').split()
inps.excludeIfgIndex = read_input_index_list(inps.excludeIfgIndex, stackFile=inps.file)
# Turn reset on if 1) no input options found to drop ifgram AND 2) there is template input
if all(not i for i in [inps.referenceFile, inps.tempBaseMax, inps.perpBaseMax, inps.connNumMax,
inps.excludeIfgIndex, inps.excludeDate, inps.coherenceBased,
inps.startDate, inps.endDate, inps.reset, inps.manual]):
print('No input option found to remove interferogram')
print('Keep all interferograms by enable --reset option')
inps.reset = True
return inps
def threshold_coherence_based_mst(date12_list, coh_list):
"""Return a minimum spanning tree of network based on the coherence inverse.
Inputs:
date12_list - list of string in YYMMDD-YYMMDD format
coh_list - list of float, average coherence for each interferogram
Output:
mst_date12_list - list of string in YYMMDD-YYMMDD format, for MST network of interferograms
"""
# coh_list --> coh_mat --> weight_mat
coh_mat = coherence_matrix(date12_list, coh_list)
mask = ~np.isnan(coh_mat)
wei_mat = np.zeros(coh_mat.shape)
wei_mat[:] = np.inf
wei_mat[mask] = 1/coh_mat[mask]
# MST path based on weight matrix
wei_mat_csr = sparse.csr_matrix(wei_mat)
mst_mat_csr = sparse.csgraph.minimum_spanning_tree(wei_mat_csr)
# Get date6_list
date12_list = ptime.yymmdd_date12(date12_list)
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date6_list = ptime.yymmdd(sorted(ptime.yyyymmdd(list(set(m_dates + s_dates)))))
# Convert MST index matrix into date12 list
[s_idx_list, m_idx_list] = [date_idx_array.tolist()
for date_idx_array in sparse.find(mst_mat_csr)[0:2]]
mst_date12_list = []
for i in range(len(m_idx_list)):
idx = sorted([m_idx_list[i], s_idx_list[i]])
date12 = date6_list[idx[0]]+'-'+date6_list[idx[1]]
mst_date12_list.append(date12)
return mst_date12_list
def select_pairs_delaunay(date_list, pbase_list, norm=True, date12_format='YYMMDD-YYMMDD'):
"""Select Pairs using Delaunay Triangulation based on temporal/perpendicular baselines
Inputs:
date_list : list of date in YYMMDD/YYYYMMDD format
pbase_list : list of float, perpendicular spatial baseline
norm : normalize temporal baseline to perpendicular baseline
Key points
1. Define a ratio between perpendicular and temporal baseline axis units (Pepe and Lanari, 2006, TGRS).
2. Pairs with too large perpendicular / temporal baseline or Doppler centroid difference should be removed
after this, using a threshold, to avoid strong decorrelations (Zebker and Villasenor, 1992, TGRS).
Reference:
Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping
of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383.
Zebker, H. A., and J. Villasenor (1992), Decorrelation in interferometric radar echoes, IEEE TGRS, 30(5), 950-959.
"""
# Get temporal baseline in days
date6_list = ptime.yymmdd(date_list)
date8_list = ptime.yyyymmdd(date_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Normalization (Pepe and Lanari, 2006, TGRS)
if norm:
temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list))
tbase_list = [tbase*temp2perp_scale for tbase in tbase_list]
# Generate Delaunay Triangulation
date12_idx_list = Triangulation(tbase_list, pbase_list).edges.tolist()
date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)]
# Convert index into date12
date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]]
for idx in date12_idx_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def threshold_temporal_baseline(date12_list, btemp_max, keep_seasonal=True, btemp_min=0.0):
"""Remove pairs/interferograms out of min/max/seasonal temporal baseline limits
Inputs:
date12_list : list of string for date12 in YYMMDD-YYMMDD format
btemp_max : float, maximum temporal baseline
btemp_min : float, minimum temporal baseline
keep_seasonal : keep interferograms with seasonal temporal baseline
Output:
date12_list_out : list of string for date12 in YYMMDD-YYMMDD format
Example:
date12_list = threshold_temporal_baseline(date12_list, 200)
date12_list = threshold_temporal_baseline(date12_list, 200, False)
"""
if not date12_list:
return []
# Get date list and tbase list
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date8_list = sorted(ptime.yyyymmdd(list(set(m_dates + s_dates))))
date6_list = ptime.yymmdd(date8_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Threshold
date12_list_out = []
for date12 in date12_list:
date1, date2 = date12.split('-')
idx1 = date6_list.index(date1)
idx2 = date6_list.index(date2)
tbase = int(abs(tbase_list[idx1] - tbase_list[idx2]))
if btemp_min <= tbase <= btemp_max:
date12_list_out.append(date12)
elif keep_seasonal and tbase/30 in [11, 12]:
date12_list_out.append(date12)
return date12_list_out
def select_master_date(date_list, pbase_list=[]):
"""Select super master date based on input temporal and/or perpendicular baseline info.
Return master date in YYYYMMDD format.
"""
date8_list = ptime.yyyymmdd(date_list)
if not pbase_list:
# Choose date in the middle
m_date8 = date8_list[int(len(date8_list) / 2)]
else:
# Get temporal baseline list
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Normalization (Pepe and Lanari, 2006, TGRS)
temp2perp_scale = (max(pbase_list) - min(pbase_list)) / (
max(tbase_list) - min(tbase_list))
tbase_list = [tbase * temp2perp_scale for tbase in tbase_list]
# Get distance matrix
ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list),
np.array(tbase_list))
ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list),
np.array(pbase_list))
ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix
ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix
# 2D distance matrix in temp/perp domain
disMat = np.sqrt(np.square(ttMat) + np.square(ppMat))
# Choose date minimize the total distance of temp/perp baseline
disMean = np.mean(disMat, 0)
m_idx = np.argmin(disMean)
m_date8 = date8_list[m_idx]
return m_date8
def threshold_doppler_overlap(date12_list,
date_list,
dop_list,
bandwidth_az,
dop_overlap_min=0.15):
"""Remove pairs/interoferogram with doppler overlap larger than critical value
Inputs:
date12_list : list of string, for date12 in YYMMDD-YYMMDD format
date_list : list of string, for date in YYMMDD/YYYYMMDD format, optional
dop_list : list of list of 3 float, for centroid Doppler frequency
bandwidth_az : float, bandwidth in azimuth direction
dop_overlap_min : float, minimum overlap of azimuth Doppler frequency
Outputs:
date12_list : list of string, for date12 in YYMMDD-YYMMDD format
"""
if not date12_list:
return []
# Get date6_list
if not date_list:
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date_list = sorted(ptime.yyyymmdd(list(set(m_dates + s_dates))))
date6_list = ptime.yymmdd(date_list)
# Threshold
date12_list_out = []
for date12 in date12_list:
date1, date2 = date12.split('-')
idx1 = date6_list.index(date1)
idx2 = date6_list.index(date2)
dop_overlap = calculate_doppler_overlap(dop_list[idx1], dop_list[idx2],
bandwidth_az)
if dop_overlap >= dop_overlap_min:
date12_list_out.append(date12)
return date12_list_out
def select_master_date(date_list, pbase_list=[]):
"""Select super master date based on input temporal and/or perpendicular baseline info.
Return master date in YYYYMMDD format.
"""
date8_list = ptime.yyyymmdd(date_list)
if not pbase_list:
# Choose date in the middle
m_date8 = date8_list[int(len(date8_list)/2)]
else:
# Get temporal baseline list
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Normalization (Pepe and Lanari, 2006, TGRS)
temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list))
tbase_list = [tbase*temp2perp_scale for tbase in tbase_list]
# Get distance matrix
ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list),
np.array(tbase_list))
ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list),
np.array(pbase_list))
ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix
ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix
# 2D distance matrix in temp/perp domain
disMat = np.sqrt(np.square(ttMat) + np.square(ppMat))
# Choose date minimize the total distance of temp/perp baseline
disMean = np.mean(disMat, 0)
m_idx = np.argmin(disMean)
m_date8 = date8_list[m_idx]
return m_date8
def exclude_dates():
global inps, dateList
if inps.ex_date_list:
input_ex_date = list(inps.ex_date_list)
inps.ex_date_list = []
if input_ex_date:
for ex_date in input_ex_date:
if os.path.isfile(ex_date):
ex_date = ptime.read_date_list(ex_date)
else:
ex_date = [ptime.yyyymmdd(ex_date)]
inps.ex_date_list += list(
set(ex_date) - set(inps.ex_date_list))
# delete dates not existed in input file
inps.ex_date_list = sorted(
list(set(inps.ex_date_list).intersection(dateList)))
inps.ex_dates = ptime.date_list2vector(inps.ex_date_list)[0]
inps.ex_idx_list = sorted(
[dateList.index(i) for i in inps.ex_date_list])
print(('exclude date:' + str(inps.ex_date_list)))
def correct_single_ifgram(dis_file, tropo_file, cor_dis_file):
print('\n------------------------------------------------------------------------------')
print('correcting relative delay for input interferogram')
print('read phase from {}'.format(dis_file))
data, atr = readfile.read(dis_file, datasetName='phase')
date1, date2 = ptime.yyyymmdd(atr['DATE12'].split('-'))
ref_y, ref_x = int(atr['REF_Y']), int(atr['REF_X'])
print('calc tropospheric delay for {}-{} from {}'.format(date1, date2, tropo_file))
tropo = readfile.read(tropo_file, datasetName=date2)[0]
tropo -= readfile.read(tropo_file, datasetName=date1)[0]
tropo *= -4. * np.pi / float(atr['WAVELENGTH'])
# apply the correction and re-referencing
data -= tropo
data -= data[ref_y, ref_x]
print('read magnitude from {}'.format(dis_file))
mag = readfile.read(dis_file, datasetName='magnitude')[0]
print('write corrected data to {}'.format(cor_dis_file))
ds_dict = {'magnitude': mag, 'phase': data}
writefile.write(ds_dict, cor_dis_file, atr)
return cor_dis_file
def select_pairs_sequential(date_list, num_connection=2, date12_format='YYMMDD-YYMMDD'):
"""Select Pairs in a Sequential way:
For each acquisition, find its num_connection nearest acquisitions in the past time.
Inputs:
date_list : list of date in YYMMDD/YYYYMMDD format
Reference:
Fattahi, H., and F. Amelung (2013), DEM Error Correction in InSAR Time Series, IEEE TGRS, 51(7), 4249-4259.
"""
date8_list = sorted(ptime.yyyymmdd(date_list))
date6_list = ptime.yymmdd(date8_list)
date_idx_list = list(range(len(date6_list)))
# Get pairs index list
date12_idx_list = []
for date_idx in date_idx_list:
for i in range(num_connection):
if date_idx-i-1 >= 0:
date12_idx_list.append([date_idx-i-1, date_idx])
date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)]
# Convert index into date12
date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]]
for idx in date12_idx_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def read_ref_date(inps):
# check input reference date
if not inps.refDate:
print('No reference date input, skip this step.')
return None
# string in digit
elif inps.refDate.isdigit():
pass
else:
if os.path.isfile(inps.refDate):
print('read reference date from file: ' + inps.refDate)
if inps.refDate.endswith('.h5'):
# HDF5 file
atr = readfile.read_attribute(inps.refDate)
inps.refDate = atr['REF_DATE']
else:
# txt file
inps.refDate = ptime.read_date_txt(inps.refDate)[0]
else:
print('input file {} does not exist, skip this step'.format(inps.refDate))
return None
inps.refDate = ptime.yyyymmdd(inps.refDate)
print('input reference date: {}'.format(inps.refDate))
# check available dates
date_list = timeseries(inps.timeseries_file[0]).get_date_list()
if inps.refDate not in date_list:
msg = 'input reference date: {} is not found.'.format(inps.refDate)
msg += '\nAll available dates:\n{}'.format(date_list)
raise Exception(msg)
return inps.refDate
def select_pairs_delaunay(date_list, pbase_list, norm=True, date12_format='YYMMDD-YYMMDD'):
"""Select Pairs using Delaunay Triangulation based on temporal/perpendicular baselines
Inputs:
date_list : list of date in YYMMDD/YYYYMMDD format
pbase_list : list of float, perpendicular spatial baseline
norm : normalize temporal baseline to perpendicular baseline
Key points
1. Define a ratio between perpendicular and temporal baseline axis units (Pepe and Lanari, 2006, TGRS).
2. Pairs with too large perpendicular / temporal baseline or Doppler centroid difference should be removed
after this, using a threshold, to avoid strong decorrelations (Zebker and Villasenor, 1992, TGRS).
Reference:
Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping
of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383.
Zebker, H. A., and J. Villasenor (1992), Decorrelation in interferometric radar echoes, IEEE TGRS, 30(5), 950-959.
"""
# Get temporal baseline in days
date6_list = ptime.yymmdd(date_list)
date8_list = ptime.yyyymmdd(date_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Normalization (Pepe and Lanari, 2006, TGRS)
if norm:
temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list))
tbase_list = [tbase*temp2perp_scale for tbase in tbase_list]
# Generate Delaunay Triangulation
date12_idx_list = Triangulation(tbase_list, pbase_list).edges.tolist()
date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)]
# Convert index into date12
date12_list = [date6_list[idx[0]]+'-'+date6_list[idx[1]]
for idx in date12_idx_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def threshold_temporal_baseline(date12_list, btemp_max, keep_seasonal=True, btemp_min=0.0):
"""Remove pairs/interferograms out of min/max/seasonal temporal baseline limits
Inputs:
date12_list : list of string for date12 in YYMMDD-YYMMDD format
btemp_max : float, maximum temporal baseline
btemp_min : float, minimum temporal baseline
keep_seasonal : keep interferograms with seasonal temporal baseline
Output:
date12_list_out : list of string for date12 in YYMMDD-YYMMDD format
Example:
date12_list = threshold_temporal_baseline(date12_list, 200)
date12_list = threshold_temporal_baseline(date12_list, 200, False)
"""
if not date12_list:
return []
# Get date list and tbase list
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date8_list = sorted(ptime.yyyymmdd(list(set(m_dates + s_dates))))
date6_list = ptime.yymmdd(date8_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Threshold
date12_list_out = []
for date12 in date12_list:
date1, date2 = date12.split('-')
idx1 = date6_list.index(date1)
idx2 = date6_list.index(date2)
tbase = int(abs(tbase_list[idx1] - tbase_list[idx2]))
if btemp_min <= tbase <= btemp_max:
date12_list_out.append(date12)
elif keep_seasonal and tbase/30 in [11, 12]:
date12_list_out.append(date12)
return date12_list_out
def select_pairs_sequential(date_list,
num_connection=2,
date12_format='YYMMDD-YYMMDD'):
"""Select Pairs in a Sequential way:
For each acquisition, find its num_connection nearest acquisitions in the past time.
Inputs:
date_list : list of date in YYMMDD/YYYYMMDD format
Reference:
Fattahi, H., and F. Amelung (2013), DEM Error Correction in InSAR Time Series, IEEE TGRS, 51(7), 4249-4259.
"""
date8_list = sorted(ptime.yyyymmdd(date_list))
date6_list = ptime.yymmdd(date8_list)
date_idx_list = list(range(len(date6_list)))
# Get pairs index list
date12_idx_list = []
for date_idx in date_idx_list:
for i in range(num_connection):
if date_idx - i - 1 >= 0:
date12_idx_list.append([date_idx - i - 1, date_idx])
date12_idx_list = [sorted(idx) for idx in sorted(date12_idx_list)]
# Convert index into date12
date12_list = [
date6_list[idx[0]] + '-' + date6_list[idx[1]]
for idx in date12_idx_list
]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def threshold_coherence_based_mst(date12_list, coh_list):
"""Return a minimum spanning tree of network based on the coherence inverse.
Inputs:
date12_list - list of string in YYMMDD-YYMMDD format
coh_list - list of float, average coherence for each interferogram
Output:
mst_date12_list - list of string in YYMMDD-YYMMDD format, for MST network of interferograms
"""
# coh_list --> coh_mat --> weight_mat
coh_mat = coherence_matrix(date12_list, coh_list)
mask = ~np.isnan(coh_mat)
wei_mat = np.zeros(coh_mat.shape)
wei_mat[:] = np.inf
wei_mat[mask] = 1/coh_mat[mask]
# MST path based on weight matrix
wei_mat_csr = sparse.csr_matrix(wei_mat)
mst_mat_csr = sparse.csgraph.minimum_spanning_tree(wei_mat_csr)
# Get date6_list
date12_list = ptime.yymmdd_date12(date12_list)
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date6_list = ptime.yymmdd(sorted(ptime.yyyymmdd(list(set(m_dates + s_dates)))))
# Convert MST index matrix into date12 list
[s_idx_list, m_idx_list] = [date_idx_array.tolist()
for date_idx_array in sparse.find(mst_mat_csr)[0:2]]
mst_date12_list = []
for i in range(len(m_idx_list)):
idx = sorted([m_idx_list[i], s_idx_list[i]])
date12 = date6_list[idx[0]]+'-'+date6_list[idx[1]]
mst_date12_list.append(date12)
return mst_date12_list
def safe2date_time(safe_file, tropo_model):
"""generate date_list and hour from safe_list"""
def seconds_UTC(seconds):
"""generate second list"""
if isinstance(seconds, list):
secondsOut = []
for second in seconds:
secondsOut.append(second)
else:
print('\nUn-recognized CENTER_LINE_UTC input!')
return None
return secondsOut
date_list = ptime.yyyymmdd(
np.loadtxt(safe_file, dtype=bytes, converters={
0: define_date
}).astype(str).tolist())
second_list = seconds_UTC(
np.loadtxt(safe_file, dtype=bytes, converters={
0: define_second
}).astype(str).tolist())
# change second into hour
hour_list = [
closest_weather_model_hour(float(second), tropo_model)
for second in second_list
]
hour = ut.most_common(hour_list)
return date_list, hour
def read_HDFEOS(inps):
"""read displacement from HDFEOS"""
print('read displacement, incidence and azimuth information')
# read metadata
HDFEOS_file = inps.input_HDFEOS[0]
atr = readfile.read_attribute(HDFEOS_file)
if inps.date == None:
date1 = atr['START_DATE']
date2 = atr['END_DATE']
else:
# date1 and date2
if '_' in "".join(inps.date):
date1, date2 = ptime.yyyymmdd("".join(inps.date).split('_'))
else:
date1 = atr['START_DATE']
date2 = ptime.yyyymmdd("".join(inps.date))
# read angle infomation
azimuth = readfile.read(HDFEOS_file, datasetName='/HDFEOS/GRIDS/timeseries/geometry/azimuthAngle')[0]
incidence = readfile.read(HDFEOS_file, datasetName='/HDFEOS/GRIDS/timeseries/geometry/incidenceAngle')[0]
if inps.velocity:
vel_file = 'velocity.h5'
iargs = [HDFEOS_file, '--start-date', date1, '--end-date', date2, '-o', vel_file, '--update']
print('\ntimeseries2velocity.py', ' '.join(iargs))
mintpy.timeseries2velocity.main(iargs)
data = readfile.read(vel_file, datasetName='velocity')[0]
os.remove(vel_file)
else:
# read / prepare data
slice_list = readfile.get_slice_list(HDFEOS_file)
# read/prepare data
dname = 'displacement'
slice_name1 = view.search_dataset_input(slice_list, '{}-{}'.format(dname, date1))[0][0]
slice_name2 = view.search_dataset_input(slice_list, '{}-{}'.format(dname, date2))[0][1]
data = readfile.read("".join(inps.input_HDFEOS), datasetName=slice_name2)[0]
data -= readfile.read("".join(inps.input_HDFEOS), datasetName=slice_name1)[0]
print("mask file")
maskfile = readfile.read(HDFEOS_file, datasetName='/HDFEOS/GRIDS/timeseries/quality/mask')[0]
data[maskfile == 0] = np.nan
azimuth[maskfile == 0] = np.nan
incidence[maskfile == 0] = np.nan
return date1, date2, data, atr, incidence, azimuth
def skip_files_with_inconsistent_size(dsPathDict,
pix_box=None,
dsName='unwrapPhase'):
"""Skip files by removing the file path from the input dsPathDict."""
atr_list = [readfile.read_attribute(fname) for fname in dsPathDict[dsName]]
length_list = [int(atr['LENGTH']) for atr in atr_list]
width_list = [int(atr['WIDTH']) for atr in atr_list]
# Check size requirements
drop_inconsistent_files = False
if any(len(set(size_list)) > 1 for size_list in [length_list, width_list]):
if pix_box is None:
drop_inconsistent_files = True
else:
# if input subset is within the min file sizes: do NOT drop
max_box_width, max_box_length = pix_box[2:4]
if max_box_length > min(length_list) or max_box_width > min(
width_list):
drop_inconsistent_files = True
# update dsPathDict
if drop_inconsistent_files:
common_length = ut.most_common(length_list)
common_width = ut.most_common(width_list)
# print out warning message
msg = '\n' + '*' * 80
msg += '\nWARNING: NOT all input unwrapped interferograms have the same row/column number!'
msg += '\nThe most common size is: ({}, {})'.format(
common_length, common_width)
msg += '\n' + '-' * 30
msg += '\nThe following dates have different size:'
dsNames = list(dsPathDict.keys())
date12_list = [atr['DATE12'] for atr in atr_list]
num_drop = 0
for i in range(len(date12_list)):
if length_list[i] != common_length or width_list[i] != common_width:
date12 = date12_list[i]
dates = ptime.yyyymmdd(date12.split('-'))
# update file list for all datasets
for dsName in dsNames:
fnames = [
i for i in dsPathDict[dsName]
if all(d[2:8] in i for d in dates)
]
if len(fnames) > 0:
dsPathDict[dsName].remove(fnames[0])
msg += '\n\t{}\t({}, {})'.format(date12, length_list[i],
width_list[i])
num_drop += 1
msg += '\n' + '-' * 30
msg += '\nSkip loading the above interferograms ({}).'.format(num_drop)
msg += '\nContinue to load the rest interferograms ({}).'.format(
len(date12_list) - num_drop)
msg += '\n' + '*' * 80 + '\n'
print(msg)
return dsPathDict
def safe_list2date_list(inps):
if os.path.isfile(inps.safe_list[0]):
print('read date list from text file: {}'.format(inps.safe_list[0]))
date_list = ptime.yyyymmdd(np.loadtxt(inps.safe_list[0],dtype=bytes,converters={0:define_date}).astype(str).tolist())
second_list = seconds_UTC(np.loadtxt(inps.safe_list[0],dtype=bytes,converters={0:define_second}).astype(str).tolist())
return date_list, second_list
def correct_local_oscilator_drift(fname, rg_dist_file=None, out_file=None):
print('-'*50)
print('correct Local Oscilator Drift for Envisat using an empirical model (Marinkovic and Larsen, 2013)')
print('-'*50)
atr = readfile.read_attribute(fname)
# Check Sensor Type
platform = atr['PLATFORM']
print('platform: '+platform)
if not platform.lower() in ['env', 'envisat']:
print('No need to correct LOD for '+platform)
return
# output file name
if not out_file:
out_file = '{}_LODcor{}'.format(os.path.splitext(fname)[0], os.path.splitext(fname)[1])
# Get LOD ramp rate from empirical model
if not rg_dist_file:
print('calculate range distance from file metadata')
rg_dist = get_relative_range_distance(atr)
else:
print('read range distance from file: %s' % (rg_dist_file))
rg_dist = readfile.read(rg_dist_file, datasetName='slantRangeDistance', print_msg=False)[0]
rg_dist -= rg_dist[int(atr['REF_Y']), int(atr['REF_X'])]
ramp_rate = np.array(rg_dist * 3.87e-7, np.float32)
# Correct LOD Ramp for Input fname
range2phase = -4*np.pi / float(atr['WAVELENGTH'])
k = atr['FILE_TYPE']
if k == 'timeseries':
# read
obj = timeseries(fname)
obj.open()
data = obj.read()
# correct LOD
diff_year = np.array(obj.yearList)
diff_year -= diff_year[obj.refIndex]
for i in range(data.shape[0]):
data[i, :, :] -= ramp_rate * diff_year[i]
# write
obj_out = timeseries(out_file)
obj_out.write2hdf5(data, refFile=fname)
elif k in ['.unw']:
data, atr = readfile.read(fname)
dates = ptime.yyyymmdd2years(ptime.yyyymmdd(atr['DATE12'].split('-')))
dt = dates[1] - dates[0]
data -= ramp_rate * range2phase * dt
writefile.write(data, out_file=out_file, metadata=atr)
else:
print('No need to correct for LOD for %s file' % (k))
return out_file
def prepare_stack(inputDir,
filePattern,
metadata=dict(),
baseline_dict=dict(),
processor='tops',
update_mode=True):
print('preparing RSC file for ', filePattern)
if processor in ['tops', 'stripmap']:
isce_files = sorted(
glob.glob(os.path.join(os.path.abspath(inputDir), '*',
filePattern)))
elif processor == 'alosStack':
isce_files = sorted(
glob.glob(os.path.join(os.path.abspath(inputDir), filePattern)))
else:
raise ValueError(
'Un-recognized ISCE stack processor: {}'.format(processor))
if len(isce_files) == 0:
raise FileNotFoundError(
'no file found in pattern: {}'.format(filePattern))
# write .rsc file for each interferogram file
num_file = len(isce_files)
prog_bar = ptime.progressBar(maxValue=num_file)
for i in range(num_file):
# prepare metadata for current file
isce_file = isce_files[i]
if processor in ['tops', 'stripmap']:
dates = os.path.basename(os.path.dirname(isce_file)).split(
'_') # to modify to YYYYMMDDTHHMMSS
elif processor == 'alosStack':
dates = os.path.basename(
os.path.dirname(os.path.dirname(isce_file))).split(
'-') # to modify to YYYYMMDDTHHMMSS
dates = ptime.yyyymmdd(dates)
else:
raise ValueError(
'Un-recognized ISCE stack processor: {}'.format(processor))
ifg_metadata = readfile.read_attribute(isce_file, metafile_ext='.xml')
ifg_metadata.update(metadata)
ifg_metadata = add_ifgram_metadata(ifg_metadata, dates, baseline_dict)
# write .rsc file
rsc_file = isce_file + '.rsc'
writefile.write_roipac_rsc(ifg_metadata,
rsc_file,
update_mode=update_mode,
print_msg=False)
prog_bar.update(i + 1, suffix='{}_{}'.format(dates[0], dates[1]))
prog_bar.close()
return
def read_baseline_file(baselineFile, exDateList=[]):
"""Read bl_list.txt without dates listed in exDateList
# Date Bperp dop0/PRF dop1/PRF dop2/PRF PRF slcDir
070106 0.0 0.03 0.0000000 0.00000000000 2155.2 /scratch/KyushuT422F650AlosA/SLC/070106/
070709 2631.9 0.07 0.0000000 0.00000000000 2155.2 /scratch/KyushuT422F650AlosA/SLC/070709/
070824 2787.3 0.07 0.0000000 0.00000000000 2155.2 /scratch/KyushuT422F650AlosA/SLC/070824/
...
Examples:
date8List, perpBaseList, dopList, prfList, slcDirList = read_baseline_file(baselineFile)
date8List, perpBaseList, dopList, prfList, slcDirList = read_baseline_file(baselineFile,['080520','100726'])
date8List, perpBaseList = read_baseline_file(baselineFile)[0:2]
"""
exDateList = ptime.yymmdd(exDateList)
if not exDateList:
exDateList = []
# Read baseline file into lines
fb = open(baselineFile)
lines = []
for line in fb:
l = str.replace(line, '\n', '').strip()
lines.append(l)
fb.close()
# Read each line and put the values into arrays
date6List = []
perpBaseList = []
dopplerList = []
slcDirList = []
for line in lines:
c = line.split() # splits on white space
date = c[0]
if not date in exDateList:
date6List.append(date)
perpBaseList.append(float(c[1]))
try:
dop = np.array([float(c[2]), float(c[3]), float(c[4])])
prf = float(c[5])
dop *= prf
dopplerList.append(dop)
except:
pass
try:
slcDirList.append(c[6])
except:
pass
date8List = ptime.yyyymmdd(date6List)
return date8List, perpBaseList, dopplerList, slcDirList
def read_baseline_file(baselineFile, exDateList=[]):
"""Read bl_list.txt without dates listed in exDateList
# Date Bperp dop0/PRF dop1/PRF dop2/PRF PRF slcDir
070106 0.0 0.03 0.0000000 0.00000000000 2155.2 /scratch/KyushuT422F650AlosA/SLC/070106/
070709 2631.9 0.07 0.0000000 0.00000000000 2155.2 /scratch/KyushuT422F650AlosA/SLC/070709/
070824 2787.3 0.07 0.0000000 0.00000000000 2155.2 /scratch/KyushuT422F650AlosA/SLC/070824/
...
Examples:
date8List, perpBaseList, dopList, prfList, slcDirList = read_baseline_file(baselineFile)
date8List, perpBaseList, dopList, prfList, slcDirList = read_baseline_file(baselineFile,['080520','100726'])
date8List, perpBaseList = read_baseline_file(baselineFile)[0:2]
"""
exDateList = ptime.yymmdd(exDateList)
if not exDateList:
exDateList = []
# Read baseline file into lines
fb = open(baselineFile)
lines = []
for line in fb:
l = str.replace(line, '\n', '').strip()
lines.append(l)
fb.close()
# Read each line and put the values into arrays
date6List = []
perpBaseList = []
dopplerList = []
slcDirList = []
for line in lines:
c = line.split() # splits on white space
date = c[0]
if date not in exDateList:
date6List.append(date)
perpBaseList.append(float(c[1]))
try:
dop = np.array([float(c[2]), float(c[3]), float(c[4])])
prf = float(c[5])
dop *= prf
dopplerList.append(dop)
except:
pass
try:
slcDirList.append(c[6])
except:
pass
date8List = ptime.yyyymmdd(date6List)
return date8List, perpBaseList, dopplerList, slcDirList
def select_pairs_all(date_list, date12_format='YYMMDD-YYMMDD'):
"""Select All Possible Pairs/Interferograms
Input : date_list - list of date in YYMMDD/YYYYMMDD format
Output: date12_list - list date12 in YYMMDD-YYMMDD format
Reference:
Berardino, P., G. Fornaro, R. Lanari, and E. Sansosti (2002), A new algorithm for surface deformation monitoring
based on small baseline differential SAR interferograms, IEEE TGRS, 40(11), 2375-2383.
"""
date8_list = sorted(ptime.yyyymmdd(date_list))
date6_list = ptime.yymmdd(date8_list)
date12_list = list(itertools.combinations(date6_list, 2))
date12_list = [date12[0]+'-'+date12[1] for date12 in date12_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def read_exclude_date(inps, dateListAll):
# Merge ex_date/startDate/endDate into ex_date
yy_list_all = ptime.yyyymmdd2years(dateListAll)
exDateList = []
# 1. template_file
if inps.template_file:
print('read option from template file: ' + inps.template_file)
inps = read_template2inps(inps.template_file, inps)
# 2. ex_date
exDateList += ptime.read_date_list(list(inps.excludeDate),
date_list_all=dateListAll)
if exDateList:
print('exclude date:' + str(exDateList))
# 3. startDate
if inps.startDate:
print('start date: ' + inps.startDate)
yy_min = ptime.yyyymmdd2years(ptime.yyyymmdd(inps.startDate))
for i in range(len(dateListAll)):
date = dateListAll[i]
if yy_list_all[i] < yy_min and date not in exDateList:
print(' remove date: ' + date)
exDateList.append(date)
# 4. endDate
if inps.endDate:
print('end date: ' + inps.endDate)
yy_max = ptime.yyyymmdd2years(ptime.yyyymmdd(inps.endDate))
for i in range(len(dateListAll)):
date = dateListAll[i]
if yy_list_all[i] > yy_max and date not in exDateList:
print(' remove date: ' + date)
exDateList.append(date)
exDateList = list(set(exDateList))
return exDateList
def select_pairs_all(date_list, date12_format='YYMMDD-YYMMDD'):
"""Select All Possible Pairs/Interferograms
Input : date_list - list of date in YYMMDD/YYYYMMDD format
Output: date12_list - list date12 in YYMMDD-YYMMDD format
Reference:
Berardino, P., G. Fornaro, R. Lanari, and E. Sansosti (2002), A new algorithm for surface deformation monitoring
based on small baseline differential SAR interferograms, IEEE TGRS, 40(11), 2375-2383.
"""
date8_list = sorted(ptime.yyyymmdd(date_list))
date6_list = ptime.yymmdd(date8_list)
date12_list = list(itertools.combinations(date6_list, 2))
date12_list = [date12[0] + '-' + date12[1] for date12 in date12_list]
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def correct_single_ifgram(dis_file, tropo_file, cor_dis_file):
print('\n------------------------------------------------------------------------------')
print('correcting relative delay for input interferogram')
print('read data from {}'.format(dis_file))
data, atr = readfile.read(dis_file, datasetName='phase')
date1, date2 = ptime.yyyymmdd(atr['DATE12'].split('-'))
print('calc tropospheric delay for {}-{} from {}'.format(date1, date2, tropo_file))
tropo = readfile.read(tropo_file, datasetName=date2)[0]
tropo -= readfile.read(tropo_file, datasetName=date1)[0]
tropo *= -4. * np.pi / float(atr['WAVELENGTH'])
print('write corrected data to {}'.format(cor_dis_file))
writefile.write(data-tropo, cor_dis_file, atr)
return cor_dis_file
def select_master_interferogram(date12_list,
date_list,
pbase_list,
m_date=None):
"""Select reference interferogram based on input temp/perp baseline info
If master_date is specified, select its closest slave_date, which is newer than master_date;
otherwise, choose the closest pair among all pairs as master interferogram.
Example:
master_date12 = pnet.select_master_ifgram(date12_list, date_list, pbase_list)
'080211-080326' = pnet.select_master_ifgram(date12_list, date_list, pbase_list, m_date='080211')
"""
pbase_array = np.array(pbase_list, dtype='float64')
# Get temporal baseline
date8_list = ptime.yyyymmdd(date_list)
date6_list = ptime.yymmdd(date8_list)
tbase_array = np.array(ptime.date_list2tbase(date8_list)[0],
dtype='float64')
# Normalization (Pepe and Lanari, 2006, TGRS)
temp2perp_scale = (max(pbase_array) - min(pbase_array)) / (
max(tbase_array) - min(tbase_array))
tbase_array *= temp2perp_scale
# Calculate sqrt of temp/perp baseline for input pairs
idx1 = np.array(
[date6_list.index(date12.split('-')[0]) for date12 in date12_list])
idx2 = np.array(
[date6_list.index(date12.split('-')[1]) for date12 in date12_list])
base_distance = np.sqrt((tbase_array[idx2] - tbase_array[idx1])**2 +
(pbase_array[idx2] - pbase_array[idx1])**2)
# Get master interferogram index
if not m_date:
# Choose pair with shortest temp/perp baseline
m_date12_idx = np.argmin(base_distance)
else:
m_date = ptime.yymmdd(m_date)
# Choose pair contains m_date with shortest temp/perp baseline
m_date12_idx_array = np.array([
date12_list.index(date12) for date12 in date12_list
if m_date + '-' in date12
])
min_base_distance = np.min(base_distance[m_date12_idx_array])
m_date12_idx = np.where(base_distance == min_base_distance)[0][0]
m_date12 = date12_list[m_date12_idx]
return m_date12
def date12_list2index(date12_list, date_list=[]):
"""Convert list of date12 string into list of index"""
# Get dateList from date12List
if not date_list:
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date_list = list(set(m_dates + s_dates))
date6_list = ptime.yymmdd(sorted(ptime.yyyymmdd(date_list)))
# Get pair index
pairs_idx = []
for date12 in date12_list:
dates = date12.split('-')
pair_idx = [date6_list.index(dates[0]), date6_list.index(dates[1])]
pairs_idx.append(pair_idx)
return pairs_idx
def date12_list2index(date12_list, date_list=[]):
"""Convert list of date12 string into list of index"""
# Get dateList from date12List
if not date_list:
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date_list = list(set(m_dates + s_dates))
date6_list = ptime.yymmdd(sorted(ptime.yyyymmdd(date_list)))
# Get pair index
pairs_idx = []
for date12 in date12_list:
dates = date12.split('-')
pair_idx = [date6_list.index(dates[0]), date6_list.index(dates[1])]
pairs_idx.append(pair_idx)
return pairs_idx
def select_pairs_mst(date_list, pbase_list, date12_format='YYMMDD-YYMMDD'):
"""Select Pairs using Minimum Spanning Tree technique
Connection Cost is calculated using the baseline distance in perp and scaled temporal baseline (Pepe and Lanari,
2006, TGRS) plane.
Inputs:
date_list : list of date in YYMMDD/YYYYMMDD format
pbase_list : list of float, perpendicular spatial baseline
References:
Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping
of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383.
Perissin D., Wang T. (2012), Repeat-pass SAR interferometry with partially coherent targets. IEEE TGRS. 271-280
"""
# Get temporal baseline in days
date6_list = ptime.yymmdd(date_list)
date8_list = ptime.yyyymmdd(date_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Normalization (Pepe and Lanari, 2006, TGRS)
temp2perp_scale = (max(pbase_list) - min(pbase_list)) / (max(tbase_list) -
min(tbase_list))
tbase_list = [tbase * temp2perp_scale for tbase in tbase_list]
# Get weight matrix
ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), np.array(tbase_list))
ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), np.array(pbase_list))
ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix
ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix
# 2D distance matrix in temp/perp domain
weightMat = np.sqrt(np.square(ttMat) + np.square(ppMat))
weightMat = sparse.csr_matrix(weightMat) # compress sparse row matrix
# MST path based on weight matrix
mstMat = sparse.csgraph.minimum_spanning_tree(weightMat)
# Convert MST index matrix into date12 list
[s_idx_list, m_idx_list] = [
date_idx_array.tolist() for date_idx_array in sparse.find(mstMat)[0:2]
]
date12_list = []
for i in range(len(m_idx_list)):
idx = sorted([m_idx_list[i], s_idx_list[i]])
date12 = date6_list[idx[0]] + '-' + date6_list[idx[1]]
date12_list.append(date12)
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def read_inps2date_time(inps):
"""Read dates and time info from input arguments.
Related options: --file OR --date-list, --hour
Parameters: inps - Namespace for the input arguments
Returns: date_list - list of str, dates in YYYYMMDD format
hour - str, hour in 2-digit with zero padding
"""
# if --file is specified
if inps.timeseries_file:
# 1) ignore --date-list and --hour
for key in ['date_list', 'hour']:
if vars(inps)[key] is not None:
vars(inps)[key] = None
msg = 'input "{:<10}" is ignored'.format(key)
msg += ', use info from file {} instead'.format(
inps.timeseries_file)
print(msg)
# 2) read dates/time from time-series file
print('read dates/time info from file: {}'.format(
inps.timeseries_file))
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open(print_msg=False)
inps.date_list = ts_obj.dateList
inps.hour = closest_weather_model_hour(
ts_obj.metadata['CENTER_LINE_UTC'], grib_source=inps.tropo_model)
# read dates if --date-list is text file
if len(inps.date_list) == 1 and os.path.isfile(inps.date_list[0]):
print('read date list from text file: {}'.format(inps.date_list[0]))
inps.date_list = np.loadtxt(inps.date_list[0],
dtype=bytes,
usecols=(0, )).astype(str).tolist()
inps.date_list = ptime.yyyymmdd(inps.date_list)
# at east 2 dates are required (for meaningful calculation)
if len(inps.date_list) < 2:
raise SystemExit('ERROR: input date list < 2')
# print time info
print('time of cloest available product: {}:00 UTC'.format(inps.hour))
return inps.date_list, inps.hour
def read_ref_date(inps):
if not inps.refDate:
print('No reference date input, skip this step.')
return inps.timeseries_file
elif os.path.isfile(inps.refDate):
print('read reference date from file: ' + inps.refDate)
inps.refDate = ptime.read_date_txt(inps.refDate)[0]
inps.refDate = ptime.yyyymmdd(inps.refDate)
print('input reference date: {}'.format(inps.refDate))
# check input reference date
date_list = timeseries(inps.timeseries_file[0]).get_date_list()
if inps.refDate not in date_list:
msg = 'input reference date: {} is not found.'.format(inps.refDate)
msg += '\nAll available dates:\n{}'.format(date_list)
raise Exception(msg)
return inps.refDate
def read_ref_date(inps):
if not inps.refDate:
print('No reference date input, skip this step.')
return inps.timeseries_file
elif os.path.isfile(inps.refDate):
print('read reference date from file: ' + inps.refDate)
inps.refDate = ptime.read_date_txt(inps.refDate)[0]
inps.refDate = ptime.yyyymmdd(inps.refDate)
print('input reference date: {}'.format(inps.refDate))
# check input reference date
date_list = timeseries(inps.timeseries_file[0]).get_date_list()
if inps.refDate not in date_list:
msg = 'input reference date: {} is not found.'.format(inps.refDate)
msg += '\nAll available dates:\n{}'.format(date_list)
raise Exception(msg)
return inps.refDate
def select_pairs_mst(date_list, pbase_list, date12_format='YYMMDD-YYMMDD'):
"""Select Pairs using Minimum Spanning Tree technique
Connection Cost is calculated using the baseline distance in perp and scaled temporal baseline (Pepe and Lanari,
2006, TGRS) plane.
Inputs:
date_list : list of date in YYMMDD/YYYYMMDD format
pbase_list : list of float, perpendicular spatial baseline
References:
Pepe, A., and R. Lanari (2006), On the extension of the minimum cost flow algorithm for phase unwrapping
of multitemporal differential SAR interferograms, IEEE TGRS, 44(9), 2374-2383.
Perissin D., Wang T. (2012), Repeat-pass SAR interferometry with partially coherent targets. IEEE TGRS. 271-280
"""
# Get temporal baseline in days
date6_list = ptime.yymmdd(date_list)
date8_list = ptime.yyyymmdd(date_list)
tbase_list = ptime.date_list2tbase(date8_list)[0]
# Normalization (Pepe and Lanari, 2006, TGRS)
temp2perp_scale = (max(pbase_list)-min(pbase_list)) / (max(tbase_list)-min(tbase_list))
tbase_list = [tbase*temp2perp_scale for tbase in tbase_list]
# Get weight matrix
ttMat1, ttMat2 = np.meshgrid(np.array(tbase_list), np.array(tbase_list))
ppMat1, ppMat2 = np.meshgrid(np.array(pbase_list), np.array(pbase_list))
ttMat = np.abs(ttMat1 - ttMat2) # temporal distance matrix
ppMat = np.abs(ppMat1 - ppMat2) # spatial distance matrix
# 2D distance matrix in temp/perp domain
weightMat = np.sqrt(np.square(ttMat) + np.square(ppMat))
weightMat = sparse.csr_matrix(weightMat) # compress sparse row matrix
# MST path based on weight matrix
mstMat = sparse.csgraph.minimum_spanning_tree(weightMat)
# Convert MST index matrix into date12 list
[s_idx_list, m_idx_list] = [date_idx_array.tolist()
for date_idx_array in sparse.find(mstMat)[0:2]]
date12_list = []
for i in range(len(m_idx_list)):
idx = sorted([m_idx_list[i], s_idx_list[i]])
date12 = date6_list[idx[0]]+'-'+date6_list[idx[1]]
date12_list.append(date12)
if date12_format == 'YYYYMMDD_YYYYMMDD':
date12_list = ptime.yyyymmdd_date12(date12_list)
return date12_list
def exclude_dates():
global inps, dateList
if inps.ex_date_list:
input_ex_date = list(inps.ex_date_list)
inps.ex_date_list = []
if input_ex_date:
for ex_date in input_ex_date:
if os.path.isfile(ex_date):
ex_date = ptime.read_date_list(ex_date)
else:
ex_date = [ptime.yyyymmdd(ex_date)]
inps.ex_date_list += list(set(ex_date) - set(inps.ex_date_list))
# delete dates not existed in input file
inps.ex_date_list = sorted(list(set(inps.ex_date_list).intersection(dateList)))
inps.ex_dates = ptime.date_list2vector(inps.ex_date_list)[0]
inps.ex_idx_list = sorted([dateList.index(i) for i in inps.ex_date_list])
print(('exclude date:' + str(inps.ex_date_list)))
def threshold_doppler_overlap(date12_list, date_list, dop_list, bandwidth_az, dop_overlap_min=0.15):
"""Remove pairs/interoferogram with doppler overlap larger than critical value
Inputs:
date12_list : list of string, for date12 in YYMMDD-YYMMDD format
date_list : list of string, for date in YYMMDD/YYYYMMDD format, optional
dop_list : list of list of 3 float, for centroid Doppler frequency
bandwidth_az : float, bandwidth in azimuth direction
dop_overlap_min : float, minimum overlap of azimuth Doppler frequency
Outputs:
date12_list : list of string, for date12 in YYMMDD-YYMMDD format
"""
if not date12_list:
return []
# Get date6_list
if not date_list:
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date_list = sorted(ptime.yyyymmdd(list(set(m_dates + s_dates))))
if not len(date_list) == len(pbase_list):
print('ERROR: number of existing dates is not equal to number of perp baseline!')
print('date list is needed for threshold filtering!')
print('skip filtering.')
return date12_list
date6_list = ptime.yymmdd(date_list)
# Threshold
date12_list_out = []
for date12 in date12_list:
date1, date2 = date12.split('-')
idx1 = date6_list.index(date1)
idx2 = date6_list.index(date2)
dop_overlap = calculate_doppler_overlap(dop_list[idx1],
dop_list[idx2],
bandwidth_az)
if dop_overlap >= dop_overlap_min:
date12_list_out.append(date12)
return date12_list_out
def threshold_perp_baseline(date12_list, date_list, pbase_list, pbase_max, pbase_min=0.0):
"""Remove pairs/interoferogram out of [pbase_min, pbase_max]
Inputs:
date12_list : list of string for date12 in YYMMDD-YYMMDD format
date_list : list of string for date in YYMMDD/YYYYMMDD format, optional
pbase_list : list of float for perpendicular spatial baseline
pbase_max : float, maximum perpendicular baseline
pbase_min : float, minimum perpendicular baseline
Output:
date12_list_out : list of string for date12 in YYMMDD-YYMMDD format
Example:
date12_list = threshold_perp_baseline(date12_list, date_list, pbase_list, 500)
"""
if not date12_list:
return []
# Get date6_list
if not date_list:
m_dates = [date12.split('-')[0] for date12 in date12_list]
s_dates = [date12.split('-')[1] for date12 in date12_list]
date_list = sorted(ptime.yyyymmdd(list(set(m_dates + s_dates))))
if not len(date_list) == len(pbase_list):
print('ERROR: number of existing dates is not equal to number of perp baseline!')
print('date list is needed for threshold filtering!')
print('skip filtering.')
return date12_list
date6_list = ptime.yymmdd(date_list)
# Threshold
date12_list_out = []
for date12 in date12_list:
date1, date2 = date12.split('-')
idx1 = date6_list.index(date1)
idx2 = date6_list.index(date2)
pbase = abs(pbase_list[idx1] - pbase_list[idx2])
if pbase_min <= pbase <= pbase_max:
date12_list_out.append(date12)
return date12_list_out
def select_master_interferogram(date12_list, date_list, pbase_list, m_date=None):
"""Select reference interferogram based on input temp/perp baseline info
If master_date is specified, select its closest slave_date, which is newer than master_date;
otherwise, choose the closest pair among all pairs as master interferogram.
Example:
master_date12 = pnet.select_master_ifgram(date12_list, date_list, pbase_list)
'080211-080326' = pnet.select_master_ifgram(date12_list, date_list, pbase_list, m_date='080211')
"""
pbase_array = np.array(pbase_list, dtype='float64')
# Get temporal baseline
date8_list = ptime.yyyymmdd(date_list)
date6_list = ptime.yymmdd(date8_list)
tbase_array = np.array(ptime.date_list2tbase(date8_list)[0], dtype='float64')
# Normalization (Pepe and Lanari, 2006, TGRS)
temp2perp_scale = (max(pbase_array)-min(pbase_array)) / (max(tbase_array)-min(tbase_array))
tbase_array *= temp2perp_scale
# Calculate sqrt of temp/perp baseline for input pairs
idx1 = np.array([date6_list.index(date12.split('-')[0]) for date12 in date12_list])
idx2 = np.array([date6_list.index(date12.split('-')[1]) for date12 in date12_list])
base_distance = np.sqrt((tbase_array[idx2] - tbase_array[idx1])**2 +
(pbase_array[idx2] - pbase_array[idx1])**2)
# Get master interferogram index
if not m_date:
# Choose pair with shortest temp/perp baseline
m_date12_idx = np.argmin(base_distance)
else:
m_date = ptime.yymmdd(m_date)
# Choose pair contains m_date with shortest temp/perp baseline
m_date12_idx_array = np.array([date12_list.index(date12) for date12 in date12_list
if m_date+'-' in date12])
min_base_distance = np.min(base_distance[m_date12_idx_array])
m_date12_idx = np.where(base_distance == min_base_distance)[0][0]
m_date12 = date12_list[m_date12_idx]
return m_date12
def main(iargs=None):
inps = cmd_line_parse(iargs)
atr = readfile.read_attribute(inps.velocity_file)
length = int(atr['LENGTH'])
width = int(atr['WIDTH'])
# Check subset input
if inps.subset_y:
inps.subset_y = sorted(inps.subset_y)
print('subset in y/azimuth direction: '+str(inps.subset_y))
else:
inps.subset_y = [0, length]
if inps.subset_x:
inps.subset_x = sorted(inps.subset_x)
print('subset in x/range direction: '+str(inps.subset_x))
else:
inps.subset_x = [0, width]
y0, y1 = inps.subset_y
x0, x1 = inps.subset_x
# Read velocity/rate
velocity = readfile.read(inps.velocity_file)[0]
print('read velocity file: '+inps.velocity_file)
k = 'interferograms'
h5 = h5py.File(inps.ifgram_file, 'r')
ifgram_list = sorted(h5[k].keys())
ifgram_num = len(ifgram_list)
date12_list = ptime.list_ifgram2date12(ifgram_list)
print('number of interferograms: '+str(ifgram_num))
# Select interferograms with unwrapping error
if inps.percentage > 0.0:
mask = readfile.read(inps.mask_file, datasetName='mask')[0]
print('read mask for pixels with unwrapping error from file: '+inps.mask_file)
unw_err_ifgram_num = int(np.rint(inps.percentage*ifgram_num))
unw_err_ifgram_idx = random.sample(list(range(ifgram_num)), unw_err_ifgram_num)
unw_err_ifgram_list = [ifgram_list[i] for i in unw_err_ifgram_idx]
unw_err_date12_list = [date12_list[i] for i in unw_err_ifgram_idx]
print('randomly choose the following %d interferograms with unwrapping error' % unw_err_ifgram_num)
print(unw_err_date12_list)
unit_unw_err = 2.0*np.pi*mask
else:
unw_err_ifgram_list = []
# Generate simulated interferograms
m_dates = ptime.yyyymmdd([i.split('_')[0] for i in date12_list])
s_dates = ptime.yyyymmdd([i.split('_')[1] for i in date12_list])
range2phase = -4.0*np.pi/float(atr['WAVELENGTH'])
print('writing simulated interferograms file: '+inps.outfile)
h5out = h5py.File(inps.outfile, 'w')
group = h5out.create_group('interferograms')
for i in range(ifgram_num):
ifgram = ifgram_list[i]
# Get temporal baseline in years
t1 = datetime.datetime(*time.strptime(m_dates[i], "%Y%m%d")[0:5])
t2 = datetime.datetime(*time.strptime(s_dates[i], "%Y%m%d")[0:5])
dt = (t2-t1)
dt = float(dt.days)/365.25
# Simuated interferograms with unwrap error
unw = velocity*dt*range2phase
if ifgram in unw_err_ifgram_list:
rand_int = random.sample(list(range(1, 10)), 1)[0]
unw += rand_int * unit_unw_err
print(ifgram+' - add unwrapping error of %d*2*pi' % rand_int)
else:
print(ifgram)
gg = group.create_group(ifgram)
dset = gg.create_dataset(ifgram, data=unw[y0:y1, x0:x1])
for key, value in h5[k][ifgram].attrs.items():
gg.attrs[key] = value
if ifgram in unw_err_ifgram_list:
gg.attrs['unwrap_error'] = 'yes'
else:
gg.attrs['unwrap_error'] = 'no'
gg.attrs['LENGTH'] = y1-y0
gg.attrs['WIDTH'] = x1-x0
h5.close()
h5out.close()
print('Done.')
return inps.outfile
def read_data(inps):
# metadata
atr = readfile.read_attribute(inps.file)
range2phase = -4 * np.pi / float(atr['WAVELENGTH'])
# change reference pixel
if inps.ref_yx:
atr['REF_Y'] = inps.ref_yx[0]
atr['REF_X'] = inps.ref_yx[1]
print('change reference point to y/x: {}'.format(inps.ref_yx))
# various file types
print('read {} from file {}'.format(inps.dset, inps.file))
k = atr['FILE_TYPE']
if k == 'velocity':
# read/prepare data
data = readfile.read(inps.file)[0] * range2phase
print("converting velocity to an interferogram with one year temporal baseline")
if inps.ref_yx:
data -= data[inps.ref_yx[0], inps.ref_yx[1]]
# metadata
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(os.path.splitext(inps.file)[0], atr['FILE_TYPE'])
elif k == 'timeseries':
# date1 and date2
if '_' in inps.dset:
date1, date2 = ptime.yyyymmdd(inps.dset.split('_'))
else:
date1 = atr['REF_DATE']
date2 = ptime.yyyymmdd(inps.dset)
# read/prepare data
data = readfile.read(inps.file, datasetName=date2)[0]
data -= readfile.read(inps.file, datasetName=date1)[0]
print('converting range to phase')
data *= range2phase
if inps.ref_yx:
data -= data[inps.ref_yx[0], inps.ref_yx[1]]
# metadata
atr['DATE'] = date1[2:8]
atr['DATE12'] = '{}-{}'.format(date1[2:8], date2[2:8])
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
# output filename
if not inps.outfile:
inps.outfile = '{}_{}.unw'.format(date1, date2)
if inps.file.startswith('geo_'):
inps.outfile = 'geo_'+inps.outfile
elif k == 'HDFEOS':
dname = inps.dset.split('-')[0]
# date1 and date2
if dname == 'displacement':
if '-' in inps.dset:
suffix = inps.dset.split('-')[1]
if '_' in suffix:
date1, date2 = ptime.yyyymmdd(suffix.split('_'))
else:
date1 = atr['REF_DATE']
date2 = ptime.yyyymmdd(suffix)
else:
raise ValueError("No '-' in input dataset! It is required for {}".format(dname))
else:
date_list = HDFEOS(inps.file).get_date_list()
date1 = date_list[0]
date2 = date_list[-1]
date12 = '{}_{}'.format(date1, date2)
# read / prepare data
slice_list = readfile.get_slice_list(inps.file)
if 'displacement' in inps.dset:
# read/prepare data
slice_name1 = view.check_dataset_input(slice_list, '{}-{}'.format(dname, date1))[0][0]
slice_name2 = view.check_dataset_input(slice_list, '{}-{}'.format(dname, date2))[0][0]
data = readfile.read(inps.file, datasetName=slice_name1)[0]
data -= readfile.read(inps.file, datasetName=slice_name2)[0]
print('converting range to phase')
data *= range2phase
if inps.ref_yx:
data -= data[inps.ref_yx[0], inps.ref_yx[1]]
else:
slice_name = view.check_dataset_input(slice_list, inps.dset)[0][0]
data = readfile.read(inps.file, datasetName=slice_name)[0]
# metadata
atr['DATE'] = date1[2:8]
atr['DATE12'] = '{}-{}'.format(date1[2:8], date2[2:8])
if dname == 'displacement':
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
elif 'coherence' in dname.lower():
atr['FILE_TYPE'] = '.cor'
atr['UNIT'] = '1'
elif dname == 'height':
atr['FILE_TYPE'] = '.dem'
atr['DATA_TYPE'] = 'int16'
else:
raise ValueError('unrecognized input dataset type: {}'.format(inps.dset))
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(date12, atr['FILE_TYPE'])
elif k == 'ifgramStack':
dname, date12 = inps.dset.split('-')
date1, date2 = date12.split('_')
# read / prepare data
data = readfile.read(inps.file, datasetName=inps.dset)[0]
if dname.startswith('unwrapPhase'):
if 'REF_X' in atr.keys():
data -= data[int(atr['REF_Y']), int(atr['REF_X'])]
print('consider reference pixel in y/x: ({}, {})'.format(atr['REF_Y'], atr['REF_X']))
else:
print('No REF_Y/X found.')
# metadata
atr['DATE'] = date1[2:8]
atr['DATE12'] = '{}-{}'.format(date1[2:8], date2[2:8])
if dname.startswith('unwrapPhase'):
atr['FILE_TYPE'] = '.unw'
atr['UNIT'] = 'radian'
elif dname == 'coherence':
atr['FILE_TYPE'] = '.cor'
atr['UNIT'] = '1'
elif dname == 'wrapPhase':
atr['FILE_TYPE'] = '.int'
atr['UNIT'] = 'radian'
else:
raise ValueError('unrecognized dataset type: {}'.format(inps.dset))
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(date12, atr['FILE_TYPE'])
if inps.file.startswith('geo_'):
inps.outfile = 'geo_'+inps.outfile
else:
# read data
data = readfile.read(inps.file, datasetName=inps.dset)[0]
# metadata
if 'coherence' in k.lower():
atr['FILE_TYPE'] = '.cor'
elif k in ['mask']:
atr['FILE_TYPE'] = '.msk'
atr['DATA_TYPE'] = 'byte'
elif k in ['geometry'] and inps.dset == 'height':
if 'Y_FIRST' in atr.keys():
atr['FILE_TYPE'] = '.dem'
atr['DATA_TYPE'] = 'int16'
else:
atr['FILE_TYPE'] = '.hgt'
atr['UNIT'] = 'm'
else:
atr['FILE_TYPE'] = '.unw'
# output filename
if not inps.outfile:
inps.outfile = '{}{}'.format(os.path.splitext(inps.file)[0], atr['FILE_TYPE'])
atr['PROCESSOR'] = 'roipac'
return data, atr, inps.outfile
def check_inputs(inps):
parser = create_parser()
# output directories/files
atr = dict()
mintpy_dir = None
if inps.timeseries_file:
atr = readfile.read_attribute(inps.timeseries_file)
mintpy_dir = os.path.dirname(inps.timeseries_file)
if not inps.outfile:
fbase = os.path.splitext(inps.timeseries_file)[0]
inps.outfile = '{}_{}.h5'.format(fbase, inps.trop_model)
elif inps.geom_file:
atr = readfile.read_attribute(inps.geom_file)
mintpy_dir = os.path.join(os.path.dirname(inps.geom_file), '..')
else:
mintpy_dir = os.path.abspath(os.getcwd())
# trop_file
inps.trop_file = os.path.join(mintpy_dir, 'inputs/{}.h5'.format(inps.trop_model))
print('output tropospheric delay file: {}'.format(inps.trop_file))
# hour
if not inps.hour:
if 'CENTER_LINE_UTC' in atr.keys():
inps.hour = ptime.closest_weather_product_time(atr['CENTER_LINE_UTC'], inps.trop_model)
else:
parser.print_usage()
raise Exception('no input for hour')
print('time of cloest available product: {}:00 UTC'.format(inps.hour))
# date list
if inps.timeseries_file:
print('read date list from timeseries file: {}'.format(inps.timeseries_file))
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open(print_msg=False)
inps.date_list = ts_obj.dateList
elif len(inps.date_list) == 1:
if os.path.isfile(inps.date_list[0]):
print('read date list from text file: {}'.format(inps.date_list[0]))
inps.date_list = ptime.yyyymmdd(np.loadtxt(inps.date_list[0],
dtype=bytes,
usecols=(0,)).astype(str).tolist())
else:
parser.print_usage()
raise Exception('ERROR: input date list < 2')
# Grib data directory
inps.grib_dir = os.path.join(inps.weather_dir, inps.trop_model)
if not os.path.isdir(inps.grib_dir):
os.makedirs(inps.grib_dir)
print('making directory: '+inps.grib_dir)
# Date list to grib file list
inps.grib_file_list = date_list2grib_file(inps.date_list,
inps.hour,
inps.trop_model,
inps.grib_dir)
if 'REF_Y' in atr.keys():
inps.ref_yx = [int(atr['REF_Y']), int(atr['REF_X'])]
print('reference pixel: {}'.format(inps.ref_yx))
# Coordinate system: geocoded or not
inps.geocoded = False
if 'Y_FIRST' in atr.keys():
inps.geocoded = True
print('geocoded: {}'.format(inps.geocoded))
# Prepare DEM, inc_angle, lat/lon file for PyAPS to read
if inps.geom_file:
geom_atr = readfile.read_attribute(inps.geom_file)
print('converting DEM/incAngle for PyAPS to read')
# DEM
data = readfile.read(inps.geom_file, datasetName='height', print_msg=False)[0]
inps.dem_file = 'pyapsDem.hgt'
writefile.write(data, inps.dem_file, metadata=geom_atr)
# inc_angle
inps.inc_angle = readfile.read(inps.geom_file, datasetName='incidenceAngle', print_msg=False)[0]
inps.inc_angle_file = 'pyapsIncAngle.flt'
writefile.write(inps.inc_angle, inps.inc_angle_file, metadata=geom_atr)
# latitude
try:
data = readfile.read(inps.geom_file, datasetName='latitude', print_msg=False)[0]
print('converting lat for PyAPS to read')
inps.lat_file = 'pyapsLat.flt'
writefile.write(data, inps.lat_file, metadata=geom_atr)
except:
inps.lat_file = None
# longitude
try:
data = readfile.read(inps.geom_file, datasetName='longitude', print_msg=False)[0]
print('converting lon for PyAPS to read')
inps.lon_file = 'pyapsLon.flt'
writefile.write(data, inps.lon_file, metadata=geom_atr)
except:
inps.lon_file = None
return inps, atr
def read_inps_dict2ifgram_stack_dict_object(inpsDict):
"""Read input arguments into dict of ifgramStackDict object"""
# inpsDict --> dsPathDict
print('-'*50)
print('searching interferometric pairs info')
print('input data files:')
maxDigit = max([len(i) for i in list(datasetName2templateKey.keys())])
dsPathDict = {}
dsNumDict = {}
for dsName in [i for i in ifgramDatasetNames
if i in datasetName2templateKey.keys()]:
key = datasetName2templateKey[dsName]
if key in inpsDict.keys():
files = sorted(glob.glob(str(inpsDict[key])))
if len(files) > 0:
dsPathDict[dsName] = files
dsNumDict[dsName] = len(files)
print('{:<{width}}: {path}'.format(dsName,
width=maxDigit,
path=inpsDict[key]))
# Check 1: required dataset
dsName0 = 'unwrapPhase'
if dsName0 not in dsPathDict.keys():
print('WARNING: No reqired {} data files found!'.format(dsName0))
return None
# Check 2: number of files for all dataset types
for key, value in dsNumDict.items():
print('number of {:<{width}}: {num}'.format(key, width=maxDigit, num=value))
dsNumList = list(dsNumDict.values())
if any(i != dsNumList[0] for i in dsNumList):
msg = 'WARNING: NOT all types of dataset have the same number of files.'
msg += ' -> skip interferograms with missing files and continue.'
print(msg)
#raise Exception(msg)
# Check 3: data dimension for all files
# dsPathDict --> pairsDict --> stackObj
dsNameList = list(dsPathDict.keys())
pairsDict = {}
for dsPath in dsPathDict[dsName0]:
dates = ptime.yyyymmdd(readfile.read_attribute(dsPath)['DATE12'].split('-'))
#####################################
# A dictionary of data files for a given pair.
# One pair may have several types of dataset.
# example ifgramPathDict = {'unwrapPhase': /pathToFile/filt.unw, 'iono':/PathToFile/iono.bil}
# All path of data file must contain the master and slave date, either in file name or folder name.
ifgramPathDict = {}
for i in range(len(dsNameList)):
dsName = dsNameList[i]
dsPath1 = dsPathDict[dsName][0]
if all(d[2:8] in dsPath1 for d in dates):
ifgramPathDict[dsName] = dsPath1
else:
dsPath2 = [i for i in dsPathDict[dsName]
if all(d[2:8] in i for d in dates)]
if len(dsPath2) > 0:
ifgramPathDict[dsName] = dsPath2[0]
else:
print('WARNING: {} file missing for pair {}'.format(dsName, dates))
ifgramObj = ifgramDict(dates=tuple(dates),
datasetDict=ifgramPathDict)
pairsDict[tuple(dates)] = ifgramObj
if len(pairsDict) > 0:
stackObj = ifgramStackDict(pairsDict=pairsDict)
else:
stackObj = None
return stackObj
def read_inps_dict2geometry_dict_object(inpsDict):
# eliminate dsName by processor
if inpsDict['processor'] in ['isce', 'doris']:
datasetName2templateKey.pop('azimuthCoord')
datasetName2templateKey.pop('rangeCoord')
elif inpsDict['processor'] in ['roipac', 'gamma']:
datasetName2templateKey.pop('latitude')
datasetName2templateKey.pop('longitude')
else:
print('Un-recognized InSAR processor: {}'.format(inpsDict['processor']))
# inpsDict --> dsPathDict
print('-'*50)
print('searching geometry files info')
print('input data files:')
maxDigit = max([len(i) for i in list(datasetName2templateKey.keys())])
dsPathDict = {}
for dsName in [i for i in geometryDatasetNames
if i in datasetName2templateKey.keys()]:
key = datasetName2templateKey[dsName]
if key in inpsDict.keys():
files = sorted(glob.glob(str(inpsDict[key])))
if len(files) > 0:
if dsName == 'bperp':
bperpDict = {}
for file in files:
date = ptime.yyyymmdd(os.path.basename(os.path.dirname(file)))
bperpDict[date] = file
dsPathDict[dsName] = bperpDict
print('{:<{width}}: {path}'.format(dsName,
width=maxDigit,
path=inpsDict[key]))
print('number of bperp files: {}'.format(len(list(bperpDict.keys()))))
else:
dsPathDict[dsName] = files[0]
print('{:<{width}}: {path}'.format(dsName,
width=maxDigit,
path=files[0]))
# Check required dataset
dsName0 = geometryDatasetNames[0]
if dsName0 not in dsPathDict.keys():
print('WARNING: No reqired {} data files found!'.format(dsName0))
# metadata
ifgramRadarMetadata = None
ifgramKey = datasetName2templateKey['unwrapPhase']
if ifgramKey in inpsDict.keys():
ifgramFiles = glob.glob(str(inpsDict[ifgramKey]))
if len(ifgramFiles) > 0:
atr = readfile.read_attribute(ifgramFiles[0])
if 'Y_FIRST' not in atr.keys():
ifgramRadarMetadata = atr.copy()
# dsPathDict --> dsGeoPathDict + dsRadarPathDict
dsNameList = list(dsPathDict.keys())
dsGeoPathDict = {}
dsRadarPathDict = {}
for dsName in dsNameList:
if dsName == 'bperp':
atr = readfile.read_attribute(next(iter(dsPathDict[dsName].values())))
else:
atr = readfile.read_attribute(dsPathDict[dsName])
if 'Y_FIRST' in atr.keys():
dsGeoPathDict[dsName] = dsPathDict[dsName]
else:
dsRadarPathDict[dsName] = dsPathDict[dsName]
geomRadarObj = None
geomGeoObj = None
if len(dsRadarPathDict) > 0:
geomRadarObj = geometryDict(processor=inpsDict['processor'],
datasetDict=dsRadarPathDict,
extraMetadata=ifgramRadarMetadata)
if len(dsGeoPathDict) > 0:
geomGeoObj = geometryDict(processor=inpsDict['processor'],
datasetDict=dsGeoPathDict,
extraMetadata=None)
return geomRadarObj, geomGeoObj
def check_inputs(inps):
parser = create_parser()
# output directories/files
atr = dict()
mintpy_dir = None
if inps.timeseries_file:
atr = readfile.read_attribute(inps.timeseries_file)
mintpy_dir = os.path.dirname(inps.timeseries_file)
if not inps.outfile:
fbase = os.path.splitext(inps.timeseries_file)[0]
inps.outfile = '{}_{}.h5'.format(fbase, inps.trop_model)
elif inps.geom_file:
atr = readfile.read_attribute(inps.geom_file)
mintpy_dir = os.path.join(os.path.dirname(inps.geom_file), '..')
else:
mintpy_dir = os.path.abspath(os.getcwd())
# trop_file
inps.trop_file = os.path.join(mintpy_dir, 'inputs/{}.h5'.format(inps.trop_model))
print('output tropospheric delay file: {}'.format(inps.trop_file))
# hour
if not inps.hour:
if 'CENTER_LINE_UTC' in atr.keys():
inps.hour = closest_weather_model_hour(atr['CENTER_LINE_UTC'], inps.trop_model)
else:
parser.print_usage()
raise Exception('no input for hour')
print('time of cloest available product: {}:00 UTC'.format(inps.hour))
# date list
if inps.timeseries_file:
print('read date list from timeseries file: {}'.format(inps.timeseries_file))
ts_obj = timeseries(inps.timeseries_file)
ts_obj.open(print_msg=False)
inps.date_list = ts_obj.dateList
elif len(inps.date_list) == 1:
if os.path.isfile(inps.date_list[0]):
print('read date list from text file: {}'.format(inps.date_list[0]))
inps.date_list = ptime.yyyymmdd(np.loadtxt(inps.date_list[0],
dtype=bytes,
usecols=(0,)).astype(str).tolist())
else:
parser.print_usage()
raise Exception('ERROR: input date list < 2')
# Grib data directory
inps.grib_dir = os.path.join(inps.weather_dir, inps.trop_model)
if not os.path.isdir(inps.grib_dir):
os.makedirs(inps.grib_dir)
print('making directory: '+inps.grib_dir)
# area extent for ERA5 grib data download
inps.snwe = get_snwe(atr)
# Date list to grib file list
inps.grib_file_list = date_list2grib_file(date_list=inps.date_list,
hour=inps.hour,
model=inps.trop_model,
grib_dir=inps.grib_dir,
snwe=inps.snwe)
if 'REF_Y' in atr.keys():
inps.ref_yx = [int(atr['REF_Y']), int(atr['REF_X'])]
print('reference pixel: {}'.format(inps.ref_yx))
return inps, atr
def main(argv):
inps = cmdLineParse()
if inps.timeseries_file:
inps.timeseries_file=ut.get_file_list([inps.timeseries_file])[0]
atr=readfile.read_attribute(inps.timeseries_file)
k = atr['FILE_TYPE']
if 'ref_y' not in list(atr.keys()) and inps.ref_yx:
print('No reference info found in input file, use input ref_yx: '+str(inps.ref_yx))
atr['ref_y'] = inps.ref_yx[0]
atr['ref_x'] = inps.ref_yx[1]
#****reading incidence angle file***/
if os.path.isfile(inps.inc_angle):
inps.inc_angle=readfile.read(inps.inc_angle, datasetName='incidenceAngle')[0]
inps.inc_angle=np.nan_to_num(inps.inc_angle)
else:
inps.inps.inc_angle = float(inps.inc_angle)
print('incidence angle: '+str(inps.inc_angle))
cinc=np.cos(inps.inc_angle*np.pi/180.0);
#****look up file****/
if inps.lookup_file:
inps.lookup_file = ut.get_file_list([inps.lookup_file])[0] #'geomap_32rlks_tight.trans'
#****GACOS****/
delay_source = 'GACOS'
# Get weather directory
if not inps.GACOS_dir:
if inps.timeseries_file:
inps.GACOS_dir = os.path.dirname(os.path.abspath(inps.timeseries_file))+'/../WEATHER/GACOS'
elif inps.lookup_file:
inps.GACOS_dir = os.path.dirname(os.path.abspath(inps.lookup_file))+'/../WEATHER/GACOS'
else:
inps.GACOS_dir = os.path.abspath(os.getcwd())
print('Store weather data into directory: '+inps.GACOS_dir)
#source_dir=os.path.dirname(os.path.abspath('timeseries_file'))+'/Agung/GACOS/data';print source_dir
#os.makedirs(GACOS_dir) -----------------------------------------------add part to copy/download weather data------#
#----get date list-----#
if not inps.date_list_file:
print('read date list info from: '+inps.timeseries_file)
h5=h5py.File(inps.timeseries_file,'r')
if 'timeseries' in list(h5.keys()):
date_list=sorted(h5[k].keys())
elif k in ['interferograms','coherence','wrapped']:
ifgram_list = sorted(h5[k].keys())
date12_list = pnet.get_date12_list(inps.timeseries_file)
m_dates = [i.split('-')[0] for i in date12_list]
s_dates = [i.split('-')[1] for i in date12_list]
date_list = ptime.yyyymmdd(sorted(list(set(m_dates + s_dates))))
else:
raise ValueError('Un-support input file type:'+k)
h5.close()
else:
date_list = ptime.yyyymmdd(np.loadtxt(inps.date_list_file, dtype=str, usecols=(0,)).tolist())
print('read date list info from: '+inps.date_list_file)
#****cheacking availability of delays****/
print('checking availability of delays')
delay_file_list=[]
for d in date_list:
if delay_source == 'GACOS': delay_file = inps.GACOS_dir+'/'+d+'.ztd';
delay_file_list.append(delay_file)
delay_file_existed = ut.get_file_list(delay_file_list)
if len(delay_file_existed)==len(date_list):
print('no missing files')
else:
print('no. of date files found:', len(delay_file_existed));
print('no. of dates:', len(date_list))
#*****Calculating delays***/
print('calculating delays')
length=int(atr['FILE_LENGTH'])
width=int(atr['WIDTH'])
#initialise delay files
date_num=len(date_list)
trop_ts=np.zeros((date_num, length, width), np.float32)
#reading wrf files for each epoch and getting delay
for i in range(date_num):
delay_file=delay_file_existed[i]
date=date_list[i]
print('calculating delay for date',date)
trop_ts[i] =get_delay(delay_file,atr,inps.lookup_file,cinc)
print('Delays Calculated')
# Convert relative phase delay on reference date
try: ref_date = atr['ref_date']
except: ref_date = date_list[0]
print('convert to relative phase delay with reference date: '+ref_date)
ref_idx = date_list.index(ref_date)
trop_ts -= np.tile(trop_ts[ref_idx,:,:], (date_num, 1, 1))
## Write tropospheric delay to HDF5
tropFile = 'GACOSdelays'+'.h5'
print('writing >>> %s' % (tropFile))
h5trop = h5py.File(tropFile, 'w')
group_trop = h5trop.create_group('timeseries')
print('number of acquisitions: '+str(date_num))
prog_bar = ptime.progress_bar(maxValue=date_num)
for i in range(date_num):
date = date_list[i]
group_trop.create_dataset(date, data=trop_ts[i], compression='gzip')
prog_bar.update(i+1, suffix=date)
prog_bar.close()
# Write Attributes
for key,value in atr.items():
group_trop.attrs[key] = value
h5trop.close()
## Write corrected Time series to HDF5
if k == 'timeseries':
if not inps.out_file:
inps.out_file = os.path.splitext(inps.timeseries_file)[0]+'_'+'GACOS'+'.h5'
print('writing trop corrected timeseries file %s' % (inps.out_file))
h5ts = h5py.File(inps.timeseries_file, 'r')
h5tsCor = h5py.File(inps.out_file, 'w')
group_tsCor = h5tsCor.create_group('timeseries')
print('number of acquisitions: '+str(date_num))
prog_bar = ptime.progress_bar(maxValue=date_num)
for i in range(date_num):
date = date_list[i];print(date)
ts = h5ts['timeseries'].get(date)[:]
group_tsCor.create_dataset(date, data=ts-trop_ts[i], compression='gzip')
prog_bar.update(i+1, suffix=date)
prog_bar.close()
h5ts.close()
# Write Attributes
for key,value in atr.items():
group_tsCor.attrs[key] = value
h5tsCor.close()
print('delays written to %s' % (inps.out_file))
print('finished')
return inps.out_file
