def skip_files_with_inconsistent_size(dsPathDict,
pix_box=None,
dsName='unwrapPhase'):
"""Skip files by removing the file path from the input dsPathDict."""
atr_list = [readfile.read_attribute(fname) for fname in dsPathDict[dsName]]
length_list = [int(atr['LENGTH']) for atr in atr_list]
width_list = [int(atr['WIDTH']) for atr in atr_list]
# Check size requirements
drop_inconsistent_files = False
if any(len(set(size_list)) > 1 for size_list in [length_list, width_list]):
if pix_box is None:
drop_inconsistent_files = True
else:
# if input subset is within the min file sizes: do NOT drop
max_box_width, max_box_length = pix_box[2:4]
if max_box_length > min(length_list) or max_box_width > min(
width_list):
drop_inconsistent_files = True
# update dsPathDict
if drop_inconsistent_files:
common_length = ut.most_common(length_list)
common_width = ut.most_common(width_list)
# print out warning message
msg = '\n' + '*' * 80
msg += '\nWARNING: NOT all input unwrapped interferograms have the same row/column number!'
msg += '\nThe most common size is: ({}, {})'.format(
common_length, common_width)
msg += '\n' + '-' * 30
msg += '\nThe following dates have different size:'
dsNames = list(dsPathDict.keys())
date12_list = [atr['DATE12'] for atr in atr_list]
num_drop = 0
for i in range(len(date12_list)):
if length_list[i] != common_length or width_list[i] != common_width:
date12 = date12_list[i]
dates = ptime.yyyymmdd(date12.split('-'))
# update file list for all datasets
for dsName in dsNames:
fnames = [
i for i in dsPathDict[dsName]
if all(d[2:8] in i for d in dates)
]
if len(fnames) > 0:
dsPathDict[dsName].remove(fnames[0])
msg += '\n\t{}\t({}, {})'.format(date12, length_list[i],
width_list[i])
num_drop += 1
msg += '\n' + '-' * 30
msg += '\nSkip loading the above interferograms ({}).'.format(num_drop)
msg += '\nContinue to load the rest interferograms ({}).'.format(
len(date12_list) - num_drop)
msg += '\n' + '*' * 80 + '\n'
print(msg)
return dsPathDict
def check_exist_grib_file(gfile_list, print_msg=True):
"""Check input list of grib files, and return the existing ones with right size."""
gfile_exist = ut.get_file_list(gfile_list)
if gfile_exist:
file_sizes = [os.path.getsize(i) for i in gfile_exist
if os.path.getsize(i) > 10e6]
if file_sizes:
comm_size = ut.most_common([i for i in file_sizes])
if print_msg:
print('common file size: {} bytes'.format(comm_size))
print('number of grib files existed : {}'.format(len(gfile_exist)))
gfile_corrupt = []
for gfile in gfile_exist:
if os.path.getsize(gfile) < comm_size * 0.9:
gfile_corrupt.append(gfile)
else:
gfile_corrupt = gfile_exist
if gfile_corrupt:
if print_msg:
print('------------------------------------------------------------------------------')
print('corrupted grib files detected! Delete them and re-download...')
print('number of grib files corrupted : {}'.format(len(gfile_corrupt)))
for i in gfile_corrupt:
rmCmd = 'rm '+i
print(rmCmd)
os.system(rmCmd)
gfile_exist.remove(i)
if print_msg:
print('------------------------------------------------------------------------------')
return gfile_exist
def check_exist_grib_file(gfile_list, print_msg=True):
"""Check input list of grib files, and return the existing ones with right size."""
gfile_exist = ut.get_file_list(gfile_list)
if gfile_exist:
file_sizes = [os.path.getsize(i) for i in gfile_exist] # if os.path.getsize(i) > 10e6]
if file_sizes:
comm_size = ut.most_common([i for i in file_sizes])
if print_msg:
print('common file size: {} bytes'.format(comm_size))
print('number of grib files existed : {}'.format(len(gfile_exist)))
gfile_corrupt = []
for gfile in gfile_exist:
if os.path.getsize(gfile) < comm_size * 0.9:
gfile_corrupt.append(gfile)
else:
gfile_corrupt = gfile_exist
if gfile_corrupt:
if print_msg:
print('------------------------------------------------------------------------------')
print('corrupted grib files detected! Delete them and re-download...')
print('number of grib files corrupted : {}'.format(len(gfile_corrupt)))
for i in gfile_corrupt:
rmCmd = 'rm '+i
print(rmCmd)
os.system(rmCmd)
gfile_exist.remove(i)
if print_msg:
print('------------------------------------------------------------------------------')
return gfile_exist
def safe2date_time(safe_file, tropo_model):
"""generate date_list and hour from safe_list"""
def seconds_UTC(seconds):
"""generate second list"""
if isinstance(seconds, list):
secondsOut = []
for second in seconds:
secondsOut.append(second)
else:
print('\nUn-recognized CENTER_LINE_UTC input!')
return None
return secondsOut
date_list = ptime.yyyymmdd(
np.loadtxt(safe_file, dtype=bytes, converters={
0: define_date
}).astype(str).tolist())
second_list = seconds_UTC(
np.loadtxt(safe_file, dtype=bytes, converters={
0: define_second
}).astype(str).tolist())
# change second into hour
hour_list = [
closest_weather_model_hour(float(second), tropo_model)
for second in second_list
]
hour = ut.most_common(hour_list)
return date_list, hour
def download_igs_tec(date_list, tec_dir, tec_sol='jpl'):
"""Download IGS TEC products for the input list of dates.
Parameters: date_list - list of str, in YYYYMMDD
tec_dir - str, path to IGS_TEC directory, e.g. ~/data/aux/IGS_TEC
tec_sol - str, TEC solution center, e.g. jpl, cod, igs
Returns: fnames - list of str, path of the downloaded TEC files
"""
print("\n------------------------------------------------------------------------------")
print("downloading GNSS-based TEC products from NASA's Archive of Space Geodesy Data (CDDIS) ...")
print('Link: https://cddis.nasa.gov/Data_and_Derived_Products/GNSS/atmospheric_products.html')
num_date = len(date_list)
n = len(str(num_date))
print(f'number of TEC files to download: {num_date}')
print(f'local TEC file directory: {tec_dir}')
# output file names/sizes
fnames = []
for date_str in date_list:
fnames.append(iono.get_igs_tec_filename(tec_dir, date_str, sol=tec_sol))
# remove all existing files
debug_mode = False
if debug_mode:
for fname in fnames:
for x in [fname, fname+'.Z']:
if os.path.isfile(x):
os.remove(x)
fsizes = [os.path.getsize(i) / 1024 if os.path.isfile(i) else 0 for i in fnames]
# download: skip existing ones
fsizec = ut.most_common(fsizes)
if fsizec < 400:
# too small, does not seem right --> download them all
date_list2dload = list(date_list)
else:
# download missing ones
date_list2dload = [d for d, s in zip(date_list, fsizes) if s < fsizec * 0.9]
num_date2dload = len(date_list2dload)
if num_date2dload == 0:
print(f'ALL files exists with consistent file size (~{fsizec:.0f} KB) --> skip re-downloading.\n')
else:
for i, date_str in enumerate(date_list2dload):
print('-'*20)
print('DATE {}/{}: {}'.format(i+1, num_date2dload, date_str))
iono.dload_igs_tec(date_str, tec_dir, sol=tec_sol, print_msg=True)
# print file size info, after downloading
fsizes = [os.path.getsize(i) / 1024 if os.path.isfile(i) else 0 for i in fnames]
for i in range(num_date):
print('[{i:0{n}d}/{N}] {f}: {s:.2f} KB'.format(n=n, i=i+1, N=num_date, f=fnames[i], s=fsizes[i]))
return fnames
def read_baseline_timeseries(baseline_dir, processor='tops'):
"""Read bperp time-series from files in baselines directory
Parameters: baseline_dir : str, path to the baselines directory
processor : str, tops for Sentinel-1/TOPS
stripmap for StripMap data
Returns: bDict : dict, in the following format:
{'20141213': [0.0, 0.0],
'20141225': [104.6, 110.1],
...
}
"""
print('read perp baseline time-series from {}'.format(baseline_dir))
# grab all existed baseline files
if processor == 'tops':
bFiles = sorted(glob.glob(os.path.join(baseline_dir, '*/*.txt')))
elif processor == 'stripmap':
bFiles = sorted(glob.glob(os.path.join(baseline_dir, '*.txt')))
else:
raise ValueError(
'Un-recognized ISCE stack processor: {}'.format(processor))
if len(bFiles) == 0:
print('WARNING: no baseline text file found in dir {}'.format(
os.path.abspath(baseline_dir)))
return None
# ignore files with different date1
# when re-run with different reference date
date1s = [os.path.basename(i).split('_')[0] for i in bFiles]
date1 = ut.most_common(date1s)
bFiles = [i for i in bFiles if os.path.basename(i).split('_')[0] == date1]
# read files into dict
bDict = {}
for bFile in bFiles:
dates = os.path.basename(bFile).split('.txt')[0].split('_')
if processor == 'tops':
bDict[dates[1]] = read_tops_baseline(bFile)
else:
bDict[dates[1]] = read_stripmap_baseline(bFile)
bDict[dates[0]] = [0, 0]
return bDict
def plot_rms_bar(ax,
date_list,
rms,
cutoff=3.,
font_size=12,
tick_year_num=1,
legend_loc='best',
disp_legend=True,
disp_side_plot=True,
disp_thres_text=False,
ylabel=r'Residual Phase $\hat \phi_{resid}$ RMS [mm]'):
""" Bar plot Phase Residual RMS
Parameters: ax : Axes object
date_list : list of string in YYYYMMDD format
rms : 1D np.array of float for RMS value in mm
cutoff : cutoff value of MAD outlier detection
tick_year_num : int, number of years per major tick
legend_loc : 'upper right' or (0.5, 0.5)
Returns: ax : Axes object
"""
dates, datevector = ptime.date_list2vector(date_list)
dates = np.array(dates)
try:
bar_width = min(ut.most_common(np.diff(dates).tolist(), k=2)) * 3 / 4
except:
bar_width = np.min(np.diff(dates).tolist()) * 3 / 4
rms = np.array(rms)
# Plot all dates
ax.bar(dates, rms, bar_width.days, color=pp.mplColors[0])
# Plot reference date
ref_idx = np.argmin(rms)
ax.bar(dates[ref_idx],
rms[ref_idx],
bar_width.days,
color=pp.mplColors[1],
label='Reference date')
# Plot exclude dates
rms_threshold = ut.median_abs_deviation_threshold(rms,
center=0.,
cutoff=cutoff)
ex_idx = rms > rms_threshold
if not np.all(ex_idx == False):
ax.bar(dates[ex_idx],
rms[ex_idx],
bar_width.days,
color='darkgray',
label='Exclude date')
# Plot rms_threshold line
(ax, xmin, xmax) = pp.auto_adjust_xaxis_date(ax,
datevector,
font_size,
every_year=tick_year_num)
ax.plot(np.array([xmin, xmax]),
np.array([rms_threshold, rms_threshold]),
'--k',
label='Median Abs Dev * {}'.format(cutoff))
# axis format
ax = pp.auto_adjust_yaxis(ax,
np.append(rms, rms_threshold),
font_size,
ymin=0.0)
ax.set_xlabel('Time [years]', fontsize=font_size)
ax.set_ylabel(ylabel, fontsize=font_size)
ax.tick_params(which='both',
direction='in',
labelsize=font_size,
bottom=True,
top=True,
left=True,
right=True)
# 2nd axes for circles
if disp_side_plot:
divider = make_axes_locatable(ax)
ax2 = divider.append_axes("right", "10%", pad="2%")
ax2.plot(np.ones(rms.shape, np.float32) * 0.5,
rms,
'o',
mfc='none',
color=pp.mplColors[0])
ax2.plot(np.ones(rms.shape, np.float32)[ref_idx] * 0.5,
rms[ref_idx],
'o',
mfc='none',
color=pp.mplColors[1])
if not np.all(ex_idx == False):
ax2.plot(np.ones(rms.shape, np.float32)[ex_idx] * 0.5,
rms[ex_idx],
'o',
mfc='none',
color='darkgray')
ax2.plot(np.array([0, 1]), np.array([rms_threshold, rms_threshold]),
'--k')
ax2.set_ylim(ax.get_ylim())
ax2.set_xlim([0, 1])
ax2.tick_params(which='both',
direction='in',
labelsize=font_size,
bottom=True,
top=True,
left=True,
right=True)
ax2.get_xaxis().set_ticks([])
ax2.get_yaxis().set_ticklabels([])
if disp_legend:
ax.legend(loc=legend_loc, frameon=False, fontsize=font_size)
# rms_threshold text
if disp_thres_text:
ymin, ymax = ax.get_ylim()
yoff = (ymax - ymin) * 0.1
if (rms_threshold - ymin) > 0.5 * (ymax - ymin):
yoff *= -1.
ax.annotate('Median Abs Dev * {}'.format(cutoff),
xy=(xmin + (xmax - xmin) * 0.05, rms_threshold + yoff),
color='k',
xycoords='data',
fontsize=font_size)
return ax
def plot_rms_bar(ax, date_list, rms, cutoff=3., font_size=12,
tick_year_num=1, legend_loc='best',
disp_legend=True, disp_side_plot=True, disp_thres_text=True,
ylabel=r'Residual Phase $\hat \phi_{resid}$ RMS [mm]'):
""" Bar plot Phase Residual RMS
Parameters: ax : Axes object
date_list : list of string in YYYYMMDD format
rms : 1D np.array of float for RMS value in mm
cutoff : cutoff value of MAD outlier detection
tick_year_num : int, number of years per major tick
legend_loc : 'upper right' or (0.5, 0.5)
Returns: ax : Axes object
"""
dates, datevector = ptime.date_list2vector(date_list)
dates = np.array(dates)
try:
bar_width = min(ut.most_common(np.diff(dates).tolist(), k=2))*3/4
except:
bar_width = np.min(np.diff(dates).tolist())*3/4
rms = np.array(rms)
# Plot all dates
ax.bar(dates, rms, bar_width.days, color=pp.mplColors[0])
# Plot reference date
ref_idx = np.argmin(rms)
ax.bar(dates[ref_idx], rms[ref_idx], bar_width.days, color=pp.mplColors[1], label='Reference date')
# Plot exclude dates
rms_threshold = ut.median_abs_deviation_threshold(rms, center=0., cutoff=cutoff)
ex_idx = rms > rms_threshold
if not np.all(ex_idx==False):
ax.bar(dates[ex_idx], rms[ex_idx], bar_width.days, color='darkgray', label='Exclude date')
# Plot rms_threshold line
(ax, xmin, xmax) = pp.auto_adjust_xaxis_date(ax, datevector, font_size, every_year=tick_year_num)
ax.plot(np.array([xmin, xmax]), np.array([rms_threshold, rms_threshold]), '--k', label='RMS threshold')
# axis format
ax = pp.auto_adjust_yaxis(ax, np.append(rms, rms_threshold), font_size, ymin=0.0)
ax.set_xlabel('Time [years]', fontsize=font_size)
ax.set_ylabel(ylabel, fontsize=font_size)
ax.tick_params(which='both', direction='in', labelsize=font_size,
bottom=True, top=True, left=True, right=True)
# 2nd axes for circles
if disp_side_plot:
divider = make_axes_locatable(ax)
ax2 = divider.append_axes("right", "10%", pad="2%")
ax2.plot(np.ones(rms.shape, np.float32) * 0.5, rms, 'o', mfc='none', color=pp.mplColors[0])
ax2.plot(np.ones(rms.shape, np.float32)[ref_idx] * 0.5, rms[ref_idx], 'o', mfc='none', color=pp.mplColors[1])
if not np.all(ex_idx==False):
ax2.plot(np.ones(rms.shape, np.float32)[ex_idx] * 0.5, rms[ex_idx], 'o', mfc='none', color='darkgray')
ax2.plot(np.array([0, 1]), np.array([rms_threshold, rms_threshold]), '--k')
ax2.set_ylim(ax.get_ylim())
ax2.set_xlim([0, 1])
ax2.tick_params(which='both', direction='in', labelsize=font_size,
bottom=True, top=True, left=True, right=True)
ax2.get_xaxis().set_ticks([])
ax2.get_yaxis().set_ticklabels([])
if disp_legend:
ax.legend(loc=legend_loc, frameon=False, fontsize=font_size)
# rms_threshold text
if disp_thres_text:
ymin, ymax = ax.get_ylim()
yoff = (ymax - ymin) * 0.1
if (rms_threshold - ymin) > 0.5 * (ymax - ymin):
yoff *= -1.
ax.annotate('Median Abs Dev * {}'.format(cutoff),
xy=(xmin + (xmax-xmin)*0.05, rms_threshold + yoff ),
color='k', xycoords='data', fontsize=font_size)
return ax