def dload_grib(date_list, hour, grib_source='ECMWF', weather_dir='./'):
'''Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
grib_source : string,
weather_dir : string,
Output:
grib_file_list : list of string
'''
## Grib data directory
weather_dir = os.path.abspath(weather_dir)
grib_dir = weather_dir+'/'+grib_source
if not os.path.isdir(grib_dir):
print 'making directory: '+grib_dir
os.makedirs(grib_dir)
## Date list to grib file list
grib_file_list = []
for d in date_list:
if grib_source == 'ECMWF': grib_file = grib_dir+'/ERA-Int_'+d+'_'+hour+'.grb'
elif grib_source == 'ERA' : grib_file = grib_dir+'/ERA_'+d+'_'+hour+'.grb'
elif grib_source == 'MERRA': grib_file = grib_dir+'/merra-'+d+'-'+hour+'.hdf'
elif grib_source == 'NARR' : grib_file = grib_dir+'/narr-a_221_'+d+'_'+hour+'00_000.grb'
grib_file_list.append(grib_file)
## Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_mode = ut.mode([os.path.getsize(i) for i in grib_file_existed])
grib_file_corrupted = [i for i in grib_file_existed if os.path.getsize(i) != grib_filesize_mode]
print 'number of grib files existed : %d' % len(grib_file_existed)
print 'file size mode: %d' % grib_filesize_mode
if grib_file_corrupted:
print '------------------------------------------------------------------------------'
print 'corrupted grib files detected! Delete them and re-download...'
print 'number of grib files corrupted : %d' % len(grib_file_corrupted)
for i in grib_file_corrupted:
rmCmd = 'rm '+i
print rmCmd
os.system(rmCmd)
grib_file_existed.remove(i)
print '------------------------------------------------------------------------------'
grib_file2download = sorted(list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [str(re.findall('\d{8}', i)[0]) for i in grib_file2download]
print 'number of grib files to download: %d' % len(date_list2download)
print '------------------------------------------------------------------------------\n'
## Download grib file using PyAPS
if grib_source == 'ECMWF': pa.ECMWFdload(date_list2download, hour, grib_dir)
elif grib_source == 'ERA' : pa.ERAdload( date_list2download, hour, grib_dir)
elif grib_source == 'MERRA': pa.MERRAdload(date_list2download, hour, grib_dir)
elif grib_source == 'NARR' : pa.NARRdload( date_list2download, hour, grib_dir)
return grib_file_existed
def main(argv):
inps = cmdLineParse()
if inps.template_file:
inps = read_template2inps(inps.template_file)
##### calculate timeseries of residual Root Mean Square
#std_list, date_list = ut.get_residual_std(inps.timeseries_file, inps.mask_file, inps.ramp_type)
rms_list, date_list = ut.get_residual_rms(inps.timeseries_file,
inps.mask_file, inps.ramp_type)
##### reference_date.txt
print '------------------------------------------------------------'
ref_idx = np.argmin(rms_list)
ref_date = date_list[ref_idx]
print 'date with minimum residual RMS: %s - %.4f' % (ref_date,
rms_list[ref_idx])
refTxtFile = 'reference_date.txt'
if (inps.save_reference_date and \
ut.update_file(refTxtFile, [inps.timeseries_file, inps.mask_file, inps.template_file],\
check_readable=False)):
f = open(refTxtFile, 'w')
f.write(ref_date + '\n')
f.close()
print 'save date to file: ' + refTxtFile
##### exclude_date.txt
print '------------------------------------------------------------'
ex_idx_list = [rms_list.index(i) for i in rms_list if i > inps.min_rms]
print 'date(s) with residual RMS > ' + str(inps.min_rms)
exTxtFile = 'exclude_date.txt'
if ex_idx_list:
if (inps.save_exclude_date and \
ut.update_file(exTxtFile, [inps.timeseries_file, inps.mask_file, inps.template_file],\
check_readable=False)):
f = open(exTxtFile, 'w')
for i in ex_idx_list:
print '%s - %.4f' % (date_list[i], rms_list[i])
f.write(date_list[i] + '\n')
f.close()
print 'save date(s) to file: ' + exTxtFile
else:
print 'None.'
##### Plot
fig_name = os.path.dirname(os.path.abspath(inps.timeseries_file))+\
'/rms_'+os.path.splitext(inps.timeseries_file)[0]
if inps.ramp_type != 'no':
fig_name += '_' + inps.ramp_type
fig_name += '.pdf'
if ut.update_file(fig_name, [exTxtFile, refTxtFile, inps.template_file],
check_readable=False):
if inps.fig_size:
fig = plt.figure(figsize=inps.fig_size)
else:
fig = plt.figure()
ax = fig.add_subplot(111)
font_size = 12
dates, datevector = ptime.date_list2vector(date_list)
try:
bar_width = ut.mode(np.diff(dates).tolist()) * 3 / 4
except:
bar_width = np.min(np.diff(dates).tolist()) * 3 / 4
x_list = [i - bar_width / 2 for i in dates]
rms_list = [i * 1000. for i in rms_list]
min_rms = inps.min_rms * 1000.
# Plot all dates
ax.bar(x_list, rms_list, bar_width.days)
#ax.bar(x_list, rms_list, bar_width.days)
# Plot reference date
#if inps.save_reference_date:
ax.bar(x_list[ref_idx],
rms_list[ref_idx],
bar_width.days,
label='Reference date')
# Plot exclude dates
#if ex_idx_list and inps.save_exclude_date:
if ex_idx_list:
ex_x_list = [x_list[i] for i in ex_idx_list]
ex_rms_list = [rms_list[i] for i in ex_idx_list]
ax.bar(ex_x_list,
ex_rms_list,
bar_width.days,
color='darkgray',
label='Exclude date(s)')
# Plot min_rms line
ax, xmin, xmax = ptime.auto_adjust_xaxis_date(
ax, datevector, font_size, every_year=inps.tick_year_num)
ax.plot(np.array([xmin, xmax]), np.array([min_rms, min_rms]), '--k')
# axis format
ax = pnet.auto_adjust_yaxis(ax,
rms_list + [min_rms],
font_size,
ymin=0.0)
ax.set_xlabel('Time [years]', fontsize=font_size)
ax.set_ylabel('Root Mean Square [mm]', fontsize=font_size)
ax.yaxis.set_ticks_position('both')
ax.tick_params(labelsize=font_size)
if inps.save_reference_date or inps.save_exclude_date:
plt.legend(fontsize=font_size)
# save figure
fig.savefig(fig_name, bbox_inches='tight', transparent=True)
print 'save figure to file: ' + fig_name
return
def main(argv):
inps = cmdLineParse()
if inps.timeseries_file:
inps.timeseries_file = ut.get_file_list([inps.timeseries_file])[0]
atr = readfile.read_attribute(inps.timeseries_file)
if inps.dem_file:
inps.dem_file = ut.get_file_list([inps.dem_file])[0]
# Convert DEM to ROIPAC format
if os.path.splitext(inps.dem_file)[1] in ['.h5']:
print 'convert DEM file to ROIPAC format'
dem, atr_dem = readfile.read(inps.dem_file)
if 'Y_FIRST' in atr_dem.keys():
atr_dem['FILE_TYPE'] = '.dem'
else:
atr_dem['FILE_TYPE'] = '.hgt'
outname = os.path.splitext(
inps.dem_file)[0] + '4pyaps' + atr_dem['FILE_TYPE']
inps.dem_file = writefile.write(dem, atr_dem, outname)
print '*******************************************************************************'
print 'Downloading weather model data ...'
## Get Grib Source
if inps.weather_model in ['ECMWF', 'ERA-Interim']:
inps.grib_source = 'ECMWF'
elif inps.weather_model == 'ERA':
inps.grib_source = 'ERA'
elif inps.weather_model == 'MERRA':
inps.grib_source = 'MERRA'
elif inps.weather_model == 'NARR':
inps.grib_source = 'NARR'
else:
raise Reception('Unrecognized weather model: ' + inps.weather_model)
print 'grib source: ' + inps.grib_source
## Grib data directory
if not inps.weather_dir:
if inps.timeseries_file:
inps.weather_dir = os.path.dirname(
os.path.abspath(inps.timeseries_file)) + '/../WEATHER'
elif inps.dem_file:
inps.weather_dir = os.path.dirname(os.path.abspath(
inps.dem_file)) + '/../WEATHER'
else:
inps.weather_dir = os.path.abspath(os.getcwd())
print 'Store weather data into directory: ' + inps.weather_dir
grib_dir = inps.weather_dir + '/' + inps.grib_source
if not os.path.isdir(grib_dir):
print 'making directory: ' + grib_dir
os.makedirs(grib_dir)
## Get Acquisition time
if not inps.hour:
inps.hour = closest_weather_product_time(atr['CENTER_LINE_UTC'],
inps.grib_source)
print 'Time of cloest available product: ' + inps.hour
## Get grib file list and date list
inps.grib_file_list = []
if not inps.date_list_file:
h5timeseries = h5py.File(inps.timeseries_file, 'r')
dateList = sorted(h5timeseries['timeseries'].keys())
h5timeseries.close()
print 'read date list info from: ' + inps.timeseries_file
else:
dateList = ptime.yyyymmdd(
np.loadtxt(inps.date_list_file, dtype=str, usecols=(0, )).tolist())
print 'read date list info from: ' + inps.date_list_file
for d in dateList:
if inps.grib_source == 'ECMWF':
grib_file = grib_dir + '/ERA-Int_' + d + '_' + inps.hour + '.grb'
elif inps.grib_source == 'ERA':
grib_file = grib_dir + '/ERA_' + d + '_' + inps.hour + '.grb'
elif inps.grib_source == 'MERRA':
grib_file = grib_dir + '/merra-' + d + '-' + inps.hour + '.hdf'
elif inps.grib_source == 'NARR':
grib_file = grib_dir + '/narr-a_221_' + d + '_' + inps.hour + '00_000.grb'
inps.grib_file_list.append(grib_file)
## Get date list to download
grib_file_existed = ut.get_file_list(inps.grib_file_list)
if grib_file_existed:
grib_filesize_mode = ut.mode(
[os.path.getsize(i) for i in grib_file_existed])
grib_file_corrupted = [
i for i in grib_file_existed
if os.path.getsize(i) != grib_filesize_mode
]
print 'number of grib files existed : %d' % len(grib_file_existed)
print 'file size mode: %d' % grib_filesize_mode
if grib_file_corrupted:
print '------------------------------------------------------------------------------'
print 'corrupted grib files detected! Delete them and re-download...'
print 'number of grib files corrupted : %d' % len(
grib_file_corrupted)
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print rmCmd
os.system(rmCmd)
grib_file_existed.remove(i)
print '------------------------------------------------------------------------------'
grib_file2download = sorted(
list(set(inps.grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print 'number of grib files to download: %d' % len(date_list2download)
print '------------------------------------------------------------------------------\n'
## Download grib file using PyAPS
if inps.grib_source == 'ECMWF':
pa.ECMWFdload(date_list2download, inps.hour, grib_dir)
elif inps.grib_source == 'ERA':
pa.ERAdload(date_list2download, inps.hour, grib_dir)
elif inps.grib_source == 'MERRA':
pa.MERRAdload(date_list2download, inps.hour, grib_dir)
elif inps.grib_source == 'NARR':
pa.NARRdload(date_list2download, inps.hour, grib_dir)
if inps.download:
print 'Download completed, exit as planned.'
return
print '*******************************************************************************'
print 'Calcualting delay for each epoch.'
## Get Incidence angle: to map the zenith delay to the slant delay
if inps.incidence_angle:
if os.path.isfile(inps.incidence_angle):
inps.incidence_angle = readfile.read(inps.incidence_angle)[0]
else:
inps.incidence_angle = float(inps.incidence_angle)
print 'incidence angle: ' + str(inps.incidence_angle)
else:
print 'calculating incidence angle ...'
inps.incidence_angle = ut.incidence_angle(atr)
inps.incidence_angle = inps.incidence_angle * np.pi / 180.0
## Create delay hdf5 file
tropFile = inps.grib_source + '.h5'
print 'writing >>> ' + tropFile
h5trop = h5py.File(tropFile, 'w')
group_trop = h5trop.create_group('timeseries')
## Create tropospheric corrected timeseries hdf5 file
if not inps.out_file:
ext = os.path.splitext(inps.timeseries_file)[1]
inps.out_file = os.path.splitext(
inps.timeseries_file)[0] + '_' + inps.grib_source + '.h5'
print 'writing >>> ' + inps.out_file
h5timeseries_tropCor = h5py.File(inps.out_file, 'w')
group_tropCor = h5timeseries_tropCor.create_group('timeseries')
## Calculate phase delay on reference date
if 'ref_date' in atr.keys():
ref_idx = dateList.index(atr['ref_date'])
else:
ref_idx = 0
print 'calculating phase delay on reference date: ' + dateList[ref_idx]
phs_ref = get_delay(inps.grib_file_list[ref_idx], atr, vars(inps))
## Loop to calculate phase delay on the other dates
h5timeseries = h5py.File(inps.timeseries_file, 'r')
for i in range(len(inps.grib_file_list)):
# Get phase delay
grib_file = inps.grib_file_list[i]
if not i == ref_idx:
print dateList[i]
phs = get_delay(grib_file, atr, vars(inps))
else:
phs = np.copy(phs_ref)
# Get relative phase delay in time
phs -= phs_ref
# Write dataset
print 'writing hdf5 file ...'
data = h5timeseries['timeseries'].get(dateList[i])[:]
dset = group_tropCor.create_dataset(dateList[i],
data=data - phs,
compression='gzip')
dset = group_trop.create_dataset(dateList[i],
data=phs,
compression='gzip')
## Write Attributes
for key, value in atr.iteritems():
group_tropCor.attrs[key] = value
group_trop.attrs[key] = value
h5timeseries.close()
h5timeseries_tropCor.close()
h5trop.close()
# Delete temporary DEM file in ROI_PAC format
if '4pyaps' in inps.dem_file:
rmCmd = 'rm ' + inps.dem_file + ' ' + inps.dem_file + '.rsc '
print rmCmd
os.system(rmCmd)
print 'Done.'
return
def dload_grib(date_list, hour, grib_source='ECMWF', weather_dir='./'):
'''Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
grib_source : string,
weather_dir : string,
Output:
grib_file_list : list of string
'''
## Grib data directory
weather_dir = os.path.abspath(weather_dir)
grib_dir = weather_dir + '/' + grib_source
if not os.path.isdir(grib_dir):
print 'making directory: ' + grib_dir
os.makedirs(grib_dir)
## Date list to grib file list
grib_file_list = date_list2grib_file(date_list, hour, grib_source,
grib_dir)
## Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_digit = ut.mode(
[len(str(os.path.getsize(i))) for i in grib_file_existed])
grib_filesize_max2 = ut.mode(
[str(os.path.getsize(i))[0:2] for i in grib_file_existed])
grib_file_corrupted = [i for i in grib_file_existed if (len(str(os.path.getsize(i))) != grib_filesize_digit or\
str(os.path.getsize(i))[0:2] != grib_filesize_max2)]
print 'file size mode: %se%d bytes' % (grib_filesize_max2,
grib_filesize_digit - 2)
print 'number of grib files existed : %d' % len(grib_file_existed)
if grib_file_corrupted:
print '------------------------------------------------------------------------------'
print 'corrupted grib files detected! Delete them and re-download...'
print 'number of grib files corrupted : %d' % len(
grib_file_corrupted)
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print rmCmd
os.system(rmCmd)
grib_file_existed.remove(i)
print '------------------------------------------------------------------------------'
grib_file2download = sorted(
list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print 'number of grib files to download: %d' % len(date_list2download)
print '------------------------------------------------------------------------------\n'
## Download grib file using PyAPS
if grib_source == 'ECMWF':
pa.ECMWFdload(date_list2download, hour, grib_dir)
elif grib_source == 'ERA':
pa.ERAdload(date_list2download, hour, grib_dir)
elif grib_source == 'NARR':
pa.NARRdload(date_list2download, hour, grib_dir)
elif grib_source == 'MERRA':
pa.MERRAdload(date_list2download, hour, grib_dir)
elif grib_source == 'MERRA1':
pa.MERRA1dload(date_list2download, hour, grib_dir)
return grib_file_existed
