def dload_grib(date_list, hour, grib_source='ECMWF', weather_dir='./'):
'''Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
grib_source : string,
weather_dir : string,
Output:
grib_file_list : list of string
'''
## Grib data directory
weather_dir = os.path.abspath(weather_dir)
grib_dir = weather_dir+'/'+grib_source
if not os.path.isdir(grib_dir):
print('making directory: '+grib_dir)
os.makedirs(grib_dir)
## Date list to grib file list
grib_file_list = []
for d in date_list:
if grib_source == 'ECMWF': grib_file = grib_dir+'/ERA-Int_'+d+'_'+hour+'.grb'
elif grib_source == 'ERA' : grib_file = grib_dir+'/ERA_'+d+'_'+hour+'.grb'
elif grib_source == 'MERRA': grib_file = grib_dir+'/merra-'+d+'-'+hour+'.hdf'
elif grib_source == 'NARR' : grib_file = grib_dir+'/narr-a_221_'+d+'_'+hour+'00_000.grb'
grib_file_list.append(grib_file)
## Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_mode = ut.mode([os.path.getsize(i) for i in grib_file_existed])
grib_file_corrupted = [i for i in grib_file_existed if os.path.getsize(i) != grib_filesize_mode]
print('number of grib files existed : %d' % len(grib_file_existed))
print('file size mode: %d' % grib_filesize_mode)
if grib_file_corrupted:
print('------------------------------------------------------------------------------')
print('corrupted grib files detected! Delete them and re-download...')
print('number of grib files corrupted : %d' % len(grib_file_corrupted))
for i in grib_file_corrupted:
rmCmd = 'rm '+i
print(rmCmd)
os.system(rmCmd)
grib_file_existed.remove(i)
print('------------------------------------------------------------------------------')
grib_file2download = sorted(list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [str(re.findall('\d{8}', i)[0]) for i in grib_file2download]
print('number of grib files to download: %d' % len(date_list2download))
print('------------------------------------------------------------------------------\n')
## Download grib file using PyAPS
if grib_source == 'ECMWF': pa.ECMWFdload(date_list2download, hour, grib_dir)
elif grib_source == 'ERA' : pa.ERAdload( date_list2download, hour, grib_dir)
elif grib_source == 'MERRA': pa.MERRAdload(date_list2download, hour, grib_dir)
elif grib_source == 'NARR' : pa.NARRdload( date_list2download, hour, grib_dir)
return grib_file_existed
def dload_grib_files(grib_file_list, trop_model='ERA5', snwe=None):
"""Download weather re-analysis grib files using PyAPS
Parameters: grib_file_list : list of string of grib files
Returns: grib_file_list : list of string
"""
print(
'\n------------------------------------------------------------------------------'
)
print('downloading weather model data using PyAPS ...')
# Get date list to download (skip already downloaded files)
grib_file_exist = check_exist_grib_file(grib_file_list, print_msg=True)
grib_file2dload = sorted(list(set(grib_file_list) - set(grib_file_exist)))
date_list2dload = [str(re.findall('\d{8}', i)[0]) for i in grib_file2dload]
print('number of grib files to download: %d' % len(date_list2dload))
print(
'------------------------------------------------------------------------------\n'
)
# Download grib file using PyAPS
if len(date_list2dload) > 0:
hour = re.findall('\d{8}[-_]\d{2}',
grib_file2dload[0])[0].replace('-',
'_').split('_')[1]
grib_dir = os.path.dirname(grib_file2dload[0])
# try 3 times to download, then use whatever downloaded to calculate delay
i = 0
while i < 3:
i += 1
try:
if trop_model in ['ERA5', 'ERAINT']:
pa.ECMWFdload(date_list2dload,
hour,
grib_dir,
model=trop_model,
snwe=snwe,
flist=grib_file2dload)
elif trop_model == 'MERRA':
pa.MERRAdload(date_list2dload, hour, grib_dir)
elif trop_model == 'NARR':
pa.NARRdload(date_list2dload, hour, grib_dir)
except:
pass
grib_file_list = check_exist_grib_file(grib_file_list, print_msg=False)
return grib_file_list
def main(argv):
DelayType = 'comb'
try:
opts, args = getopt.getopt(argv, "f:d:s:h:D:i:")
except getopt.GetoptError:
Usage()
sys.exit(1)
for opt, arg in opts:
if opt == '-f': timeSeriesFile = arg
elif opt == '-d': demFile = arg
elif opt == '-s': atmSource = arg
elif opt == '-h': hr = arg
elif opt == '-D': DelayType = arg
elif opt == '-i': inc_angle = arg
try:
timeSeriesFile
demFile
except:
Usage()
sys.exit(1)
demFile = ut.check_variable_name(demFile)
demCoord = ut.radar_or_geo(demFile)
h5timeseries = h5py.File(timeSeriesFile)
yref = h5timeseries['timeseries'].attrs['ref_y']
xref = h5timeseries['timeseries'].attrs['ref_x']
###############################################################
#incidence angle to map the zenith delay to the slant delay
try:
inc_angle
except:
print '+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++'
print 'WARNING:'
print 'incedence angle is not specified >>>> Average look angle is used ... '
print 'For more precise results use input option -i to introduce the incidence angle file or average incodence angle'
print '++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++'
input_inc_angle = 'None'
wavelength = float(h5timeseries['timeseries'].attrs['WAVELENGTH'])
inc_angle1 = float(h5timeseries['timeseries'].attrs['LOOK_REF1'])
inc_angle2 = float(h5timeseries['timeseries'].attrs['LOOK_REF2'])
inc_angle = (inc_angle1 + inc_angle2) / 2.0
print '*******************************************************************************'
print 'Near Look Angle: ' + str(inc_angle1)
print 'Far Look Angle:' + str(inc_angle2)
print 'Average Look Angle (used in pyaps to calculate delay): ' + str(
inc_angle)
print 'Acquisition time is : ' + hr
print '*******************************************************************************'
inc_angle = str(inc_angle)
if os.path.isfile(inc_angle):
incidenceFile = inc_angle
h5incidence = h5py.File(incidenceFile, 'r')
iset = h5incidence['mask'].get('mask')
inc_angle = iset[0:iset.shape[0], 0:iset.shape[1]]
h5incidence.close()
else:
inc_angle = float(inc_angle)
print 'incidence angle = ' + str(inc_angle)
inc_angle = inc_angle * pi / 180.0
################################################################
dateList = h5timeseries['timeseries'].keys()
if atmSource in ['ecmwf', 'ECMWF']:
gribSource = 'ECMWF'
if not os.path.isdir('ECMWF'):
print 'making directory: ECMWF'
os.mkdir('ECMWF')
ecmwf_file = []
for d in dateList:
ecm = './ECMWF/ERA-Int_' + d + '_' + hr + '.grb'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.ECMWFdload([d], hr, './ECMWF/')
else:
print ecm + ' already exists.'
elif atmSource in ['narr', 'NARR']:
gribSource = 'NARR'
if not os.path.isdir('NARR'):
print 'making directory: NARR'
os.mkdir('NARR')
ecmwf_file = []
for d in dateList:
ecm = './NARR/narr-a_221_' + d + '_' + hr + '00_000.grb'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.NARRdload([d], hr, './NARR/')
else:
print ecm + ' already exists.'
elif atmSource in ['era', 'ERA']:
gribSource = 'ERA'
if not os.path.isdir('ERA'):
print 'making directory: ERA'
os.mkdir('ERA')
ecmwf_file = []
for d in dateList:
ecm = './ERA/ERA_' + d + '_' + hr + '.grb'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.ERAdload([d], hr, './ERA/')
else:
print ecm + ' already exists.'
elif atmSource in ['merra', 'MERRA']:
gribSource = 'MERRA'
if not os.path.isdir('MERRA'):
print 'making directory: MERRA'
os.mkdir('MERRA')
ecmwf_file = []
for d in dateList:
ecm = './MERRA/merra-' + d + '-' + hr + '.hdf'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.MERRAdload([d], hr, './MERRA/')
else:
print ecm + ' already exists.'
else:
Usage()
sys.exit(1)
print '*******************************************************************************'
print 'Calcualting delay for each epoch.'
h5phsName = atmSource + '.h5'
h5phs = h5py.File(h5phsName, 'w')
outName = timeSeriesFile.replace(
'.h5', '_') + atmSource + '.h5' #Yunjun, Feb 15, 2015
#outName=timeSeriesFile.replace('.h5','')+'_tropCorPyAPS.h5'
h5apsCor = h5py.File(outName, 'w')
group = h5apsCor.create_group('timeseries')
group_phs = h5phs.create_group('timeseries')
#if 'X_FIRST' in h5timeseries['timeseries'].attrs.keys():
# demCoord='geo'
# print 'The coordinate system is : Geo'
#else:
# demCoord='radar'
# print 'The coordinate system is : radar'
print ecmwf_file[0]
if demCoord == 'radar':
aps1 = pa.PyAPS_rdr(str(ecmwf_file[0]),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
else:
aps1 = pa.PyAPS_geo(str(ecmwf_file[0]),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
phs1 = zeros((aps1.ny, aps1.nx))
aps1.getdelay(phs1)
phs1 = (phs1 - phs1[yref, xref]) / cos(inc_angle)
dset = group.create_dataset(dateList[0],
data=phs1 - phs1,
compression='gzip')
for i in range(1, len(ecmwf_file)):
ecm = ecmwf_file[i]
print ecm
if demCoord == 'radar':
aps = pa.PyAPS_rdr(str(ecm),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
else:
aps = pa.PyAPS_geo(str(ecm),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
phs = zeros((aps.ny, aps.nx))
aps.getdelay(phs)
phs = (phs - phs[yref, xref]) / cos(inc_angle)
phs = phs - phs1
dset = group_phs.create_dataset(dateList[i],
data=phs,
compression='gzip')
dset1 = h5timeseries['timeseries'].get(dateList[i])
data1 = dset1[0:dset1.shape[0], 0:dset1.shape[1]]
dset = group.create_dataset(dateList[i],
data=data1 + phs,
compression='gzip')
for key, value in h5timeseries['timeseries'].attrs.iteritems():
group.attrs[key] = value
group_phs.attrs[key] = value
dset1 = h5timeseries['mask'].get('mask')
Mask = dset1[0:dset1.shape[0], 0:dset1.shape[1]]
group = h5apsCor.create_group('mask')
dset = group.create_dataset('mask', data=Mask, compression='gzip')
def download_atmosphereModel(inps): #EMRE
S, N, W, E = [val for val in inps.bbox.split()] #EMRE
start_year, start_month, start_day = [
int(val) for val in inps.start_date.split('-')
] #EMRE
end_year, end_month, end_day = [
int(val) for val in inps.end_date.split('-')
] #EMRE
start_date = date(start_year, start_month, start_day) #EMRE
end_date = date(end_year, end_month, end_day) #EMRE
d1 = start_date
d2 = end_date
dateList = []
# d1 = datetime.date(2002,1,1)
# d2 = datetime.date(2002,2,1)
diff = d2 - d1
day_step = 1 #EMRE
hr = '18'
for i in range(0, diff.days + 1, day_step):
dd = (d1 + datetime.timedelta(i)).isoformat()
dateList.append(dd.replace('-', ''))
# if atmSource in ['ecmwf','ECMWF']:
if inps.data_source in ['ecmwf', 'ECMWF']: #EMRE
gribSource = 'ECMWF'
if not os.path.isdir('ECMWF'):
print 'making directory: ECMWF'
os.mkdir('ECMWF')
ecmwf_file = []
for d in dateList:
ecm = './ECMWF/ERA-Int_' + d + '_' + hr + '.grb'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.ECMWFdload([d], hr, './ECMWF/')
else:
print ecm + ' already exists.'
# elif atmSource in ['narr','NARR']:
elif inps in ['narr', 'NARR']: #EMRE
gribSource = 'NARR'
if not os.path.isdir('NARR'):
print 'making directory: NARR'
os.mkdir('NARR')
ecmwf_file = []
for d in dateList:
ecm = './NARR/narr-a_221_' + d + '_' + hr + '00_000.grb'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.NARRdload([d], hr, './NARR/')
else:
print ecm + ' already exists.'
# elif atmSource in ['era','ERA']:
elif inps in ['era', 'ERA']:
gribSource = 'ERA'
if not os.path.isdir('ERA'):
print 'making directory: ERA'
os.mkdir('ERA')
ecmwf_file = []
for d in dateList:
ecm = './ERA/ERA_' + d + '_' + hr + '.grb'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.ERAdload([d], hr, './ERA/')
else:
print ecm + ' already exists.'
# elif atmSource in ['merra','MERRA']:
elif inps in ['merra', 'MERRA']:
gribSource = 'MERRA'
if not os.path.isdir('MERRA'):
print 'making directory: MERRA'
os.mkdir('MERRA')
ecmwf_file = []
for d in dateList:
ecm = './MERRA/merra-' + d + '-' + hr + '.hdf'
ecmwf_file.append(ecm)
print[d]
if not os.path.isfile(ecm):
pa.MERRAdload([d], hr, './MERRA/')
else:
print ecm + ' already exists.'
else:
# Usage();
print 'FAILED' #EMRE
sys.exit(1)
print '*******************************************************************************'
print 'Calculating delay for each epoch.'
h5phs = h5py.File('aps.h5', 'w')
outName = 'ECMWF.h5'
#h5apsCor=h5py.File(outName,'w')
# group=h5apsCor.create_group('timeseries')
group_phs = h5phs.create_group('timeseries')
demCoord = 'geo'
print ecmwf_file[0]
#demCoord='radar'
if demCoord == 'radar':
aps1 = pa.PyAPS_rdr(str(ecmwf_file[0]),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
else:
aps1 = pa.PyAPS_geo(str(ecmwf_file[0]),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
phs1 = np.zeros((aps1.ny, aps1.nx))
print aps1.ny
print aps1.nx
# aps1.getdelay(phs1,inc=inc_angle,wvl=wavelength)
aps1.getdelay(phs1)
dset = group_phs.create_dataset(dateList[0], data=phs1, compression='gzip')
# phs1=(phs1 - phs1[yref,xref])*wavelength/(4*np.pi)
# dset = group.create_dataset(dateList[0], data= phs1- phs1, compression='gzip')
for i in range(1, len(ecmwf_file)):
ecm = ecmwf_file[i]
print ecm
if demCoord == 'radar':
aps = pa.PyAPS_rdr(str(ecm),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
else:
aps = pa.PyAPS_geo(str(ecm),
demFile,
grib=gribSource,
verb=True,
Del=DelayType)
phs = np.zeros((aps.ny, aps.nx))
# aps.getdelay(phs,inc=inc_angle,wvl=wavelength)
aps.getdelay(phs)
# phs=(phs - phs[yref,xref])*wavelength/(4*np.pi)
# phs=phs-phs1
dset = group_phs.create_dataset(dateList[i],
data=phs,
compression='gzip')
# dset1 = h5timeseries['timeseries'].get(dateList[i])
# data1 = dset1[0:dset1.shape[0],0:dset1.shape[1]]
# dset = group.create_dataset(dateList[i], data= data1+phs, compression='gzip')
for key, value in h5timeseries['timeseries'].attrs.iteritems():
# group.attrs[key] = value
group_phs.attrs[key] = value
import pyaps as pa
import numpy as np
import matplotlib.pyplot as plt
import sys
print '------------------------------------------------'
print 'You are using PyAPS from %s' % pa.__file__
print '------------------------------------------------'
print 'Testing Download Methods'
pa.ECMWFdload(['20040526', '20030426'], '12', './ECMWF/')
print 'Testing MERRA Downloads'
pa.MERRAdload(['20040526', '20030426'], '12', './MERRA/')
print 'Downloads OK'
print '------------------------------------------------'
print '------------------------------------------------'
print 'Testing ECMWF in Radar geometry, with a RMG dem'
aps1 = pa.PyAPS_rdr('ECMWF/ERA-Int_20030426_12.grb',
'dem_16rlks.hgt',
grib='ECMWF',
verb=True)
aps2 = pa.PyAPS_rdr('ECMWF/ERA-Int_20040526_12.grb',
'dem_16rlks.hgt',
grib='ECMWF',
verb=True)
print 'With Lat Lon files'
phs1 = np.zeros((aps1.ny, aps1.nx))
def main(argv):
inps = cmdLineParse()
inps.dem_file = ut.get_file_list([inps.dem_file])[0]
inps.timeseries_file = ut.get_file_list([inps.timeseries_file])[0]
atr = readfile.read_attribute(inps.timeseries_file)
print '*******************************************************************************'
print 'Downloading weather model data ...'
## Get Grib Source
if inps.weather_model in ['ECMWF', 'ERA-Interim']:
inps.grib_source = 'ECMWF'
elif inps.weather_model == 'ERA':
inps.grib_source = 'ERA'
elif inps.weather_model == 'MERRA':
inps.grib_source = 'MERRA'
elif inps.weather_model == 'NARR':
inps.grib_source = 'NARR'
else:
raise Reception('Unrecognized weather model: ' + inps.weather_model)
print 'grib source: ' + inps.grib_source
## Get Acquisition time
inps.hour = closest_weather_product_time(atr['CENTER_LINE_UTC'],
inps.grib_source)
print 'Time of cloest vailable product: ' + inps.hour
if not os.path.isdir(inps.grib_source):
print 'making directory: ' + inps.grib_source
os.mkdir(inps.grib_source)
## Loop to download
inps.grib_file_list = []
h5timeseries = h5py.File(inps.timeseries_file, 'r')
dateList = sorted(h5timeseries['timeseries'].keys())
for d in dateList:
print[d]
if inps.grib_source == 'ECMWF':
grib_file = './ECMWF/ERA-Int_' + d + '_' + inps.hour + '.grb'
elif inps.grib_source == 'ERA':
grib_file = './ERA/ERA_' + d + '_' + inps.hour + '.grb'
elif inps.grib_source == 'MERRA':
grib_file = './MERRA/merra-' + d + '-' + inps.hour + '.hdf'
elif inps.grib_source == 'NARR':
grib_file = './NARR/narr-a_221_' + d + '_' + inps.hour + '00_000.grb'
inps.grib_file_list.append(grib_file)
if os.path.isfile(grib_file):
print grib_file + ' already exists.'
else:
if inps.grib_source == 'ECMWF':
pa.ECMWFdload([d], inps.hour, './' + inps.grib_source + '/')
elif inps.grib_source == 'ERA':
pa.ERAdload([d], inps.hour, './' + inps.grib_source + '/')
elif inps.grib_source == 'MERRA':
pa.MERRAdload([d], inps.hour, './' + inps.grib_source + '/')
elif inps.grib_source == 'NARR':
pa.NARRdload([d], inps.hour, './' + inps.grib_source + '/')
print '*******************************************************************************'
print 'Calcualting delay for each epoch.'
## Get Incidence angle: to map the zenith delay to the slant delay
if inps.incidence_angle:
if os.path.isfile(inps.incidence_angle):
inps.incidence_angle = readfile.read(inps.incidence_angle)[0]
else:
inps.incidence_angle = float(inps.incidence_angle)
print 'incidence angle: ' + str(inps.incidence_angle)
else:
print 'calculating incidence angle ...'
inps.incidence_angle = ut.incidence_angle(atr)
inps.incidence_angle = inps.incidence_angle * np.pi / 180.0
## Create delay hdf5 file
tropFile = inps.grib_source + '.h5'
print 'writing >>> ' + tropFile
h5trop = h5py.File(tropFile, 'w')
group_trop = h5trop.create_group('timeseries')
## Create tropospheric corrected timeseries hdf5 file
if not inps.out_file:
ext = os.path.splitext(inps.timeseries_file)[1]
inps.out_file = os.path.splitext(
inps.timeseries_file)[0] + '_' + inps.grib_source + '.h5'
print 'writing >>> ' + inps.out_file
h5timeseries_tropCor = h5py.File(inps.out_file, 'w')
group_tropCor = h5timeseries_tropCor.create_group('timeseries')
## Calculate phase delay on reference date
if 'ref_date' in atr.keys():
ref_idx = dateList.index(atr['ref_date'])
else:
ref_idx = 0
print 'calculating phase delay on reference date: ' + dateList[ref_idx]
phs_ref = get_delay(inps.grib_file_list[ref_idx], atr, vars(inps))
## Loop to calculate phase delay on the other dates
for i in range(len(inps.grib_file_list)):
# Get phase delay
grib_file = inps.grib_file_list[i]
if not i == ref_idx:
print dateList[i]
phs = get_delay(grib_file, atr, vars(inps))
else:
phs = np.copy(phs_ref)
# Get relative phase delay in time
phs -= phs_ref
# Write dataset
print 'writing hdf5 file ...'
data = h5timeseries['timeseries'].get(dateList[i])[:]
dset = group_tropCor.create_dataset(dateList[i],
data=data + phs,
compression='gzip')
dset = group_trop.create_dataset(dateList[i],
data=phs,
compression='gzip')
## Write Attributes
for key, value in atr.iteritems():
group_tropCor.attrs[key] = value
group_trop.attrs[key] = value
h5timeseries.close()
h5timeseries_tropCor.close()
h5trop.close()
print 'Done.'
return
def main(argv):
inps = cmdLineParse()
if inps.timeseries_file:
inps.timeseries_file = ut.get_file_list([inps.timeseries_file])[0]
atr = readfile.read_attribute(inps.timeseries_file)
if inps.dem_file:
inps.dem_file = ut.get_file_list([inps.dem_file])[0]
# Convert DEM to ROIPAC format
if os.path.splitext(inps.dem_file)[1] in ['.h5']:
print 'convert DEM file to ROIPAC format'
dem, atr_dem = readfile.read(inps.dem_file)
if 'Y_FIRST' in atr_dem.keys():
atr_dem['FILE_TYPE'] = '.dem'
else:
atr_dem['FILE_TYPE'] = '.hgt'
outname = os.path.splitext(
inps.dem_file)[0] + '4pyaps' + atr_dem['FILE_TYPE']
inps.dem_file = writefile.write(dem, atr_dem, outname)
print '*******************************************************************************'
print 'Downloading weather model data ...'
## Get Grib Source
if inps.weather_model in ['ECMWF', 'ERA-Interim']:
inps.grib_source = 'ECMWF'
elif inps.weather_model == 'ERA':
inps.grib_source = 'ERA'
elif inps.weather_model == 'MERRA':
inps.grib_source = 'MERRA'
elif inps.weather_model == 'NARR':
inps.grib_source = 'NARR'
else:
raise Reception('Unrecognized weather model: ' + inps.weather_model)
print 'grib source: ' + inps.grib_source
## Grib data directory
if not inps.weather_dir:
if inps.timeseries_file:
inps.weather_dir = os.path.dirname(
os.path.abspath(inps.timeseries_file)) + '/../WEATHER'
elif inps.dem_file:
inps.weather_dir = os.path.dirname(os.path.abspath(
inps.dem_file)) + '/../WEATHER'
else:
inps.weather_dir = os.path.abspath(os.getcwd())
print 'Store weather data into directory: ' + inps.weather_dir
grib_dir = inps.weather_dir + '/' + inps.grib_source
if not os.path.isdir(grib_dir):
print 'making directory: ' + grib_dir
os.makedirs(grib_dir)
## Get Acquisition time
if not inps.hour:
inps.hour = closest_weather_product_time(atr['CENTER_LINE_UTC'],
inps.grib_source)
print 'Time of cloest available product: ' + inps.hour
## Get grib file list and date list
inps.grib_file_list = []
if not inps.date_list_file:
h5timeseries = h5py.File(inps.timeseries_file, 'r')
dateList = sorted(h5timeseries['timeseries'].keys())
h5timeseries.close()
print 'read date list info from: ' + inps.timeseries_file
else:
dateList = ptime.yyyymmdd(
np.loadtxt(inps.date_list_file, dtype=str, usecols=(0, )).tolist())
print 'read date list info from: ' + inps.date_list_file
for d in dateList:
if inps.grib_source == 'ECMWF':
grib_file = grib_dir + '/ERA-Int_' + d + '_' + inps.hour + '.grb'
elif inps.grib_source == 'ERA':
grib_file = grib_dir + '/ERA_' + d + '_' + inps.hour + '.grb'
elif inps.grib_source == 'MERRA':
grib_file = grib_dir + '/merra-' + d + '-' + inps.hour + '.hdf'
elif inps.grib_source == 'NARR':
grib_file = grib_dir + '/narr-a_221_' + d + '_' + inps.hour + '00_000.grb'
inps.grib_file_list.append(grib_file)
## Get date list to download
grib_file_existed = ut.get_file_list(inps.grib_file_list)
if grib_file_existed:
grib_filesize_mode = ut.mode(
[os.path.getsize(i) for i in grib_file_existed])
grib_file_corrupted = [
i for i in grib_file_existed
if os.path.getsize(i) != grib_filesize_mode
]
print 'number of grib files existed : %d' % len(grib_file_existed)
print 'file size mode: %d' % grib_filesize_mode
if grib_file_corrupted:
print '------------------------------------------------------------------------------'
print 'corrupted grib files detected! Delete them and re-download...'
print 'number of grib files corrupted : %d' % len(
grib_file_corrupted)
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print rmCmd
os.system(rmCmd)
grib_file_existed.remove(i)
print '------------------------------------------------------------------------------'
grib_file2download = sorted(
list(set(inps.grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print 'number of grib files to download: %d' % len(date_list2download)
print '------------------------------------------------------------------------------\n'
## Download grib file using PyAPS
if inps.grib_source == 'ECMWF':
pa.ECMWFdload(date_list2download, inps.hour, grib_dir)
elif inps.grib_source == 'ERA':
pa.ERAdload(date_list2download, inps.hour, grib_dir)
elif inps.grib_source == 'MERRA':
pa.MERRAdload(date_list2download, inps.hour, grib_dir)
elif inps.grib_source == 'NARR':
pa.NARRdload(date_list2download, inps.hour, grib_dir)
if inps.download:
print 'Download completed, exit as planned.'
return
print '*******************************************************************************'
print 'Calcualting delay for each epoch.'
## Get Incidence angle: to map the zenith delay to the slant delay
if inps.incidence_angle:
if os.path.isfile(inps.incidence_angle):
inps.incidence_angle = readfile.read(inps.incidence_angle)[0]
else:
inps.incidence_angle = float(inps.incidence_angle)
print 'incidence angle: ' + str(inps.incidence_angle)
else:
print 'calculating incidence angle ...'
inps.incidence_angle = ut.incidence_angle(atr)
inps.incidence_angle = inps.incidence_angle * np.pi / 180.0
## Create delay hdf5 file
tropFile = inps.grib_source + '.h5'
print 'writing >>> ' + tropFile
h5trop = h5py.File(tropFile, 'w')
group_trop = h5trop.create_group('timeseries')
## Create tropospheric corrected timeseries hdf5 file
if not inps.out_file:
ext = os.path.splitext(inps.timeseries_file)[1]
inps.out_file = os.path.splitext(
inps.timeseries_file)[0] + '_' + inps.grib_source + '.h5'
print 'writing >>> ' + inps.out_file
h5timeseries_tropCor = h5py.File(inps.out_file, 'w')
group_tropCor = h5timeseries_tropCor.create_group('timeseries')
## Calculate phase delay on reference date
if 'ref_date' in atr.keys():
ref_idx = dateList.index(atr['ref_date'])
else:
ref_idx = 0
print 'calculating phase delay on reference date: ' + dateList[ref_idx]
phs_ref = get_delay(inps.grib_file_list[ref_idx], atr, vars(inps))
## Loop to calculate phase delay on the other dates
h5timeseries = h5py.File(inps.timeseries_file, 'r')
for i in range(len(inps.grib_file_list)):
# Get phase delay
grib_file = inps.grib_file_list[i]
if not i == ref_idx:
print dateList[i]
phs = get_delay(grib_file, atr, vars(inps))
else:
phs = np.copy(phs_ref)
# Get relative phase delay in time
phs -= phs_ref
# Write dataset
print 'writing hdf5 file ...'
data = h5timeseries['timeseries'].get(dateList[i])[:]
dset = group_tropCor.create_dataset(dateList[i],
data=data - phs,
compression='gzip')
dset = group_trop.create_dataset(dateList[i],
data=phs,
compression='gzip')
## Write Attributes
for key, value in atr.iteritems():
group_tropCor.attrs[key] = value
group_trop.attrs[key] = value
h5timeseries.close()
h5timeseries_tropCor.close()
h5trop.close()
# Delete temporary DEM file in ROI_PAC format
if '4pyaps' in inps.dem_file:
rmCmd = 'rm ' + inps.dem_file + ' ' + inps.dem_file + '.rsc '
print rmCmd
os.system(rmCmd)
print 'Done.'
return
def dload_grib_pyaps(date_list, hour, trop_model='ECMWF', weather_dir='./'):
"""Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
trop_model : string,
weather_dir : string,
Output:
grib_file_list : list of string
"""
print(
'*' * 50 +
'\nDownloading weather model data using PyAPS (Jolivet et al., 2011, GRL) ...'
)
# Grib data directory
grib_dir = weather_dir + '/' + trop_model
if not os.path.isdir(grib_dir):
os.makedirs(grib_dir)
print('making directory: ' + grib_dir)
# Date list to grib file list
grib_file_list = date_list2grib_file(date_list, hour, trop_model, grib_dir)
# Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_digit = ut.most_common(
[len(str(os.path.getsize(i))) for i in grib_file_existed])
grib_filesize_max2 = ut.most_common(
[str(os.path.getsize(i))[0:2] for i in grib_file_existed])
grib_file_corrupted = [
i for i in grib_file_existed
if (len(str(os.path.getsize(i))) != grib_filesize_digit
or str(os.path.getsize(i))[0:2] != grib_filesize_max2)
]
print('file size mode: %se%d bytes' %
(grib_filesize_max2, grib_filesize_digit - 2))
print('number of grib files existed : %d' % len(grib_file_existed))
if grib_file_corrupted:
print(
'------------------------------------------------------------------------------'
)
print(
'corrupted grib files detected! Delete them and re-download...'
)
print('number of grib files corrupted : %d' %
len(grib_file_corrupted))
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print(rmCmd)
os.system(rmCmd)
grib_file_existed.remove(i)
print(
'------------------------------------------------------------------------------'
)
grib_file2download = sorted(
list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print('number of grib files to download: %d' % len(date_list2download))
print(
'------------------------------------------------------------------------------\n'
)
# Download grib file using PyAPS
if trop_model == 'ECMWF': pa.ECMWFdload(date_list2download, hour, grib_dir)
elif trop_model == 'MERRA':
pa.MERRAdload(date_list2download, hour, grib_dir)
elif trop_model == 'NARR':
pa.NARRdload(date_list2download, hour, grib_dir)
elif trop_model == 'ERA':
pa.ERAdload(date_list2download, hour, grib_dir)
elif trop_model == 'MERRA1':
pa.MERRA1dload(date_list2download, hour, grib_dir)
return grib_file_list
def dload_grib(date_list, hour, grib_source='ECMWF', weather_dir='./'):
'''Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
grib_source : string,
weather_dir : string,
Output:
grib_file_list : list of string
'''
## Grib data directory
weather_dir = os.path.abspath(weather_dir)
grib_dir = weather_dir + '/' + grib_source
if not os.path.isdir(grib_dir):
print 'making directory: ' + grib_dir
os.makedirs(grib_dir)
## Date list to grib file list
grib_file_list = date_list2grib_file(date_list, hour, grib_source,
grib_dir)
## Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_digit = ut.mode(
[len(str(os.path.getsize(i))) for i in grib_file_existed])
grib_filesize_max2 = ut.mode(
[str(os.path.getsize(i))[0:2] for i in grib_file_existed])
grib_file_corrupted = [i for i in grib_file_existed if (len(str(os.path.getsize(i))) != grib_filesize_digit or\
str(os.path.getsize(i))[0:2] != grib_filesize_max2)]
print 'file size mode: %se%d bytes' % (grib_filesize_max2,
grib_filesize_digit - 2)
print 'number of grib files existed : %d' % len(grib_file_existed)
if grib_file_corrupted:
print '------------------------------------------------------------------------------'
print 'corrupted grib files detected! Delete them and re-download...'
print 'number of grib files corrupted : %d' % len(
grib_file_corrupted)
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print rmCmd
os.system(rmCmd)
grib_file_existed.remove(i)
print '------------------------------------------------------------------------------'
grib_file2download = sorted(
list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print 'number of grib files to download: %d' % len(date_list2download)
print '------------------------------------------------------------------------------\n'
## Download grib file using PyAPS
if grib_source == 'ECMWF':
pa.ECMWFdload(date_list2download, hour, grib_dir)
elif grib_source == 'ERA':
pa.ERAdload(date_list2download, hour, grib_dir)
elif grib_source == 'NARR':
pa.NARRdload(date_list2download, hour, grib_dir)
elif grib_source == 'MERRA':
pa.MERRAdload(date_list2download, hour, grib_dir)
elif grib_source == 'MERRA1':
pa.MERRA1dload(date_list2download, hour, grib_dir)
return grib_file_existed
