def dload_grib_pyaps(grib_file_list):
"""Download weather re-analysis grib files using PyAPS
Parameters: grib_file_list : list of string of grib files
Returns: grib_file_list : list of string
"""
print(
'\n------------------------------------------------------------------------------'
)
print('downloading weather model data using PyAPS ...')
# Get date list to download (skip already downloaded files)
grib_file_exist = check_exist_grib_file(grib_file_list, print_msg=True)
grib_file2dload = sorted(list(set(grib_file_list) - set(grib_file_exist)))
date_list2dload = [str(re.findall('\d{8}', i)[0]) for i in grib_file2dload]
print('number of grib files to download: %d' % len(date_list2dload))
print(
'------------------------------------------------------------------------------\n'
)
# Download grib file using PyAPS
if len(date_list2dload) > 0:
hour = re.findall('\d{8}[-_]\d{2}',
grib_file2dload[0])[0].replace('-',
'_').split('_')[1]
grib_dir = os.path.dirname(grib_file2dload[0])
# try 3 times to download, then use whatever downloaded to calculate delay
trop_model = grib_file_name2trop_model_name(grib_file2dload[0])
i = 0
while i < 3:
i += 1
try:
if trop_model == 'ECMWF':
pa.ECMWFdload(date_list2dload, hour, grib_dir)
elif trop_model == 'MERRA':
pa.MERRAdload(date_list2dload, hour, grib_dir)
elif trop_model == 'NARR':
pa.NARRdload(date_list2dload, hour, grib_dir)
elif trop_model == 'ERA':
pa.ERAdload(date_list2dload, hour, grib_dir)
elif trop_model == 'MERRA1':
pa.MERRA1dload(date_list2dload, hour, grib_dir)
except:
pass
grib_file_list = check_exist_grib_file(grib_file_list, print_msg=False)
return grib_file_list
def dload_grib_pyaps(date_list, hour, trop_model='ECMWF', weather_dir='./'):
"""Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
trop_model : string,
weather_dir : string,
Output:
grib_file_list : list of string
"""
print(
'*' * 50 +
'\nDownloading weather model data using PyAPS (Jolivet et al., 2011, GRL) ...'
)
# Grib data directory
grib_dir = weather_dir + '/' + trop_model
if not os.path.isdir(grib_dir):
os.makedirs(grib_dir)
print('making directory: ' + grib_dir)
# Date list to grib file list
grib_file_list = date_list2grib_file(date_list, hour, trop_model, grib_dir)
# Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_digit = ut.most_common(
[len(str(os.path.getsize(i))) for i in grib_file_existed])
grib_filesize_max2 = ut.most_common(
[str(os.path.getsize(i))[0:2] for i in grib_file_existed])
grib_file_corrupted = [
i for i in grib_file_existed
if (len(str(os.path.getsize(i))) != grib_filesize_digit
or str(os.path.getsize(i))[0:2] != grib_filesize_max2)
]
print('file size mode: %se%d bytes' %
(grib_filesize_max2, grib_filesize_digit - 2))
print('number of grib files existed : %d' % len(grib_file_existed))
if grib_file_corrupted:
print(
'------------------------------------------------------------------------------'
)
print(
'corrupted grib files detected! Delete them and re-download...'
)
print('number of grib files corrupted : %d' %
len(grib_file_corrupted))
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print(rmCmd)
os.system(rmCmd)
grib_file_existed.remove(i)
print(
'------------------------------------------------------------------------------'
)
grib_file2download = sorted(
list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print('number of grib files to download: %d' % len(date_list2download))
print(
'------------------------------------------------------------------------------\n'
)
# Download grib file using PyAPS
if trop_model == 'ECMWF': pa.ECMWFdload(date_list2download, hour, grib_dir)
elif trop_model == 'MERRA':
pa.MERRAdload(date_list2download, hour, grib_dir)
elif trop_model == 'NARR':
pa.NARRdload(date_list2download, hour, grib_dir)
elif trop_model == 'ERA':
pa.ERAdload(date_list2download, hour, grib_dir)
elif trop_model == 'MERRA1':
pa.MERRA1dload(date_list2download, hour, grib_dir)
return grib_file_list
def dload_grib(date_list, hour, grib_source='ECMWF', weather_dir='./'):
'''Download weather re-analysis grib files using PyAPS
Inputs:
date_list : list of string in YYYYMMDD format
hour : string in HH:MM or HH format
grib_source : string,
weather_dir : string,
Output:
grib_file_list : list of string
'''
## Grib data directory
weather_dir = os.path.abspath(weather_dir)
grib_dir = weather_dir + '/' + grib_source
if not os.path.isdir(grib_dir):
print 'making directory: ' + grib_dir
os.makedirs(grib_dir)
## Date list to grib file list
grib_file_list = date_list2grib_file(date_list, hour, grib_source,
grib_dir)
## Get date list to download (skip already downloaded files)
grib_file_existed = ut.get_file_list(grib_file_list)
if grib_file_existed:
grib_filesize_digit = ut.mode(
[len(str(os.path.getsize(i))) for i in grib_file_existed])
grib_filesize_max2 = ut.mode(
[str(os.path.getsize(i))[0:2] for i in grib_file_existed])
grib_file_corrupted = [i for i in grib_file_existed if (len(str(os.path.getsize(i))) != grib_filesize_digit or\
str(os.path.getsize(i))[0:2] != grib_filesize_max2)]
print 'file size mode: %se%d bytes' % (grib_filesize_max2,
grib_filesize_digit - 2)
print 'number of grib files existed : %d' % len(grib_file_existed)
if grib_file_corrupted:
print '------------------------------------------------------------------------------'
print 'corrupted grib files detected! Delete them and re-download...'
print 'number of grib files corrupted : %d' % len(
grib_file_corrupted)
for i in grib_file_corrupted:
rmCmd = 'rm ' + i
print rmCmd
os.system(rmCmd)
grib_file_existed.remove(i)
print '------------------------------------------------------------------------------'
grib_file2download = sorted(
list(set(grib_file_list) - set(grib_file_existed)))
date_list2download = [
str(re.findall('\d{8}', i)[0]) for i in grib_file2download
]
print 'number of grib files to download: %d' % len(date_list2download)
print '------------------------------------------------------------------------------\n'
## Download grib file using PyAPS
if grib_source == 'ECMWF':
pa.ECMWFdload(date_list2download, hour, grib_dir)
elif grib_source == 'ERA':
pa.ERAdload(date_list2download, hour, grib_dir)
elif grib_source == 'NARR':
pa.NARRdload(date_list2download, hour, grib_dir)
elif grib_source == 'MERRA':
pa.MERRAdload(date_list2download, hour, grib_dir)
elif grib_source == 'MERRA1':
pa.MERRA1dload(date_list2download, hour, grib_dir)
return grib_file_existed
