def dload_grib_pyaps(grib_file_list): """Download weather re-analysis grib files using PyAPS Parameters: grib_file_list : list of string of grib files Returns: grib_file_list : list of string """ print( '\n------------------------------------------------------------------------------' ) print('downloading weather model data using PyAPS ...') # Get date list to download (skip already downloaded files) grib_file_exist = check_exist_grib_file(grib_file_list, print_msg=True) grib_file2dload = sorted(list(set(grib_file_list) - set(grib_file_exist))) date_list2dload = [str(re.findall('\d{8}', i)[0]) for i in grib_file2dload] print('number of grib files to download: %d' % len(date_list2dload)) print( '------------------------------------------------------------------------------\n' ) # Download grib file using PyAPS if len(date_list2dload) > 0: hour = re.findall('\d{8}[-_]\d{2}', grib_file2dload[0])[0].replace('-', '_').split('_')[1] grib_dir = os.path.dirname(grib_file2dload[0]) # try 3 times to download, then use whatever downloaded to calculate delay trop_model = grib_file_name2trop_model_name(grib_file2dload[0]) i = 0 while i < 3: i += 1 try: if trop_model == 'ECMWF': pa.ECMWFdload(date_list2dload, hour, grib_dir) elif trop_model == 'MERRA': pa.MERRAdload(date_list2dload, hour, grib_dir) elif trop_model == 'NARR': pa.NARRdload(date_list2dload, hour, grib_dir) elif trop_model == 'ERA': pa.ERAdload(date_list2dload, hour, grib_dir) elif trop_model == 'MERRA1': pa.MERRA1dload(date_list2dload, hour, grib_dir) except: pass grib_file_list = check_exist_grib_file(grib_file_list, print_msg=False) return grib_file_list
def dload_grib_pyaps(date_list, hour, trop_model='ECMWF', weather_dir='./'): """Download weather re-analysis grib files using PyAPS Inputs: date_list : list of string in YYYYMMDD format hour : string in HH:MM or HH format trop_model : string, weather_dir : string, Output: grib_file_list : list of string """ print( '*' * 50 + '\nDownloading weather model data using PyAPS (Jolivet et al., 2011, GRL) ...' ) # Grib data directory grib_dir = weather_dir + '/' + trop_model if not os.path.isdir(grib_dir): os.makedirs(grib_dir) print('making directory: ' + grib_dir) # Date list to grib file list grib_file_list = date_list2grib_file(date_list, hour, trop_model, grib_dir) # Get date list to download (skip already downloaded files) grib_file_existed = ut.get_file_list(grib_file_list) if grib_file_existed: grib_filesize_digit = ut.most_common( [len(str(os.path.getsize(i))) for i in grib_file_existed]) grib_filesize_max2 = ut.most_common( [str(os.path.getsize(i))[0:2] for i in grib_file_existed]) grib_file_corrupted = [ i for i in grib_file_existed if (len(str(os.path.getsize(i))) != grib_filesize_digit or str(os.path.getsize(i))[0:2] != grib_filesize_max2) ] print('file size mode: %se%d bytes' % (grib_filesize_max2, grib_filesize_digit - 2)) print('number of grib files existed : %d' % len(grib_file_existed)) if grib_file_corrupted: print( '------------------------------------------------------------------------------' ) print( 'corrupted grib files detected! Delete them and re-download...' ) print('number of grib files corrupted : %d' % len(grib_file_corrupted)) for i in grib_file_corrupted: rmCmd = 'rm ' + i print(rmCmd) os.system(rmCmd) grib_file_existed.remove(i) print( '------------------------------------------------------------------------------' ) grib_file2download = sorted( list(set(grib_file_list) - set(grib_file_existed))) date_list2download = [ str(re.findall('\d{8}', i)[0]) for i in grib_file2download ] print('number of grib files to download: %d' % len(date_list2download)) print( '------------------------------------------------------------------------------\n' ) # Download grib file using PyAPS if trop_model == 'ECMWF': pa.ECMWFdload(date_list2download, hour, grib_dir) elif trop_model == 'MERRA': pa.MERRAdload(date_list2download, hour, grib_dir) elif trop_model == 'NARR': pa.NARRdload(date_list2download, hour, grib_dir) elif trop_model == 'ERA': pa.ERAdload(date_list2download, hour, grib_dir) elif trop_model == 'MERRA1': pa.MERRA1dload(date_list2download, hour, grib_dir) return grib_file_list
def dload_grib(date_list, hour, grib_source='ECMWF', weather_dir='./'): '''Download weather re-analysis grib files using PyAPS Inputs: date_list : list of string in YYYYMMDD format hour : string in HH:MM or HH format grib_source : string, weather_dir : string, Output: grib_file_list : list of string ''' ## Grib data directory weather_dir = os.path.abspath(weather_dir) grib_dir = weather_dir + '/' + grib_source if not os.path.isdir(grib_dir): print 'making directory: ' + grib_dir os.makedirs(grib_dir) ## Date list to grib file list grib_file_list = date_list2grib_file(date_list, hour, grib_source, grib_dir) ## Get date list to download (skip already downloaded files) grib_file_existed = ut.get_file_list(grib_file_list) if grib_file_existed: grib_filesize_digit = ut.mode( [len(str(os.path.getsize(i))) for i in grib_file_existed]) grib_filesize_max2 = ut.mode( [str(os.path.getsize(i))[0:2] for i in grib_file_existed]) grib_file_corrupted = [i for i in grib_file_existed if (len(str(os.path.getsize(i))) != grib_filesize_digit or\ str(os.path.getsize(i))[0:2] != grib_filesize_max2)] print 'file size mode: %se%d bytes' % (grib_filesize_max2, grib_filesize_digit - 2) print 'number of grib files existed : %d' % len(grib_file_existed) if grib_file_corrupted: print '------------------------------------------------------------------------------' print 'corrupted grib files detected! Delete them and re-download...' print 'number of grib files corrupted : %d' % len( grib_file_corrupted) for i in grib_file_corrupted: rmCmd = 'rm ' + i print rmCmd os.system(rmCmd) grib_file_existed.remove(i) print '------------------------------------------------------------------------------' grib_file2download = sorted( list(set(grib_file_list) - set(grib_file_existed))) date_list2download = [ str(re.findall('\d{8}', i)[0]) for i in grib_file2download ] print 'number of grib files to download: %d' % len(date_list2download) print '------------------------------------------------------------------------------\n' ## Download grib file using PyAPS if grib_source == 'ECMWF': pa.ECMWFdload(date_list2download, hour, grib_dir) elif grib_source == 'ERA': pa.ERAdload(date_list2download, hour, grib_dir) elif grib_source == 'NARR': pa.NARRdload(date_list2download, hour, grib_dir) elif grib_source == 'MERRA': pa.MERRAdload(date_list2download, hour, grib_dir) elif grib_source == 'MERRA1': pa.MERRA1dload(date_list2download, hour, grib_dir) return grib_file_existed