def MetUpdate(config_file, r2c_target_path, type, RepoPath, r2c_template_path): """ Function to update r2c files with either CaPA data or Temperature data from the EC datamart Args: config_file: see class ConfigParse() r2c_target_path: path to the r2c file that is being updated type: string; currently either 'CaPA' or 'GEMTemps' RepoPath: path to the directory where the downloaded grib files are stored r2c_template_path: path to a r2c template that has the same format as the r2c_target_path, this is used because it is quicker to load Returns: NULL: """ if type == "CaPA": timestep = 6 if type == "GEMTemps": timestep = 3 #Check datamart repository and download any data that isn't in local repository if type == "CaPA": lastgribfiletime, grib_path_string = Download_Datamart_ReAnalysisHindcast(config_file,type,RepoPath) if type == "GEMTemps": lastgribfiletime, grib_path_string = Download_Datamart_GEMHindcast(config_file,type,RepoPath) #load capa template and get coordinate system template_r2c_object = pyEnSim_basics.load_r2c_template(r2c_template_path) #load r2c and get last frame and time lastindexframe, lasttimeframe = pyEnSim_basics.r2c_EndFrameData(r2c_target_path) #get the last date in the grib file repository print "The last frame is: " + str(lasttimeframe) print "the last gribfile is: " + str(lastgribfiletime) print "\n" if type == "CaPA": #starting at the next timestep, convert specified capa grib file and append to r2c file current_time = lasttimeframe current_index = lastindexframe while(current_time < lastgribfiletime): current_time = current_time + datetime.timedelta(hours = timestep) current_index = current_index + 1 current_gribpath = current_time.strftime(grib_path_string) #convert and append grib file pyEnSim_basics.grib_fastappend_r2c(grib_path = current_gribpath, template_r2c_object = template_r2c_object, r2cTargetFilePath = r2c_target_path, frameindex = current_index, frametime = current_time, convert_mult = False, convert_add = False, ensemble = False) if type == "GEMTemps": #create an ordered list of grib files to append current_time = lasttimeframe griblist = [] while(current_time < lastgribfiletime): timestamp_odd = current_time.strftime("%Y%m%d%H") current_time = current_time + datetime.timedelta(hours = timestep) timestamp_even = current_time.strftime("%Y%m%d%H") hourstamp = current_time.strftime("%H") #get relevant grib file name; this is dependent on the hour because the forecasted temps are being used if int(hourstamp) in (0,6,12,18): gribname = os.path.join(RepoPath,grib_path_string + timestamp_even + "_P000.grib2") griblist.append(gribname) if int(hourstamp) in (3,9,15,21): gribname = os.path.join(RepoPath,grib_path_string + timestamp_odd + "_P003.grib2") griblist.append(gribname) #now iterate through grib list and append each file to r2c current_time = lasttimeframe current_index = lastindexframe for i, grib_path in enumerate(griblist): current_index = current_index + 1 current_time = current_time + datetime.timedelta(hours = timestep) #convert and append grib file pyEnSim_basics.grib_fastappend_r2c(grib_path = grib_path, template_r2c_object = template_r2c_object, r2cTargetFilePath = r2c_target_path, frameindex = current_index, frametime = current_time, convert_mult = False, convert_add = -273.15, ensemble = False)
def grib_to_r2c_nomads(repos, r2c_repo, r2c_template, datestamp_object, grib_repo, silent = False): """ Function to convert the files that have been downloaded via the repo_pull_nomads function Note that the ensemble files are handled differently than the EC datamart ensemble files. NOMADS stores 1 single ensemble in 1 single grib file. There are no 'children' as in the datamart. Args: repos: #Example repos from config file, note substitution parameters (%X) in :FileName :SourceData 0:URL http://nomads.ncep.noaa.gov/cgi-bin/ 1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a 2:DeltaTimeStart 6 3:DeltaTimeEnd 240 4:DeltaTimeStep 6 5:StitchTimeStart 6 6:StitchTimeEnd 240 7:Grouping tem 8:Type NOMAD_GFS 9:Forecast 3 10:num_ensembles 20 wx_repo: wxdata folder path r2c_template: path to r2c template datestamp_object: forecast date in the datetime class grib_repo: repository where grib data is downloaded and stored Returns: NULL - but converts grib files to r2c """ #get information for source file Grouping = repos[7][0] Type = repos[8][0] num_ensembles = int(repos[10][0]) DeltaTimeStart = int(repos[2][0]) DeltaTimeEnd = int(repos[3][0]) DeltaTimeStep = int(repos[4][0]) Forecast = int(repos[9][0]) today_grib_repo = grib_repo + "/" + datestamp_object.strftime("%Y%m%d%H") + "/" r2c_dest_folder = os.path.join(r2c_repo, Grouping) r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template) print "Converting Hours: " + str(DeltaTimeStart) + " to " + str(DeltaTimeEnd) #for each ensemble member (1-20) for i in range(1,num_ensembles+1): if silent is False: pbar = i/float(num_ensembles) * 40 sys.stdout.write('\r') # the exact output you're looking for: sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100)) sys.stdout.flush() #for each timestep for j in range(DeltaTimeStart/DeltaTimeStep,DeltaTimeEnd/DeltaTimeStep + 1): #get file to convert DeltaTime = j * DeltaTimeStep TimeStamp = datestamp_object + datetime.timedelta(hours = (DeltaTime-DeltaTimeStep)) grib_filepath = today_grib_repo + Type + '_' + Grouping + '_' + "%02d" % i + '_' + "%03d" % DeltaTime + '_' + datestamp_object.strftime("%Y%m%d%H") + '.grib2' #get r2c destination filename r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-' + "%02d" % i + '.r2c' r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename) #get first file and convert to r2c if j == 1: if Grouping == 'tem': pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15) if Grouping == 'met': pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_mult = False) else: #for all grib files after the first file, append to existing r2c file if Grouping == 'tem': pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_add = -273.15) if Grouping == 'met': pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_mult = False) print '\n'
def grib2r2c_datamart(repos, wx_repo, r2c_template, datestamp_object, grib_repo, silent = False): """ Function to process the EC datamart grib files (both deterministic and ensemble) Args: repos: #Example repos from config file, note substitution parameters (%X) in :FileName :SourceData 0:URL http://nomads.ncep.noaa.gov/cgi-bin/ 1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a 2:DeltaTimeStart 6 3:DeltaTimeEnd 240 4:DeltaTimeStep 6 5:StitchTimeStart 6 6:StitchTimeEnd 240 7:Grouping tem 8:Type NOMAD_GFS 9:Forecast 3 10:num_ensembles 20 wx_repo: wxdata folder path r2c_template: path to r2c template datestamp_object: forecast date in the datetime class grib_repo: repository where grib data is downloaded and stored Returns: NULL - but creates r2c files from grib files """ #get initial info from repos Stitches = len(repos[0]) Grouping = repos[7][0] Type = repos[8][0] num_ensembles = int(repos[10][0]) DeltaTimeStart = int(repos[2][0]) DeltaTimeEnd = int(repos[3][0]) Forecast = int(repos[9][0]) #set folder paths today_grib_repo = os.path.join(grib_repo,datestamp_object.strftime("%Y%m%d%H")) r2c_dest_folder = os.path.join(wx_repo, Grouping) #Set the number of r2c files to produce based on user defined 'num_ensembles' arg from repos #note that EC datamart structure has the ensembles embedded as multiple 'children' in each grib file #the processing of these 'children' is done within the pyEnSim_basics.grib_save_r2c and #pyEnSim_basics.grib_fastappend_r2c functions frame_index = 0 if num_ensembles > 1: ensemble = num_ensembles else: ensemble = False #for each of the 'series' that are being stitched together (typically 1 or 2) for i in range(0,Stitches): StitchTimeStart = int(repos[5][i]) StitchTimeEnd = int(repos[6][i]) DeltaTimeStep = int(repos[4][i]) print "Converting Hours: " + str(StitchTimeStart) + " to " + str(StitchTimeEnd) #loop through each timestep for j in range(StitchTimeStart/DeltaTimeStep,StitchTimeEnd/DeltaTimeStep+1): if silent is False: pbar = j/float(StitchTimeEnd/DeltaTimeStep) * 40 sys.stdout.write('\r') # the exact output you're looking for: sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100)) sys.stdout.flush() frame_index = frame_index + 1 #set deltatime so that frame timestamp can be properly labelled if j == 1: DeltaTime = StitchTimeStart else: DeltaTime = DeltaTime + DeltaTimeStep #end loop if we've reached the end if DeltaTime > StitchTimeEnd: break #get first file and convert to r2c name = repos[1][i].replace('%T', str(DeltaTime).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast() grib_filepath = os.path.join(today_grib_repo,name) oldname = repos[1][i].replace('%T', str(DeltaTime-DeltaTimeStep).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast() oldgrib_filepath = os.path.join(today_grib_repo,oldname) #get r2c destination filename r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-01.r2c' r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename) r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template) frame_time = datestamp_object + datetime.timedelta(hours=int(DeltaTime)) if Grouping == "tem": if j == 1: #if this is the first pass, create the r2c file(s) pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15, ensemble = ensemble) else: #append grib data to existing r2c file(s) on all other passes pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, convert_add = -273.15, ensemble = ensemble) if Grouping == "met": if j == 1: pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, ensemble = ensemble) else: pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, grib_previous = oldgrib_filepath, ensemble = ensemble) print "\n"
def grib_to_r2c_nomads(repos, r2c_repo, r2c_template, datestamp_object, grib_repo): """ Function to convert the files that have been downloaded via the repo_pull_nomads function Note that the ensemble files are handled differently than the EC datamart ensemble files. NOMADS stores 1 single ensemble in 1 single grib file. There are no 'children' as in the datamart. Args: repos: #Example repos from config file, note substitution parameters (%X) in :FileName :SourceData 0:URL http://nomads.ncep.noaa.gov/cgi-bin/ 1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a 2:DeltaTimeStart 6 3:DeltaTimeEnd 240 4:DeltaTimeStep 6 5:StitchTimeStart 6 6:StitchTimeEnd 240 7:Grouping tem 8:Type NOMAD_GFS 9:Forecast 3 10:num_ensembles 20 wx_repo: wxdata folder path r2c_template: path to r2c template datestamp_object: forecast date in the datetime class grib_repo: repository where grib data is downloaded and stored Returns: NULL - but converts grib files to r2c """ #get information for source file Grouping = repos[7][0] Type = repos[8][0] num_ensembles = int(repos[10][0]) DeltaTimeStart = int(repos[2][0]) DeltaTimeEnd = int(repos[3][0]) DeltaTimeStep = int(repos[4][0]) Forecast = int(repos[9][0]) today_grib_repo = grib_repo + "/" + datestamp_object.strftime("%Y%m%d%H") + "/" r2c_dest_folder = os.path.join(r2c_repo, Grouping) r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template) print "Converting Hours: " + str(DeltaTimeStart) + " to " + str(DeltaTimeEnd) #for each ensemble member (1-20) for i in range(1,num_ensembles+1): pbar = i/float(num_ensembles) * 40 sys.stdout.write('\r') # the exact output you're looking for: sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100)) sys.stdout.flush() #for each timestep for j in range(DeltaTimeStart/DeltaTimeStep,DeltaTimeEnd/DeltaTimeStep + 1): #get file to convert DeltaTime = j * DeltaTimeStep TimeStamp = datestamp_object + datetime.timedelta(hours = (DeltaTime-DeltaTimeStep)) grib_filepath = today_grib_repo + Type + '_' + Grouping + '_' + "%02d" % i + '_' + "%03d" % DeltaTime + '_' + datestamp_object.strftime("%Y%m%d%H") + '.grib2' #get r2c destination filename r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-' + "%02d" % i + '.r2c' r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename) #get first file and convert to r2c if j == 1: if Grouping == 'tem': pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15) if Grouping == 'met': pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_mult = False) else: #for all grib files after the first file, append to existing r2c file if Grouping == 'tem': pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_add = -273.15) if Grouping == 'met': pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_mult = False) print '\n'
def grib2r2c_datamart(repos, wx_repo, r2c_template, datestamp_object, grib_repo): """ Function to process the EC datamart grib files (both deterministic and ensemble) Args: repos: #Example repos from config file, note substitution parameters (%X) in :FileName :SourceData 0:URL http://nomads.ncep.noaa.gov/cgi-bin/ 1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a 2:DeltaTimeStart 6 3:DeltaTimeEnd 240 4:DeltaTimeStep 6 5:StitchTimeStart 6 6:StitchTimeEnd 240 7:Grouping tem 8:Type NOMAD_GFS 9:Forecast 3 10:num_ensembles 20 wx_repo: wxdata folder path r2c_template: path to r2c template datestamp_object: forecast date in the datetime class grib_repo: repository where grib data is downloaded and stored Returns: NULL - but creates r2c files from grib files """ #get initial info from repos Stitches = len(repos[0]) Grouping = repos[7][0] Type = repos[8][0] num_ensembles = int(repos[10][0]) DeltaTimeStart = int(repos[2][0]) DeltaTimeEnd = int(repos[3][0]) Forecast = int(repos[9][0]) #set folder paths today_grib_repo = os.path.join(grib_repo,datestamp_object.strftime("%Y%m%d%H")) r2c_dest_folder = os.path.join(wx_repo, Grouping) #Set the number of r2c files to produce based on user defined 'num_ensembles' arg from repos #note that EC datamart structure has the ensembles embedded as multiple 'children' in each grib file #the processing of these 'children' is done within the pyEnSim_basics.grib_save_r2c and #pyEnSim_basics.grib_fastappend_r2c functions frame_index = 0 if num_ensembles > 1: ensemble = num_ensembles else: ensemble = False #for each of the 'series' that are being stitched together (typically 1 or 2) for i in range(0,Stitches): StitchTimeStart = int(repos[5][i]) StitchTimeEnd = int(repos[6][i]) DeltaTimeStep = int(repos[4][i]) print "Converting Hours: " + str(StitchTimeStart) + " to " + str(StitchTimeEnd) #loop through each timestep for j in range(StitchTimeStart/DeltaTimeStep,StitchTimeEnd/DeltaTimeStep+1): pbar = j/float(StitchTimeEnd/DeltaTimeStep) * 40 sys.stdout.write('\r') # the exact output you're looking for: sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100)) sys.stdout.flush() frame_index = frame_index + 1 #set deltatime so that frame timestamp can be properly labelled if j == 1: DeltaTime = StitchTimeStart else: DeltaTime = DeltaTime + DeltaTimeStep #end loop if we've reached the end if DeltaTime > StitchTimeEnd: break #get first file and convert to r2c name = repos[1][i].replace('%T', str(DeltaTime).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast() grib_filepath = os.path.join(today_grib_repo,name) oldname = repos[1][i].replace('%T', str(DeltaTime-DeltaTimeStep).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast() oldgrib_filepath = os.path.join(today_grib_repo,oldname) #get r2c destination filename r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-01.r2c' r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename) r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template) frame_time = datestamp_object + datetime.timedelta(hours=int(DeltaTime)) if Grouping == "tem": if j == 1: #if this is the first pass, create the r2c file(s) pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15, ensemble = ensemble) else: #append grib data to existing r2c file(s) on all other passes pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, convert_add = -273.15, ensemble = ensemble) if Grouping == "met": if j == 1: pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, ensemble = ensemble) else: pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, grib_previous = oldgrib_filepath, ensemble = ensemble) print "\n"