def MetUpdate(config_file, r2c_target_path, type, RepoPath, r2c_template_path):
"""
Function to update r2c files with either CaPA data or Temperature data from the EC datamart
Args:
config_file: see class ConfigParse()
r2c_target_path: path to the r2c file that is being updated
type: string; currently either 'CaPA' or 'GEMTemps'
RepoPath: path to the directory where the downloaded grib files are stored
r2c_template_path: path to a r2c template that has the same format as the r2c_target_path, this is used because it is quicker to load
Returns:
NULL:
"""
if type == "CaPA":
timestep = 6
if type == "GEMTemps":
timestep = 3
#Check datamart repository and download any data that isn't in local repository
if type == "CaPA":
lastgribfiletime, grib_path_string = Download_Datamart_ReAnalysisHindcast(config_file,type,RepoPath)
if type == "GEMTemps":
lastgribfiletime, grib_path_string = Download_Datamart_GEMHindcast(config_file,type,RepoPath)
#load capa template and get coordinate system
template_r2c_object = pyEnSim_basics.load_r2c_template(r2c_template_path)
#load r2c and get last frame and time
lastindexframe, lasttimeframe = pyEnSim_basics.r2c_EndFrameData(r2c_target_path)
#get the last date in the grib file repository
print "The last frame is: " + str(lasttimeframe)
print "the last gribfile is: " + str(lastgribfiletime)
print "\n"
if type == "CaPA":
#starting at the next timestep, convert specified capa grib file and append to r2c file
current_time = lasttimeframe
current_index = lastindexframe
while(current_time < lastgribfiletime):
current_time = current_time + datetime.timedelta(hours = timestep)
current_index = current_index + 1
current_gribpath = current_time.strftime(grib_path_string)
#convert and append grib file
pyEnSim_basics.grib_fastappend_r2c(grib_path = current_gribpath,
template_r2c_object = template_r2c_object,
r2cTargetFilePath = r2c_target_path,
frameindex = current_index,
frametime = current_time,
convert_mult = False, convert_add = False, ensemble = False)
if type == "GEMTemps":
#create an ordered list of grib files to append
current_time = lasttimeframe
griblist = []
while(current_time < lastgribfiletime):
timestamp_odd = current_time.strftime("%Y%m%d%H")
current_time = current_time + datetime.timedelta(hours = timestep)
timestamp_even = current_time.strftime("%Y%m%d%H")
hourstamp = current_time.strftime("%H")
#get relevant grib file name; this is dependent on the hour because the forecasted temps are being used
if int(hourstamp) in (0,6,12,18):
gribname = os.path.join(RepoPath,grib_path_string + timestamp_even + "_P000.grib2")
griblist.append(gribname)
if int(hourstamp) in (3,9,15,21):
gribname = os.path.join(RepoPath,grib_path_string + timestamp_odd + "_P003.grib2")
griblist.append(gribname)
#now iterate through grib list and append each file to r2c
current_time = lasttimeframe
current_index = lastindexframe
for i, grib_path in enumerate(griblist):
current_index = current_index + 1
current_time = current_time + datetime.timedelta(hours = timestep)
#convert and append grib file
pyEnSim_basics.grib_fastappend_r2c(grib_path = grib_path,
template_r2c_object = template_r2c_object,
r2cTargetFilePath = r2c_target_path,
frameindex = current_index,
frametime = current_time,
convert_mult = False, convert_add = -273.15, ensemble = False)
def grib_to_r2c_nomads(repos, r2c_repo, r2c_template, datestamp_object, grib_repo, silent = False):
"""
Function to convert the files that have been downloaded via the repo_pull_nomads function
Note that the ensemble files are handled differently than the EC datamart ensemble files.
NOMADS stores 1 single ensemble in 1 single grib file. There are no 'children' as in the datamart.
Args:
repos:
#Example repos from config file, note substitution parameters (%X) in :FileName
:SourceData
0:URL http://nomads.ncep.noaa.gov/cgi-bin/
1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a
2:DeltaTimeStart 6
3:DeltaTimeEnd 240
4:DeltaTimeStep 6
5:StitchTimeStart 6
6:StitchTimeEnd 240
7:Grouping tem
8:Type NOMAD_GFS
9:Forecast 3
10:num_ensembles 20
wx_repo: wxdata folder path
r2c_template: path to r2c template
datestamp_object: forecast date in the datetime class
grib_repo: repository where grib data is downloaded and stored
Returns:
NULL - but converts grib files to r2c
"""
#get information for source file
Grouping = repos[7][0]
Type = repos[8][0]
num_ensembles = int(repos[10][0])
DeltaTimeStart = int(repos[2][0])
DeltaTimeEnd = int(repos[3][0])
DeltaTimeStep = int(repos[4][0])
Forecast = int(repos[9][0])
today_grib_repo = grib_repo + "/" + datestamp_object.strftime("%Y%m%d%H") + "/"
r2c_dest_folder = os.path.join(r2c_repo, Grouping)
r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template)
print "Converting Hours: " + str(DeltaTimeStart) + " to " + str(DeltaTimeEnd)
#for each ensemble member (1-20)
for i in range(1,num_ensembles+1):
if silent is False:
pbar = i/float(num_ensembles) * 40
sys.stdout.write('\r')
# the exact output you're looking for:
sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100))
sys.stdout.flush()
#for each timestep
for j in range(DeltaTimeStart/DeltaTimeStep,DeltaTimeEnd/DeltaTimeStep + 1):
#get file to convert
DeltaTime = j * DeltaTimeStep
TimeStamp = datestamp_object + datetime.timedelta(hours = (DeltaTime-DeltaTimeStep))
grib_filepath = today_grib_repo + Type + '_' + Grouping + '_' + "%02d" % i + '_' + "%03d" % DeltaTime + '_' + datestamp_object.strftime("%Y%m%d%H") + '.grib2'
#get r2c destination filename
r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-' + "%02d" % i + '.r2c'
r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename)
#get first file and convert to r2c
if j == 1:
if Grouping == 'tem':
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15)
if Grouping == 'met':
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_mult = False)
else: #for all grib files after the first file, append to existing r2c file
if Grouping == 'tem':
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_add = -273.15)
if Grouping == 'met':
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_mult = False)
print '\n'
def grib2r2c_datamart(repos, wx_repo, r2c_template, datestamp_object, grib_repo, silent = False):
"""
Function to process the EC datamart grib files (both deterministic and ensemble)
Args:
repos:
#Example repos from config file, note substitution parameters (%X) in :FileName
:SourceData
0:URL http://nomads.ncep.noaa.gov/cgi-bin/
1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a
2:DeltaTimeStart 6
3:DeltaTimeEnd 240
4:DeltaTimeStep 6
5:StitchTimeStart 6
6:StitchTimeEnd 240
7:Grouping tem
8:Type NOMAD_GFS
9:Forecast 3
10:num_ensembles 20
wx_repo: wxdata folder path
r2c_template: path to r2c template
datestamp_object: forecast date in the datetime class
grib_repo: repository where grib data is downloaded and stored
Returns:
NULL - but creates r2c files from grib files
"""
#get initial info from repos
Stitches = len(repos[0])
Grouping = repos[7][0]
Type = repos[8][0]
num_ensembles = int(repos[10][0])
DeltaTimeStart = int(repos[2][0])
DeltaTimeEnd = int(repos[3][0])
Forecast = int(repos[9][0])
#set folder paths
today_grib_repo = os.path.join(grib_repo,datestamp_object.strftime("%Y%m%d%H"))
r2c_dest_folder = os.path.join(wx_repo, Grouping)
#Set the number of r2c files to produce based on user defined 'num_ensembles' arg from repos
#note that EC datamart structure has the ensembles embedded as multiple 'children' in each grib file
#the processing of these 'children' is done within the pyEnSim_basics.grib_save_r2c and
#pyEnSim_basics.grib_fastappend_r2c functions
frame_index = 0
if num_ensembles > 1:
ensemble = num_ensembles
else:
ensemble = False
#for each of the 'series' that are being stitched together (typically 1 or 2)
for i in range(0,Stitches):
StitchTimeStart = int(repos[5][i])
StitchTimeEnd = int(repos[6][i])
DeltaTimeStep = int(repos[4][i])
print "Converting Hours: " + str(StitchTimeStart) + " to " + str(StitchTimeEnd)
#loop through each timestep
for j in range(StitchTimeStart/DeltaTimeStep,StitchTimeEnd/DeltaTimeStep+1):
if silent is False:
pbar = j/float(StitchTimeEnd/DeltaTimeStep) * 40
sys.stdout.write('\r')
# the exact output you're looking for:
sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100))
sys.stdout.flush()
frame_index = frame_index + 1
#set deltatime so that frame timestamp can be properly labelled
if j == 1:
DeltaTime = StitchTimeStart
else:
DeltaTime = DeltaTime + DeltaTimeStep
#end loop if we've reached the end
if DeltaTime > StitchTimeEnd:
break
#get first file and convert to r2c
name = repos[1][i].replace('%T', str(DeltaTime).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast()
grib_filepath = os.path.join(today_grib_repo,name)
oldname = repos[1][i].replace('%T', str(DeltaTime-DeltaTimeStep).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast()
oldgrib_filepath = os.path.join(today_grib_repo,oldname)
#get r2c destination filename
r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-01.r2c'
r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename)
r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template)
frame_time = datestamp_object + datetime.timedelta(hours=int(DeltaTime))
if Grouping == "tem":
if j == 1:
#if this is the first pass, create the r2c file(s)
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15, ensemble = ensemble)
else:
#append grib data to existing r2c file(s) on all other passes
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, convert_add = -273.15, ensemble = ensemble)
if Grouping == "met":
if j == 1:
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, ensemble = ensemble)
else:
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, grib_previous = oldgrib_filepath, ensemble = ensemble)
print "\n"
def MetUpdate(config_file, r2c_target_path, type, RepoPath, r2c_template_path):
"""
Function to update r2c files with either CaPA data or Temperature data from the EC datamart
Args:
config_file: see class ConfigParse()
r2c_target_path: path to the r2c file that is being updated
type: string; currently either 'CaPA' or 'GEMTemps'
RepoPath: path to the directory where the downloaded grib files are stored
r2c_template_path: path to a r2c template that has the same format as the r2c_target_path, this is used because it is quicker to load
Returns:
NULL:
"""
if type == "CaPA":
timestep = 6
if type == "GEMTemps":
timestep = 3
#Check datamart repository and download any data that isn't in local repository
if type == "CaPA":
lastgribfiletime, grib_path_string = Download_Datamart_ReAnalysisHindcast(config_file,type,RepoPath)
if type == "GEMTemps":
lastgribfiletime, grib_path_string = Download_Datamart_GEMHindcast(config_file,type,RepoPath)
#load capa template and get coordinate system
template_r2c_object = pyEnSim_basics.load_r2c_template(r2c_template_path)
#load r2c and get last frame and time
lastindexframe, lasttimeframe = pyEnSim_basics.r2c_EndFrameData(r2c_target_path)
#get the last date in the grib file repository
print "The last frame is: " + str(lasttimeframe)
print "the last gribfile is: " + str(lastgribfiletime)
print "\n"
if type == "CaPA":
#starting at the next timestep, convert specified capa grib file and append to r2c file
current_time = lasttimeframe
current_index = lastindexframe
while(current_time < lastgribfiletime):
current_time = current_time + datetime.timedelta(hours = timestep)
current_index = current_index + 1
current_gribpath = current_time.strftime(grib_path_string)
#convert and append grib file
pyEnSim_basics.grib_fastappend_r2c(grib_path = current_gribpath,
template_r2c_object = template_r2c_object,
r2cTargetFilePath = r2c_target_path,
frameindex = current_index,
frametime = current_time,
convert_mult = False, convert_add = False, ensemble = False)
if type == "GEMTemps":
#create an ordered list of grib files to append
current_time = lasttimeframe
griblist = []
while(current_time < lastgribfiletime):
timestamp_odd = current_time.strftime("%Y%m%d%H")
current_time = current_time + datetime.timedelta(hours = timestep)
timestamp_even = current_time.strftime("%Y%m%d%H")
hourstamp = current_time.strftime("%H")
#get relevant grib file name; this is dependent on the hour because the forecasted temps are being used
if int(hourstamp) in (0,6,12,18):
gribname = os.path.join(RepoPath,grib_path_string + timestamp_even + "_P000.grib2")
griblist.append(gribname)
if int(hourstamp) in (3,9,15,21):
gribname = os.path.join(RepoPath,grib_path_string + timestamp_odd + "_P003.grib2")
griblist.append(gribname)
#now iterate through grib list and append each file to r2c
current_time = lasttimeframe
current_index = lastindexframe
for i, grib_path in enumerate(griblist):
current_index = current_index + 1
current_time = current_time + datetime.timedelta(hours = timestep)
#convert and append grib file
pyEnSim_basics.grib_fastappend_r2c(grib_path = grib_path,
template_r2c_object = template_r2c_object,
r2cTargetFilePath = r2c_target_path,
frameindex = current_index,
frametime = current_time,
convert_mult = False, convert_add = -273.15, ensemble = False)
def grib_to_r2c_nomads(repos, r2c_repo, r2c_template, datestamp_object, grib_repo):
"""
Function to convert the files that have been downloaded via the repo_pull_nomads function
Note that the ensemble files are handled differently than the EC datamart ensemble files.
NOMADS stores 1 single ensemble in 1 single grib file. There are no 'children' as in the datamart.
Args:
repos:
#Example repos from config file, note substitution parameters (%X) in :FileName
:SourceData
0:URL http://nomads.ncep.noaa.gov/cgi-bin/
1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a
2:DeltaTimeStart 6
3:DeltaTimeEnd 240
4:DeltaTimeStep 6
5:StitchTimeStart 6
6:StitchTimeEnd 240
7:Grouping tem
8:Type NOMAD_GFS
9:Forecast 3
10:num_ensembles 20
wx_repo: wxdata folder path
r2c_template: path to r2c template
datestamp_object: forecast date in the datetime class
grib_repo: repository where grib data is downloaded and stored
Returns:
NULL - but converts grib files to r2c
"""
#get information for source file
Grouping = repos[7][0]
Type = repos[8][0]
num_ensembles = int(repos[10][0])
DeltaTimeStart = int(repos[2][0])
DeltaTimeEnd = int(repos[3][0])
DeltaTimeStep = int(repos[4][0])
Forecast = int(repos[9][0])
today_grib_repo = grib_repo + "/" + datestamp_object.strftime("%Y%m%d%H") + "/"
r2c_dest_folder = os.path.join(r2c_repo, Grouping)
r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template)
print "Converting Hours: " + str(DeltaTimeStart) + " to " + str(DeltaTimeEnd)
#for each ensemble member (1-20)
for i in range(1,num_ensembles+1):
pbar = i/float(num_ensembles) * 40
sys.stdout.write('\r')
# the exact output you're looking for:
sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100))
sys.stdout.flush()
#for each timestep
for j in range(DeltaTimeStart/DeltaTimeStep,DeltaTimeEnd/DeltaTimeStep + 1):
#get file to convert
DeltaTime = j * DeltaTimeStep
TimeStamp = datestamp_object + datetime.timedelta(hours = (DeltaTime-DeltaTimeStep))
grib_filepath = today_grib_repo + Type + '_' + Grouping + '_' + "%02d" % i + '_' + "%03d" % DeltaTime + '_' + datestamp_object.strftime("%Y%m%d%H") + '.grib2'
#get r2c destination filename
r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-' + "%02d" % i + '.r2c'
r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename)
#get first file and convert to r2c
if j == 1:
if Grouping == 'tem':
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15)
if Grouping == 'met':
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_mult = False)
else: #for all grib files after the first file, append to existing r2c file
if Grouping == 'tem':
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_add = -273.15)
if Grouping == 'met':
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frameindex = j, frametime = TimeStamp, convert_mult = False)
print '\n'
def grib2r2c_datamart(repos, wx_repo, r2c_template, datestamp_object, grib_repo):
"""
Function to process the EC datamart grib files (both deterministic and ensemble)
Args:
repos:
#Example repos from config file, note substitution parameters (%X) in :FileName
:SourceData
0:URL http://nomads.ncep.noaa.gov/cgi-bin/
1:FileName filter_%S1.pl?file=%S2gep%E.t%Hz.pgrb2af%T&%query&subregion=&leftlon=-98&rightlon=-88&toplat=54&bottomlat=46&dir=%2F%S3.%Y%m%d%2F00%2Fpgrb2a
2:DeltaTimeStart 6
3:DeltaTimeEnd 240
4:DeltaTimeStep 6
5:StitchTimeStart 6
6:StitchTimeEnd 240
7:Grouping tem
8:Type NOMAD_GFS
9:Forecast 3
10:num_ensembles 20
wx_repo: wxdata folder path
r2c_template: path to r2c template
datestamp_object: forecast date in the datetime class
grib_repo: repository where grib data is downloaded and stored
Returns:
NULL - but creates r2c files from grib files
"""
#get initial info from repos
Stitches = len(repos[0])
Grouping = repos[7][0]
Type = repos[8][0]
num_ensembles = int(repos[10][0])
DeltaTimeStart = int(repos[2][0])
DeltaTimeEnd = int(repos[3][0])
Forecast = int(repos[9][0])
#set folder paths
today_grib_repo = os.path.join(grib_repo,datestamp_object.strftime("%Y%m%d%H"))
r2c_dest_folder = os.path.join(wx_repo, Grouping)
#Set the number of r2c files to produce based on user defined 'num_ensembles' arg from repos
#note that EC datamart structure has the ensembles embedded as multiple 'children' in each grib file
#the processing of these 'children' is done within the pyEnSim_basics.grib_save_r2c and
#pyEnSim_basics.grib_fastappend_r2c functions
frame_index = 0
if num_ensembles > 1:
ensemble = num_ensembles
else:
ensemble = False
#for each of the 'series' that are being stitched together (typically 1 or 2)
for i in range(0,Stitches):
StitchTimeStart = int(repos[5][i])
StitchTimeEnd = int(repos[6][i])
DeltaTimeStep = int(repos[4][i])
print "Converting Hours: " + str(StitchTimeStart) + " to " + str(StitchTimeEnd)
#loop through each timestep
for j in range(StitchTimeStart/DeltaTimeStep,StitchTimeEnd/DeltaTimeStep+1):
pbar = j/float(StitchTimeEnd/DeltaTimeStep) * 40
sys.stdout.write('\r')
# the exact output you're looking for:
sys.stdout.write("[%-40s] %d%%" % ('='*int(pbar), pbar/40*100))
sys.stdout.flush()
frame_index = frame_index + 1
#set deltatime so that frame timestamp can be properly labelled
if j == 1:
DeltaTime = StitchTimeStart
else:
DeltaTime = DeltaTime + DeltaTimeStep
#end loop if we've reached the end
if DeltaTime > StitchTimeEnd:
break
#get first file and convert to r2c
name = repos[1][i].replace('%T', str(DeltaTime).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast()
grib_filepath = os.path.join(today_grib_repo,name)
oldname = repos[1][i].replace('%T', str(DeltaTime-DeltaTimeStep).zfill(3)) #%Y%m%d%H have already been replaced in query_meteorological_forecast()
oldgrib_filepath = os.path.join(today_grib_repo,oldname)
#get r2c destination filename
r2c_dest_filename = datestamp_object.strftime("%Y%m%d") + '_' + Grouping + '_' + "%02d" % Forecast + '-01.r2c'
r2c_dest_filepath = os.path.join(r2c_dest_folder,r2c_dest_filename)
r2c_template_object = pyEnSim_basics.load_r2c_template(r2c_template)
frame_time = datestamp_object + datetime.timedelta(hours=int(DeltaTime))
if Grouping == "tem":
if j == 1:
#if this is the first pass, create the r2c file(s)
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, convert_add = -273.15, ensemble = ensemble)
else:
#append grib data to existing r2c file(s) on all other passes
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, convert_add = -273.15, ensemble = ensemble)
if Grouping == "met":
if j == 1:
pyEnSim_basics.grib_save_r2c(grib_filepath, r2c_template, r2c_dest_filepath, timestamp = datestamp_object, ensemble = ensemble)
else:
pyEnSim_basics.grib_fastappend_r2c(grib_filepath, r2c_template_object, r2c_dest_filepath, frame_index, frame_time, grib_previous = oldgrib_filepath, ensemble = ensemble)
print "\n"
