""" this routine will generate a text file that can be used to
run a python script dressing_statistics_to_netcdf.py
for multiple leads, dates, and models
"""
import numpy as np
from numpy import ma
import os, sys
from dateutils import daterange, dateshift
cmodels = ['NCEP', 'ECMWF', 'CMC']
cleades = ['24', '48', '72', '96', '120', '144', '168']
date_forecasts = daterange('2016040100', '2016070100', 24)
for cmodel in cmodels:
fname = 'blend_execute_' + cmodel + '.deck'
outfile = open(fname, 'w')
textstring = '#!/bin/tcsh \n'
outfile.write(textstring)
outfile.write(' \n')
for cleade in cleades:
outfile.write(' \n')
for date_forecast in date_forecasts:
textstring = 'blend_precip_singlemodel_dressed_gammacdf.x ' + date_forecast + ' ' + cleade + ' ' + cmodel + '\n'
outfile.write(textstring)
outfile.close()
fhour = 120
var = 't'
level = 500
vargrb = var
varnc = '%s_plev' % var
if var == 'z':
vargrb = 'gh'
varnc = 'h_plev'
datapath1 = '/scratch3/BMC/gsienkf/whitaker/%s' % expt1
datapath2 = '/scratch3/BMC/gsienkf/whitaker/%s' % expt2
analpath = '/scratch3/BMC/gsienkf/whitaker/ecanl'
if fhour > 9:
dates = dateutils.daterange(date1, date2, 24)
else:
dates = dateutils.daterange(date1, date2, 6)
ntime = None
fcsterrspect1 = None
fcsterrspect2 = None
for date in dates:
datev = dateutils.dateshift(date, fhour)
# read analysis
analfile = os.path.join(analpath, 'pgbanl.ecm.%s' % datev)
grbs = pygrib.open(analfile)
grb = grbs.select(shortName=vargrb, level=level)[0]
verif_data = grb.values[::-1, :]
grbs.close()
if fhour > 9:
fcstfile = '%s/%s/fv3longcontrol2_historyp_%s_latlon.nc' % (datapath1,
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
import numpy as np
import sys, os
from dateutils import daterange
from netCDF4 import Dataset
expt = sys.argv[1]
date1 = sys.argv[2]
date2 = sys.argv[3]
dates = daterange(date1, date2, 6)
datapath = '/scratch2/BMC/gsienkf/whitaker/%s' % expt
lats = None
#var = 'q_inc'
var = 'sphum_inc'
def getmean(data, coslats):
meancoslats = coslats.mean()
return (coslats * data).mean() / meancoslats
for date in dates:
print date
filename = os.path.join(os.path.join(datapath, date),
'control/INPUT/fv3_increment6.nc')
nc = Dataset(filename)
inc = nc[var][:].squeeze()[::-1]
print(inc.min(), inc.max())
if lats is None:
rcParams['legend.fancybox']=True rcParams['xtick.labelsize']='medium' # --- COMMAND LINE INPUT cmodel = 'NCEP' # sys.argv[1] # NCEP, ECMWF, or CMC cleade = '48' # sys.argv[2] # ending of periods lead time in hours, e.g., '24', '108' cempirical = '1' ileade = int(cleade) date_forecast = '2016050100' # sys.argv[3] date_end = dateshift(date_forecast,-ileade) date_begin = dateshift(date_end, -61*24) date_list = daterange(date_begin, date_end, 24) ndates = len(date_list) # ====================================================================== # Part 1: reading in the daily data and summing up the information over # all case days # ====================================================================== # ---- read in the array dimensions, precip values where gamma # distribution parameters and fraction zero are calculated, and # precipitation thresholds that are boundaries for light, # medium, and heavy precipitation from netcdf file. These # were previously generated in tally_gamma_statsfull_weighted.f90, # called by generate_gammadressing_stats_anymodel.f90
import dateutils
import pygrib
# compute rms error and anomaly correlation on 1 degree grid
# using native cubed-sphere history files.
def getmean(diff, coslats):
meancoslats = coslats.mean()
return (coslats * diff).mean() / meancoslats
date1 = sys.argv[1]
date2 = sys.argv[2]
exptname = sys.argv[3]
dates = dateutils.daterange(date1, date2, 12)
fhour = int(sys.argv[4])
var = sys.argv[5]
level = int(sys.argv[6])
#fhour = 120
#var = 'z'
#level = 500
vargrb = var
if var == 'z': vargrb = 'gh'
res = 384
nlons = 360
nlats = 181
latbound = 20 # boundary between tropics and extra-tropics
picklefile = 'C%s_grid.pickle' % res
analpath = '/scratch3/BMC/gsienkf/whitaker/ecanl'
from numpy import ma
import os, sys
from netCDF4 import Dataset
from verify_relia_bss_mme import verify_relia_bss_mme
import pygrib
from dateutils import daterange, dateshift
rcParams['legend.fontsize'] = 'small'
rcParams['legend.fancybox'] = True
rcParams['xtick.labelsize'] = 'medium'
cleade = sys.argv[1]
cthresh = sys.argv[2]
cyyyymmddhh_start = '2016040100'
cyyyymmddhh_end = '20160701000'
date_list = daterange(cyyyymmddhh_start, cyyyymmddhh_end, 24)
ileade = int(cleade)
ileadb = ileade - 12
cleadb = str(ileadb)
#if ileadb < 100: cleadb='0'+cleadb
if cthresh == 'POP':
rthresh = 0.254
cthresh_plot = 'POP'
else:
rthresh = float(cthresh)
cthresh_plot = r'$\geq$ ' + cthresh + ' mm'
nclasses = 21 # 0 to 100% probability by 5%
nxa = 464 # CCPA 1/8 degree grid over CONUS
nya = 224 #
rcParams['xtick.labelsize']='medium'
data_directory = '/Users/thamill/precip/ecmwf_data/'
# --- COMMAND LINE INPUT
cmodel = sys.argv[1] # NCEP, ECMWF, or CMC
cleade = sys.argv[2] # ending of periods lead time in hours, e.g., '24', '108'
ileade = int(cleade)
date_forecast = sys.argv[3] # yyyymmddhh format, date of initial condition
# --- define the previous 60 days that should have available data
date_end = dateshift(date_forecast,-ileade)
date_begin = dateshift(date_end, -61*24)
date_list = daterange(date_begin, date_end, 24) # makes a character list of dates
ndates = len(date_list)
# ======================================================================
# Part 1: reading in the daily data and summing up the information over
# all case days
# ======================================================================
# --- read a sample file to set array dimensions. These were
# previously generated by the fortran program
# generate_dressing_stats_anymodel_gammacdf.f90
infile = data_directory+'closest_hist_stats_' + \
cmodel + '_2016050100_fhour48.nc'
print infile
nc = Dataset(infile)
usethisdate = np.zeros((ndates),dtype=np.int16)
for iyear in range(2002, 2017):
# ---- find yyyymmddhh that is approximately at the center of the month
if cleadtime_begin == '12' and cleadtime_end == '00':
cyyyymmddhh_center = str(iyear) + cmono + '1500'
elif cleadtime_begin == '00' and cleadtime_end == '12':
cyyyymmddhh_center = str(iyear) + cmono + '1512'
else:
print 'invalid begin, end times for accumulation period. Stopping'
sys.exit()
cyyyymmddhh_begin = dateshift(cyyyymmddhh_center,-24*31)
cyyyymmddhh_end = dateshift(cyyyymmddhh_center,24*31)
date_list = daterange(cyyyymmddhh_begin, cyyyymmddhh_end, 24)
# ---- go thru the list of dates that are in the data set, and
# flag the elements that correspond to date_list. These
# represent the indices for the first of two times we read in.
for cdate in date_list:
idate = int(cdate)
idx = np.where(iyyyymmddhh_list == idate)
if idx >= 0: usethisdate[idx] = 1
# ---- estimate the climatology from relative frequency for selected thresholds
ktr = 0
climo_POP = np.zeros((nja,nia), dtype=np.float32) # 0.254 mm
climo_1mm = np.zeros((nja,nia), dtype=np.float32)
f = nio.open_file('/p72/reanl/whitaker/'+ecfilename)
lonsin = f.variables['lon_0'][:]
latsin = f.variables['lat_0'][:]
datesin = f.variables['initial_time0'][:]
dates = []
for date in datesin:
d = date.split('/')
mm = d[0]; dd = d[1]; yyyy = d[2][0:4]
hh = (d[2].split('(')[1]).split(':')[0]
dates.append(yyyy+mm+dd+hh)
if varshort not in ['psi','chi']:
varin = f.variables[ecvarname]
else:
varinu = f.variables[ecvarnameu]
varinv = f.variables[ecvarnamev]
datefcsts = daterange(date1,date2,12)
fcsts_2 = np.zeros((len(datefcsts),nlats,nlons),np.float32)
fcsts_1 = np.zeros((len(datefcsts),nlats,nlons),np.float32)
anals = np.zeros((len(datefcsts),nlats,nlons),np.float32)
if levels[0] is not None:
eclevels = (0.01*f.variables['lv_ISBL0'][:]).tolist()
nlevec = eclevels.index(level)
for ntime,date in enumerate(datefcsts):
dateverif = dateshift(date,int(fhr))
nt = dates.index(dateverif)
if varshort in ['chi','psi']:
if fhr=='0':
grbfile_1 =\
os.path.join(os.path.join(datapath1,date),runname1+'.t'+date[8:10]+'z.pgrbanl')
grbfile_2 =\
os.path.join(os.path.join(datapath2,date),runname2+'.t'+date[8:10]+'z.pgrbanl')
import numpy as np
import os, sys, dateutils
date1 = sys.argv[1] # date range
date2 = sys.argv[2]
datapath = sys.argv[3] # path to diag files.
runid = sys.argv[4] # suffix for diag file
hem = sys.argv[5] # NH,TR,SH,GL
outfile = sys.argv[6] # profile stats saved here
endian = 'little'
if len(sys.argv) > 7: # is diag file big endian?
endian = sys.argv[7]
sondesonly = False # use only 120,132,220,221,232 (sondes,pibals,drops)
# if sondesonly False, aircraft, pibals and surface data included also
dates = dateutils.daterange(date1,date2,6)
deltap = 50.; pbot = 975
nlevs = 23
levs = np.zeros(nlevs, np.float)
levs1 = np.zeros(nlevs, np.float)
levs2 = np.zeros(nlevs, np.float)
levs[0:18] = pbot - deltap*np.arange(18)
levs1[0:18] = levs[0:18] + 0.5*deltap
levs2[0:18] = levs[0:18] - 0.5*deltap
levs1[18] = levs2[17]
levs2[18] = 70.; levs1[19] = 70.
levs2[19] = 50.; levs1[20] = 50.
levs2[20] = 30.; levs1[21] = 30.
levs2[21] = 10.; levs1[22] = 10.
levs2[22] = 0.
levs1[0] = 1200.
# read netcdf FV3 history file, get grid info
nc = Dataset(filename_fv3)
lons_fv3 = nc['grid_xt'][:]
lats_fv3 = nc['grid_yt'][::-1,:] # flip so lats are increasing
nlats_fv3, nlons_fv3 = lons_fv3.shape
ak_fv3 = nc.ak[::-1]; bk_fv3 = nc.bk[::-1]
nlevs_fv3 = len(ak_fv3)-1
#for k in range(nlevs_fv3+1):
# print('%s %s %s' % (k,ak_fv3[k],bk_fv3[k]))
orog_fv3 = nc['hgtsfc'][:].squeeze()[::-1,:]
nc.close()
ak_ifs = None; lats_ifs = None
for date in daterange(date1,date2,hrinc):
print('processing data for %s ...' % date)
# read IFS orography and lnps from grib file.
# also get lats/lons
grbs=pygrib.open('ifsanl_2d_F400_%s.grib' % date) # orog
grb = grbs.next()
orog_ifs = grb.values[::-1,:]/grav # flip so lats are increasing
if lats_ifs is None:
lats_ifs, lons_ifs = grb.latlons()
lons_ifs = np.radians(lons_ifs)
lats_ifs = np.radians(lats_ifs[::-1,:])
nlats_ifs, nlons_ifs = lons_ifs.shape
grb = grbs.next()
ps_ifs = np.exp(grb.values[::-1,:]) # flip so lats are increasing
grbs.close()
for multiple leads, dates, and models. The python routine
generates the information that is later used to generate
fitted Gamma CDFs used for quantile mapping. This is an
alternative to the use of empirical CDFs in the
precip_forecast_CCPA_2netcdf folder.
"""
import numpy as np
from numpy import ma
import os, sys
from dateutils import daterange, dateshift
cmodels = ['NCEP', 'ECMWF', 'CMC'] # the models we seek to generate CDFs for
cleades = ['24', '48', '72', '96', '120', '144',
'168'] # ending forecast lead times
date_forecasts = daterange('2015120100', '2016070100', 24) # list of dates
fname = 'compute_singleday_gamma_stats.deck'
outfile = open(fname, 'w')
textstring = '#!/bin/tcsh \n'
outfile.write(textstring)
outfile.write(' \n')
for cmodel in cmodels:
for cleade in cleades:
outfile.write(' \n')
for date_forecast in date_forecasts:
textstring = 'python compute_singleday_gamma_stats.py ' + date_forecast + ' ' + cmodel + ' ' + cleade + '\n'
outfile.write(textstring)
outfile.close()
from get_quantiles_linear import get_quantiles_linear
from get_quantiles_linear_cmc import get_quantiles_linear_cmc
# --- queries from command line
center = sys.argv[1]
cleade = sys.argv[2]
date_end = sys.argv[3]
idaysbefore = -1 - int(int(cleade) / 24)
ihoursbefore = idaysbefore * 24
date_end_shift = dateshift(date_end, ihoursbefore)
date_begin_shift = dateshift(date_end_shift, -24 * 61)
# --- initialize stuff
date_list = daterange(date_begin_shift, date_end_shift, 24)
print 'date_list = ', date_list
ndates = len(date_list)
date_middle = date_list[ndates / 2]
cmonthno = date_middle[4:6]
imo = int(cmonthno) - 1
cmonths = [
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
'Nov', 'Dec'
]
cmonth = cmonths[imo]
if center == 'ECMWF':
nmembers = 50
else:
nmembers = 20
