vargrb = 'gh'
varnc = 'h_plev'
datapath1 = '/scratch3/BMC/gsienkf/whitaker/%s' % expt1
datapath2 = '/scratch3/BMC/gsienkf/whitaker/%s' % expt2
analpath = '/scratch3/BMC/gsienkf/whitaker/ecanl'
if fhour > 9:
dates = dateutils.daterange(date1, date2, 24)
else:
dates = dateutils.daterange(date1, date2, 6)
ntime = None
fcsterrspect1 = None
fcsterrspect2 = None
for date in dates:
datev = dateutils.dateshift(date, fhour)
# read analysis
analfile = os.path.join(analpath, 'pgbanl.ecm.%s' % datev)
grbs = pygrib.open(analfile)
grb = grbs.select(shortName=vargrb, level=level)[0]
verif_data = grb.values[::-1, :]
grbs.close()
if fhour > 9:
fcstfile = '%s/%s/fv3longcontrol2_historyp_%s_latlon.nc' % (datapath1,
date, date)
else:
fcstfile = '%s/%s/fv3control2_historyp_%s_latlon.nc' % (datapath1,
date, date)
nc = Dataset(fcstfile)
if ntime is None:
times = nc['time'][:].tolist()
ndays = ndaysomo[imonth]
if iyear % 4 == 0 and imonth == 1: ndays = 29
for iday in range(1, ndays + 1):
if iday < 10:
cday = '0' + str(iday)
else:
cday = str(iday)
cyyyymmddhh = cyear + cmonthsin[imonth] + cday + '00'
if cleadb == '00':
ishift1 = 6
ishift2 = 12
else:
ishift1 = 18
ishift2 = 24
iyyyymmddhh_first = int(dateshift(cyyyymmddhh, ishift1))
iyyyymmddhh_second = int(dateshift(cyyyymmddhh, ishift2))
#print 'will load data for ', iyyyymmddhh_first, iyyyymmddhh_second
itemindex_first = np.where(
yyyymmddhh_anal_end == iyyyymmddhh_first)
itemindex_second = np.where(
yyyymmddhh_anal_end == iyyyymmddhh_second)
#print 'itemindex_first = ',itemindex_first
if itemindex_first[0] >= 0 and itemindex_second[0] >= 0:
# ---- 12-hourly precip analysis is sum of the two 6-hourly analyses.
apcp_anal = np.squeeze(nc.variables['apcp_anal'][itemindex_first[0],:,:] + \
nc.variables['apcp_anal'][itemindex_second[0],:,:])
import scipy.stats as stats from scipy.stats import gamma from matplotlib.backends.backend_pdf import PdfPages rcParams['legend.fontsize']='small' rcParams['legend.fancybox']=True rcParams['xtick.labelsize']='medium' # --- COMMAND LINE INPUT cmodel = 'NCEP' # sys.argv[1] # NCEP, ECMWF, or CMC cleade = '48' # sys.argv[2] # ending of periods lead time in hours, e.g., '24', '108' cempirical = '1' ileade = int(cleade) date_forecast = '2016050100' # sys.argv[3] date_end = dateshift(date_forecast,-ileade) date_begin = dateshift(date_end, -61*24) date_list = daterange(date_begin, date_end, 24) ndates = len(date_list) # ====================================================================== # Part 1: reading in the daily data and summing up the information over # all case days # ====================================================================== # ---- read in the array dimensions, precip values where gamma # distribution parameters and fraction zero are calculated, and # precipitation thresholds that are boundaries for light, # medium, and heavy precipitation from netcdf file. These
nclasses = 21 # 0 to 100% probability by 5%
nxa = 464 # CCPA 1/8 degree grid over CONUS
nya = 224 #
# ---- read in precipitation analyses
date_list_anal = daterange(cyyyymmddhh_start, cyyyymmddhh_end, 24)
print 'date_list_anal = ', date_list_anal
ndates = len(date_list_anal)
apcp_anal_t = np.zeros((nxa, nya, ndates), dtype=np.float32)
mninetynine = -99.99 * np.ones((nya, nxa), dtype=np.float32)
for idate, date in zip(range(ndates), date_list_anal):
print '------------------------- getting precipitation for idate = ', idate, date
date_fearly = dateshift(date, ileade - 6)
date_flate = dateshift(date, ileade)
print 'date_fearly, date_flate = ', date_fearly, date_flate
# --- read in the first of the two precip analysis files, 6 hourly
infile = '/data/thamill/Rf2_tests/ccpa_v1/0.125d/ccpa.'+date_fearly[0:8]+\
'/18/ccpa.t18z.06h.0p125.conus.gb2'
#print infile
fexist1 = os.path.exists(infile)
if fexist1:
afile1 = pygrib.open(infile)
grb1 = afile1.select(
)[0] # --- read in the first of the two precip analysis files, 6 hourly
infile2 = '/data/thamill/Rf2_tests/ccpa_v1/0.125d/ccpa.'+date_flate[0:8]+\
from mpl_toolkits.basemap import Basemap, interp
from dateutils import hrstodate, daterange, dayofyear, \
splitdate, datetohrs, dateshift, dateto_hrs_since_day1CE
from netCDF4 import Dataset
import cPickle
# --- Read input data from command line. Get beginning and ending lead time
# in hours and the name of the file with the dates we want
# --- queries from command line
cyyyymmddhh = sys.argv[1] # the initial date/time of the forecast
# ---- read in the associated analysis
iyyyymmddhh_anal_end = int(dateshift(cyyyymmddhh,0))
iyyyymmddhh_anal_begin = int(dateshift(cyyyymmddhh,-6))
print 'iyyyymmddh_anal_end, iyyyymmddh_anal_begin = ',\
iyyyymmddhh_anal_end, iyyyymmddhh_anal_begin
# ---- read in ccpa precip analysis data for this date
# (12-h accum, generated from two 6-hourly files)
infilename = '/Projects/Reforecast2/netcdf/NationalBlend/precip_analyses_ccpa_v1_'+\
'2002010100_to_2016123100.nc'
print infilename
nc = Dataset(infilename)
yyyymmddhh_anal_list = nc.variables['yyyymmddhh_anal_end'][:]
lons_anal = nc.variables['lons_anal'][:,:]
lats_anal = nc.variables['lats_anal'][:,:]
conusmask = nc.variables['conusmask'][:,:]
ndates = len(iyyyymmddhh_list)
usethisdate = np.zeros((ndates),dtype=np.int16)
for iyear in range(2002, 2017):
# ---- find yyyymmddhh that is approximately at the center of the month
if cleadtime_begin == '12' and cleadtime_end == '00':
cyyyymmddhh_center = str(iyear) + cmono + '1500'
elif cleadtime_begin == '00' and cleadtime_end == '12':
cyyyymmddhh_center = str(iyear) + cmono + '1512'
else:
print 'invalid begin, end times for accumulation period. Stopping'
sys.exit()
cyyyymmddhh_begin = dateshift(cyyyymmddhh_center,-24*31)
cyyyymmddhh_end = dateshift(cyyyymmddhh_center,24*31)
date_list = daterange(cyyyymmddhh_begin, cyyyymmddhh_end, 24)
# ---- go thru the list of dates that are in the data set, and
# flag the elements that correspond to date_list. These
# represent the indices for the first of two times we read in.
for cdate in date_list:
idate = int(cdate)
idx = np.where(iyyyymmddhh_list == idate)
if idx >= 0: usethisdate[idx] = 1
# ---- estimate the climatology from relative frequency for selected thresholds
ktr = 0
'Nov', 'Dec'
]
cmonth = cmonths[mm - 1]
iyyyymmddhh = int(cyyyymmddhh)
# ---- read in precipitation analysis
filename = data_directory + 'precip_analyses_ccpa_v1_2002010100_to_2016123100.nc'
print 'reading ', filename
nc = Dataset(filename)
lats_anal = nc.variables['lats_anal'][:]
lons_anal = nc.variables['lons_anal'][:]
nya, nxa = np.shape(lats_anal)
iyyyymmddhh_list = nc.variables['yyyymmddhh_anal_end'][:]
cyyyymmddhh_list = str(iyyyymmddhh_list)
cdate_anal_late = dateshift(cyyyymmddhh, ileade)
cdate_anal_early = dateshift(cyyyymmddhh, ileade - 6)
idate_anal_late = int(cdate_anal_late)
idate_anal_early = int(cdate_anal_early)
idx_late = np.where(iyyyymmddhh_list == idate_anal_late)[0]
idx_early = np.where(iyyyymmddhh_list == idate_anal_early)[0]
print 'idx_late, idx_early = ', idx_late, idx_early
apcp_anal = nc.variables['apcp_anal'][idx_late[0],:,:] + \
nc.variables['apcp_anal'][idx_early[0],:,:]
mninetynine = -99.99 * np.ones((nya, nxa), dtype=np.float32)
nc.close()
# ---- read in the precipitation forecast probability at various
# stages of its production
infile = data_directory+cmodel+'/'+\
lon = linesplit[7]
if lon[-1] == 'E':
lon = 0.1 * float(lon[:-1])
else:
lon = -0.1 * float(lon[:-1])
minp = int(linesplit[9])
fcstracks[storm][fhour] = (lat, lon, minp)
except:
pass
#print 'fcst tracks'
#print fcstracks
# read observed tracks
obtracks = makehash()
for fhour in range(0, 168, 6):
datev = dateshift(date, fhour)
try:
for line in open('/lfs1/projects/gfsenkf/tcvitals/tcvitals.%s.txt' %
datev):
linesplit = line.split()
storm = linesplit[1].lstrip()
lat = linesplit[5]
lon = linesplit[6]
if lat[-1] == 'N':
lat = 0.1 * float(lat[:-1])
else:
lat = -0.1 * float(lat[:-1])
if lon[-1] == 'E':
lon = 0.1 * float(lon[:-1])
else:
lon = -0.1 * float(lon[:-1])
infilename = data_directory + 'ccpa2p5.' + cyearmo + '.nc' # 1st day in 2002 may be corrupt
nc = Dataset(infilename)
yyyymmddhh_anal_end = nc.variables['anal_date_12h'][:]
ndays = ndaysomo[imonth]
if iyear % 4 == 0 and imonth == 1: ndays = 29
for iday in range(1, ndays + 1):
if iday < 10:
cday = '0' + str(iday)
else:
cday = str(iday)
cyyyymmddhh = str(iyear) + cmonthsin[imonth] + cday + '00'
if cleadb == '00':
ishift = 12
else:
ishift = 24
iyyyymmddhh = int(dateshift(cyyyymmddhh, ishift))
itemindex = np.where(yyyymmddhh_anal_end == iyyyymmddhh)
#print 'reading precip data, index = ',itemindex[0]
if itemindex[0] >= 0:
# ---- 12-hourly precip analysis is sum of the two 6-hourly analyses.
apcp_anal = np.squeeze(
nc.variables['apcp_anal_12h'][itemindex[0], :, :])
if np.max(apcp_anal) >= 0:
#print 'apcp_anal[61,1335] = ',apcp_anal[61,1335]
sumx = np.where(apcp_anal * conusmask_in > 0,
sumx + apcp_anal, sumx)
sumlnx = np.where(apcp_anal * conusmask_in > 0,
sumlnx + np.log(apcp_anal), sumlnx)
npositive = np.where(apcp_anal * conusmask_in > 0,
center = sys.argv[2] # ECMWF, NCEP, or CMC currently
cleade = sys.argv[3] # the ending lead time of the forecast in hours, e.g. 24 = 24h
if center == 'CMCE' or center == 'CMC':
exchangeable = False # need to tally the fcst stats member by member
else:
exchangeable = True # ok to tally fcst stats over all members
ileade = int(cleade)
ileadb = ileade - 12
print 'processing day = ',cyyyymmddhh,' center = ',center,' lead = ', cleade
# ---- read in the associated analysis
iyyyymmddhh_anal_end = int(dateshift(cyyyymmddhh,ileade))
iyyyymmddhh_anal_begin = int(dateshift(cyyyymmddhh,ileade-6))
# ---- read in ccpa precip analysis data for this date
# (12-h accum, generated from two 6-hourly files)
infilename = '/Projects/Reforecast2/netcdf/NationalBlend/precip_analyses_ccpa_v1_'+\
'2002010100_to_2016123100.nc'
print infilename
nc = Dataset(infilename)
yyyymmddhh_anal_list = nc.variables['yyyymmddhh_anal_end'][:]
lons_anal_in = nc.variables['lons_anal'][:,:]
lats_anal_in = nc.variables['lats_anal'][:,:]
conusmask_in = nc.variables['conusmask'][:,:]
nya, nxa = lons_anal_in.shape
datapath2 = '/scratch2/BMC/gsienkf/whitaker/%s' % expt2
analpath = '/scratch2/BMC/gsienkf/whitaker/ecanl'
if fhour > 9:
dates = dateutils.daterange(date1, date2, 24)
else:
dates = dateutils.daterange(date1, date2, 12)
ntime = None
fcsterrspect1 = None
fcsterrspect2 = None
ncount = 0
err1 = 0
err2 = 0
for date in dates:
datei = dateutils.dateshift(date, -fhour)
# read analysis
analfile = os.path.join(analpath, 'pgbanl.ecm.%s' % date)
grbs = pygrib.open(analfile)
grb = grbs.select(shortName=vargrb, level=level)[0]
verif_data = grb.values[::-1, :]
grbs.close()
if fhour > 9:
fcstfile = '%s/%s/fv3longcontrol2_historyp_%s_latlon.nc' % (
datapath1, date, datei)
else:
fcstfile = '%s/%s/fv3ensmean_historyp_%s_latlon.nc' % (datapath1, date,
datei)
nc = Dataset(fcstfile)
if ntime is None:
times = nc['time'][:].tolist()
hh = (d[2].split('(')[1]).split(':')[0]
dates.append(yyyy+mm+dd+hh)
if varshort not in ['psi','chi']:
varin = f.variables[ecvarname]
else:
varinu = f.variables[ecvarnameu]
varinv = f.variables[ecvarnamev]
datefcsts = daterange(date1,date2,12)
fcsts_2 = np.zeros((len(datefcsts),nlats,nlons),np.float32)
fcsts_1 = np.zeros((len(datefcsts),nlats,nlons),np.float32)
anals = np.zeros((len(datefcsts),nlats,nlons),np.float32)
if levels[0] is not None:
eclevels = (0.01*f.variables['lv_ISBL0'][:]).tolist()
nlevec = eclevels.index(level)
for ntime,date in enumerate(datefcsts):
dateverif = dateshift(date,int(fhr))
nt = dates.index(dateverif)
if varshort in ['chi','psi']:
if fhr=='0':
grbfile_1 =\
os.path.join(os.path.join(datapath1,date),runname1+'.t'+date[8:10]+'z.pgrbanl')
grbfile_2 =\
os.path.join(os.path.join(datapath2,date),runname2+'.t'+date[8:10]+'z.pgrbanl')
else:
grbfile_1 =\
os.path.join(os.path.join(datapath1,date),runname1+'.t'+date[8:10]+'z.pgrbf'+fhr)
grbfile_2 =\
os.path.join(os.path.join(datapath2,date),runname2+'.t'+date[8:10]+'z.pgrbf'+fhr)
f2 = nio.open_file(grbfile_2+'.grib')
levels = f2.variables['lv_ISBL3'][:].tolist()
nlev = levels.index(level)
print 'in plotens_FIM_GFS.py: nmembers: ', nmembers
print 'in plotens_FIM_GFS.py: nanals: ', nanals
ifhr = int(fhr)
ifhrinc = 6
print 'in plotens_FIM_GFS.py: fhr: ', fhr
print 'in plotens_FIM_GFS.py: ifhr: ', ifhr
expname = os.getenv('EXPT')
if expname is None:
expname = 'hybda_realtime2014'
hr = date[8:10]
#datapath = '/lfs1/projects/rtfim/tcvitals'
run = 'gdas1'
datapathcv = '/lfs1/projects/gfsenkf/bufr/%s/' % run
yyyy, mm, dd, hh = splitdate(date)
julday = dayofyear(yyyy, mm, dd) + 1
datev = dateshift(date, int(fhr))
domains = []
tclats = makehash()
tclons = makehash()
directories = makehash()
tcvitals2 =\
datapathcv+'/bufr_%s/gdas1.t%02iz.syndata.tcvitals.tm00'%\
(date,hh)
tcvitals1 = '/pan2/projects/fim-njet/tcvitals/tcvitals.%s.txt' % date
if os.path.exists(tcvitals1):
tcvitals = tcvitals1
else:
tcvitals = tcvitals2
rcParams['xtick.labelsize']='x-small' # for colorbar tick labels
rcParams['axes.labelsize']='small' # for colorbar wording
cmonths = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
# --- from command line get the initial date and forecast lead time in hours
cyyyymmddhh_init = sys.argv[1] # YYYYMMDDHH format of initial forecast date
iyyyymmddhh_init = int(cyyyymmddhh_init)
cleadb = sys.argv[2] # enter beginning lead time in hours
cleade = sys.argv[3] # enter endlead time in hours
cthresh = sys.argv[4] # threshold values must take one from below:
# 'POP', '1mm', '2p5mm', '5mm', '10mm', '25mm', '50mm', 'q50', 'q67', 'q80', 'q90', 'q95'
csupp = sys.argv[5] # number of supplemental locations, 01 to 20
ilead = int(cleade)
cyyyymmddhh_fcst = dateshift(cyyyymmddhh_init,ilead)
iday = ilead/24
clead_dayb = str(int(cleadb)/24)
clead_daye = str(iday)
cmonth = cyyyymmddhh_init[4:6]
imonth = int(cmonth)
cmonthname = cmonths[imonth-1]
# --- set up inputs and constants
input_data_directory = '/Projects/Reforecast2/netcdf' # where input CCPA and reforecast data stored
analog_data_directory = '/Rf2_tests/ccpa' # where analog forecast output was stored
nxa = 464 # ccpa grid dimensions
nya = 224
nxf = 128 # reforecast grid surrounding CONUS
nyf = 61
from ecmwfapi import ECMWFService
import datetime, sys
from dateutils import dateshift, splitdate
# date to retrieve
date = sys.argv[1]
dateref = dateshift(date, 12) # wait till 12-h after
if dateref > datetime.datetime.utcnow().strftime('%Y%m%d%H'):
raise ValueError('data for %s not yet available' % date)
else:
print "retrieving for %s" % date
# number of vertical levels.
nlevs = 137
levelist = []
for nlev in range(1, nlevs + 1):
levelist.append('%s/' % nlev)
levelist137 = ''.join(levelist)[:-1]
# output grid
#grid = '0.075/0.075' # lat/lon spacing
grid = 'F640' # 1280 latitude gaussian grid
#parameters="phis/lnps/tmp/spfh/ugrd/vgrd/clwmr/icwmr/rnwmr/snwmr/vvel"
#parameters="129/152/130/133/131/132/246/247/75/76/135"
# phis/lnps/tmp/spfh/ugrd/vgrd/o3mr
parameters = "129/152/130/133/131/132/203"
# get data one date at a time, all fields at once.
server = ECMWFService('mars')
import time as timey
from get_cdf_precip_anal import get_cdf_precip_anal
from get_cdf_cmc import get_cdf_cmc
from get_cdf_ecmwf_ncep import get_cdf_ecmwf_ncep
from get_quantiles_linear import get_quantiles_linear
from get_quantiles_linear_cmc import get_quantiles_linear_cmc
# --- queries from command line
center = sys.argv[1]
cleade = sys.argv[2]
date_end = sys.argv[3]
idaysbefore = -1 - int(int(cleade) / 24)
ihoursbefore = idaysbefore * 24
date_end_shift = dateshift(date_end, ihoursbefore)
date_begin_shift = dateshift(date_end_shift, -24 * 61)
# --- initialize stuff
date_list = daterange(date_begin_shift, date_end_shift, 24)
print 'date_list = ', date_list
ndates = len(date_list)
date_middle = date_list[ndates / 2]
cmonthno = date_middle[4:6]
imo = int(cmonthno) - 1
cmonths = [
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct',
'Nov', 'Dec'
]
cmonth = cmonths[imo]
