vargrb = 'gh' varnc = 'h_plev' datapath1 = '/scratch3/BMC/gsienkf/whitaker/%s' % expt1 datapath2 = '/scratch3/BMC/gsienkf/whitaker/%s' % expt2 analpath = '/scratch3/BMC/gsienkf/whitaker/ecanl' if fhour > 9: dates = dateutils.daterange(date1, date2, 24) else: dates = dateutils.daterange(date1, date2, 6) ntime = None fcsterrspect1 = None fcsterrspect2 = None for date in dates: datev = dateutils.dateshift(date, fhour) # read analysis analfile = os.path.join(analpath, 'pgbanl.ecm.%s' % datev) grbs = pygrib.open(analfile) grb = grbs.select(shortName=vargrb, level=level)[0] verif_data = grb.values[::-1, :] grbs.close() if fhour > 9: fcstfile = '%s/%s/fv3longcontrol2_historyp_%s_latlon.nc' % (datapath1, date, date) else: fcstfile = '%s/%s/fv3control2_historyp_%s_latlon.nc' % (datapath1, date, date) nc = Dataset(fcstfile) if ntime is None: times = nc['time'][:].tolist()
ndays = ndaysomo[imonth] if iyear % 4 == 0 and imonth == 1: ndays = 29 for iday in range(1, ndays + 1): if iday < 10: cday = '0' + str(iday) else: cday = str(iday) cyyyymmddhh = cyear + cmonthsin[imonth] + cday + '00' if cleadb == '00': ishift1 = 6 ishift2 = 12 else: ishift1 = 18 ishift2 = 24 iyyyymmddhh_first = int(dateshift(cyyyymmddhh, ishift1)) iyyyymmddhh_second = int(dateshift(cyyyymmddhh, ishift2)) #print 'will load data for ', iyyyymmddhh_first, iyyyymmddhh_second itemindex_first = np.where( yyyymmddhh_anal_end == iyyyymmddhh_first) itemindex_second = np.where( yyyymmddhh_anal_end == iyyyymmddhh_second) #print 'itemindex_first = ',itemindex_first if itemindex_first[0] >= 0 and itemindex_second[0] >= 0: # ---- 12-hourly precip analysis is sum of the two 6-hourly analyses. apcp_anal = np.squeeze(nc.variables['apcp_anal'][itemindex_first[0],:,:] + \ nc.variables['apcp_anal'][itemindex_second[0],:,:])
import scipy.stats as stats from scipy.stats import gamma from matplotlib.backends.backend_pdf import PdfPages rcParams['legend.fontsize']='small' rcParams['legend.fancybox']=True rcParams['xtick.labelsize']='medium' # --- COMMAND LINE INPUT cmodel = 'NCEP' # sys.argv[1] # NCEP, ECMWF, or CMC cleade = '48' # sys.argv[2] # ending of periods lead time in hours, e.g., '24', '108' cempirical = '1' ileade = int(cleade) date_forecast = '2016050100' # sys.argv[3] date_end = dateshift(date_forecast,-ileade) date_begin = dateshift(date_end, -61*24) date_list = daterange(date_begin, date_end, 24) ndates = len(date_list) # ====================================================================== # Part 1: reading in the daily data and summing up the information over # all case days # ====================================================================== # ---- read in the array dimensions, precip values where gamma # distribution parameters and fraction zero are calculated, and # precipitation thresholds that are boundaries for light, # medium, and heavy precipitation from netcdf file. These
nclasses = 21 # 0 to 100% probability by 5% nxa = 464 # CCPA 1/8 degree grid over CONUS nya = 224 # # ---- read in precipitation analyses date_list_anal = daterange(cyyyymmddhh_start, cyyyymmddhh_end, 24) print 'date_list_anal = ', date_list_anal ndates = len(date_list_anal) apcp_anal_t = np.zeros((nxa, nya, ndates), dtype=np.float32) mninetynine = -99.99 * np.ones((nya, nxa), dtype=np.float32) for idate, date in zip(range(ndates), date_list_anal): print '------------------------- getting precipitation for idate = ', idate, date date_fearly = dateshift(date, ileade - 6) date_flate = dateshift(date, ileade) print 'date_fearly, date_flate = ', date_fearly, date_flate # --- read in the first of the two precip analysis files, 6 hourly infile = '/data/thamill/Rf2_tests/ccpa_v1/0.125d/ccpa.'+date_fearly[0:8]+\ '/18/ccpa.t18z.06h.0p125.conus.gb2' #print infile fexist1 = os.path.exists(infile) if fexist1: afile1 = pygrib.open(infile) grb1 = afile1.select( )[0] # --- read in the first of the two precip analysis files, 6 hourly infile2 = '/data/thamill/Rf2_tests/ccpa_v1/0.125d/ccpa.'+date_flate[0:8]+\
from mpl_toolkits.basemap import Basemap, interp from dateutils import hrstodate, daterange, dayofyear, \ splitdate, datetohrs, dateshift, dateto_hrs_since_day1CE from netCDF4 import Dataset import cPickle # --- Read input data from command line. Get beginning and ending lead time # in hours and the name of the file with the dates we want # --- queries from command line cyyyymmddhh = sys.argv[1] # the initial date/time of the forecast # ---- read in the associated analysis iyyyymmddhh_anal_end = int(dateshift(cyyyymmddhh,0)) iyyyymmddhh_anal_begin = int(dateshift(cyyyymmddhh,-6)) print 'iyyyymmddh_anal_end, iyyyymmddh_anal_begin = ',\ iyyyymmddhh_anal_end, iyyyymmddhh_anal_begin # ---- read in ccpa precip analysis data for this date # (12-h accum, generated from two 6-hourly files) infilename = '/Projects/Reforecast2/netcdf/NationalBlend/precip_analyses_ccpa_v1_'+\ '2002010100_to_2016123100.nc' print infilename nc = Dataset(infilename) yyyymmddhh_anal_list = nc.variables['yyyymmddhh_anal_end'][:] lons_anal = nc.variables['lons_anal'][:,:] lats_anal = nc.variables['lats_anal'][:,:] conusmask = nc.variables['conusmask'][:,:]
ndates = len(iyyyymmddhh_list) usethisdate = np.zeros((ndates),dtype=np.int16) for iyear in range(2002, 2017): # ---- find yyyymmddhh that is approximately at the center of the month if cleadtime_begin == '12' and cleadtime_end == '00': cyyyymmddhh_center = str(iyear) + cmono + '1500' elif cleadtime_begin == '00' and cleadtime_end == '12': cyyyymmddhh_center = str(iyear) + cmono + '1512' else: print 'invalid begin, end times for accumulation period. Stopping' sys.exit() cyyyymmddhh_begin = dateshift(cyyyymmddhh_center,-24*31) cyyyymmddhh_end = dateshift(cyyyymmddhh_center,24*31) date_list = daterange(cyyyymmddhh_begin, cyyyymmddhh_end, 24) # ---- go thru the list of dates that are in the data set, and # flag the elements that correspond to date_list. These # represent the indices for the first of two times we read in. for cdate in date_list: idate = int(cdate) idx = np.where(iyyyymmddhh_list == idate) if idx >= 0: usethisdate[idx] = 1 # ---- estimate the climatology from relative frequency for selected thresholds ktr = 0
'Nov', 'Dec' ] cmonth = cmonths[mm - 1] iyyyymmddhh = int(cyyyymmddhh) # ---- read in precipitation analysis filename = data_directory + 'precip_analyses_ccpa_v1_2002010100_to_2016123100.nc' print 'reading ', filename nc = Dataset(filename) lats_anal = nc.variables['lats_anal'][:] lons_anal = nc.variables['lons_anal'][:] nya, nxa = np.shape(lats_anal) iyyyymmddhh_list = nc.variables['yyyymmddhh_anal_end'][:] cyyyymmddhh_list = str(iyyyymmddhh_list) cdate_anal_late = dateshift(cyyyymmddhh, ileade) cdate_anal_early = dateshift(cyyyymmddhh, ileade - 6) idate_anal_late = int(cdate_anal_late) idate_anal_early = int(cdate_anal_early) idx_late = np.where(iyyyymmddhh_list == idate_anal_late)[0] idx_early = np.where(iyyyymmddhh_list == idate_anal_early)[0] print 'idx_late, idx_early = ', idx_late, idx_early apcp_anal = nc.variables['apcp_anal'][idx_late[0],:,:] + \ nc.variables['apcp_anal'][idx_early[0],:,:] mninetynine = -99.99 * np.ones((nya, nxa), dtype=np.float32) nc.close() # ---- read in the precipitation forecast probability at various # stages of its production infile = data_directory+cmodel+'/'+\
lon = linesplit[7] if lon[-1] == 'E': lon = 0.1 * float(lon[:-1]) else: lon = -0.1 * float(lon[:-1]) minp = int(linesplit[9]) fcstracks[storm][fhour] = (lat, lon, minp) except: pass #print 'fcst tracks' #print fcstracks # read observed tracks obtracks = makehash() for fhour in range(0, 168, 6): datev = dateshift(date, fhour) try: for line in open('/lfs1/projects/gfsenkf/tcvitals/tcvitals.%s.txt' % datev): linesplit = line.split() storm = linesplit[1].lstrip() lat = linesplit[5] lon = linesplit[6] if lat[-1] == 'N': lat = 0.1 * float(lat[:-1]) else: lat = -0.1 * float(lat[:-1]) if lon[-1] == 'E': lon = 0.1 * float(lon[:-1]) else: lon = -0.1 * float(lon[:-1])
infilename = data_directory + 'ccpa2p5.' + cyearmo + '.nc' # 1st day in 2002 may be corrupt nc = Dataset(infilename) yyyymmddhh_anal_end = nc.variables['anal_date_12h'][:] ndays = ndaysomo[imonth] if iyear % 4 == 0 and imonth == 1: ndays = 29 for iday in range(1, ndays + 1): if iday < 10: cday = '0' + str(iday) else: cday = str(iday) cyyyymmddhh = str(iyear) + cmonthsin[imonth] + cday + '00' if cleadb == '00': ishift = 12 else: ishift = 24 iyyyymmddhh = int(dateshift(cyyyymmddhh, ishift)) itemindex = np.where(yyyymmddhh_anal_end == iyyyymmddhh) #print 'reading precip data, index = ',itemindex[0] if itemindex[0] >= 0: # ---- 12-hourly precip analysis is sum of the two 6-hourly analyses. apcp_anal = np.squeeze( nc.variables['apcp_anal_12h'][itemindex[0], :, :]) if np.max(apcp_anal) >= 0: #print 'apcp_anal[61,1335] = ',apcp_anal[61,1335] sumx = np.where(apcp_anal * conusmask_in > 0, sumx + apcp_anal, sumx) sumlnx = np.where(apcp_anal * conusmask_in > 0, sumlnx + np.log(apcp_anal), sumlnx) npositive = np.where(apcp_anal * conusmask_in > 0,
center = sys.argv[2] # ECMWF, NCEP, or CMC currently cleade = sys.argv[3] # the ending lead time of the forecast in hours, e.g. 24 = 24h if center == 'CMCE' or center == 'CMC': exchangeable = False # need to tally the fcst stats member by member else: exchangeable = True # ok to tally fcst stats over all members ileade = int(cleade) ileadb = ileade - 12 print 'processing day = ',cyyyymmddhh,' center = ',center,' lead = ', cleade # ---- read in the associated analysis iyyyymmddhh_anal_end = int(dateshift(cyyyymmddhh,ileade)) iyyyymmddhh_anal_begin = int(dateshift(cyyyymmddhh,ileade-6)) # ---- read in ccpa precip analysis data for this date # (12-h accum, generated from two 6-hourly files) infilename = '/Projects/Reforecast2/netcdf/NationalBlend/precip_analyses_ccpa_v1_'+\ '2002010100_to_2016123100.nc' print infilename nc = Dataset(infilename) yyyymmddhh_anal_list = nc.variables['yyyymmddhh_anal_end'][:] lons_anal_in = nc.variables['lons_anal'][:,:] lats_anal_in = nc.variables['lats_anal'][:,:] conusmask_in = nc.variables['conusmask'][:,:] nya, nxa = lons_anal_in.shape
datapath2 = '/scratch2/BMC/gsienkf/whitaker/%s' % expt2 analpath = '/scratch2/BMC/gsienkf/whitaker/ecanl' if fhour > 9: dates = dateutils.daterange(date1, date2, 24) else: dates = dateutils.daterange(date1, date2, 12) ntime = None fcsterrspect1 = None fcsterrspect2 = None ncount = 0 err1 = 0 err2 = 0 for date in dates: datei = dateutils.dateshift(date, -fhour) # read analysis analfile = os.path.join(analpath, 'pgbanl.ecm.%s' % date) grbs = pygrib.open(analfile) grb = grbs.select(shortName=vargrb, level=level)[0] verif_data = grb.values[::-1, :] grbs.close() if fhour > 9: fcstfile = '%s/%s/fv3longcontrol2_historyp_%s_latlon.nc' % ( datapath1, date, datei) else: fcstfile = '%s/%s/fv3ensmean_historyp_%s_latlon.nc' % (datapath1, date, datei) nc = Dataset(fcstfile) if ntime is None: times = nc['time'][:].tolist()
hh = (d[2].split('(')[1]).split(':')[0] dates.append(yyyy+mm+dd+hh) if varshort not in ['psi','chi']: varin = f.variables[ecvarname] else: varinu = f.variables[ecvarnameu] varinv = f.variables[ecvarnamev] datefcsts = daterange(date1,date2,12) fcsts_2 = np.zeros((len(datefcsts),nlats,nlons),np.float32) fcsts_1 = np.zeros((len(datefcsts),nlats,nlons),np.float32) anals = np.zeros((len(datefcsts),nlats,nlons),np.float32) if levels[0] is not None: eclevels = (0.01*f.variables['lv_ISBL0'][:]).tolist() nlevec = eclevels.index(level) for ntime,date in enumerate(datefcsts): dateverif = dateshift(date,int(fhr)) nt = dates.index(dateverif) if varshort in ['chi','psi']: if fhr=='0': grbfile_1 =\ os.path.join(os.path.join(datapath1,date),runname1+'.t'+date[8:10]+'z.pgrbanl') grbfile_2 =\ os.path.join(os.path.join(datapath2,date),runname2+'.t'+date[8:10]+'z.pgrbanl') else: grbfile_1 =\ os.path.join(os.path.join(datapath1,date),runname1+'.t'+date[8:10]+'z.pgrbf'+fhr) grbfile_2 =\ os.path.join(os.path.join(datapath2,date),runname2+'.t'+date[8:10]+'z.pgrbf'+fhr) f2 = nio.open_file(grbfile_2+'.grib') levels = f2.variables['lv_ISBL3'][:].tolist() nlev = levels.index(level)
print 'in plotens_FIM_GFS.py: nmembers: ', nmembers print 'in plotens_FIM_GFS.py: nanals: ', nanals ifhr = int(fhr) ifhrinc = 6 print 'in plotens_FIM_GFS.py: fhr: ', fhr print 'in plotens_FIM_GFS.py: ifhr: ', ifhr expname = os.getenv('EXPT') if expname is None: expname = 'hybda_realtime2014' hr = date[8:10] #datapath = '/lfs1/projects/rtfim/tcvitals' run = 'gdas1' datapathcv = '/lfs1/projects/gfsenkf/bufr/%s/' % run yyyy, mm, dd, hh = splitdate(date) julday = dayofyear(yyyy, mm, dd) + 1 datev = dateshift(date, int(fhr)) domains = [] tclats = makehash() tclons = makehash() directories = makehash() tcvitals2 =\ datapathcv+'/bufr_%s/gdas1.t%02iz.syndata.tcvitals.tm00'%\ (date,hh) tcvitals1 = '/pan2/projects/fim-njet/tcvitals/tcvitals.%s.txt' % date if os.path.exists(tcvitals1): tcvitals = tcvitals1 else: tcvitals = tcvitals2
rcParams['xtick.labelsize']='x-small' # for colorbar tick labels rcParams['axes.labelsize']='small' # for colorbar wording cmonths = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec'] # --- from command line get the initial date and forecast lead time in hours cyyyymmddhh_init = sys.argv[1] # YYYYMMDDHH format of initial forecast date iyyyymmddhh_init = int(cyyyymmddhh_init) cleadb = sys.argv[2] # enter beginning lead time in hours cleade = sys.argv[3] # enter endlead time in hours cthresh = sys.argv[4] # threshold values must take one from below: # 'POP', '1mm', '2p5mm', '5mm', '10mm', '25mm', '50mm', 'q50', 'q67', 'q80', 'q90', 'q95' csupp = sys.argv[5] # number of supplemental locations, 01 to 20 ilead = int(cleade) cyyyymmddhh_fcst = dateshift(cyyyymmddhh_init,ilead) iday = ilead/24 clead_dayb = str(int(cleadb)/24) clead_daye = str(iday) cmonth = cyyyymmddhh_init[4:6] imonth = int(cmonth) cmonthname = cmonths[imonth-1] # --- set up inputs and constants input_data_directory = '/Projects/Reforecast2/netcdf' # where input CCPA and reforecast data stored analog_data_directory = '/Rf2_tests/ccpa' # where analog forecast output was stored nxa = 464 # ccpa grid dimensions nya = 224 nxf = 128 # reforecast grid surrounding CONUS nyf = 61
from ecmwfapi import ECMWFService import datetime, sys from dateutils import dateshift, splitdate # date to retrieve date = sys.argv[1] dateref = dateshift(date, 12) # wait till 12-h after if dateref > datetime.datetime.utcnow().strftime('%Y%m%d%H'): raise ValueError('data for %s not yet available' % date) else: print "retrieving for %s" % date # number of vertical levels. nlevs = 137 levelist = [] for nlev in range(1, nlevs + 1): levelist.append('%s/' % nlev) levelist137 = ''.join(levelist)[:-1] # output grid #grid = '0.075/0.075' # lat/lon spacing grid = 'F640' # 1280 latitude gaussian grid #parameters="phis/lnps/tmp/spfh/ugrd/vgrd/clwmr/icwmr/rnwmr/snwmr/vvel" #parameters="129/152/130/133/131/132/246/247/75/76/135" # phis/lnps/tmp/spfh/ugrd/vgrd/o3mr parameters = "129/152/130/133/131/132/203" # get data one date at a time, all fields at once. server = ECMWFService('mars')
import time as timey from get_cdf_precip_anal import get_cdf_precip_anal from get_cdf_cmc import get_cdf_cmc from get_cdf_ecmwf_ncep import get_cdf_ecmwf_ncep from get_quantiles_linear import get_quantiles_linear from get_quantiles_linear_cmc import get_quantiles_linear_cmc # --- queries from command line center = sys.argv[1] cleade = sys.argv[2] date_end = sys.argv[3] idaysbefore = -1 - int(int(cleade) / 24) ihoursbefore = idaysbefore * 24 date_end_shift = dateshift(date_end, ihoursbefore) date_begin_shift = dateshift(date_end_shift, -24 * 61) # --- initialize stuff date_list = daterange(date_begin_shift, date_end_shift, 24) print 'date_list = ', date_list ndates = len(date_list) date_middle = date_list[ndates / 2] cmonthno = date_middle[4:6] imo = int(cmonthno) - 1 cmonths = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ] cmonth = cmonths[imo]