def poolStatPDFs(paramDict):
"""
...This should be used with
the map() method in the multiprocessing library (Pool object)
Call
from multiprocessing import Pool, cpu_count, Process
p = Pool(nCores)
compareDat = prepForPool(ref, test)
p.map(poolStatPDFs, compareDat)
Input
paramDict -- dictionary, output from prepForPool
Output
PDF...
"""
# unpack dictionary
plotVars = paramDict['plotVars']
outFile = paramDict['pdf']
wnRange = paramDict['wnRange']
band = paramDict['band']
nCol = paramDict['numberCol']
shortWave = paramDict['sw']
sys.exit('This needs a lot of work')
rrtmgp.statPDF(refFile, testFile, singlePDF=True, \
tPauseP=paramDict['tPauseP'], xTitle=xt, yTitle=yt, \
prefix=statPrefix, atmType=aType, statCSV=statCSV)
return
def plotStat(self):
"""
Plot test and reference flux and HR statistics (max diff, RMS
diff, diff spread, troposphere/stratosphere)
"""
COMPARE.statPDF(self.refFile, self.testFile, singlePDF=True, \
xTitle=self.xTitle, yTitle=self.yTitle, forcing=self.forcing, \
prefix=self.statPrefix, atmType=self.atmType)
tmpPDF = '%s_all_bands.pdf' % self.statPrefix
os.rename(tmpPDF, '%s/%s' % (self.outDir, tmpPDF))
def plotProf(self):
"""
Plot test and reference flux and heating rate profiles and the
test-ref differences
"""
if self.broadOnly:
# broadband plotting
COMPARE.profPDFs(self.refFile, self.testFile, self.yTitle, \
prefix=self.profPrefix, atmType=self.atmType, \
inBand=None, yLog=self.yLog, broadOnly=True)
tmpPDF = '%s_broadband.pdf' % self.profPrefix
os.rename(tmpPDF, '%s/%s' % (self.outDir, tmpPDF))
else:
# by-band plotting
for iBand in self.bands:
COMPARE.profPDFs(self.refFile, self.testFile, self.yTitle, \
prefix=self.profPrefix, atmType=self.atmType, \
inBand=iBand, yLog=self.yLog)
tmpPDF = '%s_%02d.pdf' % (self.profPrefix, iBand + 1)
os.rename(tmpPDF, '%s/%s' % (self.outDir, tmpPDF))
def parseConfig(inFile):
"""
Read configuration file input into main() and extract necessary
information from it
Input
inFile -- string, full path to input configuration file
Output
dictionary with the following keys:
ref -- string, full path to reference model netCDF (LBLRTM)
test -- string, full path to test model netCDF (RRTMGP)
stat -- string, prefix for stats PDF file
prof -- string, prefix for profiles PDF file
x -- string, x-axis label
y -- string, y-axis label
atm -- string, atmosphere type (e.g., Garand)
forcing -- boolean, are forcing files being compared?
"""
# extract metadata from configuration file
cParse = configparser.ConfigParser()
cParse.read(inFile)
cRefName = cParse.get('Plot Params', 'reference_model')
cRefMD = cParse.get('Plot Params', 'reference_description')
cTestMD = cParse.get('Plot Params', 'test_description')
cTestName = cParse.get('Plot Params', 'test_model')
aType = cParse.get('Plot Params', 'atmosphere')
# flux files
refFile = cParse.get('Filename Params', 'reference_path')
testFile = cParse.get('Filename Params', 'test_path')
utils.file_check(refFile); utils.file_check(testFile)
# forcing files
rForceFile = \
cParse.get('Filename Params', 'reference_force_path')
tForceFile = cParse.get('Filename Params', 'test_force_path')
if len(rForceFile) > 0 and len(tForceFile) > 0:
forcing = True
else:
forcing = False
if forcing:
utils.file_check(rForceFile); utils.file_check(tForceFile)
refFile, testFile = compare.forcingDiff(refFile, testFile, \
rForceFile, tForceFile, \
repVars=['flux_dir_dn', 'band_flux_dir_dn'])
cRefForceName = \
cParse.get('Plot Params', 'reference_forcing_model')
cRefForceMD = cParse.get('Plot Params', 'reference_description')
cTestForceName = cParse.get('Plot Params', 'test_forcing_model')
cTestForceMD = cParse.get('Plot Params', 'test_description')
xt = cRefForceName
yt = '%s - %s' % (cTestForceName, cRefForceName)
else:
xt = cRefName
yt = '%s - %s' % (cTestName, cRefName)
# endif forcing
statPrefix = cParse.get('Filename Params', 'stats_prefix')
profPrefix = cParse.get('Filename Params', 'profiles_prefix')
return {'ref': refFile, 'test': testFile, 'x': xt, 'y': yt, \
'atm': aType, 'stat': statPrefix, 'prof': profPrefix, \
'forcing': forcing}
def varSetup(ref, test, diffuse=False, broadband=False, charts=False):
"""
Extract the variables we need for further calculation and plotting
Input
ref -- string, path to reference netCDF file
test -- string, path to test netCDF file
Output
refDict -- dictionary with reference values for flux (direct or
diffuse), heating rate, pressure (level and layer), and TSI
(total solar irradiance)
testDict -- like refDict, but with test values
numBands -- int, number of SW bands
numLev -- int, number of levels per profile
numProf -- int, number of columns per band
plotVars -- string list of ordinate variables to be plotted
fluxPlotLab -- string, used for flux (direct or diffuse) plot
labels; one of either [DIRSTR, DIFSTR] global variables
fluxFileStr -- string, either "direct" or "diffuse"; used in
output PDF files
Keywords
diffuse -- boolean, calculate differences in diffuse flux rather
than direct beam
broadband -- boolean, extract and plot broadband parameters
charts -- boolean, extract parameters for CHARTS plots
"""
if broadband:
dnParam = 'flux_dif_dn' if diffuse else 'flux_dir_dn'
else:
dnParam = 'band_flux_dif_dn' if diffuse else 'band_flux_dir_dn'
# endif broadband
# get the output for plotting
if broadband:
if charts:
plotVars = ['flux_dn', 'flux_dir_dn', 'flux_up', 'flux_dif_dn']
else:
plotVars = [dnParam, 'flux_up', 'heating_rate']
# endif charts
else:
if charts:
plotVars = ['band_flux_dn', 'band_flux_dir_dn', \
'band_flux_up', 'band_flux_dif_dn']
else:
plotVars = [dnParam, 'band_flux_up', 'band_heating_rate']
# endif charts
# endif broadband
plotVars += ['']
plotVars += ['p_lay', 'p_lev', 'band_lims_wvn', \
'total_solar_irradiance']
refDict = compare.getVars(ref, attrList=plotVars)
testDict = compare.getVars(test, attrList=plotVars)
#refDict['p_lev'] *= -1
# just for now, until we correct LBLRTM pressure levels
#refDict['p_lev'] = np.array(testDict['p_lev'])
iNegRow, iNegCol = np.where(refDict['p_lev'] < 0)
if iNegRow.size != 0:
refDict['p_lev'][iNegRow, iNegCol] = \
testDict['p_lev'][iNegRow, iNegCol]
print('Replacing LBLRTM pressures with RRTMGP pressures ' + \
'for (lev, profile): ')
for row, col in zip(iNegRow, iNegCol): print(row, col)
# endif iNeg
# for SW, should only be plotting fluxes and heating rate
# WARNING: for the rest of the module, it is assumed that this
# order/convention (heating rate, down flux, up flux) is followed
nVar = 4 if charts else 3
plotVars = plotVars[:nVar]
# only because i had insane RRTMGP band_flux_dif_dn values (inf)
if charts:
testDict[plotVars[3]] = testDict[plotVars[0]] - testDict[plotVars[1]]
# endif charts
# some quality control (consistency check)
# also some perturbations to test file for script verification
for dum in plotVars:
if refDict[dum].shape != testDict[dum].shape:
errMsg = 'Returning: '
errMsg += '%s and %s do not have equal dimensions for %s' % \
(ref, test, dum)
sys.exit(errMsg)
# endif
# for SW, had to introduce some pertubations to the test files so
# we could see test-ref differences
# doing a Gaussian noise distribution (mu=0, sigma=1)
# testDict[dum] += np.random.normal(size=testDict[dum].shape)
# end QC loop
# grab dimensions of flux variables (HR is on layers, not levels)
varShape = refDict[plotVars[0]].shape
if broadband:
numLev = varShape[0]; numProf = varShape[1]
numBands = 1
else:
numLev = varShape[0]; numProf = varShape[1]
numBands = varShape[2]
# endif broadband
fluxPlotLab = str(DIFSTR) if diffuse else str(DIRSTR)
fluxFileStr = 'diffuse' if diffuse else 'direct'
if charts: fluxFileStr = 'both'
return refDict, testDict, numBands, numLev, numProf, plotVars, \
fluxPlotLab, fluxFileStr
def readConfig(self):
"""
Read in the configuration file parameters that will be used with
the COMPARE module
Kinda works like a second constructor
"""
# all of the valid affirmative entries for booleans in .ini file
yes = ['y', 'ye', 'yes', 't', 'tr', 'tru', 'true']
print('Reading %s' % self.iniFile)
cParse = ConfigParser.ConfigParser()
cParse.read(self.iniFile)
self.bandAvg = cParse.get('Computation', 'band_average')
self.doPWV = cParse.get('Computation', 'pwv')
self.refName = cParse.get('Plot Params', 'reference_model')
self.refDesc = cParse.get('Plot Params', 'reference_description')
self.testName = cParse.get('Plot Params', 'test_model')
self.TestDesc = cParse.get('Plot Params', 'test_description')
self.atmType = cParse.get('Plot Params', 'atmosphere')
self.yLog = cParse.get('Plot Params', 'log')
self.bands = cParse.get('Plot Params', 'bands')
self.doStats = cParse.get('Plot Params', 'stats_plots')
self.doProfs = cParse.get('Plot Params', 'prof_plots')
self.refFile = cParse.get('Filename Params', 'reference_path')
self.testFile = cParse.get('Filename Params', 'test_path')
self.profPrefix = cParse.get('Filename Params', 'profiles_prefix')
self.statPrefix = cParse.get('Filename Params', 'stats_prefix')
self.csvFile = cParse.get('Filename Params', 'coefficients_file')
self.outDir = cParse.get('Filename Params', 'output_dir')
self.exe = cParse.get('Filename Params', 'solver')
self.ncDir = cParse.get('Filename Params', 'netcdf_dir')
# handle forcing parameters
self.forcing = False
self.rForceFile = \
cParse.get('Filename Params', 'reference_force_path')
self.tForceFile = cParse.get('Filename Params', 'test_force_path')
self.rForceName = \
cParse.get('Plot Params', 'reference_forcing_model')
self.rForceDesc = \
cParse.get('Plot Params', 'reference_description')
self.tForceName = \
cParse.get('Plot Params', 'test_forcing_model')
self.tForceDesc = \
cParse.get('Plot Params', 'test_description')
if os.path.exists(self.rForceFile) and \
os.path.exists(self.tForceFile):
self.forcing = True
self.xTitle = self.rForceName
yt = '%s - %s' % (cTestForceName, cRefForceName)
self.refFile, self.testFile = \
COMPARE.forcingDiff(self.refFile, self.testFile, \
self.rForceFile, self.tForceFile)
else:
forceFile = self.rForceFile if not \
os.path.exists(self.rForceFile) else self.tForceFile
if forceFile != '':
print('Could not find %s, forcing not done' % forceFile)
# end forcing
# check that everything that is required exists
paths = [self.refFile, self.testFile, self.csvFile, \
self.exe, self.ncDir]
for path in paths:
utils.file_check(path)
if not os.path.exists(self.outDir): os.makedirs(self.outDir)
# for consistency with bandmerge() method -- the bandmerged
# file is the same as the test netCDF file
self.mergeNC = self.testFile
# standardize boolean inputs
self.doPWV = True if self.doPWV in yes else False
self.bandAvg = True if self.bandAvg in yes else False
self.yLog = True if self.yLog.lower() in yes else False
self.doStats = True if self.doStats in yes else False
self.doProfs = True if self.doProfs in yes else False
# by default, plot all bands
self.bands = np.arange(self.nBands) if self.bands == '' else \
np.array(self.bands.split()).astype(int)-1
self.xTitle = self.refName
self.yTitle = '%s - %s' % (self.testName, self.refName)
# coefficients from RRTMG, which used PWV instead of transmittance
# in the equation a0(ibnd) + a1(ibnd)*exp(a2(ibnd)*pwvcm)
if self.doPWV:
a0 = [1.66, 1.55, 1.58, 1.66, 1.54, 1.454, 1.89, 1.33, \
1.668, 1.66, 1.66, 1.66, 1.66, 1.66, 1.66, 1.66]
a1 = [0.00, 0.25, 0.22, 0.00, 0.13, 0.446, -0.10, 0.40, \
-0.006, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]
a2 = [0.00, -12.0, -11.7, 0.00, -0.72,-0.243, 0.19,-0.062, \
0.414, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00]
self.coeffs = np.array([a0, a1, a2]).T
# RRTMG hard codes the secant range to be between 1.5 and 1.8
self.secMM = np.array([1.5, 1.8])
def __init__(self, inDict, outDir=os.getcwd(), tPauseP=100.0, \
bands=None, yLog=False, broadband=False, convertHR=False):
"""
Extract the variables we need for further calculation and plotting
It starts with just fluxes, then heating rates are derived from
those fluxes.
Input
inDict -- dictionary generated with sw_compare.parseConfig()
Keywords
outDir -- string, top level directory for output figures
tPauseP -- float, tropopause pressure threshold (mbar)
bands -- int array (or list) of bands to plot
yLog -- boolean, plot the test-model differences on a log scale
broadband -- boolean, plot differences integrated over all
bands
convertHR -- boolean; LBLRTM and RRTMGP HR units are
expected in K/s; setting this keyword converts HR to K/day.
"""
ref = inDict['ref']
test = inDict['test']
self.plotVars = \
['flux_dn', 'flux_dir_dn', 'flux_dif_dn', 'flux_up', \
'flux_net', 'heating_rate', \
'band_flux_dn', 'band_flux_dir_dn', 'band_flux_dif_dn', \
'band_flux_up', 'band_flux_net', 'band_heating_rate', \
'p_lay', 'p_lev', 'band_lims_wvn', 'total_solar_irradiance']
# generate dictionaries with arrays corresponding to all plotVars
self.refDict = compare.getVars(ref,
attrList=self.plotVars,
convertHR=convertHR)
self.testDict = compare.getVars(test,
attrList=self.plotVars,
convertHR=convertHR)
# only because i had insane RRTMGP band_flux_dif_dn values (inf)
self.testDict['flux_dif_dn'] = \
self.testDict['flux_dn'] - self.testDict['flux_dir_dn']
self.testDict['band_flux_dif_dn'] = \
self.testDict['band_flux_dn'] - self.testDict['band_flux_dir_dn']
# some quality control (consistency check)
# also some perturbations to test file for script verification
for dum in self.plotVars:
if self.refDict[dum].shape != self.testDict[dum].shape:
errMsg = 'Returning: '
errMsg += '%s and %s do not have equal dimensions for %s' % \
(ref, test, dum)
sys.exit(errMsg)
# endif
# end QC loop
# grab dimensions of flux variables (using a by-band variable
# instead of broadband so we can deduce number of bands)
varShape = self.refDict['band_flux_dn'].shape
self.numLev = varShape[0]
self.numLay = varShape[0] - 1
self.numProf = varShape[1]
self.numBands = varShape[2]
self.tPauseP = float(tPauseP)
# other parameters extracted from the configuration file
self.statPrefix = inDict['stat']
self.profPrefix = inDict['prof']
self.xTitle = inDict['x']
self.yTitle = inDict['y']
self.atmType = inDict['atm']
self.forcing = inDict['forcing']
self.outDir = str(outDir)
self.broadBand = bool(broadband)
self.yLog = bool(yLog)
self.bands = None if bands is None else np.array(bands)
def prepForPool(refFile, testFile, outDir='.', shortWave=False, \
tPauseP=100.0, plotMean=False, yLog=False, inBand=None, \
atmType='Garand Atmospheres', \
prefix='PROFS_sens_key_cnt_add_min_04_dbl_r472_trynewn2opres'):
"""
Call
broadbandList = prepForPool(ref, test)
Input
refFile -- string, path to netCDF for reference model
testFile -- string, path to netCDF for test model
Output
List of dictionaries that will be used in poolProfPDFs() -- each
list element is a dictionary for a single band
Keywords
outDir -- string, top level directory for output figures
shortWave -- boolean, specifies that short-wave radiation flux
units should be specified in axis labels
tPauseP -- float, tropopause pressure threshold (mbar)
prefix -- string that will be placed in front of the band
number in the name of the output PDF file
plotMean -- boolean, plot mean of variable for a given pressure
and band over all columns
yLog -- boolean, plot ordinate axis on log scale instead of linear
atmType -- string, atmosphere type for profiles
"""
plotVars = ['band_flux_up', 'band_flux_dn', 'band_heating_rate', \
'p_lay', 'p_lev', 'band_wavenumbers']
refDict = rrtmgp.getVars(refFile, attrList=plotVars)
testDict = rrtmgp.getVars(testFile, attrList=plotVars)
# some quality control (consistency check)
dum = plotVars[0]
if refDict[dum].shape != testDict[dum].shape:
sys.exit('%s and %s do not have equal dimensions, returning' % \
refFile, testFile)
# grab dimensions of variables
varShape = refDict[dum].shape
nBand = varShape[2]; nCol = varShape[1]
broadbandList = []
bandList = range(nBand) if inBand is None else [inBand]
refWN = refDict['band_wavenumbers']
for band in bandList:
outFile = '%s/%s_%02d.pdf' % (outDir, prefix, band+1)
wnRange = refWN[:, band]
bandDict = {'plotVars': plotVars, 'wnRange': wnRange, \
'pdf': outFile, 'band': band, 'numberCol': nCol, \
'sw': shortWave, 'pLay': refDict['p_lay'], \
'pLev': refDict['p_lev'], \
'band_flux_up_ref': refDict['band_flux_up'][:, :, band], \
'band_flux_dn_ref': refDict['band_flux_dn'][:, :, band], \
'band_heating_rate_ref': \
refDict['band_heating_rate'][:, :, band], \
'band_flux_up_test': testDict['band_flux_up'][:, :, band], \
'band_flux_dn_test': testDict['band_flux_dn'][:, :, band], \
'band_heating_rate_test': \
testDict['band_heating_rate'][:, :, band], \
'tPauseP': tPauseP, 'plot_mean': plotMean, 'log_y': yLog, \
'atmType': atmType, 'doBroadband': False}
broadbandList.append(bandDict)
# end bands
# for summing arrays:
axNum = 2
# and now the broadband dictionary
outFile = '%s/%s_broadband.pdf' % (outDir, prefix)
bandDict = {'plotVars': plotVars, \
'wnRange': [refWN[0, 0], refWN[1, -1]], \
'pdf': outFile, 'band': 'Broad', 'numberCol': nCol, \
'sw': shortWave, 'pLay': refDict['p_lay'], \
'pLev': refDict['p_lev'], \
'band_flux_up_ref': np.sum(refDict['band_flux_up'], axis=axNum), \
'band_flux_dn_ref': np.sum(refDict['band_flux_dn'], axis=axNum), \
'band_heating_rate_ref': \
np.sum(refDict['band_heating_rate'], axis=axNum), \
'band_flux_up_test': \
np.sum(testDict['band_flux_up'], axis=axNum), \
'band_flux_dn_test': \
np.sum(testDict['band_flux_dn'], axis=axNum), \
'band_heating_rate_test': \
np.sum(testDict['band_heating_rate'], axis=axNum), \
'tPauseP': tPauseP, 'plot_mean': plotMean, 'log_y': yLog, \
'atmType': atmType, 'doBroadband': True}
broadbandList.append(bandDict)
return broadbandList
def poolProfPDFs(paramDict):
"""
Plot upward flux, downward flux, and heating rate as well as the
test-ref difference for each of those three variables on a single
page (3x2 array of figures) for each column and a single band
specified in ref and test. Fluxes and HR are plotted as functions
of pressure, so we are plotting profiles. This should be used with
the map() method in the multiprocessing library (Pool object)
Call
from multiprocessing import Pool, cpu_count, Process
p = Pool(nCores)
compareDat = prepForPool(ref, test)
p.map(poolProfPDFs, compareDat)
Input
paramDict -- dictionary, output from prepForPool
Output
PDF for a single waveband as specified in paramDict
"""
# unpack dictionary
plotVars = paramDict['plotVars']
outFile = paramDict['pdf']
wnRange = paramDict['wnRange']
band = paramDict['band']
nCol = paramDict['numberCol']
shortWave = paramDict['sw']
atmType = paramDict['atmType']
doBroad = paramDict['doBroadband']
plotTitle = ['Upward Flux', 'Downward Flux', 'Heating Rate', \
'Pressure (mbar)', 'Pressure (mbar)', 'Wavenumber Range']
pdf = PdfPages(outFile)
for aCol in range(nCol):
# plot 1 atm column per page, 3x2 array of subfigures
print('Column %d of %d' % (aCol+1, nCol))
figTitle = '%s Column %d, %.0f-%.0f cm$^{-1}$' % \
(atmType, aCol+1, wnRange[0], wnRange[1])
fig = plot.figure()
fig.set_size_inches(8.5, 11)
fig.suptitle(figTitle, fontweight='bold')
ctr = 1
# space between plots
plot.subplots_adjust(hspace=hspace, wspace=hspace)
# now loop over subplot rows and columns
for iRow in range(3):
pVar = plotVars[iRow]
pTitle = plotTitle[iRow]
for iCol in range(2):
plot.subplot(3, 2, ctr)
# fluxes are by level, heat rate are by layer
pStr = 'pLay' if iRow == 2 else 'pLev'
pressure = paramDict[pStr]
# some other plotting params
delta = True if iCol % 2 == 1 else False
if 'Flux' in pTitle:
units = '[W m$^{-2}$]' if shortWave else '[W m$^{-2}$]'
tempVar = 'Flux'
else:
units = '[K day$^{-1}$]'
tempVar = 'HR'
# end units and tempVar
if delta:
xt = 'RRTMGP-LBLRTM %s Difference %s' % (tempVar, units)
yt = ''
else:
xt = '%s %s' % (tempVar, units)
yt = 'Pressure [mbar]'
# end delta
varRef = paramDict[pVar + '_ref']
varTest = paramDict[pVar + '_test']
rrtmgp.plotProfiles(varRef[:, aCol], varTest[:, aCol], \
pressure[:, aCol], pTitle=pTitle, xTitle=xt, yTitle=yt, \
plotDelta=delta, tPauseP=paramDict['tPauseP'], \
plotMean=paramDict['plot_mean'], yLog=paramDict['log_y'])
ctr += 1
# end iCol
# end iRow
# write page for column
pdf.savefig()
plot.close()
# end column
pdf.close()
bandStr = 'broad' if doBroad else '%d' % (band+1)
print('Processed band %s' % bandStr)
return 0
testFile = cParse.get('Filename Params', 'test_path')
profPrefix = cParse.get('Filename Params', 'profiles_prefix')
statPrefix = cParse.get('Filename Params', 'stats_prefix')
xt = cRefName
yt = '%s - %s' % (cTestName, cRefName)
totCores = cpu_count()
if args.available_cores:
print('User can use up to {} cores'.format(totCores))
sys.exit(0)
# endif ac
# plot profile statistics
if args.stats:
rrtmgp.statPDF(refFile, testFile, forcing=False, singlePDF=True, \
xTitle=xt, yTitle=yt)
else:
compareDat = prepForPool(refFile, testFile, plotMean=args.mean, \
yLog=args.log_y, inBand=inBand, atmType=aType, \
prefix=profPrefix)
nCores = args.cores
nCores = nCores if nCores < totCores else totCores-1
p = Pool(nCores)
if args.profiles: p.map(poolProfPDFs, compareDat)
# end plot_stats
# end main()