def pltMethods(plumePath, plume, plm, modelStats, saveMethod=True):
''' Methods grabbed from postMINX for an individual plume.
Cycles through imageMethods and numMethods for the MINX retrieved plume,
has the option to use imageMethods for the model retrieved plume as well.
Defaults to saveMethod = True to save the images and number methods.
To view the results without saving, set saveMethod = False and run:
plt.show()
Note that this is unadvised as all plots will show up at once.
To view individual methods, please use postMINX directly.
'''
import pandas as pd
saveDir = os.path.join(currentDir, 'postAltOutputs', plume[7:-4])
nu.makeDir(saveDir)
os.chdir(saveDir)
for im in imageMethods:
saveName = plume[7:-4] + im + '.png'
try:
eval('plm.{}(save={},filename="{}")'.format(
im, saveMethod, saveName))
plt.close('all')
except:
plt.close('all')
continue
print('Save {} success!'.format(saveName))
# number methods for minx plume only
numSave = plume[7:-4] + '.txt'
option = 'filtered_height'
numList = []
for nu in numMethods:
numVal = eval('plm.{}(option="{}")'.format(nu, option))
numRow = [nu, numVal]
numList += [numRow]
numDF = pd.DataFrame(numList, columns=('method', option))
numDF.to_csv(numSave, sep=',', index=False)
if modelStats:
options = ['filtered_height', 'model_height']
for im in imageMethods:
try:
saveName = 'Mod_{}{}.png'.format(plume[7:-4], im)
eval('plm.{}(save={}, option={}, filename="{}")'.format(
im, saveMethod, options, saveName))
plt.close('all')
print('Save {} success!'.format(saveName))
except:
plt.close('all')
continue
numSave = 'Mod_{}.txt'.format(plume[7:-4])
numList = []
option1 = 'filtered_height'
option2 = 'model_height' # note that this won't actually get the model
opt2 = plm.Model.height
os.chdir(currentDir)
def plotFuels(fuelDir, plmFuels, threshVal, pt, modifier=''):
''' Get results by fuel type. '''
nu.makeDir(fuelDir)
distRanges = [[0, 10], [10, 30], [30, 60], [60, 500]]
distDir = os.path.join(fuelDir, 'distances')
for fuel in plmFuels:
try:
plmHeights = plmFuels[fuel]['plmHeights']
plmModHeights = plmFuels[fuel]['plmModHeights']
plmDists = plmFuels[fuel]['plmDists']
plmCount = plmFuels[fuel]['plmCount']
cleanPlm, cleanMod, cleanDist = clean_inv(plmHeights,
plmModHeights, plmDists)
overallImages(plmDists,
plmHeights,
plmModHeights,
saveDir=fuelDir,
modifier=modifier + '_fuel{}'.format(fuel))
saveStatsLite(fuelDir,
plmCount,
plmHeights,
plmModHeights,
modifier=modifier + '_fuel{}'.format(fuel))
for dRange in distRanges:
try:
dPlmHeights, dPlmModHeights, dPlmDists = filterPlmRange(
dRange, plmHeights, plmModHeights, plmDists)
overallImages(
dPlmDists,
dPlmHeights,
dPlmModHeights,
distDir,
modifier=modifier +
'_{}_dist{}-{}'.format(fuel, dRange[0], dRange[1]))
saveStatsLite(
distDir,
'N/A',
dPlmHeights,
dPlmModHeights,
modifier=modifier +
'{}_dist{}-{}'.format(fuel, dRange[0], dRange[1]))
except:
pass
cleanModPlmRatio = []
for pInd, (plm, modPlm) in enumerate(zip(cleanPlm, cleanMod)):
modPlmRatio = modPlm / plm
cleanModPlmRatio += [modPlmRatio]
pltDistRatio(cleanDist,
cleanModPlmRatio,
fuelDir,
modifier=modifier + '_fuel{}'.format(fuel))
pltPlmHeightComp('fuel', fuel, cleanPlm, cleanMod, fuelDir,
modifier)
nu.plotFuelMap(fuelDir,
plmOrigins=plmFuels[fuel]['plmXYOrigins'],
modifier=modifier + '_{}'.format(fuel))
print('Fuel {} map plotted'.format(fuel))
except:
pass
def scatter_3D(lons,
lats,
calHeights,
modHeights,
fireBoundaries,
savePath=os.getcwd(),
title='',
showPlot=showPlot):
from mpl_toolkits.mplot3d import Axes3D
import copy
#fireBoundaries is a polygon
if os.path.exists(savePath) == False:
nu.makeDir(savePath)
print('Adding new save path {}'.format(savePath))
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.autoscale()
ax.plot(lons, lats, calHeights, 'o', label='CALIOP')
ax.plot(lons, lats, modHeights, 'o', label='Model')
x, y = fireBoundaries.exterior.xy
ax.plot(x, y, color='r', label='FireBound')
ax.legend()
ax.set_title(title)
try:
plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
except:
pass
if showPlot == True:
plt.show()
#name and save the image to png and pkl file
filename = "".join(title.split())
filename = '_'.join(filename.split(':'))
pklfig = copy.copy(fig)
fig.savefig(os.path.join(savePath, filename + '.png'))
pklfile = '{}.pkl'.format(filename)
with open(os.path.join(savePath, pklfile), 'wb') as f:
pickle.dump(pklfig, f, protocol=2)
return
def plotBiomes(biomeDir, plmBiomes, modifier=''):
''' Get results by biome. '''
nu.makeDir(biomeDir)
for biome in plmBiomes:
try:
cleanPlm, cleanMod, cleanDist = clean_inv(
plmBiomes[biome]['plmHeights'],
plmBiomes[biome]['plmModHeights'],
plmBiomes[biome]['plmDists'])
cleanModPlmRatio = []
for pInd, (plm, modPlm) in enumerate(zip(cleanPlm, cleanMod)):
modPlmRatio = modPlm / plm # is this proper division?
cleanModPlmRatio += [modPlmRatio]
pltDistRatio(cleanDist,
cleanModPlmRatio,
biomeDir,
modifier=modifier + '_biome{}'.format(biome))
pltPlmHeightComp('biome', biome, cleanPlm, cleanMod, biomeDir,
modifier)
except:
pass
def plotFSTs(season=season, spcs=spcs, spcsFiles=spcsFiles, outputName = outputName, saveDir=saveDir):
# print minimum outputs
rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST)
mInds = []
for m in season:
mInds += [monthList.index(m)]
if os.path.exists(saveDir) == False:
nu.makeDir(saveDir)
for spcInd, nomvar in enumerate(spcs):
try:
filename = os.path.join(saveDir, 'output_file_{0}_{1}.fst'.format(outputName, nomvar))
print('Creating and saving to {}'.format(filename))
tmp = open(filename, 'w+'); tmp.close()
output_file = filename
file_id = rmn.fnom(output_file)
open_fst = rmn.fstouv(file_id, rmn.FST_RW)
open_file = spcsFiles[spcInd]
print "Parameter: " + nomvar
seaSpcData = get_var(pyg.open(open_file), nomvar, mInds)
nc_lnsp = pyg.open(lnsp_file)
pressures = get_pressures(nc_lnsp, mInds)
timelen, levlen, latlen, lonlen = seaSpcData.shape
#NOTE: uncomment the following three lines to prep data for basemap use
#lonShiftSSData = shift_lon(seaSpcData)
#vertInterpSSData = vert_interp(pressures, lonShiftSSData)
#meanSSData = np.mean(vertInterpSSData, axis=0)
#NOTE: uncommment the following four liness to use for fst plotting
vertInterpSSData = vert_interp(pressures, seaSpcData)
meanSSData = np.mean(vertInterpSSData, axis=0) # temp
for lvl, ray in enumerate(meanSSData):
meanSSData[lvl] = np.flipud(ray)
scaleFac = scaleSpcs[allSpcs.index(nomvar)]
scaledSSData = meanSSData*scaleFac
#define grid for this file - note that the MACC grid in the file is
#defined for lons -180 to 180, but the python defGrid_L can't deal
#with that and defines the grid from 0 to 360 so will have to reorder
#the MACC fields a bit, or they end up 180 deg out of phase
# Also necessary to add one more longitude to wrap around
dlatlon = 360./lonlen # this is equal to the resolution of the grid
params0 = {
'grtyp' : 'Z',
'grref' : 'L',
'nj' : latlen,
'ni' : lonlen,
'lat0' : -90.,
'lon0' : 0.,
'dlat' : dlatlon,
'dlon' : dlatlon
}
MACC_grid= rmn.encodeGrid(params0)
print("Grids created.")
print 'Grid Shape:' + str(MACC_grid['shape'])
# copies the default record
new_record = rmn.FST_RDE_META_DEFAULT.copy()
tic_record = rmn.FST_RDE_META_DEFAULT.copy()
tac_record = rmn.FST_RDE_META_DEFAULT.copy()
try:
rmn.writeGrid(file_id, MACC_grid)
tac = rmn.fstinl(file_id, nomvar='>>')[0]
tic = rmn.fstinl(file_id, nomvar='^^')[0]
tic_record.update(rmn.fstprm(tic))
tac_record.update(rmn.fstprm(tac))
tic_record.update({'datyp' : rmn.FST_DATYP_LIST['float']})
tac_record.update({'datyp' : rmn.FST_DATYP_LIST['float']})
rmn.fsteff(tic)
rmn.fsteff(tac)
tic_record.update({'d': MACC_grid['ay']})
tac_record.update({'d': MACC_grid['ax']})
toc_record = vgd.vgd_new_pres(const_pressure, ip1=MACC_grid['ig1'], ip2=MACC_grid['ig2'])
rmn.fstecr(file_id, tic_record) # write the dictionary record to the file as a new record
rmn.fstecr(file_id, tac_record) # write the dictionary record to the file as a new record
vgd.vgd_write(toc_record, file_id)
except:
raise
for rp1 in xrange(len(const_pressure)): # writes a record for every level (as a different ip1)
try:
# converts rp1 into a ip1 with pressure kind
ip1 = rmn.convertIp(rmn.CONVIP_ENCODE, const_pressure[rp1], rmn.KIND_PRESSURE)
new_record.update(MACC_grid)
new_record.update({ # Update with specific meta
'nomvar': nomvar,
'typvar': 'C',
'etiket': 'MACCRean',
'ni' : MACC_grid['ni'],
'nj' : MACC_grid['nj'],
'ig1' : tic_record['ip1'],
'ig2' : tic_record['ip2'],
'ig3' : tic_record['ip3'],
'ig4' : tic_record['ig4'],
'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20120101, 0000000),
'deet' : 0, # Timestep in sec
'ip1' : ip1
})
#tmp_nparray = np.asfortranarray(monthly_mean[rp1])
tmp = scaledSSData[rp1]
tmp = np.transpose(tmp)
# data array is structured as tmp = monthly_mean[level] where monthly_mean is [level, lat, lon]
new_record.update({'d': tmp.astype(np.float32)}) # Updates with data array in the form (lon x lat)
print "Defined a new record with dimensions ({0}, {1})".format(new_record['ni'], new_record['nj'])
rmn.fstecr(file_id, new_record) # write the dictionary record to the file as a new record
except:
#rmn.closeall(file_id)
rmn.fstfrm(file_id)
rmn.fclos(file_id)
raise
rmn.fstfrm(file_id)
rmn.fclos(file_id)
print('{} complete~'.format(filename))
except:
rmn.fstfrm(file_id)
rmn.fclos(file_id)
raise
print('Finished plotting all FSTs. ')
def pltPlume(plumePath,
plume,
minFRP,
mod_fire,
mod_nofire=None,
threshVal=0,
startTime=12,
toSave=savePlumePlts,
modelStats=True,
picklePath=picklePath):
'''
pltPlume() returns a plume object with FRP, filtered_height, model height, model values(AF at height), and distance from plume origin.
Additional plotting methods and stats can be turned on by setting toSave=True and modelStats=True.
See postMINX.py documentation for other parameters.
'''
nu.makeDir(picklePath)
try:
# check to see if the pickle file exists first
plm = pickle.load(
open(os.path.join(picklePath, plume[0:5] + plume[6:-3] + 'pkl'),
'rb'))
if plm['frp'] >= minFRP:
if toSave == True:
try:
pltMethods(path, plume, plm, modelStats=True)
except:
pass
return plm
else:
#print('{} FRP too low, plume will not be used.'.format(plume))
return None
except:
plm = pm.Plume.from_filename(os.path.join(plumePath, plume))
if plm['frp'] >= minFRP:
try:
plm.get_model(fst_dir=fireDir,
ctrl_dir=noFireDir,
threshold=threshVal,
filestart=startTime)
plm.save_plume(
os.path.join(picklePath,
'Plume_{}.pkl'.format(plume[7:26])))
print('{} has been pickled.'.format(plume))
if toSave == True:
try:
pltMethods(plumePath, plume, plm, modelStats=True)
except:
pass
try:
return plm
except:
#print('Plume and Model height were not returned.')
return None
except:
return None
else:
print('{} FRP too low, plume will not be used.'.format(plume))
return None
import postMINX as pm
import datetime
import niceUtilities as nu
try:
import cPickle as pickle
except ImportError:
import pickle
# for now do the september cases
picklePath = "/fs/site2/dev/eccc/aq/r1/eld001/MINX/Working/apr_oct/FW-GM_start00_pklPlmsOps/"
savePklPath = './FW-GM_CAL_start00_pklPlmsOps'
plumeList = os.listdir(picklePath)
septInts = "/fs/site2/dev/eccc/aq/r1/eld001/MINX/CALIOP/intercepts_s20170901_e20170930.p"
nu.makeDir(savePklPath)
septArray = []
for p in plumeList:
plm=pickle.load(open(os.path.join(picklePath, p), 'rb'))
print(plm)
plmDate =plm['datetime']
mnth = plmDate.month
if mnth == 9:
septArray +=[plm]
firePath = '/fs/site2/dev/eccc/aq/r1/eld001/MINX/CALIOP/intercepts_s20170901_e20170930.p'
fires = pickle.load(open(firePath,'rb'))
firePasses = []
fireBounds = []
keptFires = []
fDays = []
plmBandList += 'B'
else:
plmBandList += 'R'
# get the FRPS from all of the retrieved firePixels.
# NOTE: did not factor in confidence level here
pwrInd = 5
pixList = os.listdir(pixDir)
pwrs = []
for t in pixList:
tLines = nu.readFile(os.path.join(pixDir, t))
for rowInd, row in enumerate(tLines, 4):
if rowInd < len(tLines):
pwrs += [nu.toFloats(tLines[rowInd])[pwrInd]]
nu.makeDir(saveDir)
percentiles = [50, 95, 98]
distRanges = [[0, 10], [10, 30], [30, 50], [50, 500]]
distBins = [0, 5, 10, 20, 30, 50, 100]
# pass percentiles as the modifier
for pt in percentiles:
savePtDir = os.path.join(saveDir, '{}_percentile'.format(pt))
nu.makeDir(savePtDir)
biomeDir = os.path.join(savePtDir, 'biomeBreakdown')
fuelDir = os.path.join(savePtDir, 'fuelBreakdown')
pwrMin = np.percentile(pwrs, pt)
print('minFRP: {}'.format(pwrMin))
plumes, plmHeights, plmModHeights, plmDists, plmBiomes, plmFuels, threshVal = pa.processPlumes(
plumeDir,
list(plumeList),