def getPeriodGridString(period, grid, exp=None, beginyear=None):
''' utility function to check period and grid and return valid and usable strings '''
# period
if period is None: pass
elif isinstance(period,(int,np.integer)):
if beginyear is None: beginyear = int(exp.begindate[0:4]) # most datasets begin in 1979
period = (beginyear, beginyear+period)
elif len(period) != 2 and all(isInt(period)): raise DateError
periodstr = '{0:4d}-{1:4d}'.format(*period) if period else ''
# grid
gridstr = grid if grid else ''
# return
return periodstr, gridstr
def getPeriodGridString(period, grid, exp=None, beginyear=None):
''' utility function to check period and grid and return valid and usable strings '''
# period
if period is None: pass
elif isinstance(period, (int, np.integer)):
if beginyear is None:
beginyear = int(exp.begindate[0:4]) # most datasets begin in 1979
period = (beginyear, beginyear + period)
elif len(period) != 2 and all(isInt(period)):
raise DateError
periodstr = '{0:4d}-{1:4d}'.format(*period) if period else ''
# grid
gridstr = grid if grid else ''
# return
return periodstr, gridstr
def computeClimatology(experiment, filetype, domain, periods=None, offset=0, griddef=None, varlist=None,
ldebug=False, loverwrite=False, lparallel=False, pidstr='', logger=None):
''' worker function to compute climatologies for given file parameters. '''
# input type checks
if not isinstance(experiment,Exp): raise TypeError
if not isinstance(filetype,basestring): raise TypeError
if not isinstance(domain,(np.integer,int)): raise TypeError
if periods is not None and not (isinstance(periods,(tuple,list)) and isInt(periods)): raise TypeError
if not isinstance(offset,(np.integer,int)): raise TypeError
if not isinstance(loverwrite,(bool,np.bool)): raise TypeError
if griddef is not None and not isinstance(griddef,GridDefinition): raise TypeError
#if pidstr == '[proc01]': raise TypeError # to test error handling
# load source
dataset_name = experiment.name
fileclass = fileclasses[filetype] # used for target file name
tsfile = fileclass.tsfile.format(domain,'')
expfolder = experiment.avgfolder
filepath = '{:s}/{:s}'.format(expfolder, tsfile)
logger.info('\n\n{0:s} *** Processing Experiment {1:<15s} *** '.format(pidstr,"'{:s}'".format(dataset_name)) +
'\n{0:s} *** {1:^37s} *** \n'.format(pidstr,"'{:s}'".format(tsfile)))
# check file and read begin/enddates
if not os.path.exists(filepath):
#raise IOError, "Source file '{:s}' does not exist!".format(filepath)
# print message and skip
skipmsg = "\n{:s} >>> File '{:s}' in dataset '{:s}' is missing --- skipping!".format(pidstr,tsfile,dataset_name)
skipmsg += "\n{:s} >>> ('{:s}')\n".format(pidstr,filepath)
logger.warning(skipmsg)
# N.B.: this can cause a lot of error messages, when not all files are present
else: # if monthly source file exists
import netCDF4 as nc
ncfile = nc.Dataset(filepath,mode='r')
begintuple = ncfile.begin_date.split('-')
endtuple = ncfile.end_date.split('-')
ncfile.close()
# N.B.: at this point we don't want to initialize a full GDAL-enabled dataset, since we don't even
# know if we need it, and it creates a lot of overhead
# determine age of source file
if not loverwrite: sourceage = datetime.fromtimestamp(os.path.getmtime(filepath))
# figure out start date
filebegin = int(begintuple[0]) # first element is the year
fileend = int(endtuple[0]) # first element is the year
begindate = offset + filebegin
if not ( filebegin <= begindate <= fileend ): raise DateError
# handle cases where the first month in the record is not January
firstmonth = int(begintuple[1]) # second element is the month
shift = firstmonth-1 # will be zero for January (01)
## loop over periods
if periods is None: periods = [begindate-fileend]
# periods.sort(reverse=True) # reverse, so that largest chunk is done first
source = None # will later be assigned to the source dataset
for period in periods:
# figure out period
enddate = begindate + period
if filebegin > enddate: raise DateError, 'End date earlier than begin date.'
if enddate-1 > fileend: # if filebegin is 1979 and the simulation is 10 years, fileend will be 1988, not 1989!
# if end date is not available, skip period
endmsg = "\n{:s} --- Invalid Period for '{:s}': End Date {:4d} not in File! --- \n".format(pidstr,dataset_name,enddate)
endmsg += "{:s} --- ('{:s}')\n".format(pidstr,filepath)
logger.info(endmsg)
else: ## perform averaging for selected period
# determine if sink file already exists, and what to do about it
periodstr = '{0:4d}-{1:4d}'.format(begindate,enddate)
gridstr = '' if griddef is None or griddef.name is 'WRF' else '_'+griddef.name
filename = fileclass.climfile.format(domain,gridstr,'_'+periodstr)
if ldebug: filename = 'test_' + filename
if lparallel: tmppfx = 'tmp_wrfavg_{:s}_'.format(pidstr[1:-1])
else: tmppfx = 'tmp_wrfavg_'.format(pidstr[1:-1])
tmpfilename = tmppfx + filename
assert os.path.exists(expfolder)
filepath = expfolder+filename
tmpfilepath = expfolder+tmpfilename
lskip = False # else just go ahead
if os.path.exists(filepath):
if not loverwrite:
age = datetime.fromtimestamp(os.path.getmtime(filepath))
# if sink file is newer than source file, skip (do not recompute)
if age > sourceage and os.path.getsize(filepath) > 1e6: lskip = True
# N.B.: NetCDF files smaller than 1MB are usually incomplete header fragments from a previous crash
#print sourceage, age
if not lskip: os.remove(filepath)
# depending on last modification time of file or overwrite setting, start computation, or skip
if lskip:
# print message
skipmsg = "\n{:s} >>> Skipping: file '{:s}' in dataset '{:s}' already exists and is newer than source file.".format(pidstr,filename,dataset_name)
skipmsg += "\n{:s} >>> ('{:s}')\n".format(pidstr,filepath)
logger.info(skipmsg)
else:
## begin actual computation
beginmsg = "\n{:s} <<< Computing '{:s}' (d{:02d}) Climatology from {:s}".format(
pidstr,dataset_name,domain,periodstr)
if griddef is None: beginmsg += " >>> \n"
else: beginmsg += " ('{:s}' grid) >>> \n".format(griddef.name)
logger.info(beginmsg)
## actually load datasets
if source is None:
source = loadWRF_TS(experiment=experiment, filetypes=[filetype], domains=domain) # comes out as a tuple...
if not lparallel and ldebug: logger.info('\n'+str(source)+'\n')
# prepare sink
if os.path.exists(tmpfilepath): os.remove(tmpfilepath) # remove old temp files
sink = DatasetNetCDF(name='WRF Climatology', folder=expfolder, filelist=[tmpfilename], atts=source.atts.copy(), mode='w')
sink.atts.period = periodstr
# initialize processing
if griddef is None: lregrid = False
else: lregrid = True
CPU = CentralProcessingUnit(source, sink, varlist=varlist, tmp=lregrid, feedback=ldebug) # no need for lat/lon
# start processing climatology
if shift != 0:
logger.info('{0:s} (shifting climatology by {1:d} month, to start with January) \n'.format(pidstr,shift))
CPU.Climatology(period=period, offset=offset, shift=shift, flush=False)
# N.B.: immediate flushing should not be necessary for climatologies, since they are much smaller!
# reproject and resample (regrid) dataset
if lregrid:
CPU.Regrid(griddef=griddef, flush=True)
logger.info('%s --- '+str(griddef.geotansform)+' --- \n'%(pidstr))
# sync temporary storage with output dataset (sink)
CPU.sync(flush=True)
# add Geopotential Height Variance
if 'GHT_Var' in sink and 'Z_var' not in sink:
data_array = ( sink['GHT_Var'].data_array - sink['Z'].data_array**2 )**0.5
atts = dict(name='Z_var',units='m',long_name='Square Root of Geopotential Height Variance')
sink += Variable(axes=sink['Z'].axes, data=data_array, atts=atts)
# add (relative) Vorticity Variance
if 'Vorticity_Var' in sink and 'zeta_var' not in sink:
data_array = ( sink['Vorticity_Var'].data_array - sink['zeta'].data_array**2 )**0.5
atts = dict(name='zeta_var',units='1/s',long_name='Square Root of Relative Vorticity Variance')
sink += Variable(axes=sink['zeta'].axes, data=data_array, atts=atts)
# add names and length of months
sink.axisAnnotation('name_of_month', name_of_month, 'time',
atts=dict(name='name_of_month', units='', long_name='Name of the Month'))
if not sink.hasVariable('length_of_month'):
sink += Variable(name='length_of_month', units='days', axes=(sink.time,), data=days_per_month,
atts=dict(name='length_of_month',units='days',long_name='Length of Month'))
# close... and write results to file
sink.sync()
sink.close()
writemsg = "\n{:s} >>> Writing to file '{:s}' in dataset {:s}".format(pidstr,filename,dataset_name)
writemsg += "\n{:s} >>> ('{:s}')\n".format(pidstr,filepath)
logger.info(writemsg)
# rename file to proper name
if os.path.exists(filepath): os.remove(filepath) # remove old file
os.rename(tmpfilepath,filepath) # this will overwrite the old file
# print dataset
if not lparallel and ldebug:
logger.info('\n'+str(sink)+'\n')
# clean up (not sure if this is necessary, but there seems to be a memory leak...
del sink, CPU; gc.collect() # get rid of these guys immediately
# clean up and return
if source is not None: source.unload(); del source
# N.B.: source is only loaded once for all periods
# N.B.: garbage is collected in multi-processing wrapper as well
# return
return 0 # so far, there is no measure of success, hence, if there is no crash...
def getFigAx(subplot, name=None, title=None, figsize=None, mpl=None, margins=None,
sharex=None, sharey=None, AxesGrid=False, ngrids=None, direction='row',
axes_pad = None, add_all=True, share_all=None, aspect=False,
label_mode='L', cbar_mode=None, cbar_location='right',
cbar_pad=None, cbar_size='5%', axes_class=None, lreduce=True):
# configure matplotlib
warn('Deprecated function: use Figure or Axes class methods.')
if mpl is None: import matplotlib as mpl
elif isinstance(mpl,dict): mpl = loadMPL(**mpl) # there can be a mplrc, but also others
elif not isinstance(mpl,ModuleType): raise TypeError
from plotting.figure import MyFigure # prevent circular reference
# figure out subplots
if isinstance(subplot,(np.integer,int)):
if subplot == 1: subplot = (1,1)
elif subplot == 2: subplot = (1,2)
elif subplot == 3: subplot = (1,3)
elif subplot == 4: subplot = (2,2)
elif subplot == 6: subplot = (2,3)
elif subplot == 9: subplot = (3,3)
else: raise NotImplementedError
elif not (isinstance(subplot,(tuple,list)) and len(subplot) == 2) and all(isInt(subplot)): raise TypeError
# create figure
if figsize is None:
if subplot == (1,1): figsize = (3.75,3.75)
elif subplot == (1,2) or subplot == (1,3): figsize = (6.25,3.75)
elif subplot == (2,1) or subplot == (3,1): figsize = (3.75,6.25)
else: figsize = (6.25,6.25)
#elif subplot == (2,2) or subplot == (3,3): figsize = (6.25,6.25)
#else: raise NotImplementedError
# figure out margins
if margins is None:
# N.B.: the rectangle definition is presumably left, bottom, width, height
if subplot == (1,1): margins = (0.09,0.09,0.88,0.88)
elif subplot == (1,2) or subplot == (1,3): margins = (0.06,0.1,0.92,0.87)
elif subplot == (2,1) or subplot == (3,1): margins = (0.09,0.11,0.88,0.82)
elif subplot == (2,2) or subplot == (3,3): margins = (0.055,0.055,0.925,0.925)
else: margins = (0.09,0.11,0.88,0.82)
#elif subplot == (2,2) or subplot == (3,3): margins = (0.09,0.11,0.88,0.82)
#else: raise NotImplementedError
if title is not None: margins = margins[:3]+(margins[3]-0.03,) # make room for title
if AxesGrid:
if share_all is None: share_all = True
if axes_pad is None: axes_pad = 0.05
# create axes using the Axes Grid package
fig = mpl.pylab.figure(facecolor='white', figsize=figsize, FigureClass=MyFigure)
if axes_class is None:
from plotting.axes import MyLocatableAxes
axes_class=(MyLocatableAxes,{})
from mpl_toolkits.axes_grid1 import ImageGrid
# AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html
grid = ImageGrid(fig, margins, nrows_ncols = subplot, ngrids=ngrids, direction=direction,
axes_pad=axes_pad, add_all=add_all, share_all=share_all, aspect=aspect,
label_mode=label_mode, cbar_mode=cbar_mode, cbar_location=cbar_location,
cbar_pad=cbar_pad, cbar_size=cbar_size, axes_class=axes_class)
# return figure and axes
axes = tuple([ax for ax in grid]) # this is already flattened
if lreduce and len(axes) == 1: axes = axes[0] # return a bare axes instance, if there is only one axes
else:
# create axes using normal subplot routine
if axes_pad is None: axes_pad = 0.03
wspace = hspace = axes_pad
if share_all:
sharex='all'; sharey='all'
if sharex is True or sharex is None: sharex = 'col' # default
if sharey is True or sharey is None: sharey = 'row'
if sharex: hspace -= 0.015
if sharey: wspace -= 0.015
# create figure
from matplotlib.pyplot import subplots
# GridSpec: http://matplotlib.org/users/gridspec.html
fig, axes = subplots(subplot[0], subplot[1], sharex=sharex, sharey=sharey,
squeeze=lreduce, facecolor='white', figsize=figsize, FigureClass=MyFigure)
# there is also a subplot_kw=dict() and fig_kw=dict()
# just adjust margins
margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0]+margins[2],
top=margins[1]+margins[3], wspace=wspace, hspace=hspace)
fig.subplots_adjust(**margin_dict)
# add figure title
if name is not None: fig.canvas.set_window_title(name) # window title
if title is not None: fig.suptitle(title) # title on figure (printable)
# return Figure/ImageGrid and tuple of axes
#if AxesGrid: fig = grid # return ImageGrid instead of figure
return fig, axes
def getFigAx(subplot, name=None, title=None, title_font='large', figsize=None, stylesheet=None,
variable_plotargs=None, dataset_plotargs=None, plot_labels=None, yright=False, xtop=False,
sharex=None, sharey=None, AxesGrid=False, ngrids=None, direction='row',
axes_pad = None, add_all=True, share_all=None, aspect=False, margins=None,
label_mode='L', cbar_mode=None, cbar_location='right', lreduce=True,
cbar_pad=None, cbar_size='5%', axes_class=None, axes_args=None,
lpresentation=False, lpublication=False, figure_class=None, **figure_args):
# load stylesheet
if stylesheet is not None:
loadStyleSheet(stylesheet, lpresentation=lpresentation, lpublication=lpublication)
if stylesheet in ('myggplot','ggplot'):
warn("Rewriting built-in color definitions to GG-plot defaults.")
if dataset_plotargs is not None: dataset_plotargs = toGGcolors(dataset_plotargs) # modifies in-place!
# default figure class
if figure_class is None: figure_class = MyFigure
elif not issubclass(figure_class, Figure): raise TypeError
# figure out subplots
if isinstance(subplot,(np.integer,int)):
if subplot == 1: subplot = (1,1)
elif subplot == 2: subplot = (1,2)
elif subplot == 3: subplot = (1,3)
elif subplot == 4: subplot = (2,2)
elif subplot == 6: subplot = (2,3)
elif subplot == 9: subplot = (3,3)
else: raise NotImplementedError
elif not (isinstance(subplot,(tuple,list)) and len(subplot) == 2) and all(isInt(subplot)): raise TypeError
# create figure
if figsize is None:
if lpublication:
if subplot == (1,1): figsize = (3.75,3.75)
elif subplot == (1,2) or subplot == (1,3): figsize = (6.25,3.75)
elif subplot == (2,1) or subplot == (3,1): figsize = (3.75,7)
else: figsize = (6.25,6.25)
elif lpresentation:
if subplot == (1,2) or subplot == (1,3): figsize = (5,3)
elif subplot == (2,1) or subplot == (3,1): figsize = (3,5)
else: figsize = (5,5)
else:
if subplot == (1,1): figsize = (5,5)
elif subplot == (1,2) or subplot == (1,3): figsize = (9,5)
elif subplot == (2,1) or subplot == (3,1): figsize = (5,9)
else: figsize = (9,9)
# figure out margins
if margins is None:
# N.B.: the rectangle definition is presumably left, bottom, width, height
if subplot == (1,1): margins = (0.1,0.1,0.85,0.85)
elif subplot == (1,2) or subplot == (1,3): margins = (0.06,0.1,0.92,0.87)
elif subplot == (2,1) or subplot == (3,1): margins = (0.09,0.11,0.88,0.82)
elif subplot == (2,2) or subplot == (3,3): margins = (0.06,0.08,0.92,0.92)
else: margins = (0.09,0.11,0.88,0.82)
#elif subplot == (2,2) or subplot == (3,3): margins = (0.09,0.11,0.88,0.82)
#else: raise NotImplementedError
title_height = getattr(figure_class, 'title_height', 0.05) # use default from figure
if title is not None: margins = margins[:3]+(margins[3]-title_height,) # make room for title
# # some style sheets have different label sizes
# if stylesheet.lower() in ('myggplot','ggplot'):
# margins = list(margins)
# margins[0] += 0.015; margins[1] -= 0.01 # left, bottom
# margins[2] += 0.02; margins[3] += 0.02 # width, height
if AxesGrid:
if share_all is None: share_all = True
if axes_pad is None: axes_pad = 0.05
# adjust margins for ignored label pads
margins = list(margins)
margins[0] += 0.005; margins[1] -= 0.02 # left, bottom
margins[2] -= 0.005; margins[3] -= 0.00 # width, height
# create axes using the Axes Grid package
if axes_class is None: axes_class=MyLocatableAxes
fig = mpl.pylab.figure(facecolor='white', figsize=figsize, axes_class=axes_class,
FigureClass=MyFigure, **figure_args)
if axes_args is None: axes_class = (axes_class,{})
elif isinstance(axes_args,dict): axes_class = (axes_class,axes_args)
else: raise TypeError
from mpl_toolkits.axes_grid1 import ImageGrid
# AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html
grid = ImageGrid(fig, margins, nrows_ncols = subplot, ngrids=ngrids, direction=direction,
axes_pad=axes_pad, add_all=add_all, share_all=share_all, aspect=aspect,
label_mode=label_mode, cbar_mode=cbar_mode, cbar_location=cbar_location,
cbar_pad=cbar_pad, cbar_size=cbar_size, axes_class=axes_class)
# return figure and axes
axes = np.asarray(grid).reshape(subplot) # don't want flattened array
#axes = tuple([ax for ax in grid]) # this is already flattened
if lreduce and len(axes) == 1: axes = axes[0] # return a bare axes instance, if there is only one axes
else:
# create axes using normal subplot routine
if axes_pad is None: axes_pad = 0.03
wspace = hspace = axes_pad
if share_all:
sharex='all'; sharey='all'
if sharex is True or sharex is None: sharex = 'col' # default
if sharey is True or sharey is None: sharey = 'row'
if sharex: hspace -= 0.015
if sharey: wspace -= 0.015
# other axes arguments
if axes_class is None: axes_class=MyAxes
if axes_args is not None and not isinstance(axes_args,dict): raise TypeError
# create figure
from matplotlib.pyplot import subplots
# GridSpec: http://matplotlib.org/users/gridspec.html
fig, axes = subplots(subplot[0], subplot[1], sharex=sharex, sharey=sharey,squeeze=lreduce,
facecolor='white', figsize=figsize, FigureClass=MyFigure,
subplot_kw=axes_args, axes_class=axes_class, **figure_args)
# there is also a subplot_kw=dict() and fig_kw=dict()
# just adjust margins
margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0]+margins[2],
top=margins[1]+margins[3], wspace=wspace, hspace=hspace)
fig.subplots_adjust(**margin_dict)
## set label positions
# X-/Y-labels and -ticks
yright = not sharey and subplot[0]==2 if yright is None else yright
xtop = not sharex and subplot[1]==2 if xtop is None else xtop
if isinstance(axes, Axes):
axes.yright = yright
axes.xtop = xtop
else:
if axes.ndim == 1:
if subplot[0] == 2: axes[-1].yright = yright # right panel
if subplot[1] == 2: axes[0].xtop = xtop # top panel
elif axes.ndim == 2:
for ax in axes[:,-1]: ax.yright = yright # right column
for ax in axes[0,:]: ax.xtop = xtop # top row
else: raise ValueError
# add figure title
if name is None: name = title
if name is not None: fig.canvas.set_window_title(name) # window title
if isinstance(title_font,basestring): title_font = dict(fontsize=title_font)
if title is not None: fig.suptitle(title, **title_font) # title on figure (printable)
# add default line styles for variables and datasets to axes (figure doesn't need to know)
if isinstance(axes, np.ndarray):
for ax in axes.ravel():
ax.variable_plotargs = variable_plotargs
ax.dataset_plotargs = dataset_plotargs
ax.plot_labels = plot_labels
else:
axes.variable_plotargs = variable_plotargs
axes.dataset_plotargs = dataset_plotargs
axes.plot_labels = plot_labels
# return Figure/ImageGrid and tuple of axes
#if AxesGrid: fig = grid # return ImageGrid instead of figure
return fig, axes
def computeClimatology(experiment,
filetype,
domain,
periods=None,
offset=0,
griddef=None,
varlist=None,
ldebug=False,
loverwrite=False,
lparallel=False,
pidstr='',
logger=None):
''' worker function to compute climatologies for given file parameters. '''
# input type checks
if not isinstance(experiment, Exp): raise TypeError
if not isinstance(filetype, basestring): raise TypeError
if not isinstance(domain, (np.integer, int)): raise TypeError
if periods is not None and not (isinstance(periods, (tuple, list))
and isInt(periods)):
raise TypeError
if not isinstance(offset, (np.integer, int)): raise TypeError
if not isinstance(loverwrite, (bool, np.bool)): raise TypeError
if griddef is not None and not isinstance(griddef, GridDefinition):
raise TypeError
#if pidstr == '[proc01]': raise TypeError # to test error handling
# load source
dataset_name = experiment.name
fileclass = fileclasses[filetype] # used for target file name
tsfile = fileclass.tsfile.format(domain, '')
expfolder = experiment.avgfolder
filepath = '{:s}/{:s}'.format(expfolder, tsfile)
logger.info('\n\n{0:s} *** Processing Experiment {1:<15s} *** '.
format(pidstr, "'{:s}'".format(dataset_name)) +
'\n{0:s} *** {1:^37s} *** \n'.format(
pidstr, "'{:s}'".format(tsfile)))
# check file and read begin/enddates
if not os.path.exists(filepath):
#raise IOError, "Source file '{:s}' does not exist!".format(filepath)
# print message and skip
skipmsg = "\n{:s} >>> File '{:s}' in dataset '{:s}' is missing --- skipping!".format(
pidstr, tsfile, dataset_name)
skipmsg += "\n{:s} >>> ('{:s}')\n".format(pidstr, filepath)
logger.warning(skipmsg)
# N.B.: this can cause a lot of error messages, when not all files are present
else: # if monthly source file exists
import netCDF4 as nc
ncfile = nc.Dataset(filepath, mode='r')
begintuple = ncfile.begin_date.split('-')
endtuple = ncfile.end_date.split('-')
ncfile.close()
# N.B.: at this point we don't want to initialize a full GDAL-enabled dataset, since we don't even
# know if we need it, and it creates a lot of overhead
# determine age of source file
if not loverwrite:
sourceage = datetime.fromtimestamp(os.path.getmtime(filepath))
# figure out start date
filebegin = int(begintuple[0]) # first element is the year
fileend = int(endtuple[0]) # first element is the year
begindate = offset + filebegin
if not (filebegin <= begindate <= fileend): raise DateError
# handle cases where the first month in the record is not January
firstmonth = int(begintuple[1]) # second element is the month
shift = firstmonth - 1 # will be zero for January (01)
## loop over periods
if periods is None: periods = [begindate - fileend]
# periods.sort(reverse=True) # reverse, so that largest chunk is done first
source = None # will later be assigned to the source dataset
for period in periods:
# figure out period
enddate = begindate + period
if filebegin > enddate:
raise DateError, 'End date earlier than begin date.'
if enddate - 1 > fileend: # if filebegin is 1979 and the simulation is 10 years, fileend will be 1988, not 1989!
# if end date is not available, skip period
endmsg = "\n{:s} --- Invalid Period for '{:s}': End Date {:4d} not in File! --- \n".format(
pidstr, dataset_name, enddate)
endmsg += "{:s} --- ('{:s}')\n".format(pidstr, filepath)
logger.info(endmsg)
else: ## perform averaging for selected period
# determine if sink file already exists, and what to do about it
periodstr = '{0:4d}-{1:4d}'.format(begindate, enddate)
gridstr = '' if griddef is None or griddef.name is 'WRF' else '_' + griddef.name
filename = fileclass.climfile.format(domain, gridstr,
'_' + periodstr)
if ldebug: filename = 'test_' + filename
if lparallel: tmppfx = 'tmp_wrfavg_{:s}_'.format(pidstr[1:-1])
else: tmppfx = 'tmp_wrfavg_'.format(pidstr[1:-1])
tmpfilename = tmppfx + filename
assert os.path.exists(expfolder)
filepath = expfolder + filename
tmpfilepath = expfolder + tmpfilename
lskip = False # else just go ahead
if os.path.exists(filepath):
if not loverwrite:
age = datetime.fromtimestamp(
os.path.getmtime(filepath))
# if sink file is newer than source file, skip (do not recompute)
if age > sourceage and os.path.getsize(filepath) > 1e6:
lskip = True
# N.B.: NetCDF files smaller than 1MB are usually incomplete header fragments from a previous crash
#print sourceage, age
if not lskip: os.remove(filepath)
# depending on last modification time of file or overwrite setting, start computation, or skip
if lskip:
# print message
skipmsg = "\n{:s} >>> Skipping: file '{:s}' in dataset '{:s}' already exists and is newer than source file.".format(
pidstr, filename, dataset_name)
skipmsg += "\n{:s} >>> ('{:s}')\n".format(
pidstr, filepath)
logger.info(skipmsg)
else:
if griddef is None: lregrid = False
else: lregrid = True
## begin actual computation
beginmsg = "\n{:s} <<< Computing '{:s}' (d{:02d}) Climatology from {:s}".format(
pidstr, dataset_name, domain, periodstr)
if not lregrid: beginmsg += " >>> \n"
else:
beginmsg += " ('{:s}' grid) >>> \n".format(
griddef.name)
logger.info(beginmsg)
## actually load datasets
if source is None:
source = loadWRF_TS(
experiment=experiment,
filetypes=[filetype],
domains=domain) # comes out as a tuple...
if not lparallel and ldebug:
logger.info('\n' + str(source) + '\n')
# prepare sink
if os.path.exists(tmpfilepath):
os.remove(tmpfilepath) # remove old temp files
sink = DatasetNetCDF(name='WRF Climatology',
folder=expfolder,
filelist=[tmpfilename],
atts=source.atts.copy(),
mode='w')
sink.atts.period = periodstr
# if lregrid: addGDALtoDataset(sink, griddef=griddef)
# initialize processing
CPU = CentralProcessingUnit(
source,
sink,
varlist=varlist,
tmp=lregrid,
feedback=ldebug) # no need for lat/lon
# start processing climatology
if shift != 0:
logger.info(
'{0:s} (shifting climatology by {1:d} month, to start with January) \n'
.format(pidstr, shift))
CPU.Climatology(period=period,
offset=offset,
shift=shift,
flush=False)
# N.B.: immediate flushing should not be necessary for climatologies, since they are much smaller!
# reproject and resample (regrid) dataset
if lregrid:
CPU.Regrid(griddef=griddef, flush=True)
logger.info('{:s} --- {:s} --- \n'.format(
pidstr, griddef.name))
logger.debug('{:s} --- {:s} --- \n'.format(
pidstr, str(griddef)))
# sync temporary storage with output dataset (sink)
CPU.sync(flush=True)
# add Geopotential Height Variance
if 'GHT_Var' in sink and 'Z_var' not in sink:
data_array = (sink['GHT_Var'].data_array -
sink['Z'].data_array**2)**0.5
atts = dict(
name='Z_var',
units='m',
long_name=
'Square Root of Geopotential Height Variance')
sink += Variable(axes=sink['Z'].axes,
data=data_array,
atts=atts)
# add (relative) Vorticity Variance
if 'Vorticity_Var' in sink and 'zeta_var' not in sink:
data_array = (sink['Vorticity_Var'].data_array -
sink['zeta'].data_array**2)**0.5
atts = dict(
name='zeta_var',
units='1/s',
long_name=
'Square Root of Relative Vorticity Variance')
sink += Variable(axes=sink['zeta'].axes,
data=data_array,
atts=atts)
# add names and length of months
sink.axisAnnotation('name_of_month',
name_of_month,
'time',
atts=dict(
name='name_of_month',
units='',
long_name='Name of the Month'))
if not sink.hasVariable('length_of_month'):
sink += Variable(name='length_of_month',
units='days',
axes=(sink.time, ),
data=days_per_month,
atts=dict(
name='length_of_month',
units='days',
long_name='Length of Month'))
# close... and write results to file
sink.sync()
sink.close()
writemsg = "\n{:s} >>> Writing to file '{:s}' in dataset {:s}".format(
pidstr, filename, dataset_name)
writemsg += "\n{:s} >>> ('{:s}')\n".format(
pidstr, filepath)
logger.info(writemsg)
# rename file to proper name
if os.path.exists(filepath):
os.remove(filepath) # remove old file
os.rename(tmpfilepath,
filepath) # this will overwrite the old file
# print dataset
if not lparallel and ldebug:
logger.info('\n' + str(sink) + '\n')
# clean up (not sure if this is necessary, but there seems to be a memory leak...
del sink, CPU
gc.collect() # get rid of these guys immediately
# clean up and return
if source is not None:
source.unload()
del source
# N.B.: source is only loaded once for all periods
# N.B.: garbage is collected in multi-processing wrapper as well
# return
return 0 # so far, there is no measure of success, hence, if there is no crash...
def getMetaData(dataset, mode, dataargs):
''' determine dataset type and meta data, as well as path to main source file '''
# determine dataset mode
lclim = False
lts = False
if mode == 'climatology': lclim = True
elif mode == 'time-series': lts = True
else: raise NotImplementedError, "Unrecognized Mode: '{:s}'".format(mode)
# defaults for specific variables
obs_res = None
domain = None
filetype = None
varlist = dataargs.get('varlist', None)
# determine meta data based on dataset type
if dataset == 'WRF':
# WRF datasets
module = import_module('datasets.WRF')
exp = dataargs['experiment']
dataset_name = exp.name
domain = dataargs['domain']
grid = dataargs.get('grid', None)
# figure out period
period = dataargs['period']
if period is None: pass
elif isinstance(period, (int, np.integer)):
beginyear = int(exp.begindate[0:4])
period = (beginyear, beginyear + period)
elif len(period) != 2 and all(isInt(period)):
raise DateError
if period is None: periodstr = ''
else: periodstr = '{0:4d}-{1:4d}'.format(*period)
gridstr = grid if grid is not None else ''
# identify file and domain
if len(dataargs['filetypes']) > 1:
raise DatasetError # process only one file at a time
filetype = dataargs['filetypes'][0]
if isinstance(domain, (list, tuple)): domain = domain[0]
if not isinstance(domain, (np.integer, int)): raise DatasetError
datamsgstr = "Processing WRF '{:s}'-file from Experiment '{:s}' (d{:02d})".format(
filetype, dataset_name, domain)
# assemble filename to check modification dates (should be only one file)
fileclass = module.fileclasses[
filetype] # avoid WRF & CESM name collision
pstr = '_' + periodstr if periodstr else ''
gstr = '_' + gridstr if gridstr else ''
if lclim:
filename = fileclass.climfile.format(
domain, gstr, pstr) # insert domain number, grid, and period
elif lts:
filename = fileclass.tsfile.format(
domain, gstr) # insert domain number, and grid
avgfolder = exp.avgfolder
# load source data
if lclim:
loadfct = functools.partial(
loadWRF,
experiment=exp,
name=None,
domains=domain,
grid=None,
varlist=varlist,
period=period,
filetypes=[filetype],
varatts=None,
lconst=True) # still want topography...
elif lts:
loadfct = functools.partial(
loadWRF_TS,
experiment=exp,
name=None,
domains=domain,
grid=None,
varlist=varlist,
filetypes=[filetype],
varatts=None,
lconst=True) # still want topography...
filepath = '{:s}/{:s}'.format(avgfolder, filename)
elif dataset == 'CESM':
# CESM datasets
module = import_module('datasets.CESM')
exp = dataargs['experiment']
dataset_name = exp.name
# figure out period
period = dataargs['period']
if period is None: pass
elif isinstance(period, (int, np.integer)):
beginyear = int(exp.begindate[0:4])
period = (beginyear, beginyear + period)
elif len(period) != 2 and all(isInt(period)):
raise DateError
# identify file
if len(dataargs['filetypes']) > 1:
raise DatasetError # process only one file at a time
filetype = dataargs['filetypes'][0]
# check period
if period is None: periodstr = ''
else: periodstr = '{0:4d}-{1:4d}'.format(*period)
datamsgstr = "Processing CESM '{:s}'-file from Experiment '{:s}'".format(
filetype, dataset_name)
# assemble filename to check modification dates (should be only one file)
fileclass = module.fileclasses[
filetype] # avoid WRF & CESM name collision
pstr = '_' + periodstr if periodstr else ''
if lclim:
filename = fileclass.climfile.format(
'', pstr) # insert domain number, grid, and period
elif lts:
filename = fileclass.tsfile.format(
'') # insert domain number, and grid
avgfolder = exp.avgfolder
# load source data
load3D = dataargs.pop(
'load3D', None) # if 3D fields should be loaded (default: False)
if lclim:
loadfct = functools.partial(loadCESM,
experiment=exp,
name=None,
grid=None,
period=period,
varlist=varlist,
filetypes=[filetype],
varatts=None,
load3D=load3D,
translateVars=None)
elif lts:
loadfct = functools.partial(loadCESM_TS,
experiment=exp,
name=None,
grid=None,
varlist=varlist,
filetypes=[filetype],
varatts=None,
load3D=load3D,
translateVars=None)
filepath = '{:s}/{:s}'.format(avgfolder, filename)
elif dataset == dataset.upper() or dataset == 'Unity':
# observational datasets
module = import_module('datasets.{0:s}'.format(dataset))
dataset_name = module.dataset_name
resolution = dataargs['resolution']
if resolution: obs_res = '{0:s}_{1:s}'.format(dataset_name, resolution)
else: obs_res = dataset_name
# figure out period
period = dataargs['period']
if period is None: pass
elif isinstance(period, (int, np.integer)):
period = (1979, 1979 + period) # they all begin in 1979
elif len(period) != 2 and not all(isInt(period)):
raise DateError
datamsgstr = "Processing Dataset '{:s}'".format(dataset_name)
# check period
if period is None:
if mode == 'climatology': periodstr = 'Long-Term Mean'
else: periodstr = ''
else: periodstr = '{0:4d}-{1:4d}'.format(*period)
# assemble filename to check modification dates (should be only one file)
filename = getFileName(grid=None,
period=period,
name=obs_res,
filetype=mode)
avgfolder = module.avgfolder
# load pre-processed climatology
if lclim:
loadfct = functools.partial(module.loadClimatology,
name=dataset_name,
period=period,
grid=None,
varlist=varlist,
resolution=resolution,
varatts=None,
folder=module.avgfolder,
filelist=None)
elif lts:
loadfct = functools.partial(module.loadTimeSeries,
name=dataset_name,
grid=None,
varlist=varlist,
resolution=resolution,
varatts=None,
folder=None,
filelist=None)
# check if the source file is actually correct
filepath = '{:s}/{:s}'.format(avgfolder, filename)
if not os.path.exists(filepath):
source = loadfct(
) # no varlist - obs don't have many variables anyways
filepath = source.filelist[0]
# N.B.: it would be nice to print a message, but then we would have to make the logger available,
# which would be too much trouble
else:
raise DatasetError, "Dataset '{:s}' not found!".format(dataset)
## assemble and return meta data
if not os.path.exists(filepath):
raise IOError, "Source file '{:s}' does not exist!".format(filepath)
dataargs = namedTuple(dataset_name=dataset_name,
period=period,
periodstr=periodstr,
avgfolder=avgfolder,
filetype=filetype,
domain=domain,
obs_res=obs_res,
varlist=varlist)
# return meta data
return module, dataargs, loadfct, filepath, datamsgstr
def getFigAx(subplot, name=None, title=None, figsize=None, mpl=None, margins=None,
sharex=None, sharey=None, AxesGrid=False, ngrids=None, direction='row',
axes_pad = None, add_all=True, share_all=None, aspect=False,
label_mode='L', cbar_mode=None, cbar_location='right',
cbar_pad=None, cbar_size='5%', axes_class=None, lreduce=True):
# configure matplotlib
warn('Deprecated function: use Figure or Axes class methods.')
if mpl is None: import matplotlib as mpl
elif isinstance(mpl,dict): mpl = loadMPL(**mpl) # there can be a mplrc, but also others
elif not isinstance(mpl,ModuleType): raise TypeError
from plotting.figure import MyFigure # prevent circular reference
# figure out subplots
if isinstance(subplot,(np.integer,int)):
if subplot == 1: subplot = (1,1)
elif subplot == 2: subplot = (1,2)
elif subplot == 3: subplot = (1,3)
elif subplot == 4: subplot = (2,2)
elif subplot == 6: subplot = (2,3)
elif subplot == 9: subplot = (3,3)
else: raise NotImplementedError
elif not (isinstance(subplot,(tuple,list)) and len(subplot) == 2) and all(isInt(subplot)): raise TypeError
# create figure
if figsize is None:
if subplot == (1,1): figsize = (3.75,3.75)
elif subplot == (1,2) or subplot == (1,3): figsize = (6.25,3.75)
elif subplot == (2,1) or subplot == (3,1): figsize = (3.75,6.25)
else: figsize = (6.25,6.25)
#elif subplot == (2,2) or subplot == (3,3): figsize = (6.25,6.25)
#else: raise NotImplementedError
# figure out margins
if margins is None:
# N.B.: the rectangle definition is presumably left, bottom, width, height
if subplot == (1,1): margins = (0.09,0.09,0.88,0.88)
elif subplot == (1,2) or subplot == (1,3): margins = (0.06,0.1,0.92,0.87)
elif subplot == (2,1) or subplot == (3,1): margins = (0.09,0.11,0.88,0.82)
elif subplot == (2,2) or subplot == (3,3): margins = (0.055,0.055,0.925,0.925)
else: margins = (0.09,0.11,0.88,0.82)
#elif subplot == (2,2) or subplot == (3,3): margins = (0.09,0.11,0.88,0.82)
#else: raise NotImplementedError
if title is not None: margins = margins[:3]+(margins[3]-0.03,) # make room for title
if AxesGrid:
if share_all is None: share_all = True
if axes_pad is None: axes_pad = 0.05
# create axes using the Axes Grid package
fig = mpl.pylab.figure(facecolor='white', figsize=figsize, FigureClass=MyFigure)
if axes_class is None:
from plotting.axes import MyLocatableAxes
axes_class=(MyLocatableAxes,{})
from mpl_toolkits.axes_grid1 import ImageGrid
# AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html
grid = ImageGrid(fig, margins, nrows_ncols = subplot, ngrids=ngrids, direction=direction,
axes_pad=axes_pad, add_all=add_all, share_all=share_all, aspect=aspect,
label_mode=label_mode, cbar_mode=cbar_mode, cbar_location=cbar_location,
cbar_pad=cbar_pad, cbar_size=cbar_size, axes_class=axes_class)
# return figure and axes
axes = tuple([ax for ax in grid]) # this is already flattened
if lreduce and len(axes) == 1: axes = axes[0] # return a bare axes instance, if there is only one axes
else:
# create axes using normal subplot routine
if axes_pad is None: axes_pad = 0.03
wspace = hspace = axes_pad
if share_all:
sharex='all'; sharey='all'
if sharex is True or sharex is None: sharex = 'col' # default
if sharey is True or sharey is None: sharey = 'row'
if sharex: hspace -= 0.015
if sharey: wspace -= 0.015
# create figure
from matplotlib.pyplot import subplots
# GridSpec: http://matplotlib.org/users/gridspec.html
fig, axes = subplots(subplot[0], subplot[1], sharex=sharex, sharey=sharey,
squeeze=lreduce, facecolor='white', figsize=figsize, FigureClass=MyFigure)
# there is also a subplot_kw=dict() and fig_kw=dict()
# just adjust margins
margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0]+margins[2],
top=margins[1]+margins[3], wspace=wspace, hspace=hspace)
fig.subplots_adjust(**margin_dict)
# add figure title
if name is not None: fig.canvas.set_window_title(name) # window title
if title is not None: fig.suptitle(title) # title on figure (printable)
# return Figure/ImageGrid and tuple of axes
#if AxesGrid: fig = grid # return ImageGrid instead of figure
return fig, axes
def getFigAx(subplot,
name=None,
title=None,
title_font='x-large',
title_height=None,
figsize=None,
variable_plotargs=None,
dataset_plotargs=None,
plot_labels=None,
yright=False,
xtop=False,
sharex=None,
sharey=None,
lAxesGrid=False,
ngrids=None,
direction='row',
lPolarAxes=False,
lTaylor=False,
axes_pad=None,
add_all=True,
share_all=None,
aspect=False,
margins=None,
label_mode='L',
cbar_mode=None,
cbar_location='right',
lreduce=True,
cbar_pad=None,
cbar_size='5%',
axes_class=None,
axes_args=None,
stylesheet=None,
lpresentation=False,
lpublication=False,
figure_class=None,
**figure_args):
# load stylesheet
if stylesheet is not None:
loadStyleSheet(stylesheet,
lpresentation=lpresentation,
lpublication=lpublication)
if stylesheet in ('myggplot', 'ggplot'):
warn("Rewriting built-in color definitions to GG-plot defaults.")
if dataset_plotargs is not None:
dataset_plotargs = toGGcolors(
dataset_plotargs) # modifies in-place!
# default figure class
if figure_class is None: figure_class = MyFigure
elif not issubclass(figure_class, Figure): raise TypeError
# figure out subplots
if isinstance(subplot, (np.integer, int)):
if subplot == 1: subplot = (1, 1)
elif subplot == 2: subplot = (1, 2)
elif subplot == 3: subplot = (1, 3)
elif subplot == 4: subplot = (2, 2)
elif subplot == 6: subplot = (2, 3)
elif subplot == 9: subplot = (3, 3)
else: raise NotImplementedError
elif not (isinstance(subplot,
(tuple, list)) and len(subplot) == 2) and all(
isInt(subplot)):
raise TypeError
# create figure
if figsize is None:
if lpublication:
if subplot == (1, 1): figsize = (3.75, 3.75)
elif subplot == (1, 2) or subplot == (1, 3): figsize = (6.25, 3.75)
elif subplot == (2, 1) or subplot == (3, 1): figsize = (3.75, 7)
else: figsize = (6.25, 6.25)
elif lpresentation:
if subplot == (1, 2) or subplot == (1, 3): figsize = (5, 3)
elif subplot == (2, 1) or subplot == (3, 1): figsize = (3, 5)
else: figsize = (5, 5)
else:
if subplot == (1, 1): figsize = (5, 5)
elif subplot == (1, 2) or subplot == (1, 3): figsize = (9, 5)
elif subplot == (2, 1) or subplot == (3, 1): figsize = (5, 9)
else: figsize = (9, 9)
# figure out margins
if margins is None:
# N.B.: the rectangle definition is presumably left, bottom, width, height
if subplot == (1, 1): margins = (0.1, 0.1, 0.85, 0.85)
elif subplot == (1, 2) or subplot == (1, 3):
margins = (0.06, 0.1, 0.92, 0.87)
elif subplot == (2, 1) or subplot == (3, 1):
margins = (0.09, 0.11, 0.88, 0.82)
elif subplot == (2, 2) or subplot == (3, 3):
margins = (0.06, 0.08, 0.92, 0.92)
else:
margins = (0.09, 0.11, 0.88, 0.82)
if title_height is None:
title_height = getattr(figure_class, 'title_height',
0.05) # use default from figure
if title is not None:
margins = margins[:3] + (margins[3] - title_height,
) # make room for title
# # some style sheets have different label sizes
# if stylesheet.lower() in ('myggplot','ggplot'):
# margins = list(margins)
# margins[0] += 0.015; margins[1] -= 0.01 # left, bottom
# margins[2] += 0.02; margins[3] += 0.02 # width, height
# handle special TaylorPlot axes
if lTaylor:
if not lPolarAxes: lPolarAxes = True
if not axes_class: axes_class = TaylorAxes
# handle mixed Polar/Axes
if isinstance(axes_class, (list, tuple, np.ndarray)):
for i, axcls in enumerate(axes_class):
if axcls is None:
if lTaylor: axes_class[i] = TaylorAxes
elif lPolarAxes: axes_class[i] = MyPolarAxes
else: axes_class[i] = MyAxes
elif axcls.lower() == 'taylor': axes_class[i] = TaylorAxes
elif axcls.lower() == 'polar': axes_class[i] = MyPolarAxes
elif axcls.lower() in ('regular', 'default'):
axes_class[i] = MyAxes
if not issubclass(axcls, Axes): raise TypeError(axcls)
# create axes
if lAxesGrid:
if share_all is None: share_all = True
if axes_pad is None: axes_pad = 0.05
# adjust margins for ignored label pads
margins = list(margins)
margins[0] += 0.005
margins[1] -= 0.02 # left, bottom
margins[2] -= 0.005
margins[3] -= 0.00 # width, height
# create axes using the Axes Grid package
if axes_class is None: axes_class = MyLocatableAxes
fig = mpl.pylab.figure(facecolor='white',
figsize=figsize,
axes_class=axes_class,
FigureClass=figure_class,
**figure_args)
if axes_args is None: axes_class = (axes_class, {})
elif isinstance(axes_args, dict): axes_class = (axes_class, axes_args)
else: raise TypeError
from mpl_toolkits.axes_grid1 import ImageGrid
# AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html
grid = ImageGrid(fig,
margins,
nrows_ncols=subplot,
ngrids=ngrids,
direction=direction,
axes_pad=axes_pad,
add_all=add_all,
share_all=share_all,
aspect=aspect,
label_mode=label_mode,
cbar_mode=cbar_mode,
cbar_location=cbar_location,
cbar_pad=cbar_pad,
cbar_size=cbar_size,
axes_class=axes_class)
# return figure and axes
axes = np.asarray(grid).reshape(subplot) # don't want flattened array
#axes = tuple([ax for ax in grid]) # this is already flattened
elif isinstance(axes_class, (list, tuple, np.ndarray)):
# PolarAxes can't share axes and by default don't have labels
if figure_args is None: figure_args = dict()
fig = figure(facecolor='white',
figsize=figsize,
FigureClass=figure_class,
**figure_args)
# now create list of axes
if axes_args is None: axes_args = dict()
axes = np.empty(subplot, dtype=object)
n = 0
for i in range(subplot[0]):
for j in range(subplot[1]):
n += 1
axes[i, j] = fig.add_subplot(subplot[0],
subplot[1],
n,
axes_class=axes_class[n - 1],
aspect=aspect,
**axes_args)
# just adjust margins
if axes_pad is None: axes_pad = 0.03
wspace = hspace = 0.1
margin_dict = dict(left=margins[0],
bottom=margins[1],
right=margins[0] + margins[2],
top=margins[1] + margins[3],
wspace=wspace,
hspace=hspace)
fig.subplots_adjust(**margin_dict)
else:
# select default axes based on other arguments
if axes_class is None:
if lPolarAxes:
axes_class = MyPolarAxes
share_all = sharex = sharey = False
# N.B.: PolarAxes does not support sharing of axes, and
# default behavior is to hide labels
else:
axes_class = MyAxes
# create axes using normal subplot routine
if axes_pad is None: axes_pad = 0.03
wspace = hspace = axes_pad
if share_all:
sharex = 'all'
sharey = 'all'
if sharex is True or sharex is None: sharex = 'col' # default
if sharey is True or sharey is None: sharey = 'row'
if sharex: hspace -= 0.015
if sharey: wspace -= 0.015
# other axes arguments
if axes_args is not None and not isinstance(axes_args, dict):
raise TypeError
# create figure
from matplotlib.pyplot import subplots
# GridSpec: http://matplotlib.org/users/gridspec.html
fig, axes = subplots(subplot[0],
subplot[1],
sharex=sharex,
sharey=sharey,
squeeze=lreduce,
facecolor='white',
figsize=figsize,
FigureClass=figure_class,
subplot_kw=axes_args,
axes_class=axes_class,
**figure_args)
# there is also a subplot_kw=dict() and fig_kw=dict()
# just adjust margins
margin_dict = dict(left=margins[0],
bottom=margins[1],
right=margins[0] + margins[2],
top=margins[1] + margins[3],
wspace=wspace,
hspace=hspace)
fig.subplots_adjust(**margin_dict)
# apply reduction
if lreduce:
if isinstance(axes, np.ndarray):
axes = axes.squeeze() # remove singleton dimensions
if axes.ndim == 0: axes = axes.item()
if isinstance(axes, (list, tuple)) and len(axes) == 1:
axes = axes[
0] # return a bare axes instance, if there is only one axes
## set label positions
if not lPolarAxes:
# X-/Y-labels and -ticks
yright = not sharey and subplot[0] == 2 if yright is None else yright
xtop = not sharex and subplot[1] == 2 if xtop is None else xtop
if isinstance(axes, Axes):
axes.yright = yright
axes.xtop = xtop
else:
if axes.ndim == 1:
if subplot[0] == 2: axes[-1].yright = yright # right panel
if subplot[1] == 2: axes[0].xtop = xtop # top panel
elif axes.ndim == 2:
for ax in axes[:, -1]:
ax.yright = yright # right column
for ax in axes[0, :]:
ax.xtop = xtop # top row
else:
raise ValueError
# add figure title
if name is None: name = title
if name is not None: fig.canvas.set_window_title(name) # window title
if title is not None:
y = 1. - (title_height / (5. if 'x' in title_font else 8.)
) # smaller title closer to the top
if isinstance(title_font, basestring):
title_font = dict(fontsize=title_font, y=y)
fig.suptitle(title, **title_font) # title on figure (printable)
fig.title_height = title_height # save value
# add default line styles for variables and datasets to axes (figure doesn't need to know)
if isinstance(axes, np.ndarray):
for ax in axes.ravel():
ax.variable_plotargs = variable_plotargs
ax.dataset_plotargs = dataset_plotargs
ax.plot_labels = plot_labels
else:
axes.variable_plotargs = variable_plotargs
axes.dataset_plotargs = dataset_plotargs
axes.plot_labels = plot_labels
# return Figure/ImageGrid and tuple of axes
#if AxesGrid: fig = grid # return ImageGrid instead of figure
return fig, axes
def getMetaData(dataset, mode, dataargs):
''' determine dataset type and meta data, as well as path to main source file '''
# determine dataset mode
lclim = False; lts = False
if mode == 'climatology': lclim = True
elif mode == 'time-series': lts = True
else: raise NotImplementedError, "Unrecognized Mode: '{:s}'".format(mode)
# defaults for specific variables
obs_res = None; domain = None; filetype = None
varlist = dataargs.get('varlist',None)
# determine meta data based on dataset type
if dataset == 'WRF':
# WRF datasets
module = import_module('datasets.WRF')
exp = dataargs['experiment']
dataset_name = exp.name
domain = dataargs['domain']
grid = dataargs.get('grid',None)
# figure out period
period = dataargs['period']
if period is None: pass
elif isinstance(period,(int,np.integer)):
beginyear = int(exp.begindate[0:4])
period = (beginyear, beginyear+period)
elif len(period) != 2 and all(isInt(period)): raise DateError
if period is None: periodstr = ''
else: periodstr = '{0:4d}-{1:4d}'.format(*period)
gridstr = grid if grid is not None else ''
# identify file and domain
if len(dataargs['filetypes']) > 1: raise DatasetError # process only one file at a time
filetype = dataargs['filetypes'][0]
if isinstance(domain,(list,tuple)): domain = domain[0]
if not isinstance(domain, (np.integer,int)): raise DatasetError
datamsgstr = "Processing WRF '{:s}'-file from Experiment '{:s}' (d{:02d})".format(filetype, dataset_name, domain)
# assemble filename to check modification dates (should be only one file)
fileclass = module.fileclasses[filetype] # avoid WRF & CESM name collision
pstr = '_'+periodstr if periodstr else ''
gstr = '_'+gridstr if gridstr else ''
if lclim: filename = fileclass.climfile.format(domain,gstr,pstr) # insert domain number, grid, and period
elif lts: filename = fileclass.tsfile.format(domain,gstr) # insert domain number, and grid
avgfolder = exp.avgfolder
# load source data
if lclim:
loadfct = functools.partial(loadWRF, experiment=exp, name=None, domains=domain, grid=None, varlist=varlist,
period=period, filetypes=[filetype], varatts=None, lconst=True) # still want topography...
elif lts:
loadfct = functools.partial(loadWRF_TS, experiment=exp, name=None, domains=domain, grid=None, varlist=varlist,
filetypes=[filetype], varatts=None, lconst=True) # still want topography...
filepath = '{:s}/{:s}'.format(avgfolder,filename)
elif dataset == 'CESM':
# CESM datasets
module = import_module('datasets.CESM')
exp = dataargs['experiment']
dataset_name = exp.name
# figure out period
period = dataargs['period']
if period is None: pass
elif isinstance(period,(int,np.integer)):
beginyear = int(exp.begindate[0:4])
period = (beginyear, beginyear+period)
elif len(period) != 2 and all(isInt(period)): raise DateError
# identify file
if len(dataargs['filetypes']) > 1: raise DatasetError # process only one file at a time
filetype = dataargs['filetypes'][0]
# check period
if period is None: periodstr = ''
else: periodstr = '{0:4d}-{1:4d}'.format(*period)
datamsgstr = "Processing CESM '{:s}'-file from Experiment '{:s}'".format(filetype, dataset_name)
# assemble filename to check modification dates (should be only one file)
fileclass = module.fileclasses[filetype] # avoid WRF & CESM name collision
pstr = '_'+periodstr if periodstr else ''
if lclim: filename = fileclass.climfile.format('',pstr) # insert domain number, grid, and period
elif lts: filename = fileclass.tsfile.format('') # insert domain number, and grid
avgfolder = exp.avgfolder
# load source data
load3D = dataargs.pop('load3D',None) # if 3D fields should be loaded (default: False)
if lclim:
loadfct = functools.partial(loadCESM, experiment=exp, name=None, grid=None, period=period, varlist=varlist,
filetypes=[filetype], varatts=None, load3D=load3D, translateVars=None)
elif lts:
loadfct = functools.partial(loadCESM_TS, experiment=exp, name=None, grid=None, varlist=varlist,
filetypes=[filetype], varatts=None, load3D=load3D, translateVars=None)
filepath = '{:s}/{:s}'.format(avgfolder,filename)
elif dataset == dataset.upper() or dataset == 'Unity':
# observational datasets
module = import_module('datasets.{0:s}'.format(dataset))
dataset_name = module.dataset_name
resolution = dataargs['resolution']
if resolution: obs_res = '{0:s}_{1:s}'.format(dataset_name,resolution)
else: obs_res = dataset_name
# figure out period
period = dataargs['period']
if period is None: pass
elif isinstance(period,(int,np.integer)):
period = (1979, 1979+period) # they all begin in 1979
elif len(period) != 2 and not all(isInt(period)): raise DateError
datamsgstr = "Processing Dataset '{:s}'".format(dataset_name)
# check period
if period is None:
if mode == 'climatology': periodstr = 'Long-Term Mean'
else: periodstr = ''
else: periodstr = '{0:4d}-{1:4d}'.format(*period)
# assemble filename to check modification dates (should be only one file)
filename = getFileName(grid=None, period=period, name=obs_res, filetype=mode)
avgfolder = module.avgfolder
# load pre-processed climatology
if lclim:
loadfct = functools.partial(module.loadClimatology, name=dataset_name, period=period, grid=None, varlist=varlist,
resolution=resolution, varatts=None, folder=module.avgfolder, filelist=None)
elif lts:
loadfct = functools.partial(module.loadTimeSeries, name=dataset_name, grid=None, varlist=varlist,
resolution=resolution, varatts=None, folder=None, filelist=None)
# check if the source file is actually correct
filepath = '{:s}/{:s}'.format(avgfolder,filename)
if not os.path.exists(filepath):
source = loadfct() # no varlist - obs don't have many variables anyways
filepath = source.filelist[0]
# N.B.: it would be nice to print a message, but then we would have to make the logger available,
# which would be too much trouble
else:
raise DatasetError, "Dataset '{:s}' not found!".format(dataset)
## assemble and return meta data
if not os.path.exists(filepath): raise IOError, "Source file '{:s}' does not exist!".format(filepath)
dataargs = namedTuple(dataset_name=dataset_name, period=period, periodstr=periodstr, avgfolder=avgfolder,
filetype=filetype, domain=domain, obs_res=obs_res, varlist=varlist)
# return meta data
return module, dataargs, loadfct, filepath, datamsgstr
