def getPeriodGridString(period, grid, exp=None, beginyear=None): ''' utility function to check period and grid and return valid and usable strings ''' # period if period is None: pass elif isinstance(period,(int,np.integer)): if beginyear is None: beginyear = int(exp.begindate[0:4]) # most datasets begin in 1979 period = (beginyear, beginyear+period) elif len(period) != 2 and all(isInt(period)): raise DateError periodstr = '{0:4d}-{1:4d}'.format(*period) if period else '' # grid gridstr = grid if grid else '' # return return periodstr, gridstr
def getPeriodGridString(period, grid, exp=None, beginyear=None): ''' utility function to check period and grid and return valid and usable strings ''' # period if period is None: pass elif isinstance(period, (int, np.integer)): if beginyear is None: beginyear = int(exp.begindate[0:4]) # most datasets begin in 1979 period = (beginyear, beginyear + period) elif len(period) != 2 and all(isInt(period)): raise DateError periodstr = '{0:4d}-{1:4d}'.format(*period) if period else '' # grid gridstr = grid if grid else '' # return return periodstr, gridstr
def computeClimatology(experiment, filetype, domain, periods=None, offset=0, griddef=None, varlist=None, ldebug=False, loverwrite=False, lparallel=False, pidstr='', logger=None): ''' worker function to compute climatologies for given file parameters. ''' # input type checks if not isinstance(experiment,Exp): raise TypeError if not isinstance(filetype,basestring): raise TypeError if not isinstance(domain,(np.integer,int)): raise TypeError if periods is not None and not (isinstance(periods,(tuple,list)) and isInt(periods)): raise TypeError if not isinstance(offset,(np.integer,int)): raise TypeError if not isinstance(loverwrite,(bool,np.bool)): raise TypeError if griddef is not None and not isinstance(griddef,GridDefinition): raise TypeError #if pidstr == '[proc01]': raise TypeError # to test error handling # load source dataset_name = experiment.name fileclass = fileclasses[filetype] # used for target file name tsfile = fileclass.tsfile.format(domain,'') expfolder = experiment.avgfolder filepath = '{:s}/{:s}'.format(expfolder, tsfile) logger.info('\n\n{0:s} *** Processing Experiment {1:<15s} *** '.format(pidstr,"'{:s}'".format(dataset_name)) + '\n{0:s} *** {1:^37s} *** \n'.format(pidstr,"'{:s}'".format(tsfile))) # check file and read begin/enddates if not os.path.exists(filepath): #raise IOError, "Source file '{:s}' does not exist!".format(filepath) # print message and skip skipmsg = "\n{:s} >>> File '{:s}' in dataset '{:s}' is missing --- skipping!".format(pidstr,tsfile,dataset_name) skipmsg += "\n{:s} >>> ('{:s}')\n".format(pidstr,filepath) logger.warning(skipmsg) # N.B.: this can cause a lot of error messages, when not all files are present else: # if monthly source file exists import netCDF4 as nc ncfile = nc.Dataset(filepath,mode='r') begintuple = ncfile.begin_date.split('-') endtuple = ncfile.end_date.split('-') ncfile.close() # N.B.: at this point we don't want to initialize a full GDAL-enabled dataset, since we don't even # know if we need it, and it creates a lot of overhead # determine age of source file if not loverwrite: sourceage = datetime.fromtimestamp(os.path.getmtime(filepath)) # figure out start date filebegin = int(begintuple[0]) # first element is the year fileend = int(endtuple[0]) # first element is the year begindate = offset + filebegin if not ( filebegin <= begindate <= fileend ): raise DateError # handle cases where the first month in the record is not January firstmonth = int(begintuple[1]) # second element is the month shift = firstmonth-1 # will be zero for January (01) ## loop over periods if periods is None: periods = [begindate-fileend] # periods.sort(reverse=True) # reverse, so that largest chunk is done first source = None # will later be assigned to the source dataset for period in periods: # figure out period enddate = begindate + period if filebegin > enddate: raise DateError, 'End date earlier than begin date.' if enddate-1 > fileend: # if filebegin is 1979 and the simulation is 10 years, fileend will be 1988, not 1989! # if end date is not available, skip period endmsg = "\n{:s} --- Invalid Period for '{:s}': End Date {:4d} not in File! --- \n".format(pidstr,dataset_name,enddate) endmsg += "{:s} --- ('{:s}')\n".format(pidstr,filepath) logger.info(endmsg) else: ## perform averaging for selected period # determine if sink file already exists, and what to do about it periodstr = '{0:4d}-{1:4d}'.format(begindate,enddate) gridstr = '' if griddef is None or griddef.name is 'WRF' else '_'+griddef.name filename = fileclass.climfile.format(domain,gridstr,'_'+periodstr) if ldebug: filename = 'test_' + filename if lparallel: tmppfx = 'tmp_wrfavg_{:s}_'.format(pidstr[1:-1]) else: tmppfx = 'tmp_wrfavg_'.format(pidstr[1:-1]) tmpfilename = tmppfx + filename assert os.path.exists(expfolder) filepath = expfolder+filename tmpfilepath = expfolder+tmpfilename lskip = False # else just go ahead if os.path.exists(filepath): if not loverwrite: age = datetime.fromtimestamp(os.path.getmtime(filepath)) # if sink file is newer than source file, skip (do not recompute) if age > sourceage and os.path.getsize(filepath) > 1e6: lskip = True # N.B.: NetCDF files smaller than 1MB are usually incomplete header fragments from a previous crash #print sourceage, age if not lskip: os.remove(filepath) # depending on last modification time of file or overwrite setting, start computation, or skip if lskip: # print message skipmsg = "\n{:s} >>> Skipping: file '{:s}' in dataset '{:s}' already exists and is newer than source file.".format(pidstr,filename,dataset_name) skipmsg += "\n{:s} >>> ('{:s}')\n".format(pidstr,filepath) logger.info(skipmsg) else: ## begin actual computation beginmsg = "\n{:s} <<< Computing '{:s}' (d{:02d}) Climatology from {:s}".format( pidstr,dataset_name,domain,periodstr) if griddef is None: beginmsg += " >>> \n" else: beginmsg += " ('{:s}' grid) >>> \n".format(griddef.name) logger.info(beginmsg) ## actually load datasets if source is None: source = loadWRF_TS(experiment=experiment, filetypes=[filetype], domains=domain) # comes out as a tuple... if not lparallel and ldebug: logger.info('\n'+str(source)+'\n') # prepare sink if os.path.exists(tmpfilepath): os.remove(tmpfilepath) # remove old temp files sink = DatasetNetCDF(name='WRF Climatology', folder=expfolder, filelist=[tmpfilename], atts=source.atts.copy(), mode='w') sink.atts.period = periodstr # initialize processing if griddef is None: lregrid = False else: lregrid = True CPU = CentralProcessingUnit(source, sink, varlist=varlist, tmp=lregrid, feedback=ldebug) # no need for lat/lon # start processing climatology if shift != 0: logger.info('{0:s} (shifting climatology by {1:d} month, to start with January) \n'.format(pidstr,shift)) CPU.Climatology(period=period, offset=offset, shift=shift, flush=False) # N.B.: immediate flushing should not be necessary for climatologies, since they are much smaller! # reproject and resample (regrid) dataset if lregrid: CPU.Regrid(griddef=griddef, flush=True) logger.info('%s --- '+str(griddef.geotansform)+' --- \n'%(pidstr)) # sync temporary storage with output dataset (sink) CPU.sync(flush=True) # add Geopotential Height Variance if 'GHT_Var' in sink and 'Z_var' not in sink: data_array = ( sink['GHT_Var'].data_array - sink['Z'].data_array**2 )**0.5 atts = dict(name='Z_var',units='m',long_name='Square Root of Geopotential Height Variance') sink += Variable(axes=sink['Z'].axes, data=data_array, atts=atts) # add (relative) Vorticity Variance if 'Vorticity_Var' in sink and 'zeta_var' not in sink: data_array = ( sink['Vorticity_Var'].data_array - sink['zeta'].data_array**2 )**0.5 atts = dict(name='zeta_var',units='1/s',long_name='Square Root of Relative Vorticity Variance') sink += Variable(axes=sink['zeta'].axes, data=data_array, atts=atts) # add names and length of months sink.axisAnnotation('name_of_month', name_of_month, 'time', atts=dict(name='name_of_month', units='', long_name='Name of the Month')) if not sink.hasVariable('length_of_month'): sink += Variable(name='length_of_month', units='days', axes=(sink.time,), data=days_per_month, atts=dict(name='length_of_month',units='days',long_name='Length of Month')) # close... and write results to file sink.sync() sink.close() writemsg = "\n{:s} >>> Writing to file '{:s}' in dataset {:s}".format(pidstr,filename,dataset_name) writemsg += "\n{:s} >>> ('{:s}')\n".format(pidstr,filepath) logger.info(writemsg) # rename file to proper name if os.path.exists(filepath): os.remove(filepath) # remove old file os.rename(tmpfilepath,filepath) # this will overwrite the old file # print dataset if not lparallel and ldebug: logger.info('\n'+str(sink)+'\n') # clean up (not sure if this is necessary, but there seems to be a memory leak... del sink, CPU; gc.collect() # get rid of these guys immediately # clean up and return if source is not None: source.unload(); del source # N.B.: source is only loaded once for all periods # N.B.: garbage is collected in multi-processing wrapper as well # return return 0 # so far, there is no measure of success, hence, if there is no crash...
def getFigAx(subplot, name=None, title=None, figsize=None, mpl=None, margins=None, sharex=None, sharey=None, AxesGrid=False, ngrids=None, direction='row', axes_pad = None, add_all=True, share_all=None, aspect=False, label_mode='L', cbar_mode=None, cbar_location='right', cbar_pad=None, cbar_size='5%', axes_class=None, lreduce=True): # configure matplotlib warn('Deprecated function: use Figure or Axes class methods.') if mpl is None: import matplotlib as mpl elif isinstance(mpl,dict): mpl = loadMPL(**mpl) # there can be a mplrc, but also others elif not isinstance(mpl,ModuleType): raise TypeError from plotting.figure import MyFigure # prevent circular reference # figure out subplots if isinstance(subplot,(np.integer,int)): if subplot == 1: subplot = (1,1) elif subplot == 2: subplot = (1,2) elif subplot == 3: subplot = (1,3) elif subplot == 4: subplot = (2,2) elif subplot == 6: subplot = (2,3) elif subplot == 9: subplot = (3,3) else: raise NotImplementedError elif not (isinstance(subplot,(tuple,list)) and len(subplot) == 2) and all(isInt(subplot)): raise TypeError # create figure if figsize is None: if subplot == (1,1): figsize = (3.75,3.75) elif subplot == (1,2) or subplot == (1,3): figsize = (6.25,3.75) elif subplot == (2,1) or subplot == (3,1): figsize = (3.75,6.25) else: figsize = (6.25,6.25) #elif subplot == (2,2) or subplot == (3,3): figsize = (6.25,6.25) #else: raise NotImplementedError # figure out margins if margins is None: # N.B.: the rectangle definition is presumably left, bottom, width, height if subplot == (1,1): margins = (0.09,0.09,0.88,0.88) elif subplot == (1,2) or subplot == (1,3): margins = (0.06,0.1,0.92,0.87) elif subplot == (2,1) or subplot == (3,1): margins = (0.09,0.11,0.88,0.82) elif subplot == (2,2) or subplot == (3,3): margins = (0.055,0.055,0.925,0.925) else: margins = (0.09,0.11,0.88,0.82) #elif subplot == (2,2) or subplot == (3,3): margins = (0.09,0.11,0.88,0.82) #else: raise NotImplementedError if title is not None: margins = margins[:3]+(margins[3]-0.03,) # make room for title if AxesGrid: if share_all is None: share_all = True if axes_pad is None: axes_pad = 0.05 # create axes using the Axes Grid package fig = mpl.pylab.figure(facecolor='white', figsize=figsize, FigureClass=MyFigure) if axes_class is None: from plotting.axes import MyLocatableAxes axes_class=(MyLocatableAxes,{}) from mpl_toolkits.axes_grid1 import ImageGrid # AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html grid = ImageGrid(fig, margins, nrows_ncols = subplot, ngrids=ngrids, direction=direction, axes_pad=axes_pad, add_all=add_all, share_all=share_all, aspect=aspect, label_mode=label_mode, cbar_mode=cbar_mode, cbar_location=cbar_location, cbar_pad=cbar_pad, cbar_size=cbar_size, axes_class=axes_class) # return figure and axes axes = tuple([ax for ax in grid]) # this is already flattened if lreduce and len(axes) == 1: axes = axes[0] # return a bare axes instance, if there is only one axes else: # create axes using normal subplot routine if axes_pad is None: axes_pad = 0.03 wspace = hspace = axes_pad if share_all: sharex='all'; sharey='all' if sharex is True or sharex is None: sharex = 'col' # default if sharey is True or sharey is None: sharey = 'row' if sharex: hspace -= 0.015 if sharey: wspace -= 0.015 # create figure from matplotlib.pyplot import subplots # GridSpec: http://matplotlib.org/users/gridspec.html fig, axes = subplots(subplot[0], subplot[1], sharex=sharex, sharey=sharey, squeeze=lreduce, facecolor='white', figsize=figsize, FigureClass=MyFigure) # there is also a subplot_kw=dict() and fig_kw=dict() # just adjust margins margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0]+margins[2], top=margins[1]+margins[3], wspace=wspace, hspace=hspace) fig.subplots_adjust(**margin_dict) # add figure title if name is not None: fig.canvas.set_window_title(name) # window title if title is not None: fig.suptitle(title) # title on figure (printable) # return Figure/ImageGrid and tuple of axes #if AxesGrid: fig = grid # return ImageGrid instead of figure return fig, axes
def getFigAx(subplot, name=None, title=None, title_font='large', figsize=None, stylesheet=None, variable_plotargs=None, dataset_plotargs=None, plot_labels=None, yright=False, xtop=False, sharex=None, sharey=None, AxesGrid=False, ngrids=None, direction='row', axes_pad = None, add_all=True, share_all=None, aspect=False, margins=None, label_mode='L', cbar_mode=None, cbar_location='right', lreduce=True, cbar_pad=None, cbar_size='5%', axes_class=None, axes_args=None, lpresentation=False, lpublication=False, figure_class=None, **figure_args): # load stylesheet if stylesheet is not None: loadStyleSheet(stylesheet, lpresentation=lpresentation, lpublication=lpublication) if stylesheet in ('myggplot','ggplot'): warn("Rewriting built-in color definitions to GG-plot defaults.") if dataset_plotargs is not None: dataset_plotargs = toGGcolors(dataset_plotargs) # modifies in-place! # default figure class if figure_class is None: figure_class = MyFigure elif not issubclass(figure_class, Figure): raise TypeError # figure out subplots if isinstance(subplot,(np.integer,int)): if subplot == 1: subplot = (1,1) elif subplot == 2: subplot = (1,2) elif subplot == 3: subplot = (1,3) elif subplot == 4: subplot = (2,2) elif subplot == 6: subplot = (2,3) elif subplot == 9: subplot = (3,3) else: raise NotImplementedError elif not (isinstance(subplot,(tuple,list)) and len(subplot) == 2) and all(isInt(subplot)): raise TypeError # create figure if figsize is None: if lpublication: if subplot == (1,1): figsize = (3.75,3.75) elif subplot == (1,2) or subplot == (1,3): figsize = (6.25,3.75) elif subplot == (2,1) or subplot == (3,1): figsize = (3.75,7) else: figsize = (6.25,6.25) elif lpresentation: if subplot == (1,2) or subplot == (1,3): figsize = (5,3) elif subplot == (2,1) or subplot == (3,1): figsize = (3,5) else: figsize = (5,5) else: if subplot == (1,1): figsize = (5,5) elif subplot == (1,2) or subplot == (1,3): figsize = (9,5) elif subplot == (2,1) or subplot == (3,1): figsize = (5,9) else: figsize = (9,9) # figure out margins if margins is None: # N.B.: the rectangle definition is presumably left, bottom, width, height if subplot == (1,1): margins = (0.1,0.1,0.85,0.85) elif subplot == (1,2) or subplot == (1,3): margins = (0.06,0.1,0.92,0.87) elif subplot == (2,1) or subplot == (3,1): margins = (0.09,0.11,0.88,0.82) elif subplot == (2,2) or subplot == (3,3): margins = (0.06,0.08,0.92,0.92) else: margins = (0.09,0.11,0.88,0.82) #elif subplot == (2,2) or subplot == (3,3): margins = (0.09,0.11,0.88,0.82) #else: raise NotImplementedError title_height = getattr(figure_class, 'title_height', 0.05) # use default from figure if title is not None: margins = margins[:3]+(margins[3]-title_height,) # make room for title # # some style sheets have different label sizes # if stylesheet.lower() in ('myggplot','ggplot'): # margins = list(margins) # margins[0] += 0.015; margins[1] -= 0.01 # left, bottom # margins[2] += 0.02; margins[3] += 0.02 # width, height if AxesGrid: if share_all is None: share_all = True if axes_pad is None: axes_pad = 0.05 # adjust margins for ignored label pads margins = list(margins) margins[0] += 0.005; margins[1] -= 0.02 # left, bottom margins[2] -= 0.005; margins[3] -= 0.00 # width, height # create axes using the Axes Grid package if axes_class is None: axes_class=MyLocatableAxes fig = mpl.pylab.figure(facecolor='white', figsize=figsize, axes_class=axes_class, FigureClass=MyFigure, **figure_args) if axes_args is None: axes_class = (axes_class,{}) elif isinstance(axes_args,dict): axes_class = (axes_class,axes_args) else: raise TypeError from mpl_toolkits.axes_grid1 import ImageGrid # AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html grid = ImageGrid(fig, margins, nrows_ncols = subplot, ngrids=ngrids, direction=direction, axes_pad=axes_pad, add_all=add_all, share_all=share_all, aspect=aspect, label_mode=label_mode, cbar_mode=cbar_mode, cbar_location=cbar_location, cbar_pad=cbar_pad, cbar_size=cbar_size, axes_class=axes_class) # return figure and axes axes = np.asarray(grid).reshape(subplot) # don't want flattened array #axes = tuple([ax for ax in grid]) # this is already flattened if lreduce and len(axes) == 1: axes = axes[0] # return a bare axes instance, if there is only one axes else: # create axes using normal subplot routine if axes_pad is None: axes_pad = 0.03 wspace = hspace = axes_pad if share_all: sharex='all'; sharey='all' if sharex is True or sharex is None: sharex = 'col' # default if sharey is True or sharey is None: sharey = 'row' if sharex: hspace -= 0.015 if sharey: wspace -= 0.015 # other axes arguments if axes_class is None: axes_class=MyAxes if axes_args is not None and not isinstance(axes_args,dict): raise TypeError # create figure from matplotlib.pyplot import subplots # GridSpec: http://matplotlib.org/users/gridspec.html fig, axes = subplots(subplot[0], subplot[1], sharex=sharex, sharey=sharey,squeeze=lreduce, facecolor='white', figsize=figsize, FigureClass=MyFigure, subplot_kw=axes_args, axes_class=axes_class, **figure_args) # there is also a subplot_kw=dict() and fig_kw=dict() # just adjust margins margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0]+margins[2], top=margins[1]+margins[3], wspace=wspace, hspace=hspace) fig.subplots_adjust(**margin_dict) ## set label positions # X-/Y-labels and -ticks yright = not sharey and subplot[0]==2 if yright is None else yright xtop = not sharex and subplot[1]==2 if xtop is None else xtop if isinstance(axes, Axes): axes.yright = yright axes.xtop = xtop else: if axes.ndim == 1: if subplot[0] == 2: axes[-1].yright = yright # right panel if subplot[1] == 2: axes[0].xtop = xtop # top panel elif axes.ndim == 2: for ax in axes[:,-1]: ax.yright = yright # right column for ax in axes[0,:]: ax.xtop = xtop # top row else: raise ValueError # add figure title if name is None: name = title if name is not None: fig.canvas.set_window_title(name) # window title if isinstance(title_font,basestring): title_font = dict(fontsize=title_font) if title is not None: fig.suptitle(title, **title_font) # title on figure (printable) # add default line styles for variables and datasets to axes (figure doesn't need to know) if isinstance(axes, np.ndarray): for ax in axes.ravel(): ax.variable_plotargs = variable_plotargs ax.dataset_plotargs = dataset_plotargs ax.plot_labels = plot_labels else: axes.variable_plotargs = variable_plotargs axes.dataset_plotargs = dataset_plotargs axes.plot_labels = plot_labels # return Figure/ImageGrid and tuple of axes #if AxesGrid: fig = grid # return ImageGrid instead of figure return fig, axes
def computeClimatology(experiment, filetype, domain, periods=None, offset=0, griddef=None, varlist=None, ldebug=False, loverwrite=False, lparallel=False, pidstr='', logger=None): ''' worker function to compute climatologies for given file parameters. ''' # input type checks if not isinstance(experiment, Exp): raise TypeError if not isinstance(filetype, basestring): raise TypeError if not isinstance(domain, (np.integer, int)): raise TypeError if periods is not None and not (isinstance(periods, (tuple, list)) and isInt(periods)): raise TypeError if not isinstance(offset, (np.integer, int)): raise TypeError if not isinstance(loverwrite, (bool, np.bool)): raise TypeError if griddef is not None and not isinstance(griddef, GridDefinition): raise TypeError #if pidstr == '[proc01]': raise TypeError # to test error handling # load source dataset_name = experiment.name fileclass = fileclasses[filetype] # used for target file name tsfile = fileclass.tsfile.format(domain, '') expfolder = experiment.avgfolder filepath = '{:s}/{:s}'.format(expfolder, tsfile) logger.info('\n\n{0:s} *** Processing Experiment {1:<15s} *** '. format(pidstr, "'{:s}'".format(dataset_name)) + '\n{0:s} *** {1:^37s} *** \n'.format( pidstr, "'{:s}'".format(tsfile))) # check file and read begin/enddates if not os.path.exists(filepath): #raise IOError, "Source file '{:s}' does not exist!".format(filepath) # print message and skip skipmsg = "\n{:s} >>> File '{:s}' in dataset '{:s}' is missing --- skipping!".format( pidstr, tsfile, dataset_name) skipmsg += "\n{:s} >>> ('{:s}')\n".format(pidstr, filepath) logger.warning(skipmsg) # N.B.: this can cause a lot of error messages, when not all files are present else: # if monthly source file exists import netCDF4 as nc ncfile = nc.Dataset(filepath, mode='r') begintuple = ncfile.begin_date.split('-') endtuple = ncfile.end_date.split('-') ncfile.close() # N.B.: at this point we don't want to initialize a full GDAL-enabled dataset, since we don't even # know if we need it, and it creates a lot of overhead # determine age of source file if not loverwrite: sourceage = datetime.fromtimestamp(os.path.getmtime(filepath)) # figure out start date filebegin = int(begintuple[0]) # first element is the year fileend = int(endtuple[0]) # first element is the year begindate = offset + filebegin if not (filebegin <= begindate <= fileend): raise DateError # handle cases where the first month in the record is not January firstmonth = int(begintuple[1]) # second element is the month shift = firstmonth - 1 # will be zero for January (01) ## loop over periods if periods is None: periods = [begindate - fileend] # periods.sort(reverse=True) # reverse, so that largest chunk is done first source = None # will later be assigned to the source dataset for period in periods: # figure out period enddate = begindate + period if filebegin > enddate: raise DateError, 'End date earlier than begin date.' if enddate - 1 > fileend: # if filebegin is 1979 and the simulation is 10 years, fileend will be 1988, not 1989! # if end date is not available, skip period endmsg = "\n{:s} --- Invalid Period for '{:s}': End Date {:4d} not in File! --- \n".format( pidstr, dataset_name, enddate) endmsg += "{:s} --- ('{:s}')\n".format(pidstr, filepath) logger.info(endmsg) else: ## perform averaging for selected period # determine if sink file already exists, and what to do about it periodstr = '{0:4d}-{1:4d}'.format(begindate, enddate) gridstr = '' if griddef is None or griddef.name is 'WRF' else '_' + griddef.name filename = fileclass.climfile.format(domain, gridstr, '_' + periodstr) if ldebug: filename = 'test_' + filename if lparallel: tmppfx = 'tmp_wrfavg_{:s}_'.format(pidstr[1:-1]) else: tmppfx = 'tmp_wrfavg_'.format(pidstr[1:-1]) tmpfilename = tmppfx + filename assert os.path.exists(expfolder) filepath = expfolder + filename tmpfilepath = expfolder + tmpfilename lskip = False # else just go ahead if os.path.exists(filepath): if not loverwrite: age = datetime.fromtimestamp( os.path.getmtime(filepath)) # if sink file is newer than source file, skip (do not recompute) if age > sourceage and os.path.getsize(filepath) > 1e6: lskip = True # N.B.: NetCDF files smaller than 1MB are usually incomplete header fragments from a previous crash #print sourceage, age if not lskip: os.remove(filepath) # depending on last modification time of file or overwrite setting, start computation, or skip if lskip: # print message skipmsg = "\n{:s} >>> Skipping: file '{:s}' in dataset '{:s}' already exists and is newer than source file.".format( pidstr, filename, dataset_name) skipmsg += "\n{:s} >>> ('{:s}')\n".format( pidstr, filepath) logger.info(skipmsg) else: if griddef is None: lregrid = False else: lregrid = True ## begin actual computation beginmsg = "\n{:s} <<< Computing '{:s}' (d{:02d}) Climatology from {:s}".format( pidstr, dataset_name, domain, periodstr) if not lregrid: beginmsg += " >>> \n" else: beginmsg += " ('{:s}' grid) >>> \n".format( griddef.name) logger.info(beginmsg) ## actually load datasets if source is None: source = loadWRF_TS( experiment=experiment, filetypes=[filetype], domains=domain) # comes out as a tuple... if not lparallel and ldebug: logger.info('\n' + str(source) + '\n') # prepare sink if os.path.exists(tmpfilepath): os.remove(tmpfilepath) # remove old temp files sink = DatasetNetCDF(name='WRF Climatology', folder=expfolder, filelist=[tmpfilename], atts=source.atts.copy(), mode='w') sink.atts.period = periodstr # if lregrid: addGDALtoDataset(sink, griddef=griddef) # initialize processing CPU = CentralProcessingUnit( source, sink, varlist=varlist, tmp=lregrid, feedback=ldebug) # no need for lat/lon # start processing climatology if shift != 0: logger.info( '{0:s} (shifting climatology by {1:d} month, to start with January) \n' .format(pidstr, shift)) CPU.Climatology(period=period, offset=offset, shift=shift, flush=False) # N.B.: immediate flushing should not be necessary for climatologies, since they are much smaller! # reproject and resample (regrid) dataset if lregrid: CPU.Regrid(griddef=griddef, flush=True) logger.info('{:s} --- {:s} --- \n'.format( pidstr, griddef.name)) logger.debug('{:s} --- {:s} --- \n'.format( pidstr, str(griddef))) # sync temporary storage with output dataset (sink) CPU.sync(flush=True) # add Geopotential Height Variance if 'GHT_Var' in sink and 'Z_var' not in sink: data_array = (sink['GHT_Var'].data_array - sink['Z'].data_array**2)**0.5 atts = dict( name='Z_var', units='m', long_name= 'Square Root of Geopotential Height Variance') sink += Variable(axes=sink['Z'].axes, data=data_array, atts=atts) # add (relative) Vorticity Variance if 'Vorticity_Var' in sink and 'zeta_var' not in sink: data_array = (sink['Vorticity_Var'].data_array - sink['zeta'].data_array**2)**0.5 atts = dict( name='zeta_var', units='1/s', long_name= 'Square Root of Relative Vorticity Variance') sink += Variable(axes=sink['zeta'].axes, data=data_array, atts=atts) # add names and length of months sink.axisAnnotation('name_of_month', name_of_month, 'time', atts=dict( name='name_of_month', units='', long_name='Name of the Month')) if not sink.hasVariable('length_of_month'): sink += Variable(name='length_of_month', units='days', axes=(sink.time, ), data=days_per_month, atts=dict( name='length_of_month', units='days', long_name='Length of Month')) # close... and write results to file sink.sync() sink.close() writemsg = "\n{:s} >>> Writing to file '{:s}' in dataset {:s}".format( pidstr, filename, dataset_name) writemsg += "\n{:s} >>> ('{:s}')\n".format( pidstr, filepath) logger.info(writemsg) # rename file to proper name if os.path.exists(filepath): os.remove(filepath) # remove old file os.rename(tmpfilepath, filepath) # this will overwrite the old file # print dataset if not lparallel and ldebug: logger.info('\n' + str(sink) + '\n') # clean up (not sure if this is necessary, but there seems to be a memory leak... del sink, CPU gc.collect() # get rid of these guys immediately # clean up and return if source is not None: source.unload() del source # N.B.: source is only loaded once for all periods # N.B.: garbage is collected in multi-processing wrapper as well # return return 0 # so far, there is no measure of success, hence, if there is no crash...
def getMetaData(dataset, mode, dataargs): ''' determine dataset type and meta data, as well as path to main source file ''' # determine dataset mode lclim = False lts = False if mode == 'climatology': lclim = True elif mode == 'time-series': lts = True else: raise NotImplementedError, "Unrecognized Mode: '{:s}'".format(mode) # defaults for specific variables obs_res = None domain = None filetype = None varlist = dataargs.get('varlist', None) # determine meta data based on dataset type if dataset == 'WRF': # WRF datasets module = import_module('datasets.WRF') exp = dataargs['experiment'] dataset_name = exp.name domain = dataargs['domain'] grid = dataargs.get('grid', None) # figure out period period = dataargs['period'] if period is None: pass elif isinstance(period, (int, np.integer)): beginyear = int(exp.begindate[0:4]) period = (beginyear, beginyear + period) elif len(period) != 2 and all(isInt(period)): raise DateError if period is None: periodstr = '' else: periodstr = '{0:4d}-{1:4d}'.format(*period) gridstr = grid if grid is not None else '' # identify file and domain if len(dataargs['filetypes']) > 1: raise DatasetError # process only one file at a time filetype = dataargs['filetypes'][0] if isinstance(domain, (list, tuple)): domain = domain[0] if not isinstance(domain, (np.integer, int)): raise DatasetError datamsgstr = "Processing WRF '{:s}'-file from Experiment '{:s}' (d{:02d})".format( filetype, dataset_name, domain) # assemble filename to check modification dates (should be only one file) fileclass = module.fileclasses[ filetype] # avoid WRF & CESM name collision pstr = '_' + periodstr if periodstr else '' gstr = '_' + gridstr if gridstr else '' if lclim: filename = fileclass.climfile.format( domain, gstr, pstr) # insert domain number, grid, and period elif lts: filename = fileclass.tsfile.format( domain, gstr) # insert domain number, and grid avgfolder = exp.avgfolder # load source data if lclim: loadfct = functools.partial( loadWRF, experiment=exp, name=None, domains=domain, grid=None, varlist=varlist, period=period, filetypes=[filetype], varatts=None, lconst=True) # still want topography... elif lts: loadfct = functools.partial( loadWRF_TS, experiment=exp, name=None, domains=domain, grid=None, varlist=varlist, filetypes=[filetype], varatts=None, lconst=True) # still want topography... filepath = '{:s}/{:s}'.format(avgfolder, filename) elif dataset == 'CESM': # CESM datasets module = import_module('datasets.CESM') exp = dataargs['experiment'] dataset_name = exp.name # figure out period period = dataargs['period'] if period is None: pass elif isinstance(period, (int, np.integer)): beginyear = int(exp.begindate[0:4]) period = (beginyear, beginyear + period) elif len(period) != 2 and all(isInt(period)): raise DateError # identify file if len(dataargs['filetypes']) > 1: raise DatasetError # process only one file at a time filetype = dataargs['filetypes'][0] # check period if period is None: periodstr = '' else: periodstr = '{0:4d}-{1:4d}'.format(*period) datamsgstr = "Processing CESM '{:s}'-file from Experiment '{:s}'".format( filetype, dataset_name) # assemble filename to check modification dates (should be only one file) fileclass = module.fileclasses[ filetype] # avoid WRF & CESM name collision pstr = '_' + periodstr if periodstr else '' if lclim: filename = fileclass.climfile.format( '', pstr) # insert domain number, grid, and period elif lts: filename = fileclass.tsfile.format( '') # insert domain number, and grid avgfolder = exp.avgfolder # load source data load3D = dataargs.pop( 'load3D', None) # if 3D fields should be loaded (default: False) if lclim: loadfct = functools.partial(loadCESM, experiment=exp, name=None, grid=None, period=period, varlist=varlist, filetypes=[filetype], varatts=None, load3D=load3D, translateVars=None) elif lts: loadfct = functools.partial(loadCESM_TS, experiment=exp, name=None, grid=None, varlist=varlist, filetypes=[filetype], varatts=None, load3D=load3D, translateVars=None) filepath = '{:s}/{:s}'.format(avgfolder, filename) elif dataset == dataset.upper() or dataset == 'Unity': # observational datasets module = import_module('datasets.{0:s}'.format(dataset)) dataset_name = module.dataset_name resolution = dataargs['resolution'] if resolution: obs_res = '{0:s}_{1:s}'.format(dataset_name, resolution) else: obs_res = dataset_name # figure out period period = dataargs['period'] if period is None: pass elif isinstance(period, (int, np.integer)): period = (1979, 1979 + period) # they all begin in 1979 elif len(period) != 2 and not all(isInt(period)): raise DateError datamsgstr = "Processing Dataset '{:s}'".format(dataset_name) # check period if period is None: if mode == 'climatology': periodstr = 'Long-Term Mean' else: periodstr = '' else: periodstr = '{0:4d}-{1:4d}'.format(*period) # assemble filename to check modification dates (should be only one file) filename = getFileName(grid=None, period=period, name=obs_res, filetype=mode) avgfolder = module.avgfolder # load pre-processed climatology if lclim: loadfct = functools.partial(module.loadClimatology, name=dataset_name, period=period, grid=None, varlist=varlist, resolution=resolution, varatts=None, folder=module.avgfolder, filelist=None) elif lts: loadfct = functools.partial(module.loadTimeSeries, name=dataset_name, grid=None, varlist=varlist, resolution=resolution, varatts=None, folder=None, filelist=None) # check if the source file is actually correct filepath = '{:s}/{:s}'.format(avgfolder, filename) if not os.path.exists(filepath): source = loadfct( ) # no varlist - obs don't have many variables anyways filepath = source.filelist[0] # N.B.: it would be nice to print a message, but then we would have to make the logger available, # which would be too much trouble else: raise DatasetError, "Dataset '{:s}' not found!".format(dataset) ## assemble and return meta data if not os.path.exists(filepath): raise IOError, "Source file '{:s}' does not exist!".format(filepath) dataargs = namedTuple(dataset_name=dataset_name, period=period, periodstr=periodstr, avgfolder=avgfolder, filetype=filetype, domain=domain, obs_res=obs_res, varlist=varlist) # return meta data return module, dataargs, loadfct, filepath, datamsgstr
def getFigAx(subplot, name=None, title=None, title_font='x-large', title_height=None, figsize=None, variable_plotargs=None, dataset_plotargs=None, plot_labels=None, yright=False, xtop=False, sharex=None, sharey=None, lAxesGrid=False, ngrids=None, direction='row', lPolarAxes=False, lTaylor=False, axes_pad=None, add_all=True, share_all=None, aspect=False, margins=None, label_mode='L', cbar_mode=None, cbar_location='right', lreduce=True, cbar_pad=None, cbar_size='5%', axes_class=None, axes_args=None, stylesheet=None, lpresentation=False, lpublication=False, figure_class=None, **figure_args): # load stylesheet if stylesheet is not None: loadStyleSheet(stylesheet, lpresentation=lpresentation, lpublication=lpublication) if stylesheet in ('myggplot', 'ggplot'): warn("Rewriting built-in color definitions to GG-plot defaults.") if dataset_plotargs is not None: dataset_plotargs = toGGcolors( dataset_plotargs) # modifies in-place! # default figure class if figure_class is None: figure_class = MyFigure elif not issubclass(figure_class, Figure): raise TypeError # figure out subplots if isinstance(subplot, (np.integer, int)): if subplot == 1: subplot = (1, 1) elif subplot == 2: subplot = (1, 2) elif subplot == 3: subplot = (1, 3) elif subplot == 4: subplot = (2, 2) elif subplot == 6: subplot = (2, 3) elif subplot == 9: subplot = (3, 3) else: raise NotImplementedError elif not (isinstance(subplot, (tuple, list)) and len(subplot) == 2) and all( isInt(subplot)): raise TypeError # create figure if figsize is None: if lpublication: if subplot == (1, 1): figsize = (3.75, 3.75) elif subplot == (1, 2) or subplot == (1, 3): figsize = (6.25, 3.75) elif subplot == (2, 1) or subplot == (3, 1): figsize = (3.75, 7) else: figsize = (6.25, 6.25) elif lpresentation: if subplot == (1, 2) or subplot == (1, 3): figsize = (5, 3) elif subplot == (2, 1) or subplot == (3, 1): figsize = (3, 5) else: figsize = (5, 5) else: if subplot == (1, 1): figsize = (5, 5) elif subplot == (1, 2) or subplot == (1, 3): figsize = (9, 5) elif subplot == (2, 1) or subplot == (3, 1): figsize = (5, 9) else: figsize = (9, 9) # figure out margins if margins is None: # N.B.: the rectangle definition is presumably left, bottom, width, height if subplot == (1, 1): margins = (0.1, 0.1, 0.85, 0.85) elif subplot == (1, 2) or subplot == (1, 3): margins = (0.06, 0.1, 0.92, 0.87) elif subplot == (2, 1) or subplot == (3, 1): margins = (0.09, 0.11, 0.88, 0.82) elif subplot == (2, 2) or subplot == (3, 3): margins = (0.06, 0.08, 0.92, 0.92) else: margins = (0.09, 0.11, 0.88, 0.82) if title_height is None: title_height = getattr(figure_class, 'title_height', 0.05) # use default from figure if title is not None: margins = margins[:3] + (margins[3] - title_height, ) # make room for title # # some style sheets have different label sizes # if stylesheet.lower() in ('myggplot','ggplot'): # margins = list(margins) # margins[0] += 0.015; margins[1] -= 0.01 # left, bottom # margins[2] += 0.02; margins[3] += 0.02 # width, height # handle special TaylorPlot axes if lTaylor: if not lPolarAxes: lPolarAxes = True if not axes_class: axes_class = TaylorAxes # handle mixed Polar/Axes if isinstance(axes_class, (list, tuple, np.ndarray)): for i, axcls in enumerate(axes_class): if axcls is None: if lTaylor: axes_class[i] = TaylorAxes elif lPolarAxes: axes_class[i] = MyPolarAxes else: axes_class[i] = MyAxes elif axcls.lower() == 'taylor': axes_class[i] = TaylorAxes elif axcls.lower() == 'polar': axes_class[i] = MyPolarAxes elif axcls.lower() in ('regular', 'default'): axes_class[i] = MyAxes if not issubclass(axcls, Axes): raise TypeError(axcls) # create axes if lAxesGrid: if share_all is None: share_all = True if axes_pad is None: axes_pad = 0.05 # adjust margins for ignored label pads margins = list(margins) margins[0] += 0.005 margins[1] -= 0.02 # left, bottom margins[2] -= 0.005 margins[3] -= 0.00 # width, height # create axes using the Axes Grid package if axes_class is None: axes_class = MyLocatableAxes fig = mpl.pylab.figure(facecolor='white', figsize=figsize, axes_class=axes_class, FigureClass=figure_class, **figure_args) if axes_args is None: axes_class = (axes_class, {}) elif isinstance(axes_args, dict): axes_class = (axes_class, axes_args) else: raise TypeError from mpl_toolkits.axes_grid1 import ImageGrid # AxesGrid: http://matplotlib.org/mpl_toolkits/axes_grid/users/overview.html grid = ImageGrid(fig, margins, nrows_ncols=subplot, ngrids=ngrids, direction=direction, axes_pad=axes_pad, add_all=add_all, share_all=share_all, aspect=aspect, label_mode=label_mode, cbar_mode=cbar_mode, cbar_location=cbar_location, cbar_pad=cbar_pad, cbar_size=cbar_size, axes_class=axes_class) # return figure and axes axes = np.asarray(grid).reshape(subplot) # don't want flattened array #axes = tuple([ax for ax in grid]) # this is already flattened elif isinstance(axes_class, (list, tuple, np.ndarray)): # PolarAxes can't share axes and by default don't have labels if figure_args is None: figure_args = dict() fig = figure(facecolor='white', figsize=figsize, FigureClass=figure_class, **figure_args) # now create list of axes if axes_args is None: axes_args = dict() axes = np.empty(subplot, dtype=object) n = 0 for i in range(subplot[0]): for j in range(subplot[1]): n += 1 axes[i, j] = fig.add_subplot(subplot[0], subplot[1], n, axes_class=axes_class[n - 1], aspect=aspect, **axes_args) # just adjust margins if axes_pad is None: axes_pad = 0.03 wspace = hspace = 0.1 margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0] + margins[2], top=margins[1] + margins[3], wspace=wspace, hspace=hspace) fig.subplots_adjust(**margin_dict) else: # select default axes based on other arguments if axes_class is None: if lPolarAxes: axes_class = MyPolarAxes share_all = sharex = sharey = False # N.B.: PolarAxes does not support sharing of axes, and # default behavior is to hide labels else: axes_class = MyAxes # create axes using normal subplot routine if axes_pad is None: axes_pad = 0.03 wspace = hspace = axes_pad if share_all: sharex = 'all' sharey = 'all' if sharex is True or sharex is None: sharex = 'col' # default if sharey is True or sharey is None: sharey = 'row' if sharex: hspace -= 0.015 if sharey: wspace -= 0.015 # other axes arguments if axes_args is not None and not isinstance(axes_args, dict): raise TypeError # create figure from matplotlib.pyplot import subplots # GridSpec: http://matplotlib.org/users/gridspec.html fig, axes = subplots(subplot[0], subplot[1], sharex=sharex, sharey=sharey, squeeze=lreduce, facecolor='white', figsize=figsize, FigureClass=figure_class, subplot_kw=axes_args, axes_class=axes_class, **figure_args) # there is also a subplot_kw=dict() and fig_kw=dict() # just adjust margins margin_dict = dict(left=margins[0], bottom=margins[1], right=margins[0] + margins[2], top=margins[1] + margins[3], wspace=wspace, hspace=hspace) fig.subplots_adjust(**margin_dict) # apply reduction if lreduce: if isinstance(axes, np.ndarray): axes = axes.squeeze() # remove singleton dimensions if axes.ndim == 0: axes = axes.item() if isinstance(axes, (list, tuple)) and len(axes) == 1: axes = axes[ 0] # return a bare axes instance, if there is only one axes ## set label positions if not lPolarAxes: # X-/Y-labels and -ticks yright = not sharey and subplot[0] == 2 if yright is None else yright xtop = not sharex and subplot[1] == 2 if xtop is None else xtop if isinstance(axes, Axes): axes.yright = yright axes.xtop = xtop else: if axes.ndim == 1: if subplot[0] == 2: axes[-1].yright = yright # right panel if subplot[1] == 2: axes[0].xtop = xtop # top panel elif axes.ndim == 2: for ax in axes[:, -1]: ax.yright = yright # right column for ax in axes[0, :]: ax.xtop = xtop # top row else: raise ValueError # add figure title if name is None: name = title if name is not None: fig.canvas.set_window_title(name) # window title if title is not None: y = 1. - (title_height / (5. if 'x' in title_font else 8.) ) # smaller title closer to the top if isinstance(title_font, basestring): title_font = dict(fontsize=title_font, y=y) fig.suptitle(title, **title_font) # title on figure (printable) fig.title_height = title_height # save value # add default line styles for variables and datasets to axes (figure doesn't need to know) if isinstance(axes, np.ndarray): for ax in axes.ravel(): ax.variable_plotargs = variable_plotargs ax.dataset_plotargs = dataset_plotargs ax.plot_labels = plot_labels else: axes.variable_plotargs = variable_plotargs axes.dataset_plotargs = dataset_plotargs axes.plot_labels = plot_labels # return Figure/ImageGrid and tuple of axes #if AxesGrid: fig = grid # return ImageGrid instead of figure return fig, axes
def getMetaData(dataset, mode, dataargs): ''' determine dataset type and meta data, as well as path to main source file ''' # determine dataset mode lclim = False; lts = False if mode == 'climatology': lclim = True elif mode == 'time-series': lts = True else: raise NotImplementedError, "Unrecognized Mode: '{:s}'".format(mode) # defaults for specific variables obs_res = None; domain = None; filetype = None varlist = dataargs.get('varlist',None) # determine meta data based on dataset type if dataset == 'WRF': # WRF datasets module = import_module('datasets.WRF') exp = dataargs['experiment'] dataset_name = exp.name domain = dataargs['domain'] grid = dataargs.get('grid',None) # figure out period period = dataargs['period'] if period is None: pass elif isinstance(period,(int,np.integer)): beginyear = int(exp.begindate[0:4]) period = (beginyear, beginyear+period) elif len(period) != 2 and all(isInt(period)): raise DateError if period is None: periodstr = '' else: periodstr = '{0:4d}-{1:4d}'.format(*period) gridstr = grid if grid is not None else '' # identify file and domain if len(dataargs['filetypes']) > 1: raise DatasetError # process only one file at a time filetype = dataargs['filetypes'][0] if isinstance(domain,(list,tuple)): domain = domain[0] if not isinstance(domain, (np.integer,int)): raise DatasetError datamsgstr = "Processing WRF '{:s}'-file from Experiment '{:s}' (d{:02d})".format(filetype, dataset_name, domain) # assemble filename to check modification dates (should be only one file) fileclass = module.fileclasses[filetype] # avoid WRF & CESM name collision pstr = '_'+periodstr if periodstr else '' gstr = '_'+gridstr if gridstr else '' if lclim: filename = fileclass.climfile.format(domain,gstr,pstr) # insert domain number, grid, and period elif lts: filename = fileclass.tsfile.format(domain,gstr) # insert domain number, and grid avgfolder = exp.avgfolder # load source data if lclim: loadfct = functools.partial(loadWRF, experiment=exp, name=None, domains=domain, grid=None, varlist=varlist, period=period, filetypes=[filetype], varatts=None, lconst=True) # still want topography... elif lts: loadfct = functools.partial(loadWRF_TS, experiment=exp, name=None, domains=domain, grid=None, varlist=varlist, filetypes=[filetype], varatts=None, lconst=True) # still want topography... filepath = '{:s}/{:s}'.format(avgfolder,filename) elif dataset == 'CESM': # CESM datasets module = import_module('datasets.CESM') exp = dataargs['experiment'] dataset_name = exp.name # figure out period period = dataargs['period'] if period is None: pass elif isinstance(period,(int,np.integer)): beginyear = int(exp.begindate[0:4]) period = (beginyear, beginyear+period) elif len(period) != 2 and all(isInt(period)): raise DateError # identify file if len(dataargs['filetypes']) > 1: raise DatasetError # process only one file at a time filetype = dataargs['filetypes'][0] # check period if period is None: periodstr = '' else: periodstr = '{0:4d}-{1:4d}'.format(*period) datamsgstr = "Processing CESM '{:s}'-file from Experiment '{:s}'".format(filetype, dataset_name) # assemble filename to check modification dates (should be only one file) fileclass = module.fileclasses[filetype] # avoid WRF & CESM name collision pstr = '_'+periodstr if periodstr else '' if lclim: filename = fileclass.climfile.format('',pstr) # insert domain number, grid, and period elif lts: filename = fileclass.tsfile.format('') # insert domain number, and grid avgfolder = exp.avgfolder # load source data load3D = dataargs.pop('load3D',None) # if 3D fields should be loaded (default: False) if lclim: loadfct = functools.partial(loadCESM, experiment=exp, name=None, grid=None, period=period, varlist=varlist, filetypes=[filetype], varatts=None, load3D=load3D, translateVars=None) elif lts: loadfct = functools.partial(loadCESM_TS, experiment=exp, name=None, grid=None, varlist=varlist, filetypes=[filetype], varatts=None, load3D=load3D, translateVars=None) filepath = '{:s}/{:s}'.format(avgfolder,filename) elif dataset == dataset.upper() or dataset == 'Unity': # observational datasets module = import_module('datasets.{0:s}'.format(dataset)) dataset_name = module.dataset_name resolution = dataargs['resolution'] if resolution: obs_res = '{0:s}_{1:s}'.format(dataset_name,resolution) else: obs_res = dataset_name # figure out period period = dataargs['period'] if period is None: pass elif isinstance(period,(int,np.integer)): period = (1979, 1979+period) # they all begin in 1979 elif len(period) != 2 and not all(isInt(period)): raise DateError datamsgstr = "Processing Dataset '{:s}'".format(dataset_name) # check period if period is None: if mode == 'climatology': periodstr = 'Long-Term Mean' else: periodstr = '' else: periodstr = '{0:4d}-{1:4d}'.format(*period) # assemble filename to check modification dates (should be only one file) filename = getFileName(grid=None, period=period, name=obs_res, filetype=mode) avgfolder = module.avgfolder # load pre-processed climatology if lclim: loadfct = functools.partial(module.loadClimatology, name=dataset_name, period=period, grid=None, varlist=varlist, resolution=resolution, varatts=None, folder=module.avgfolder, filelist=None) elif lts: loadfct = functools.partial(module.loadTimeSeries, name=dataset_name, grid=None, varlist=varlist, resolution=resolution, varatts=None, folder=None, filelist=None) # check if the source file is actually correct filepath = '{:s}/{:s}'.format(avgfolder,filename) if not os.path.exists(filepath): source = loadfct() # no varlist - obs don't have many variables anyways filepath = source.filelist[0] # N.B.: it would be nice to print a message, but then we would have to make the logger available, # which would be too much trouble else: raise DatasetError, "Dataset '{:s}' not found!".format(dataset) ## assemble and return meta data if not os.path.exists(filepath): raise IOError, "Source file '{:s}' does not exist!".format(filepath) dataargs = namedTuple(dataset_name=dataset_name, period=period, periodstr=periodstr, avgfolder=avgfolder, filetype=filetype, domain=domain, obs_res=obs_res, varlist=varlist) # return meta data return module, dataargs, loadfct, filepath, datamsgstr