def __init__(self, shmem): AggregationTask.__init__(self, shmem) reader = wire.PackedMessageReader(shmem) # TODO: may wish to merge more than 2 at a time? # Unpack header self.left_inst = pickle_load(reader.string()) self.right_inst = pickle_load(reader.string())
def plot_fluxnet_LAI_CHT_AGB(directory, sites_list, timeframe, linestyle): #=============================================================================== """ This function plots the LAI, CHT and AGB auxilliary data from FluxNet sites Function arguments: ------------------ directory: path to the directory where the daily variables files ('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1') are stored sites_list: list of fluxnet sites names time frame: list of two years (e.g. [2005, 2007]) gives the time frame to plot linestyle: line style of the plot to make (can be '-', '--', '+', etc) """ plt.close('all') print "\nPlotting timeseries of LAI, CHT and AGB\n" for site in sites_list: print site + '\n' filepath = os.path.join(directory, 'timeseries_%s.pickle' % site) series = pickle_load(open(filepath, 'rb')) # settings of the plot: figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8, 10)) figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.) variables = ['crop_no', 'LAI', 'CHT', 'AGB'] axlabels = [ 'crop ID', r'LAI (m$^{-2}$ m$^{-2}$)', 'CHT (m)', r'AGB (kg$_{DM}$ m$^{-2}$)' ] ylims = [(0., 14.), (0., 7.5), (0., 3.), (0., 3.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) lab = '' ls = linestyle fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims): if var == 'crop_no': ls = '-' else: ls = linestyle series[site][var][start:end].plot(ax=ax, lw=2, style=ls, label=lab, fontsize=fsz) ax.axhline(y=0., c='k') ax.set_ylim(ylim) ax.set_ylabel(axlabel) if var != 'AGB': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) plt.show() return None
def test_merge(self): klass = self._get_mean_uda() lcol = self.col_fragments[0] rcol = self.col_fragments[1] left = self._update(klass, [lcol]) right = self._update(klass, [rcol]) task, mm = self._make_merge_task(left, right) _execute_task(task, self.lock) mm.seek(0) reader = wire.PackedMessageReader(mm) # success if not reader.uint8(): raise Exception(reader.string()) result = pickle_load(reader.string()) larr = lcol.to_numpy_for_pandas() rarr = rcol.to_numpy_for_pandas() assert larr is not None ex_total = (pd.Series(larr).sum() + pd.Series(rarr).sum()) assert result.total == ex_total
def init_session(remote_address, cookie_data=None): if cookie_data is not None: cookie = SessionCookie.load(cookie_data) else: cookie = None # Default sid for the session sid = sha224('%s-%s' % (remote_address, datetime.utcnow())).hexdigest() if cookie is None: cookie = SessionCookie(sid) else: try: cookie.get_sid() except KeyError: # For some reason the cookie did not contain a SID, set to default cookie.set_sid(sid) # Set the session singleton (there can be only one!) global CURRENT_SESSION ppath = get_session_pickle_path(cookie.get_sid()) if isfile(ppath): # Load our old session data and initialise the cookie try: with open(ppath, 'rb') as session_pickle: CURRENT_SESSION = pickle_load(session_pickle) CURRENT_SESSION.init_cookie(CURRENT_SESSION.get_sid()) except Exception, e: # On any error, just create a new session CURRENT_SESSION = Session(cookie)
def load_pickle(filename): file = open(filename + '.pck', 'r') res = pickle_load(file) file.close() return res
def learning_curve_avg(classifiers, datasets, outdir, pickle_name='learning'): with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r') as results_file: results = pickle_load(results_file) for dataset in datasets: print 'Dataset:', dataset for classifier in classifiers: print 'Classifier:', classifier macro_avg = mean([ res_tup[0] for res_tup in results[dataset][classifier].itervalues() ]) * 100 macro_tip = sorted((size, res_tup[0]) for size, res_tup in results[dataset] [classifier].iteritems())[-1][1] * 100 amb_avg = mean([ res_tup[11] for res_tup in results[dataset][classifier].itervalues() ]) amb_tip = sorted((size, res_tup[11]) for size, res_tup in results[dataset][classifier].iteritems())[-1][1] rec_avg = mean([ res_tup[13] for res_tup in results[dataset][classifier].itervalues() ]) * 100 rec_tip = sorted((size, res_tup[13]) for size, res_tup in results[dataset] [classifier].iteritems())[-1][1] * 100 print( '{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f} ' 'MACROAVG/MACROTIP/AMBAVG/AMBTIP/RECAVG/RECTIP').format( macro_avg, macro_tip, amb_avg, amb_tip, rec_avg, rec_tip)
def plot_learning_curve_results(classifiers, datasets, outdir, pickle_name='learning', verbose=False): from matplotlib import pyplot as plt if not classifiers and not datasets: print >> stderr, 'No classifiers or datasets specified, exiting' return # Load the results to be plotted if verbose: print >> stderr, 'Loading results...', with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r') as results_file: results = pickle_load(results_file) if verbose: print >> stderr, 'Done!' image_formats = ('svg', ) for fig_name, fig in _ambiguity_plot_gen(results, classifiers, datasets): for image_format in image_formats: plt.savefig(path_join(outdir, fig_name + '.' + image_format), format=image_format, figure=fig)
def learning_curve_avg(classifiers, datasets, outdir, pickle_name='learning'): with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r') as results_file: results = pickle_load(results_file) for dataset in datasets: print 'Dataset:', dataset for classifier in classifiers: print 'Classifier:', classifier macro_avg = mean([res_tup[0] for res_tup in results[dataset][classifier].itervalues()]) * 100 macro_tip = sorted((size, res_tup[0]) for size, res_tup in results[dataset][classifier].iteritems())[-1][1] * 100 amb_avg = mean([res_tup[11] for res_tup in results[dataset][classifier].itervalues()]) amb_tip = sorted((size, res_tup[11]) for size, res_tup in results[dataset][classifier].iteritems())[-1][1] rec_avg = mean([res_tup[13] for res_tup in results[dataset][classifier].itervalues()]) * 100 rec_tip = sorted((size, res_tup[13]) for size, res_tup in results[dataset][classifier].iteritems())[-1][1] * 100 print ('{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f} ' 'MACROAVG/MACROTIP/AMBAVG/AMBTIP/RECAVG/RECTIP').format( macro_avg, macro_tip, amb_avg, amb_tip, rec_avg, rec_tip)
def main(args): argp = ARGPARSER.parse_args(args[1:]) if not argp.no_cache: # We can't do it iteratively listening to stdin, read it all doc = Document('<classify>', [], [], '<classify>') for _string in (l.rstrip('\n') for l in argp.input): doc.abstract.append(_string_to_ann_sent(_string)) docs = (doc, ) else: docs = (Document('Line: %s' % i, [], [_string_to_ann_sent(_string)], '<stdin>') for i, _string in enumerate( (l.rstrip('\n') for l in argp.input), start=1)) # Cache the strings for speed if not argp.no_cache: cache_simstring((docs, ), verbose=argp.verbose) with open(argp.model_path, 'r') as model_file: classifier = pickle_load(model_file) # TODO: Faster to do it in a batch instead for doc in docs: for sent in doc: for ann in sent: print '%s\t%s' % (sent.annotation_text(ann), str(classifier.classify(doc, sent, ann, ranked=True)))
def _load_simstring_cache(): global SIMSTRING_QUERY_CACHE if not isfile(SIMSTRING_QUERY_CACHE_PATH): #XXX: TODO: We need a check-sum test if we are to keep a specific db SIMSTRING_QUERY_CACHE = {} else: with open(SIMSTRING_QUERY_CACHE_PATH, 'rb') as cache_file: SIMSTRING_QUERY_CACHE = pickle_load(cache_file)
def _load_pickle(pickle_path, save=True): if isfile(pickle_path): try: with open(pickle_path, 'rb') as pickle_file: loaded_pickle = pickle_load(pickle_file) except EOFError, e: print >> stderr, "failed to load pickle %s: %s" % (pickle_path, e) raise
def getAnnObject2(collection,document): '''newest version of the getAnnObject methode''' try: from os.path import join as path_join from document import real_directory real_dir = real_directory(collection) except: real_dir=collection app_path = WORK_DIR + "/application/" ann = None full_name = collection + document full_name = full_name.replace("/","") if( isfile(app_path+full_name)): temp=open (app_path+full_name , 'rb') ann = pickle_load(temp) temp.close() else: ann = TextAnnotations(real_dir+document) ann = SimpleAnnotations(ann) ann.folia = {} try: #TODO:good error message ann.folia=get_extra_info(collection,document) except Exception as e: ann.folia = {} Messager.error('Error: get extra folia info() failed: %s' % e) #Validation: try: import os import simplejson as json import session docdir = os.path.dirname(ann._document) string = session.load_conf()["config"] val = json.loads(string)["validationOn"] #validate if config enables it and if it's not already done. if val: if not ann.validated: from verify_annotations import verify_annotation projectconf = ProjectConfiguration(docdir) issues = [] issues = verify_annotation(ann, projectconf) else: issues = ann.issues else: ann.validated = False issues = [] except session.NoSessionError: issues = [] except KeyError: issues = [] except Exception as e: # TODO add an issue about the failure? issues = [] ann.issues = issues temp=open (app_path+full_name , 'wb') pickle_dump(ann, temp) temp.close() return ann
def run(self): if self.prior_state is not None: agg_inst = self.prior_state else: klass = pickle_load(self.agg_class_pickled) agg_inst = klass() args = self._deserialize_args() agg_inst.update(*args) self._write_response(agg_inst)
def test_update(self): klass = self._get_mean_uda() col = self.col_fragments[0] task, mm = self._make_update_task(klass, [col]) _execute_task(task, self.lock) mm.seek(0) reader = wire.PackedMessageReader(mm) # success if not reader.uint8(): raise Exception(reader.string()) result = pickle_load(reader.string()) ex_total = pd.Series(col.to_numpy_for_pandas()).sum() assert result.total == ex_total # Test with prior state col = self.col_fragments[1] task, mm = self._make_update_task(klass, [col], prior_state=result) # Executor's turn again self.lock.release() _execute_task(task, self.lock) mm.seek(0) reader = wire.PackedMessageReader(mm) # success if not reader.uint8(): raise Exception(reader.string()) result = pickle_load(reader.string()) ex_total += pd.Series(col.to_numpy_for_pandas()).sum() # pandas will yield 0 on None input strangely assert ex_total != 0 assert result.total == ex_total
def plot_fluxnet_LAI_CHT_AGB(directory, sites_list, timeframe, linestyle): #=============================================================================== """ This function plots the LAI, CHT and AGB auxilliary data from FluxNet sites Function arguments: ------------------ directory: path to the directory where the daily variables files ('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1') are stored sites_list: list of fluxnet sites names time frame: list of two years (e.g. [2005, 2007]) gives the time frame to plot linestyle: line style of the plot to make (can be '-', '--', '+', etc) """ plt.close('all') print "\nPlotting timeseries of LAI, CHT and AGB\n" for site in sites_list: print site+'\n' filepath = os.path.join(directory,'timeseries_%s.pickle'%site) series = pickle_load(open(filepath,'rb')) # settings of the plot: figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8,10)) figs.subplots_adjust(0.1,0.07,0.98,0.95,0.,0.) variables = ['crop_no','LAI','CHT','AGB'] axlabels = ['crop ID',r'LAI (m$^{-2}$ m$^{-2}$)','CHT (m)',r'AGB (kg$_{DM}$ m$^{-2}$)'] ylims = [(0.,14.),(0.,7.5),(0.,3.),(0.,3.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) lab = '' ls = linestyle fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes,variables,axlabels,ylims): if var=='crop_no': ls='-' else: ls=linestyle series[site][var][start:end].plot(ax=ax, lw=2, style=ls, label=lab, fontsize=fsz) ax.axhline(y=0., c='k') ax.set_ylim(ylim) ax.set_ylabel(axlabel) if var != 'AGB': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) plt.show() return None
def _read_header(self): reader = wire.PackedMessageReader(self.shmem) # Unpack header self.agg_class_pickled = reader.string() has_prior_state = reader.uint8() != 0 if has_prior_state: self.prior_state = pickle_load(reader.string()) else: self.prior_state = None
def load(filename): """Load a template from ``filename``, return ``Templater`` object. This method must be used in pair with ``Templater.dump`` - it loads the template definition from a file using cPickle, creates a ``Templater`` object with the definition and returns it. """ fp = open(filename) processed_template = pickle_load(fp) fp.close() return processed_template
def _load_pickle(pickle_path, save=True): if isfile(pickle_path): with open(pickle_path, 'rb') as pickle_file: loaded_pickle = pickle_load(pickle_file) else: loaded_pickle = IdMapper() if save: atexit_register(_save_pickle_func(loaded_pickle, pickle_path)) return loaded_pickle
def import_parser(path, auto=True, from_raw=True): qpt = QPTokenizer() if (auto and path.endswith('qpy')) or (not auto and not from_raw): cfile = open(path, 'rb') assert cfile.read(len(COMPILED_SIGNATURE)) == COMPILED_SIGNATURE, "Wrong file format" dump = pickle_load(cfile) return {proto.name: proto.make() for proto in dump.inner_scopes} elif (auto and path.endswith('qp')) or (not auto and from_raw): return qpt.parse_file(path) else: raise ValueError("auto parser path should end with .qp for raw parsers, " "or .qpy for compiled ones.")
def import_parser(path, auto=True, from_raw=True): qpt = QPTokenizer() if (auto and path.endswith('qpy')) or (not auto and not from_raw): cfile = open(path, 'rb') assert cfile.read( len(COMPILED_SIGNATURE)) == COMPILED_SIGNATURE, "Wrong file format" dump = pickle_load(cfile) return {proto.name: proto.make() for proto in dump.inner_scopes} elif (auto and path.endswith('qp')) or (not auto and from_raw): return qpt.parse_file(path) else: raise ValueError( "auto parser path should end with .qp for raw parsers, " "or .qpy for compiled ones.")
def getAnnObject(collection, document): try: real_dir = real_directory(collection) except: real_dir = collection app_path = WORK_DIR + "/application/" full_name = collection + document full_name = full_name.replace("/", "") if (os.path.isfile(app_path + full_name)): temp = open(app_path + full_name, 'rb') ann = pickle_load(temp) temp.close() else: ann = TextAnnotations(real_dir + document) ann = SimpleAnnotations(ann) ann.folia = {} try: #TODO:good error message ann.folia = get_extra_info(collection, document) except Exception as e: ann.folia = {} Messager.error('Error: get extra folia info() failed: %s' % e) #Validation: try: docdir = os.path.dirname(ann._document) string = session.load_conf()["config"] val = json.loads(string)["validationOn"] #validate if config enables it and if it's not already done. if val: if not ann.validated: projectconf = ProjectConfiguration(docdir) issues = verify_annotation(ann, projectconf) else: issues = ann.issues else: ann.validated = False issues = [] except session.NoSessionError: issues = [] except KeyError: issues = [] except Exception as e: # TODO add an issue about the failure? issues = [] Messager.error('Error: validation failed: %s' % e) ann.issues = issues temp = open(app_path + full_name, 'wb') pickle_dump(ann, temp) temp.close() return ann
def load(cls, filename=PICKLE_CACHE_PATH): from cPickle import UnpicklingError from cPickle import load as pickle_load try: with open(filename, 'rb') as cache_file: map_ = unordall(pickle_load(cache_file)) return cls(map_) except UnpicklingError: print >> sys.stderr, 'warning: failed to read cache file.' raise except IOError: print >> sys.stderr, 'note: cache file not found.' raise except: print >> sys.stderr, 'warning: unexpected error loading cache.' raise
def load_cache(fn): from cPickle import UnpicklingError from cPickle import load as pickle_load try: with open(fn, 'rb') as cache_file: data = pickle_load(cache_file) return data except UnpicklingError: print >> sys.stderr, "rewritetex: warning: failed to read cache file." raise except IOError: print >> sys.stderr, "rewritetex: note: cache file not found." raise except: print >> sys.stderr, "rewritetex: warning: unexpected error loading cache." raise
def test_finalize(self): klass = self._get_mean_uda() col = self.col_fragments[0] result = self._update(klass, [col]) task, mm = self._make_finalize_task(result) _execute_task(task, self.lock) mm.seek(0) reader = wire.PackedMessageReader(mm) # success if not reader.uint8(): raise Exception(reader.string()) result = pickle_load(reader.string()) arr = col.to_numpy_for_pandas() ex_result = pd.Series(arr).mean() assert result == ex_result
def plot_learning_curve_results(classifiers, datasets, outdir, pickle_name='learning', verbose=False): from matplotlib import pyplot as plt if not classifiers and not datasets: print >> stderr, 'No classifiers or datasets specified, exiting' return # Load the results to be plotted if verbose: print >> stderr, 'Loading results...', with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r' ) as results_file: results = pickle_load(results_file) if verbose: print >> stderr, 'Done!' image_formats = ('svg', ) for fig_name, fig in _ambiguity_plot_gen(results, classifiers, datasets): for image_format in image_formats: plt.savefig(path_join(outdir, fig_name + '.' + image_format), format=image_format, figure=fig)
def main(): #=============================================================================== global wofostdir, sibcasadir, obsdir #------------------------------------------------------------------------------- # ================================= USER INPUT ================================= # read the settings from the rc file (mostly directory paths) rcdict = rc.read('settings.rc') sites = [s.strip(' ') for s in rcdict['sites'].split(',')] years = [int(s.strip(' ')) for s in rcdict['years'].split(',')] TER_method = rcdict['TER_method'] R10 = rcdict['R10'] resolution = rcdict['resolution'] # specific plotting options: #TER_method = 'grow-only' # this is to select the corresponding WOFOST output file #R10 = '0.08' # this is to select the corresponding WOFOST output file #=============================================================================== #------------------------------------------------------------------------------- # extract the needed information # input data directory paths rootdir = rcdict['rootdir'] sibcasadir = os.path.join(rootdir, 'intercomparison_study/SiBCASA_runs') wofostdir = rcdict['outputdir'] obsdir = rcdict['obsdir'] figdir = os.path.join(rootdir, 'intercomparison_study/figures') #------------------------------------------------------------------------------- # Start a directory to store OBS, SIMW (wofost), SIMB (SiBCASA) # recover the FluxNet observed data from pickle files res_timeseries = dict() res_timeseries['OBS'] = dict() res_timeseries['SIMB'] = dict() res_timeseries['SIMW'] = dict() filename = os.path.join(obsdir, '%s_timeseries_OBS.pickle' % resolution) try: res_timeseries['OBS'] = pickle_load(open(filename, 'rb')) except IOError: print 'could not find the observations output file %s' % filename res_timeseries['OBS'] = None # recover the SiBCASA runs filename = os.path.join(sibcasadir, '%s_timeseries_SiBCASA.pickle' % resolution) try: res_timeseries['SIMB'] = pickle_load(open(filename, 'rb')) except IOError: print 'could not find the SiBCASA output file %s' % filename res_timeseries['SIMB'] = None # recover the WOFOST runs filename = os.path.join(wofostdir, '%s_timeseries_'%resolution +\ '%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10)) try: res_timeseries['SIMC'] = pickle_load(open(filename, 'rb')) except IOError: print 'could not find the WOFOST output file %s' % filename res_timeseries['SIMC'] = None #------------------------------------------------------------------------------- # plot the observed and simulated timeseries with pandas library # with pandas we plot all years one after another, and can zoom in on one # particular year plt.close('all') # create figure sub-folder if it doesn't already exists figsubdir = os.path.join(figdir,'R10=%s/TER_%s/'%(R10,TER_method)+\ '3-hourly_fluxes_perf') if not os.path.exists(figsubdir): print 'creating new directory %s' % figsubdir os.makedirs(figsubdir) #------------------------------------------------------------------------------- # we plot the 3-hourly fluxes of simulations versus observations for the years # and sites that perform well on the daily scale # we plot years 2005, 2009, 2013 for site BE-Lon, which showed an extremely # good result on the SIM vs OBS daily fluxes comparison years = [2005, 2009, 2013] variables = ['GPP', 'TER', 'NEE'] axlabels = [ r'GPP (g m$^{-2}$ d$^{-1}$)', r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)' ] ylims = [(-60., 5.), (0., 20.), (-50., 15.)] one_to_one = np.arange(-100, 100, 10) for site in ['BE-Lon']: if site != 'IT-BCi': for year in years: timeframe = [year, year] start = str(int(timeframe[0])) + '-05-01' end = str(int(timeframe[1])) + '-07-01' print site for var, axlabel, lim in zip(variables, axlabels, ylims): fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5, 5)) fig.subplots_adjust(0.15, 0.15, 0.85, 0.85, 0., 0.) # select every 6 half-hourly flux in the observations, # to get to 3-hourly frequency OBS = res_timeseries['OBS'][site][var][::6][ start:end].dropna() # convert the 3-hourly simulated fluxes from gC.m-2.s-1 # to micromol CO2 .m-2.s-1 SIM = res_timeseries['SIMC'][site][var][start:end].dropna() SIM = SIM * 1000000. / 12.01 #micro mol CO2 per m2 per sec # the observed GPP needs a minus sign for convention if var == 'GPP': OBS = -OBS # use the min and max to frame the figure print var, min(min(OBS), min(SIM)), max(max(OBS), max(SIM)) varmin = math.floor(min(min(OBS), min(SIM))) varmax = math.ceil(max(max(OBS), max(SIM))) ax.scatter(OBS, SIM, marker='o') # fit a linear regression line in OBS/SIM scatter plot # and plot line and R2 mask = ~np.isnan(SIM) z = np.polyfit(OBS[mask], SIM[mask], 1) p = np.poly1d(z) ax.plot(one_to_one, p(one_to_one), 'r-') slope, intercept, r_value, p_value, std_err = \ linreg(OBS[mask], SIM[mask]) ax.annotate(r'r$^2$ = %.2f' % r_value**2, xy=(0.95, 0.15), xytext=(0.15, 0.9), xycoords='axes fraction', ha='center', va='center', color='r') ax.plot(one_to_one, one_to_one, c='k', lw=1) ax.set_xlabel('obs') ax.set_ylabel('sim') ax.set_xlim(varmin, varmax) ax.set_ylim(varmin, varmax) fig.suptitle(r'%s 3-hourly %s fluxes ($\mu$' % (site, var) + r'mol m$^{-2}$ s$^{-1}$)' + '\nfrom %s to %s\n' % (start, end)) fig.savefig(os.path.join( figsubdir, '%s_%s_%s.png' % (site, year, var)), dpi=300)
except NoSessionError: pass # New "fresh" cookie session check init_session('127.0.0.1') try: session = get_session() session['foo'] = 'bar' except NoSessionError: assert False # Pickle check init_session('127.0.0.1') tmp_file_path = None try: tmp_file_fh, tmp_file_path = mkstemp() os_close(tmp_file_fh) session = get_session() session['foo'] = 'bar' with open(tmp_file_path, 'wb') as tmp_file: pickle_dump(session, tmp_file) del session with open(tmp_file_path, 'rb') as tmp_file: session = pickle_load(tmp_file) assert session['foo'] == 'bar' finally: if tmp_file_path is not None: remove(tmp_file_path)
def optimize_fgap(site, crop_no, crop_name, year, NUTS_no, selec_method, ncells, nsoils, weather, force_optimization): #=============================================================================== # Temporarily add code directory to python path, to be able to import pcse # modules sys.path.insert(0, codedir) sys.path.insert(0, os.path.join(codedir,'carbon_cycle')) #------------------------------------------------------------------------------- import glob from maries_toolbox import define_opti_years,\ select_cells, select_soils #------------------------------------------------------------------------------- # if the optimization has already been performed and we don't want # to redo it, we skip that region filepath = os.path.join(optimidir,'fgap_%s_optimized.pickle'%NUTS_no) if (os.path.exists(filepath) and force_optimization==False): optimum = pickle_load(open(filepath,'rb')) print "We have already calculated the optimum fgap for that "+\ "year and crop: fgap=%.2f"%optimum[2] return optimum[2] #------------------------------------------------------------------------------- # we select the grid cell of the FluxNet site gridlist = pickle_load(open(os.path.join(CGMSdir, 'gridlist_objects/shortgridlist.pickle'),'rb')) selected_grid_cells = gridlist[NUTS_no] #------------------------------------------------------------------------------- # where possible, we retrieve local information about yield and sowing date local_sowda = None local_yield = None for row in custom_yns: if row[0]==site and row[1]==year and row[2]==crop_no: local_sowda = row[3] local_yield = row[4] print 'We recovered local info from site %s:'%site print 'sowing date of %s:'%crop_name, local_sowda, 'grain yield: %.3f'%local_yield break if local_sowda==None and local_yield==None: print 'No local information on sowing date and yield.' #------------------------------------------------------------------------------- # we retrieve the EUROSTAT pre-processed yield observations: if local_sowda==None and local_yield==None: try: filename1 = os.path.join(EUROSTATdir, 'preprocessed_yields.pickle') yields_dict = pickle_load(open(filename1,'rb')) except IOError: print '\nYou have not preprocessed the EUROSTAT observations' print 'Run the script 03_preprocess_obs.py first!\n' sys.exit() # NB: we do NOT detrend the yields anymore, since fgap is not supposed to be # representative of multi-annual gap obs_yields = yields_dict[crop_name][NUTS_no] return None #------------------------------------------------------------------------------- # if there were no reported yield on the year X, we skip that region if (year not in obs_yields[1]): print 'No reported yield, we have to gap-fill later' filename = os.path.join(optimidir,'fgap_%s_tobegapfilled.pickle'%NUTS_no) outlist = [NUTS_no, 2, 1., selected_grid_cells] pickle_dump(outlist, open(filename,'wb')) return 1. #------------------------------------------------------------------------------- # NB: in the optimization routine, we use the observed cultivation # fraction of the crop to calculate the soil cultivated areas, and # thus to compute the regional yields (= weighted average of yields # using soil cultivated areas) # if the observed cultivated fraction is zero, we skip that region selected_soil_types = select_soils(crop_no,[g for g,a in selected_grid_cells], CGMSdir, method=selec_method, n=nsoils) print 'we selected grid cell %i, top %i soil types, for optimization'%( selected_grid_cells[0][0], nsoils) #------------------------------------------------------------------------------- # we set the optimization code (gives us info on how we optimize) opti_code = 1 # 1= observations are available for optimization # 2= no obs available #print obs_yields[1], obs_yields[0] # in all other cases, we optimize the yield gap factor optimum = optimize_regional_yldgapf_dyn(NUTS_no, obs_yields, crop_no, selected_grid_cells, selected_soil_types, weather, CGMSdir, [year], obs_type='yield', plot_rmse=False) # pickle the information per NUTS region outlist = [NUTS_no, opti_code, optimum, selected_grid_cells] filename = os.path.join(optimidir,'fgap_%s_optimized.pickle'%NUTS_no) pickle_dump(outlist, open(filename,'wb')) return optimum #=============================================================================== # Function to optimize the regional yield gap factor using the difference # between the regional simulated and the observed harvest or yield (ie. 1 gap to # optimize per NUTS region). This function iterates dynamically to find the # optimum YLDGAPF. def optimize_regional_yldgapf_dyn(NUTS_no_, detrend, crop_no_, selected_grid_cells_, selected_soil_types_, weather, inputdir, opti_years_, obs_type='yield', plot_rmse=False): #=============================================================================== import math from operator import itemgetter as operator_itemgetter from matplotlib import pyplot as plt from pcse.models import Wofost71_WLP_FD from pcse.base_classes import WeatherDataProvider from pcse.fileinput.cabo_weather import CABOWeatherDataProvider # aggregated yield method: # 2- we construct a 2D array with same dimensions as TSO_regional, # containing the observed yields row = [] # this list will become the row of the 2D array for y,year in enumerate(opti_years_): index_year = np.argmin(np.absolute(detrend[1]-year)) row = row + [detrend[0][index_year]] OBS = np.tile(row, (5,1)) # repeats the list as a row 3 times, to get a # 2D array # 3- we calculate all the individual yields from the selected grid cells x # soils combinations # NB: we explore the range of yldgapf between 0.1 and 1. f0 = 0. f2 = 0.5 f4 = 1. f_step = 0.25 # Until the precision of the yield gap factor is good enough (i.e. < 0.02) # we loop over it. We do 12 iterations in total with this method. iter_no = 0 RMSE_stored = list() while (f_step >= 0.02): iter_no = iter_no + 1 # sub-method: looping over the yield gap factors # we build a range of 3 yield gap factors to explore one low bound, one # high bound, one in the middle f_step = (f4 - f0)/4. f1 = f0 + f_step f3 = f2 + f_step f_range = [f0, f1, f2, f3, f4] RES = [] # list in which we will store the yields of the combinations counter=0 for grid, arable_land in selected_grid_cells_: frac_arable = arable_land / 625000000. # Retrieve the weather data of one grid cell (all years are in one # file) if (weather == 'CGMS'): filename = os.path.join(inputdir,'weather_objects/', 'weatherobject_g%d.pickle'%grid) weatherdata = WeatherDataProvider() weatherdata._load(filename) if (weather == 'ECMWF'): weatherdata = CABOWeatherDataProvider('%i'%grid,fpath=ECMWFdir) # Retrieve the soil data of one grid cell (all possible soil types) filename = os.path.join(inputdir,'soildata_objects/', 'soilobject_g%d.pickle'%grid) soil_iterator = pickle_load(open(filename,'rb')) for smu, stu_no, weight, soildata in selected_soil_types_[grid]: # TSO will store all the yields of one grid cell x soil # combination, for all years and all 3 yldgapf values TSO = np.zeros((len(f_range), len(opti_years_))) counter +=1 for y, year in enumerate(opti_years_): # Retrieve yearly data filename = os.path.join(inputdir, 'timerdata_objects/%i/c%i/'%(year,crop_no_), 'timerobject_g%d_c%d_y%d.pickle'\ %(grid,crop_no_,year)) timerdata = pickle_load(open(filename,'rb')) filename = os.path.join(inputdir, 'cropdata_objects/%i/c%i/'%(year,crop_no_), 'cropobject_g%d_c%d_y%d.pickle'\ %(grid,crop_no_,year)) cropdata = pickle_load(open(filename,'rb')) if str(grid).startswith('1'): dum = str(grid)[0:2] else: dum = str(grid)[0] filename = os.path.join(inputdir, 'sitedata_objects/%i/c%i/grid_%s/' %(year,crop_no_,dum), 'siteobject_g%d_c%d_y%d_s%d.pickle'\ %(grid,crop_no_,year,stu_no)) sitedata = pickle_load(open(filename,'rb')) for f,factor in enumerate(f_range): cropdata['YLDGAPF']=factor # run WOFOST wofost_object = Wofost71_WLP_FD(sitedata, timerdata, soildata, cropdata, weatherdata) wofost_object.run_till_terminate() # get the yield (in kgDM.ha-1) TSO[f,y] = wofost_object.get_variable('TWSO') #print grid, stu_no, year, counter, [y[0] for y in TSO], OBS[0] RES = RES + [(grid, stu_no, weight*frac_arable, TSO)] # 4- we aggregate the yield or harvest into the regional one with array # operations sum_weighted_vals = np.zeros((len(f_range), len(opti_years_))) # empty 2D array with same dimension as TSO sum_weights = 0. for grid, stu_no, weight, TSO in RES: # adding weighted 2D-arrays in the empty array sum_weighted_yields # NB: variable 'weight' is actually the cultivated area in m2 sum_weighted_vals = sum_weighted_vals + (weight/10000.)*TSO # computing the total sum of the cultivated area in ha sum_weights = sum_weights + (weight/10000.) if (obs_type == 'harvest'): TSO_regional = sum_weighted_vals / 1000000. # sum of the individual # harvests in 1000 tDM elif (obs_type == 'yield'): TSO_regional = sum_weighted_vals / sum_weights # weighted average of # all yields in kgDM/ha # 5- we compute the (sim-obs) differences. DIFF = TSO_regional - OBS if (TSO_regional[-1][0] <= 0.): print 'WARNING: no simulated crop growth. We set the optimum fgap to 1.' return 1. if (TSO_regional[-1] <= OBS[-1]): print 'WARNING: obs yield > sim yield. We set optimum to 1.' return 1. # 6- we calculate the RMSE (root mean squared error) of the 3 yldgapf # The RMSE of each yldgapf is based on N obs-sim differences for the N # years looped over RMSE = np.zeros(len(f_range)) for f,factor in enumerate(f_range): list_of_DIFF = [] for y, year in enumerate(opti_years_): list_of_DIFF = list_of_DIFF + [DIFF[f,y]] RMSE[f] = np.sqrt(np.mean( [ math.pow(j,2) for j in list_of_DIFF ] )) #print RMSE, f_range # We store the value of the RMSE for plotting purposes RMSE_stored = RMSE_stored + [(f_range[1], RMSE[1]), (f_range[3], RMSE[3])] if (iter_no == 1): RMSE_stored = RMSE_stored + [(f_range[0], RMSE[0]), (f_range[2], RMSE[2]), (f_range[4], RMSE[4])] # 7- We update the yldgapf range to explore for the next iteration. # For this we do a linear interpolation of RMSE between the 3 yldgapf # explored here, and the next range to explore is the one having the # smallest interpolated RMSE index_new_center = RMSE.argmin() # if the optimum is close to 1: if index_new_center == len(f_range)-1: f0 = f_range[index_new_center-2] f2 = f_range[index_new_center-1] f4 = f_range[index_new_center] # if the optimum is close to 0: elif index_new_center == 0: f0 = f_range[index_new_center] f2 = f_range[index_new_center+1] f4 = f_range[index_new_center+2] else: f0 = f_range[index_new_center-1] f2 = f_range[index_new_center] f4 = f_range[index_new_center+1] # when we are finished iterating on the yield gap factor range, we plot the # RMSE as a function of the yield gap factor if (plot_rmse == True): RMSE_stored = sorted(RMSE_stored, key=operator_itemgetter(0)) x,y = zip(*RMSE_stored) fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5,5)) fig.subplots_adjust(0.15,0.16,0.95,0.96,0.4,0.) ax.plot(x, y, c='k', marker='o') ax.set_xlabel('yldgapf (-)') ax.set_ylabel('RMSE') fig.savefig('%s_opti_fgap.png'%NUTS_no_) #pickle_dump(RMSE_stored,open('%s_RMSE.pickle'%NUTS_no_,'wb')) # 8- when we are finished iterating on the yield gap factor range, we return # the optimum value. We look for the yldgapf with the lowest RMSE index_optimum = RMSE.argmin() optimum_yldgapf = f_range[index_optimum] print 'optimum found: %.2f +/- %.2f'%(optimum_yldgapf, f_step) # 10- we return the optimized YLDGAPF return optimum_yldgapf #=============================================================================== def str_to_bool(s): #=============================================================================== if s.strip(' ') == 'True': return True elif s.strip(' ') == 'False': return False else: raise ValueError #=============================================================================== if __name__=='__main__': main()
# Has config.py been changed? or getmtime(get_config_py_path()) > cache_mtime # Any file has changed in the dir since the cache was generated or any(True for f in listdir(directory) if (getmtime(path_join(directory, f)) > cache_mtime # Ignore hidden files and not f.startswith('.'))) # The configuration is newer than the cache or getmtime(get_config_path(directory)) > cache_mtime): generate = True docstats = [] else: generate = False try: with open(cache_file_path, 'rb') as cache_file: docstats = pickle_load(cache_file) except UnpicklingError: # Corrupt data, re-generate Messager.warning('Stats cache %s was corrupted; regenerating' % cache_file_path, -1) generate = True except EOFError: # Corrupt data, re-generate generate = True except OSError, e: Messager.warning('Failed checking file modification times for stats cache check; regenerating') generate = True # "header" and types ####stat_types = [("Entities", "int"), ("Relations", "int"), ("Events", "int")] ####if options_get_validation(directory) != 'none':
def plot_fluxnet_daily_c_fluxes(directory, sites_list, timeframe, linestyle): #=============================================================================== """ This function plots the average daytime Tair, average SWin, average VPD, average Tsoil and average volumetric soil moisture content, all taken from the formatted 2015 FluxNet DD (daily data) files. NB: Daily averages have been computed from the half-hourly data. Function arguments: ------------------ directory: path to the directory where the daily variables files ('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1') are stored sites_list: list of fluxnet sites names time frame: list of two years (e.g. [2005, 2007]) gives the time frame to plot linestyle: line style of the plot to make (can be '-', '--', '+', etc) """ plt.close('all') print "\nPlotting carbon fluxes timeseries\n" for site in sites_list: print site + '\n' filepath = os.path.join(directory, 'timeseries_%s.pickle' % site) series = pickle_load(open(filepath, 'rb')) # settings of the plot: figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8, 10)) figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.) variables = ['crop_no', 'GPP', 'TER', 'NEE'] axlabels = [ 'crop ID', r'GPP (g m$^{-2}$ d$^{-1}$)', r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)' ] ylims = [(0., 14.), (-30., 5.), (-5., 20.), (-30., 30.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) lab = '' ls = linestyle fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims): if (var == 'TER' or var == 'GPP'): lab = '-' series[site][var][start:end].plot(ax=ax, lw=2, style=ls, label=lab, fontsize=fsz) if var == 'GPP': series[site]['GPP_day'][start:end].plot(ax=ax, lw=2, style=ls, label='day partition', fontsize=fsz) series[site]['GPP_night'][start:end].plot( ax=ax, lw=2, style=ls, label='night partition', fontsize=fsz) if var == 'TER': series[site]['TER_day'][start:end].plot(ax=ax, lw=2, style=ls, label='day partition', fontsize=fsz) series[site]['TER_night'][start:end].plot( ax=ax, lw=2, style=ls, label='night partition', fontsize=fsz) ax.axhline(y=0., c='k') ax.set_ylim(ylim) if (var == 'GPP_day'): ax.legend(loc='lower left', prop={'size': 12}) if (var == 'TER_day'): ax.legend(loc='upper left', prop={'size': 12}) ax.set_ylabel(axlabel) if var != 'NEE': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) plt.show() return None
def plot_fluxnet_micromet(directory, sites_list, timeframe, linestyle): #=============================================================================== """ This function plots the average daytime Tair, average SWin, average VPD, average Tsoil and average volumetric soil moisture content, all taken from the formatted 2015 FluxNet DD (daily data) files. NB: Daily averages have been computed from the half-hourly data. Function arguments: ------------------ directory: path to the directory where the daily variables files ('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1') are stored sites_list: list of fluxnet sites names time frame: list of two years (e.g. [2005, 2007]) gives the time frame to plot linestyle: line style of the plot to make (can be '-', '--', '+', etc) """ plt.close('all') for site in sites_list: print site + '\n' filepath = os.path.join(directory, 'timeseries_%s.pickle' % site) series = pickle_load(open(filepath, 'rb')) # settings of the plot: figs, axes = plt.subplots(nrows=5, ncols=1, figsize=(8, 9)) figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.) variables = ['Ta_day', 'SWin', 'VPD', 'Ts_1', 'SWC_1'] axlabels = [ r'T$_{air}$ day (deg C)', r'SW$_{in}$ (W m$^{-2}$)', 'VPD (hPa)', r'T$_{soil}$ (deg C)', 'SWC (%)' ] ylims = [(-20., 40.), (0., 400.), (0., 17.), (-5., 35.), (0., 60.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) ls = linestyle lab = 'bullshit' fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims): if var.endswith('_1'): lab = 'layer 1' series[site][var][start:end].plot(ax=ax, lw=2, style=ls, label=lab, fontsize=fsz) if var == 'Ts_1': series[site]['Ts_2'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 2', fontsize=fsz) series[site]['Ts_3'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 3', fontsize=fsz) if var == 'SWC_1': series[site]['SWC_2'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 2', fontsize=fsz) series[site]['SWC_3'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 3', fontsize=fsz) if (var == 'Ts_1'): ax.legend(loc='upper left', prop={'size': 12}) if (var == 'SWC_1'): ax.legend(loc='lower left', prop={'size': 12}) ax.axhline(y=0., c='k') ax.set_ylim(ylim) ax.set_ylabel(axlabel) if var == 'Ta_day': ax.set_yticks([-10., 0., 10., 20., 30., 40.]) if var == 'SWin': ax.set_yticks([0., 100., 200., 300., 400.]) if var == 'VPD': ax.set_yticks([0., 5., 10., 15.]) if var == 'Ts_1': ax.set_yticks([0., 10., 20., 30.]) if var != 'SWC_1': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) plt.show() return None
def main(): #=============================================================================== global inputdir, codedir, outputdir, CGMSdir, obsdir\ #------------------------------------------------------------------------------- import cx_Oracle import sqlalchemy as sa from datetime import datetime #------------------------------------------------------------------------------- # ================================= USER INPUT ================================= # read the settings from the rc file rcdict = rc.read('settings.rc') #=============================================================================== #------------------------------------------------------------------------------- # extract the needed information from the rc file sites = [s.strip(' ') for s in rcdict['sites'].split(',')] crops = [s.strip(' ') for s in rcdict['crops'].split(',')] crop_nos = [int(s.strip(' ')) for s in rcdict['crop_nos'].split(',')] years = [int(s.strip(' ')) for s in rcdict['years'].split(',')] obsdir = rcdict['obsdir'] inputdir = rcdict['inputdir'] CGMSdir = os.path.join(inputdir, 'CGMS') codedir = rcdict['codedir'] #------------------------------------------------------------------------------- # get the closest CGMS grid cell id number for each FluxNet site # get the sites longitude and latitudes sitdict = open_csv(os.path.join(obsdir,'regrouped_data'), 'sites_info.txt', convert_to_float=False) site_lons = sitdict['site_lons'] site_lats = sitdict['site_lats'] # we read the CGMS grid cells coordinates from file CGMS_cells = open_csv(CGMSdir, 'CGMS_grid_list.csv', convert_to_float=True) all_grids = CGMS_cells['GRID_NO'] all_lons = CGMS_cells['LONGITUDE'] all_lats = CGMS_cells['LATITUDE'] flux_gri = dict() for i,site in enumerate(sitdict['sites']): lon = float(site_lons[i]) lat = float(site_lats[i]) # compute the distance to site for all CGMS grid cells dist_list = list() for j,grid_no in enumerate(all_grids): distance = ((all_lons[j]-lon)**2. + (all_lats[j]-lat)**2.)**(1./2.) dist_list += [distance] # select the closest grid cell indx = np.argmin(np.array(dist_list)) flux_gri[site] = all_grids[indx] print 'FluxNet site %s with lon=%5.2f, lat=%5.2f: closest grid cell is %i'%(site, lon, lat, all_grids[indx]) #------------------------------------------------------------------------------- # create new file with grid cell number in it filename = os.path.join(inputdir,'sites_info2.csv') newres = open(filename,'wb') oldres = open(os.path.join(obsdir,'regrouped_data/sites_info.txt'),'rU') reader = oldres.readlines() oldres.close() for l,line in enumerate(reader): site = line.split(',')[0].strip(' ') if l==0: line = line.strip('\n')+', gridcells\n' else: line = line.strip('\n') + ',%10i'%int(flux_gri[site]) + '\n' newres.write(line) newres.close() print '\nWe successfully created the input file with grid cell IDs:\n%s'%filename #------------------------------------------------------------------------------- # retrieve the necessary input data for all sites # settings of the connection user = "******" password = "******" tns = "EURDAS.WORLD" dsn = "oracle+cx_oracle://{user}:{pw}@{tns}".format(user=user,pw=password,tns=tns) engine = sa.create_engine(dsn) print engine # test the connection: try: connection = cx_Oracle.connect("cgms12eu_select/[email protected]") except cx_Oracle.DatabaseError: print '\nBEWARE!! The Oracle database is not responding. Probably, you are' print 'not using a computer wired within the Wageningen University network.' print '--> Get connected with ethernet cable before trying again!' sys.exit() for c,crop in enumerate(crops): crop_no = crop_nos[c] print '\nRetrieving input data for %s (CGMS id=%i)'%(crop,crop_no) # We add a timestamp at start of the retrieval start_timestamp = datetime.utcnow() # We retrieve the list of suitable soil types for the selected crop # species filename = os.path.join(CGMSdir, 'soildata_objects/', 'suitablesoilsobject_c%d.pickle'%(crop_no)) if os.path.exists(filename): suitable_stu = pickle_load(open(filename,'rb')) else: from pcse.db.cgms11 import STU_Suitability suitable_stu = STU_Suitability(engine, crop_no) suitable_stu_list = [] for item in suitable_stu: suitable_stu_list = suitable_stu_list + [item] suitable_stu = suitable_stu_list pickle_dump(suitable_stu,open(filename,'wb')) print 'retrieving suitable soils for %s'%crop # WE LOOP OVER ALL YEARS: for y, year in enumerate(years): print '\n######################## Year %i ##############'%year+\ '##########\n' # if we do a serial iteration, we loop over the grid cells that # contain arable land for grid in flux_gri.values(): retrieve_CGMS_input(grid, year, crop_no, suitable_stu, engine) # We add a timestamp at end of the retrieval, to time the process end_timestamp = datetime.utcnow() print '\nDuration of the retrieval:', end_timestamp-start_timestamp
def retrieve_CGMS_input(grid, year, crop_no, suitable_stu, engine, retrieve_weather=False): #=============================================================================== # Temporarily add code directory to python path, to be able to import pcse # modules sys.path.insert(0, codedir) #------------------------------------------------------------------------------- from pcse.exceptions import PCSEError from pcse.db.cgms11 import TimerDataProvider, SoilDataIterator, \ CropDataProvider, STU_Suitability, \ SiteDataProvider, WeatherObsGridDataProvider # if the retrieval does not raise an error, the crop was cultivated that year print ' - grid cell no %i'%grid try: # We retrieve the crop calendar (timerdata) filename = os.path.join(CGMSdir, 'timerdata_objects/%i/c%i/'%(year,crop_no), 'timerobject_g%d_c%d_y%d.pickle'%(grid, crop_no, year)) if os.path.exists(filename): pass else: timerdata = TimerDataProvider(engine, grid, crop_no, year) pickle_dump(timerdata,open(filename,'wb')) # If required by the user, we retrieve the weather data if retrieve_weather == True: filename = os.path.join(CGMSdir, 'weather_objects/', 'weatherobject_g%d.pickle'%(grid)) if os.path.exists(filename): pass else: weatherdata = WeatherObsGridDataProvider(engine, grid) weatherdata._dump(filename) # We retrieve the soil data (soil_iterator) filename = os.path.join(CGMSdir, 'soildata_objects/', 'soilobject_g%d.pickle'%(grid)) if os.path.exists(filename): soil_iterator = pickle_load(open(filename,'rb')) else: soil_iterator = SoilDataIterator(engine, grid) pickle_dump(soil_iterator,open(filename,'wb')) # We retrieve the crop variety info (crop_data) filename = os.path.join(CGMSdir, 'cropdata_objects/%i/c%i/'%(year,crop_no), 'cropobject_g%d_c%d_y%d.pickle'%(grid,crop_no,year)) if os.path.exists(filename): pass else: cropdata = CropDataProvider(engine, grid, crop_no, year) pickle_dump(cropdata,open(filename,'wb')) # WE LOOP OVER ALL SOIL TYPES LOCATED IN THE GRID CELL: for smu_no, area_smu, stu_no, percentage, soildata in soil_iterator: # NB: we remove all unsuitable soils from the iteration if (stu_no not in suitable_stu): pass else: print ' soil type no %i'%stu_no # We retrieve the site data (site management) if (str(grid)).startswith('1'): dum = str(grid)[0:2] else: dum = str(grid)[0] filename = os.path.join(CGMSdir, 'sitedata_objects/%i/c%i/grid_%s/'%(year,crop_no,dum), 'siteobject_g%d_c%d_y%d_s%d.pickle'%(grid, crop_no, year, stu_no)) if os.path.exists(filename): pass else: sitedata = SiteDataProvider(engine,grid,crop_no,year,stu_no) pickle_dump(sitedata,open(filename,'wb')) # if an error is raised, the crop was not grown that year except PCSEError: print ' the crop was not grown that year in that grid cell' except Exception as e: print ' Unexpected error', e#sys.exc_info()[0] return None
def main(): #=============================================================================== global wofostdir, sibcasadir, obsdir #------------------------------------------------------------------------------- # ================================= USER INPUT ================================= # read the settings from the rc file (mostly directory paths) rcdict = rc.read('settings.rc') sites = [s.strip(' ') for s in rcdict['sites'].split(',')] years = [int(s.strip(' ')) for s in rcdict['years'].split(',')] TER_method = rcdict['TER_method'] R10 = rcdict['R10'] resolution = rcdict['resolution'] # specific plotting options: #TER_method = 'grow-only' # this is to select the corresponding WOFOST output file #R10 = '0.08' # this is to select the corresponding WOFOST output file #=============================================================================== #------------------------------------------------------------------------------- # extract the needed information # input data directory paths rootdir = rcdict['rootdir'] sibcasadir = os.path.join(rootdir,'intercomparison_study/SiBCASA_runs') wofostdir = rcdict['outputdir'] obsdir = rcdict['obsdir'] figdir = os.path.join(rootdir,'intercomparison_study/figures') #------------------------------------------------------------------------------- # Start a directory to store OBS, SIMW (wofost), SIMB (SiBCASA) # recover the FluxNet observed data from pickle files res_timeseries = dict() res_timeseries['OBS'] = dict() res_timeseries['SIMB'] = dict() res_timeseries['SIMW'] = dict() filename = os.path.join(obsdir, '%s_timeseries_OBS.pickle'%resolution) try: res_timeseries['OBS'] = pickle_load(open(filename,'rb')) except IOError: print 'could not find the observations output file %s'%filename res_timeseries['OBS'] = None # recover the SiBCASA runs filename = os.path.join(sibcasadir, '%s_timeseries_SiBCASA.pickle'%resolution) try: res_timeseries['SIMB'] = pickle_load(open(filename,'rb')) except IOError: print 'could not find the SiBCASA output file %s'%filename res_timeseries['SIMB'] = None # recover the WOFOST runs filename = os.path.join(wofostdir, '%s_timeseries_'%resolution +\ '%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10)) try: res_timeseries['SIMC'] = pickle_load(open(filename,'rb')) except IOError: print 'could not find the WOFOST output file %s'%filename res_timeseries['SIMC'] = None #------------------------------------------------------------------------------- # plot the observed and simulated timeseries with pandas library # with pandas we plot all years one after another, and can zoom in on one # particular year plt.close('all') # create figure sub-folder if it doesn't already exists figsubdir = os.path.join(figdir,'R10=%s/TER_%s/'%(R10,TER_method)+\ '3-hourly_fluxes_perf') if not os.path.exists(figsubdir): print 'creating new directory %s'%figsubdir os.makedirs(figsubdir) #------------------------------------------------------------------------------- # we plot the 3-hourly fluxes of simulations versus observations for the years # and sites that perform well on the daily scale # we plot years 2005, 2009, 2013 for site BE-Lon, which showed an extremely # good result on the SIM vs OBS daily fluxes comparison years = [2005,2009,2013] variables = ['GPP','TER','NEE'] axlabels = [r'GPP (g m$^{-2}$ d$^{-1}$)', r'TER (g m$^{-2}$ d$^{-1}$)',r'NEE (g m$^{-2}$ d$^{-1}$)'] ylims = [(-60.,5.),(0.,20.),(-50.,15.)] one_to_one = np.arange(-100,100,10) for site in ['BE-Lon']: if site != 'IT-BCi': for year in years: timeframe = [year,year] start = str(int(timeframe[0]))+'-05-01' end = str(int(timeframe[1]))+'-07-01' print site for var, axlabel, lim in zip(variables,axlabels,ylims): fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5,5)) fig.subplots_adjust(0.15,0.15,0.85,0.85,0.,0.) # select every 6 half-hourly flux in the observations, # to get to 3-hourly frequency OBS = res_timeseries['OBS'][site][var][::6][start:end].dropna() # convert the 3-hourly simulated fluxes from gC.m-2.s-1 # to micromol CO2 .m-2.s-1 SIM = res_timeseries['SIMC'][site][var][start:end].dropna() SIM = SIM * 1000000. / 12.01 #micro mol CO2 per m2 per sec # the observed GPP needs a minus sign for convention if var=='GPP': OBS=-OBS # use the min and max to frame the figure print var, min(min(OBS), min(SIM)), max(max(OBS), max(SIM)) varmin = math.floor(min(min(OBS), min(SIM))) varmax = math.ceil(max(max(OBS), max(SIM))) ax.scatter(OBS, SIM, marker='o') # fit a linear regression line in OBS/SIM scatter plot # and plot line and R2 mask = ~np.isnan(SIM) z = np.polyfit(OBS[mask], SIM[mask], 1) p = np.poly1d(z) ax.plot(one_to_one,p(one_to_one),'r-') slope, intercept, r_value, p_value, std_err = \ linreg(OBS[mask], SIM[mask]) ax.annotate(r'r$^2$ = %.2f'%r_value**2, xy=(0.95, 0.15), xytext=(0.15, 0.9), xycoords='axes fraction', ha='center', va='center', color='r') ax.plot(one_to_one,one_to_one, c='k', lw=1) ax.set_xlabel('obs') ax.set_ylabel('sim') ax.set_xlim(varmin,varmax) ax.set_ylim(varmin,varmax) fig.suptitle(r'%s 3-hourly %s fluxes ($\mu$'%(site, var)+ r'mol m$^{-2}$ s$^{-1}$)'+'\nfrom %s to %s\n'%(start,end)) fig.savefig(os.path.join(figsubdir, '%s_%s_%s.png'%(site,year,var)), dpi=300)
def main(): #=============================================================================== global outputdir, obsdir #------------------------------------------------------------------------------- # ================================= USER INPUT ================================= # read the settings from the rc file rcdict = rc.read('settings.rc') #=============================================================================== #------------------------------------------------------------------------------- # extract the needed information for that script sites = [s.strip(' ') for s in rcdict['sites'].split(',')] years = [s.strip(' ') for s in rcdict['years'].split(',')] TER_method = rcdict['TER_method'] R10 = rcdict['R10'] resolution = rcdict['resolution'] # can be hourly or daily if resolution=='daily': res='1d' elif resolution=='3-hourly': res='3H' # directory paths outputdir = rcdict['outputdir'] obsdir = rcdict['obsdir'] forwardir = os.path.join(outputdir, 'forward_runs') #------------------------------------------------------------------------------- # load the WOFOST runs of all crops # we store the two pandas series in one pickle file filepath = os.path.join(forwardir,'%s_timeseries_'%resolution+\ '%s_WOFOST.pickle'%TER_method) series = pickle_load(open(filepath,'rb')) filepath = os.path.join(obsdir,'daily_timeseries_OBS.pickle') obs = pickle_load(open(filepath,'rb')) final_series = dict() for s,site in enumerate(sites): print site print obs[site].keys() final_series[site] = dict() # read the crop rotation from FluxNet file rotation = obs[site]['crop_no'] # slice each year's required time series, append to final series for varname in ['GPP','TER','Raut','Rhet','NEE']: print 'variable %s'%varname var = [] for year in years: # get the crop number for that year if site != 'IT-BCi': try: crop_no = rotation[year:year][0] except IndexError: # index error occurs when the year is # not in the rotation time series startdate = '%s-01-01 00:00:00'%year enddate = '%s-12-31 23:59:59'%year dtimes = pd.date_range(startdate, enddate, freq=res) na_vals = np.array(len(dtimes)*[np.nan]) var += [pd.Series(na_vals, index=dtimes)] print ' ',site, year, 'unknown crop cover: skip.' continue elif site == 'IT-BCi': if int(year) not in np.arange(2004,2010,1): startdate = '%s-01-01 00:00:00'%year enddate = '%s-12-31 23:59:59'%year dtimes = pd.date_range(startdate, enddate, freq=res) na_vals = np.array(len(dtimes)*[np.nan]) var += [pd.Series(na_vals, index=dtimes)] print ' ',site, year, 'unknown crop cover: skip.' continue else: crop_no = 2 # try slicing and concatenating that year's timeseries from file try: # if the GPP = 0 (failed growing season), we set TER and # NEE to zero as well if np.mean(series[site]['c%i'%crop_no]['GPP'][year:year]) == 0.: startdate = '%s-01-01 00:00:00'%year enddate = '%s-12-31 23:59:59'%year dtimes = pd.date_range(startdate, enddate, freq=res) zeros = np.array(len(dtimes)*[0.]) var += [pd.Series(zeros, index=dtimes)] else: var += [series[site]['c%i'%crop_no][varname][year:year]] print ' ',site, year, '%2i'%crop_no, 'slicing' except KeyError: # key error occurs when we haven't ran a crop # or a year with WOFOST startdate = '%s-01-01 00:00:00'%year enddate = '%s-12-31 23:59:59'%year dtimes = pd.date_range(startdate, enddate, freq=res) na_vals = np.array(len(dtimes)*[np.nan]) var += [pd.Series(na_vals, index=dtimes)] print ' ',site, year, '%2i'%crop_no, 'skip.' final_series[site][varname] = pd.concat(var) #final_series[site]['GPP'].plot() #plt.show() # store the final WOFOST timeseries filepath = os.path.join(outputdir,'%s_timeseries_'%resolution+\ '%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10)) pickle_dump(final_series, open(filepath,'wb')) print 'successfully dumped %s'%filepath
def _load(self, f): self.store = pickle_load(f)
def main(): #=============================================================================== global outputdir, obsdir #------------------------------------------------------------------------------- # ================================= USER INPUT ================================= # read the settings from the rc file rcdict = rc.read('settings.rc') #=============================================================================== #------------------------------------------------------------------------------- # extract the needed information for that script sites = [s.strip(' ') for s in rcdict['sites'].split(',')] years = [s.strip(' ') for s in rcdict['years'].split(',')] TER_method = rcdict['TER_method'] R10 = rcdict['R10'] resolution = rcdict['resolution'] # can be hourly or daily if resolution == 'daily': res = '1d' elif resolution == '3-hourly': res = '3H' # directory paths outputdir = rcdict['outputdir'] obsdir = rcdict['obsdir'] forwardir = os.path.join(outputdir, 'forward_runs') #------------------------------------------------------------------------------- # load the WOFOST runs of all crops # we store the two pandas series in one pickle file filepath = os.path.join(forwardir,'%s_timeseries_'%resolution+\ '%s_WOFOST.pickle'%TER_method) series = pickle_load(open(filepath, 'rb')) filepath = os.path.join(obsdir, 'daily_timeseries_OBS.pickle') obs = pickle_load(open(filepath, 'rb')) final_series = dict() for s, site in enumerate(sites): print site print obs[site].keys() final_series[site] = dict() # read the crop rotation from FluxNet file rotation = obs[site]['crop_no'] # slice each year's required time series, append to final series for varname in ['GPP', 'TER', 'Raut', 'Rhet', 'NEE']: print 'variable %s' % varname var = [] for year in years: # get the crop number for that year if site != 'IT-BCi': try: crop_no = rotation[year:year][0] except IndexError: # index error occurs when the year is # not in the rotation time series startdate = '%s-01-01 00:00:00' % year enddate = '%s-12-31 23:59:59' % year dtimes = pd.date_range(startdate, enddate, freq=res) na_vals = np.array(len(dtimes) * [np.nan]) var += [pd.Series(na_vals, index=dtimes)] print ' ', site, year, 'unknown crop cover: skip.' continue elif site == 'IT-BCi': if int(year) not in np.arange(2004, 2010, 1): startdate = '%s-01-01 00:00:00' % year enddate = '%s-12-31 23:59:59' % year dtimes = pd.date_range(startdate, enddate, freq=res) na_vals = np.array(len(dtimes) * [np.nan]) var += [pd.Series(na_vals, index=dtimes)] print ' ', site, year, 'unknown crop cover: skip.' continue else: crop_no = 2 # try slicing and concatenating that year's timeseries from file try: # if the GPP = 0 (failed growing season), we set TER and # NEE to zero as well if np.mean(series[site]['c%i' % crop_no]['GPP'][year:year]) == 0.: startdate = '%s-01-01 00:00:00' % year enddate = '%s-12-31 23:59:59' % year dtimes = pd.date_range(startdate, enddate, freq=res) zeros = np.array(len(dtimes) * [0.]) var += [pd.Series(zeros, index=dtimes)] else: var += [ series[site]['c%i' % crop_no][varname][year:year] ] print ' ', site, year, '%2i' % crop_no, 'slicing' except KeyError: # key error occurs when we haven't ran a crop # or a year with WOFOST startdate = '%s-01-01 00:00:00' % year enddate = '%s-12-31 23:59:59' % year dtimes = pd.date_range(startdate, enddate, freq=res) na_vals = np.array(len(dtimes) * [np.nan]) var += [pd.Series(na_vals, index=dtimes)] print ' ', site, year, '%2i' % crop_no, 'skip.' final_series[site][varname] = pd.concat(var) #final_series[site]['GPP'].plot() #plt.show() # store the final WOFOST timeseries filepath = os.path.join(outputdir,'%s_timeseries_'%resolution+\ '%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10)) pickle_dump(final_series, open(filepath, 'wb')) print 'successfully dumped %s' % filepath
def main(): #=============================================================================== global wofostdir, sibcasadir, obsdir #------------------------------------------------------------------------------- # ================================= USER INPUT ================================= # read the settings from the rc file (mostly directory paths) rcdict = rc.read('settings.rc') sites = [s.strip(' ') for s in rcdict['sites'].split(',')] years = [int(s.strip(' ')) for s in rcdict['years'].split(',')] TER_method = rcdict['TER_method'] R10 = rcdict['R10'] # specific plotting options: #TER_method = 'grow-only' # this is to select the corresponding WOFOST output file #R10 = '0.08' # this is to select the corresponding WOFOST output file #=============================================================================== #------------------------------------------------------------------------------- # extract the needed information # input data directory paths rootdir = rcdict['rootdir'] sibcasadir = os.path.join(rootdir, 'intercomparison_study/SiBCASA_runs') wofostdir = rcdict['outputdir'] obsdir = rcdict['obsdir'] figdir = os.path.join(rootdir, 'intercomparison_study/figures') #------------------------------------------------------------------------------- # Start a directory to store OBS, SIMW (wofost), SIMB (SiBCASA) # recover the FluxNet observed data from pickle files res_timeseries = dict() res_timeseries['OBS'] = dict() res_timeseries['SIMB'] = dict() res_timeseries['SIMW'] = dict() filename = os.path.join(obsdir, 'timeseries_OBS.pickle') try: res_timeseries['OBS'] = pickle_load(open(filename, 'rb')) except IOError: print 'could not find the observations output file %s' % filename res_timeseries['OBS'] = None # recover the SiBCASA runs filename = os.path.join(sibcasadir, 'timeseries_SiBCASA.pickle') try: res_timeseries['SIMB'] = pickle_load(open(filename, 'rb')) except IOError: print 'could not find the SiBCASA output file %s' % filename res_timeseries['SIMB'] = None # recover the WOFOST runs filename = os.path.join(wofostdir, 'timeseries_%s_R10=%s_'%(TER_method,R10) +\ 'WOFOST_crop_rotation.pickle') try: print 'opening the WOFOST output file %s' % filename res_timeseries['SIMC'] = pickle_load(open(filename, 'rb')) except IOError: print 'could not find the WOFOST output file %s' % filename res_timeseries['SIMC'] = None #------------------------------------------------------------------------------- # plot the observed and simulated timeseries with pandas library # with pandas we plot all years one after another, and can zoom in on one # particular year plt.close('all') # create figure sub-folder if it doesn't already exists figsubdir = os.path.join(figdir, 'R10=%s/TER_%s/' % (R10, TER_method)) if not os.path.exists(figsubdir): print 'creating new directory %s' % figsubdir os.makedirs(figsubdir) #------------------------------------------------------------------------------- years = np.arange(2004, 2015, 1) for site in sites: for year in years: timeframe = [year, year] print site figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8, 10)) figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.) variables = ['crop_no', 'GPP', 'TER', 'NEE'] axlabels = [ 'crop ID', r'GPP (g m$^{-2}$ d$^{-1}$)', r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)' ] ylims = [(0., 14.), (-18., 2.), (-1., 12.), (-10., 10.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) print '[%s:%s]' % (start, end) fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims): if (var == 'crop_no'): try: OBS = res_timeseries['OBS'][site][var][ start:end].dropna() OBS[~(OBS == -9999.)].plot( ax=ax, lw=2, #OBS.plot(ax=ax, lw=2, style='-', label='obs', fontsize=fsz) crop_no = OBS[0] minobs = OBS[~(OBS == -9999.)].min() maxobs = OBS[~(OBS == -9999.)].max() except TypeError: minobs = 0. maxobs = 0. minwof = 1. maxwof = 1. elif (var == 'TER'): # observations try: OBS = res_timeseries['OBS'][site][var][ start:end].dropna() OBS[~(OBS == -9999.)].plot( ax=ax, lw=2, #OBS.plot(ax=ax, lw=2, c='b', style='+', label='obs', fontsize=fsz) minobs = OBS[~(OBS == -9999.)].min() maxobs = OBS[~(OBS == -9999.)].max() except TypeError: minobs = 0. maxobs = 0. # SIBCASA sims try: #res_timeseries['SIMB'][site]['Raut'][start:end].plot(ax=ax, #lw=2, c='g', style=':', label='SiBCASA Raut', fontsize=fsz) res_timeseries['SIMB'][site][var][start:end].plot( ax=ax, lw=2, c='g', style='--', label='SiBCASA TER', fontsize=fsz) except TypeError: pass # WOFOST sims try: #WOF = res_timeseries['SIMC'][site]['Raut'][start:end].dropna() #WOF.plot(ax=ax, lw=2, c='r', #style='_', label='WOFOST Raut', fontsize=fsz) WOF = res_timeseries['SIMC'][site][var][ start:end].dropna() WOF.plot(ax=ax, lw=2, c='r', style='x', label='WOFOST TER', fontsize=fsz) minwof = WOF.min() maxwof = WOF.max() except TypeError: minwof = 0. maxwof = 0. WOF = 0. else: # observations try: OBS = res_timeseries['OBS'][site][var][ start:end].dropna() OBS[~(OBS == -9999.)].plot( ax=ax, lw=2, #OBS.plot(ax=ax, lw=2, c='b', style='+', label='obs', fontsize=fsz) minobs = OBS[~(OBS == -9999.)].min() maxobs = OBS[~(OBS == -9999.)].max() except TypeError: minobs = 0. maxobs = 0. # SIBCASA sims try: res_timeseries['SIMB'][site][var][start:end].plot( ax=ax, lw=2, c='g', style='--', label='SiBCASA', fontsize=fsz) except TypeError: pass # WOFOST simsA try: WOF = res_timeseries['SIMC'][site][var][ start:end].dropna() #WOF[~(WOF==-9999.)].plot(ax=ax, lw=2, WOF.plot(ax=ax, lw=2, c='r', style='x', label='WOFOST', fontsize=fsz) minwof = WOF.min() maxwof = WOF.max() except TypeError: minwof = 0. maxwof = 0. WOF = 0. ax.axhline(y=0., c='k') minvar = math.floor(min(minobs, minwof)) - 1. maxvar = math.ceil(max(maxobs, maxwof)) + 1. ax.set_ylim(minvar, maxvar) #ax.set_ylim(ylim) if (var == 'GPP'): ax.legend(loc='lower left', prop={'size': 12}) #if (var=='TER'): ax.legend(loc='upper left',prop={'size':10}) ax.set_ylabel(axlabel) if var != 'NEE': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) figs.savefig( os.path.join( figsubdir, 'crop%i_%s_%i.png' % (crop_no, site, timeframe[0]))) plt.close('all') #plt.show() #------------------------------------------------------------------------------- timeframe = [2004, 2014] for site in sites: print site figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(15, 10)) figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.) variables = ['crop_no', 'GPP', 'TER', 'NEE'] axlabels = [ 'crop ID', r'GPP (g m$^{-2}$ d$^{-1}$)', r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)' ] ylims = [(0., 14.), (-30., 2.), (-2., 20.), (-20., 10.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) print '[%s:%s]' % (start, end) fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims): if (var == 'crop_no'): try: OBS = res_timeseries['OBS'][site][var][start:end].dropna() OBS[~(OBS == -9999.)].plot( ax=ax, lw=2, #OBS.plot(ax=ax, lw=2, style='-', label='obs', fontsize=fsz) crop_no = OBS[0] minobs = OBS[~(OBS == -9999.)].min() maxobs = OBS[~(OBS == -9999.)].max() except TypeError: minobs = 0. maxobs = 0. minwof = 1. maxwof = 1. else: # observations try: OBS = res_timeseries['OBS'][site][var][start:end].dropna() OBS[~(OBS == -9999.)].plot( ax=ax, lw=2, #OBS.plot(ax=ax, lw=2, c='b', style='+', label='obs', fontsize=fsz) minobs = OBS[~(OBS == -9999.)].min() maxobs = OBS[~(OBS == -9999.)].max() except TypeError: minobs = 0. maxobs = 0. # SIBCASA sims try: res_timeseries['SIMB'][site][var][start:end].plot( ax=ax, lw=2, c='g', style='--', label='SiBCASA', fontsize=fsz) except TypeError: pass # WOFOST simsA try: WOF = res_timeseries['SIMC'][site][var][start:end].dropna() #WOF[~(WOF==-9999.)].plot(ax=ax, lw=2, WOF.plot(ax=ax, lw=2, c='r', style='x', label='WOFOST', fontsize=fsz) minwof = WOF.min() maxwof = WOF.max() except TypeError: minwof = 0. maxwof = 0. WOF = 0. ax.axhline(y=0., c='k') minvar = math.floor(min(minobs, minwof)) - 1. maxvar = math.ceil(max(maxobs, maxwof)) + 1. #ax.set_ylim(minvar,maxvar) ax.set_ylim(ylim) if (var == 'GPP'): ax.legend(loc='lower left', prop={'size': 12}) ax.set_ylabel(axlabel) if var != 'NEE': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) figs.savefig( os.path.join( figsubdir, 'timeseries_crop%i_%s_%i-%i.png' % (crop_no, site, timeframe[0], timeframe[1]))) plt.close('all')
def plot_fluxnet_micromet(directory, sites_list, timeframe, linestyle): #=============================================================================== """ This function plots the average daytime Tair, average SWin, average VPD, average Tsoil and average volumetric soil moisture content, all taken from the formatted 2015 FluxNet DD (daily data) files. NB: Daily averages have been computed from the half-hourly data. Function arguments: ------------------ directory: path to the directory where the daily variables files ('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1') are stored sites_list: list of fluxnet sites names time frame: list of two years (e.g. [2005, 2007]) gives the time frame to plot linestyle: line style of the plot to make (can be '-', '--', '+', etc) """ plt.close('all') for site in sites_list: print site+'\n' filepath = os.path.join(directory,'timeseries_%s.pickle'%site) series = pickle_load(open(filepath,'rb')) # settings of the plot: figs, axes = plt.subplots(nrows=5, ncols=1, figsize=(8,9)) figs.subplots_adjust(0.1,0.07,0.98,0.95,0.,0.) variables = ['Ta_day','SWin','VPD','Ts_1','SWC_1'] axlabels = [r'T$_{air}$ day (deg C)', r'SW$_{in}$ (W m$^{-2}$)', 'VPD (hPa)',r'T$_{soil}$ (deg C)','SWC (%)'] ylims = [(-20.,40.),(0.,400.),(0.,17.),(-5.,35.),(0.,60.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) ls = linestyle lab = 'bullshit' fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes,variables,axlabels,ylims): if var.endswith('_1'): lab='layer 1' series[site][var][start:end].plot(ax=ax, lw=2, style=ls, label=lab, fontsize=fsz) if var=='Ts_1': series[site]['Ts_2'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 2', fontsize=fsz) series[site]['Ts_3'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 3', fontsize=fsz) if var=='SWC_1': series[site]['SWC_2'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 2', fontsize=fsz) series[site]['SWC_3'][start:end].plot(ax=ax, lw=2, style=ls, label='layer 3', fontsize=fsz) if (var=='Ts_1'): ax.legend(loc='upper left',prop={'size':12}) if (var=='SWC_1'): ax.legend(loc='lower left',prop={'size':12}) ax.axhline(y=0., c='k') ax.set_ylim(ylim) ax.set_ylabel(axlabel) if var=='Ta_day': ax.set_yticks([-10.,0.,10.,20.,30.,40.]) if var=='SWin': ax.set_yticks([0.,100.,200.,300.,400.]) if var=='VPD': ax.set_yticks([0.,5.,10.,15.]) if var=='Ts_1': ax.set_yticks([0.,10.,20.,30.]) if var != 'SWC_1': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) plt.show() return None
def optimize_regional_yldgapf_dyn(NUTS_no_, detrend, crop_no_, selected_grid_cells_, selected_soil_types_, weather, inputdir, opti_years_, obs_type='yield', plot_rmse=False): #=============================================================================== import math from operator import itemgetter as operator_itemgetter from matplotlib import pyplot as plt from pcse.models import Wofost71_WLP_FD from pcse.base_classes import WeatherDataProvider from pcse.fileinput.cabo_weather import CABOWeatherDataProvider # aggregated yield method: # 2- we construct a 2D array with same dimensions as TSO_regional, # containing the observed yields row = [] # this list will become the row of the 2D array for y,year in enumerate(opti_years_): index_year = np.argmin(np.absolute(detrend[1]-year)) row = row + [detrend[0][index_year]] OBS = np.tile(row, (5,1)) # repeats the list as a row 3 times, to get a # 2D array # 3- we calculate all the individual yields from the selected grid cells x # soils combinations # NB: we explore the range of yldgapf between 0.1 and 1. f0 = 0. f2 = 0.5 f4 = 1. f_step = 0.25 # Until the precision of the yield gap factor is good enough (i.e. < 0.02) # we loop over it. We do 12 iterations in total with this method. iter_no = 0 RMSE_stored = list() while (f_step >= 0.02): iter_no = iter_no + 1 # sub-method: looping over the yield gap factors # we build a range of 3 yield gap factors to explore one low bound, one # high bound, one in the middle f_step = (f4 - f0)/4. f1 = f0 + f_step f3 = f2 + f_step f_range = [f0, f1, f2, f3, f4] RES = [] # list in which we will store the yields of the combinations counter=0 for grid, arable_land in selected_grid_cells_: frac_arable = arable_land / 625000000. # Retrieve the weather data of one grid cell (all years are in one # file) if (weather == 'CGMS'): filename = os.path.join(inputdir,'weather_objects/', 'weatherobject_g%d.pickle'%grid) weatherdata = WeatherDataProvider() weatherdata._load(filename) if (weather == 'ECMWF'): weatherdata = CABOWeatherDataProvider('%i'%grid,fpath=ECMWFdir) # Retrieve the soil data of one grid cell (all possible soil types) filename = os.path.join(inputdir,'soildata_objects/', 'soilobject_g%d.pickle'%grid) soil_iterator = pickle_load(open(filename,'rb')) for smu, stu_no, weight, soildata in selected_soil_types_[grid]: # TSO will store all the yields of one grid cell x soil # combination, for all years and all 3 yldgapf values TSO = np.zeros((len(f_range), len(opti_years_))) counter +=1 for y, year in enumerate(opti_years_): # Retrieve yearly data filename = os.path.join(inputdir, 'timerdata_objects/%i/c%i/'%(year,crop_no_), 'timerobject_g%d_c%d_y%d.pickle'\ %(grid,crop_no_,year)) timerdata = pickle_load(open(filename,'rb')) filename = os.path.join(inputdir, 'cropdata_objects/%i/c%i/'%(year,crop_no_), 'cropobject_g%d_c%d_y%d.pickle'\ %(grid,crop_no_,year)) cropdata = pickle_load(open(filename,'rb')) if str(grid).startswith('1'): dum = str(grid)[0:2] else: dum = str(grid)[0] filename = os.path.join(inputdir, 'sitedata_objects/%i/c%i/grid_%s/' %(year,crop_no_,dum), 'siteobject_g%d_c%d_y%d_s%d.pickle'\ %(grid,crop_no_,year,stu_no)) sitedata = pickle_load(open(filename,'rb')) for f,factor in enumerate(f_range): cropdata['YLDGAPF']=factor # run WOFOST wofost_object = Wofost71_WLP_FD(sitedata, timerdata, soildata, cropdata, weatherdata) wofost_object.run_till_terminate() # get the yield (in kgDM.ha-1) TSO[f,y] = wofost_object.get_variable('TWSO') #print grid, stu_no, year, counter, [y[0] for y in TSO], OBS[0] RES = RES + [(grid, stu_no, weight*frac_arable, TSO)] # 4- we aggregate the yield or harvest into the regional one with array # operations sum_weighted_vals = np.zeros((len(f_range), len(opti_years_))) # empty 2D array with same dimension as TSO sum_weights = 0. for grid, stu_no, weight, TSO in RES: # adding weighted 2D-arrays in the empty array sum_weighted_yields # NB: variable 'weight' is actually the cultivated area in m2 sum_weighted_vals = sum_weighted_vals + (weight/10000.)*TSO # computing the total sum of the cultivated area in ha sum_weights = sum_weights + (weight/10000.) if (obs_type == 'harvest'): TSO_regional = sum_weighted_vals / 1000000. # sum of the individual # harvests in 1000 tDM elif (obs_type == 'yield'): TSO_regional = sum_weighted_vals / sum_weights # weighted average of # all yields in kgDM/ha # 5- we compute the (sim-obs) differences. DIFF = TSO_regional - OBS if (TSO_regional[-1][0] <= 0.): print 'WARNING: no simulated crop growth. We set the optimum fgap to 1.' return 1. if (TSO_regional[-1] <= OBS[-1]): print 'WARNING: obs yield > sim yield. We set optimum to 1.' return 1. # 6- we calculate the RMSE (root mean squared error) of the 3 yldgapf # The RMSE of each yldgapf is based on N obs-sim differences for the N # years looped over RMSE = np.zeros(len(f_range)) for f,factor in enumerate(f_range): list_of_DIFF = [] for y, year in enumerate(opti_years_): list_of_DIFF = list_of_DIFF + [DIFF[f,y]] RMSE[f] = np.sqrt(np.mean( [ math.pow(j,2) for j in list_of_DIFF ] )) #print RMSE, f_range # We store the value of the RMSE for plotting purposes RMSE_stored = RMSE_stored + [(f_range[1], RMSE[1]), (f_range[3], RMSE[3])] if (iter_no == 1): RMSE_stored = RMSE_stored + [(f_range[0], RMSE[0]), (f_range[2], RMSE[2]), (f_range[4], RMSE[4])] # 7- We update the yldgapf range to explore for the next iteration. # For this we do a linear interpolation of RMSE between the 3 yldgapf # explored here, and the next range to explore is the one having the # smallest interpolated RMSE index_new_center = RMSE.argmin() # if the optimum is close to 1: if index_new_center == len(f_range)-1: f0 = f_range[index_new_center-2] f2 = f_range[index_new_center-1] f4 = f_range[index_new_center] # if the optimum is close to 0: elif index_new_center == 0: f0 = f_range[index_new_center] f2 = f_range[index_new_center+1] f4 = f_range[index_new_center+2] else: f0 = f_range[index_new_center-1] f2 = f_range[index_new_center] f4 = f_range[index_new_center+1] # when we are finished iterating on the yield gap factor range, we plot the # RMSE as a function of the yield gap factor if (plot_rmse == True): RMSE_stored = sorted(RMSE_stored, key=operator_itemgetter(0)) x,y = zip(*RMSE_stored) fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5,5)) fig.subplots_adjust(0.15,0.16,0.95,0.96,0.4,0.) ax.plot(x, y, c='k', marker='o') ax.set_xlabel('yldgapf (-)') ax.set_ylabel('RMSE') fig.savefig('%s_opti_fgap.png'%NUTS_no_) #pickle_dump(RMSE_stored,open('%s_RMSE.pickle'%NUTS_no_,'wb')) # 8- when we are finished iterating on the yield gap factor range, we return # the optimum value. We look for the yldgapf with the lowest RMSE index_optimum = RMSE.argmin() optimum_yldgapf = f_range[index_optimum] print 'optimum found: %.2f +/- %.2f'%(optimum_yldgapf, f_step) # 10- we return the optimized YLDGAPF return optimum_yldgapf
def _load(self,f): self.store = pickle_load(f)
def compute_timeseries_fluxes(crop_no, grid_no, lon, lat, year, R10, Eact0, selec_method, nsoils, TER_method='grow-only', scale='daily'): # possible methods: 'grow-only': NEE = GPP + Rgrow + Rsoil # 'rauto': NEE = GPP + Rgrow + Rmaint + Rsoil #=============================================================================== import math import pandas as pd import datetime as dt from maries_toolbox import open_pcse_csv_output, select_soils print '- grid cell no %i: lon = %.2f , lat = %.2f' % (grid_no, lon, lat) prod_figure = False # we retrieve the tsurf and rad variables from ECMWF filename_rad = 'rad_ecmwf_%i_lon%.2f_lat%.2f.pickle' % (year, lon, lat) path_rad = os.path.join(ECMWFdir, filename_rad) rad = pickle_load(open(path_rad, 'rb')) filename_ts = 'ts_ecmwf_%i_lon%.2f_lat%.2f.pickle' % (year, lon, lat) path_ts = os.path.join(ECMWFdir, filename_ts) ts = pickle_load(open(path_ts, 'rb')) # we initialize the timeseries for the grid cell # time list for timeseries time_cell_persec_timeseries = rad[0] time_cell_perday_timeseries = rad[0][::8] / (3600. * 24.) # length of all carbon fluxes timeseries len_persec = len(rad[0]) len_perday = len(rad[0][::8]) # GPP timeseries gpp_cell_persec_timeseries = np.array([0.] * len_persec) gpp_cell_perday_timeseries = np.array([0.] * len_perday) # autotrophic respiration timeseries raut_cell_persec_timeseries = np.array([0.] * len_persec) raut_cell_perday_timeseries = np.array([0.] * len_perday) # heterotrophic respiration timeseries rhet_cell_persec_timeseries = np.array([0.] * len_persec) rhet_cell_perday_timeseries = np.array([0.] * len_perday) # we initialize some variables sum_stu_areas = 0. # sum of soil types areas delta = 3600. * 3. # number of seconds in delta (here 3 hours) if (prod_figure == True): from matplotlib import pyplot as plt plt.close('all') fig1, axes = plt.subplots(nrows=3, ncols=1, figsize=(14, 10)) fig1.subplots_adjust(0.1, 0.1, 0.98, 0.9, 0.2, 0.2) # Select soil types to loop over soilist = select_soils(crop_no, [grid_no], CGMSdir, method=selec_method, n=nsoils) #--------------------------------------------------------------- # loop over soil types #--------------------------------------------------------------- for smu, stu_no, stu_area, soildata in soilist[grid_no]: # We open the WOFOST results file filename = 'wofost_g%i_s%i.txt' % (grid_no, stu_no) results_set = open_pcse_csv_output(forwardir, [filename]) wofost_data = results_set[0] # We apply the short wave radiation diurnal cycle on the GPP # and R_auto # we create empty time series for this specific stu gpp_cycle_timeseries = np.array([]) raut_cycle_timeseries = np.array([]) gpp_perday_timeseries = np.array([]) raut_perday_timeseries = np.array([]) # we compile the sum of the stu areas to do a weighted average of # GPP and Rauto later on sum_stu_areas += stu_area #----------------------------------------------------------- # loop over days of the year #----------------------------------------------------------- for DOY, timeinsec in enumerate(time_cell_persec_timeseries[::8]): # conversion of current time in seconds into date time = dt.date(year, 1, 1) + dt.timedelta(DOY) #print 'date:', time # we test to see if we are within the growing season test_sow = (time - wofost_data[filename]['day'][0]).total_seconds() test_rip = (time - wofost_data[filename]['day'][-1]).total_seconds() #print 'tests:', test_sow, test_rip # if the day of the time series is before sowing date: plant # fluxes are set to zero if test_sow < 0.: gpp_day = 0. raut_day = 0. # or if the day of the time series is after the harvest date: # plant fluxes are set to zero elif test_rip > 0.: gpp_day = 0. raut_day = 0. # else we get the daily total GPP and Raut in kgCH2O/ha/day # from wofost, and we weigh it with the stu area to later on # calculate the weighted average GPP and Raut in the grid cell else: # index of the sowing date in the time_cell_timeseries: if (test_sow == 0.): DOY_sowing = DOY if (test_rip == 0.): DOY_harvest = DOY #print 'DOY sowing:', DOY_sowing # translation of cell to stu timeseries index index_day_w = DOY - DOY_sowing #print 'index of day in wofost record:', index_day_w # unit conversion: from kgCH2O/ha/day to gC/m2/day gpp_day = - wofost_data[filename]['GASS'][index_day_w] * \ (mmC / mmCH2O) * 0.1 maint_resp = wofost_data[filename]['MRES'][index_day_w] * \ (mmC / mmCH2O) * 0.1 try: # if there are any available assimilates for growth growth_fac = (wofost_data[filename]['DMI'][index_day_w]) / \ (wofost_data[filename]['GASS'][index_day_w] - wofost_data[filename]['MRES'][index_day_w]) growth_resp = (1. - growth_fac) * (-gpp_day - maint_resp) except ZeroDivisionError: # otherwise there is no crop growth growth_resp = 0. if TER_method == 'rauto': raut_day = growth_resp + maint_resp elif TER_method == 'grow-only': raut_day = growth_resp # we select the radiation diurnal cycle for that date # NB: the last index is ignored in the selection, so we DO have # 8 time steps selected only (it's a 3-hourly dataset) rad_cycle = rad[1][DOY * 8:DOY * 8 + 8] # we apply the radiation cycle on the GPP and Rauto # and we transform the daily integral into a rate weights = rad_cycle / sum(rad_cycle) # the sum of the 8 rates is equal to total/delta: sum_gpp_rates = gpp_day / delta sum_raut_rates = raut_day / delta # the day's 8 values of actual gpp and raut rates per second: gpp_cycle = weights * sum_gpp_rates raut_cycle = weights * sum_raut_rates # NB: we check if the applied diurnal cycle is correct assert (sum(weights) - 1. < 0.000001), "wrong radiation kernel" assert (len(gpp_cycle) * int(delta) == 86400), "wrong delta in diurnal cycle" assert ((sum(gpp_cycle)*delta-gpp_day) < 0.00001), "wrong diurnal cycle "+\ "applied on GPP: residual=%.2f "%(sum(gpp_cycle)*delta-gpp_day) +\ "on DOY %i"%DOY assert ((sum(raut_cycle)*delta-raut_day) < 0.00001), "wrong diurnal cycle "+\ "applied on Rauto: residual=%.2f "%(sum(raut_cycle)*delta-raut_day) +\ "on DOY %i"%DOY # if the applied diurnal cycle is ok, we append that day's cycle # to the yearly record of the stu gpp_cycle_timeseries = np.concatenate( (gpp_cycle_timeseries, gpp_cycle), axis=0) raut_cycle_timeseries = np.concatenate( (raut_cycle_timeseries, raut_cycle), axis=0) # we also store the carbon fluxes per day, for comparison with fluxnet gpp_perday_timeseries = np.concatenate( (gpp_perday_timeseries, [gpp_day]), axis=0) raut_perday_timeseries = np.concatenate( (raut_perday_timeseries, [raut_day]), axis=0) #----------------------------------------------------------- # end of day nb loop #----------------------------------------------------------- # plot the soil type timeseries if requested by the user if (prod_figure == True): for ax, var, name, lims in zip(axes.flatten(), [ gpp_perday_timeseries, raut_perday_timeseries, gpp_perday_timeseries + raut_perday_timeseries ], ['GPP', 'Rauto', 'NPP'], [[-20., 0.], [0., 10.], [-15., 0.]]): ax.plot(time_cell_perday_timeseries, var, label='stu %i' % stu_no) #ax.set_xlim([40.,170.]) #ax.set_ylim(lims) ax.set_ylabel(name + r' (g$_{C}$ m$^{-2}$ d$^{-1}$)', fontsize=14) # We compile time series of carbon fluxes in units per day and per second # a- sum the PER SECOND timeseries gpp_cell_persec_timeseries = gpp_cell_persec_timeseries + \ gpp_cycle_timeseries*stu_area raut_cell_persec_timeseries = raut_cell_persec_timeseries + \ raut_cycle_timeseries*stu_area # b- sum the PER DAY timeseries gpp_cell_perday_timeseries = gpp_cell_perday_timeseries + \ gpp_perday_timeseries*stu_area raut_cell_perday_timeseries = raut_cell_perday_timeseries + \ raut_perday_timeseries*stu_area #--------------------------------------------------------------- # end of soil type loop #--------------------------------------------------------------- # finish ploting the soil type timeseries if requested by the user if (prod_figure == True): plt.xlabel('time (DOY)', fontsize=14) plt.legend(loc='upper left', ncol=2, fontsize=10) fig1.suptitle('Daily carbon fluxes of %s for all '%crop+\ 'soil types of grid cell %i in %i'%(grid_no, year), fontsize=18) figname = 'GPP_allsoils_%i_c%i_g%i.png'%(year,crop_no,\ grid_no) #plt.show() fig1.savefig(os.path.join(analysisdir, figname)) # compute the weighted average of GPP, Rauto over the grid cell # a- PER SECOND gpp_cell_persec_timeseries = gpp_cell_persec_timeseries / sum_stu_areas raut_cell_persec_timeseries = raut_cell_persec_timeseries / sum_stu_areas # b- PER DAY gpp_cell_perday_timeseries = gpp_cell_perday_timeseries / sum_stu_areas raut_cell_perday_timeseries = raut_cell_perday_timeseries / sum_stu_areas # compute the heterotrophic respiration with the A-gs equation # NB: we assume here Rhet only dependant on tsurf, not soil moisture #fw = Cw * wsmax / (wg + wsmin) tsurf_inter = Eact0 / (283.15 * 8.314) * (1 - 283.15 / ts[1]) # a- PER SEC: rhet_cell_persec_timeseries = R10 * np.array( [math.exp(t) for t in tsurf_inter]) # b- PER DAY: for i in range(len(rhet_cell_perday_timeseries)): rhet_cell_perday_timeseries[i] = rhet_cell_persec_timeseries[i*8] * delta +\ rhet_cell_persec_timeseries[i*8+1] * delta +\ rhet_cell_persec_timeseries[i*8+2] * delta +\ rhet_cell_persec_timeseries[i*8+3] * delta +\ rhet_cell_persec_timeseries[i*8+4] * delta +\ rhet_cell_persec_timeseries[i*8+5] * delta +\ rhet_cell_persec_timeseries[i*8+6] * delta +\ rhet_cell_persec_timeseries[i*8+7] * delta # conversion from mgCO2 to gC conversion_fac = (mmC / mmCO2) * 0.001 rhet_cell_persec_timeseries = rhet_cell_persec_timeseries * conversion_fac rhet_cell_perday_timeseries = rhet_cell_perday_timeseries * conversion_fac # calculate TER: ter_cell_persec_timeseries = raut_cell_persec_timeseries + \ rhet_cell_persec_timeseries ter_cell_perday_timeseries = raut_cell_perday_timeseries + \ rhet_cell_perday_timeseries # calculate NEE: nee_cell_persec_timeseries = gpp_cell_persec_timeseries + \ raut_cell_persec_timeseries + \ rhet_cell_persec_timeseries nee_cell_perday_timeseries = gpp_cell_perday_timeseries + \ raut_cell_perday_timeseries + \ rhet_cell_perday_timeseries # here we choose to return the carbon fluxes PER DAY if scale == 'daily': return time_cell_perday_timeseries, gpp_cell_perday_timeseries, \ raut_cell_perday_timeseries, rhet_cell_perday_timeseries, \ ter_cell_perday_timeseries, nee_cell_perday_timeseries elif scale == '3-hourly': return time_cell_persec_timeseries, gpp_cell_persec_timeseries, \ raut_cell_persec_timeseries, rhet_cell_persec_timeseries, \ ter_cell_persec_timeseries, nee_cell_persec_timeseries
def plot_fluxnet_daily_c_fluxes(directory, sites_list, timeframe, linestyle): #=============================================================================== """ This function plots the average daytime Tair, average SWin, average VPD, average Tsoil and average volumetric soil moisture content, all taken from the formatted 2015 FluxNet DD (daily data) files. NB: Daily averages have been computed from the half-hourly data. Function arguments: ------------------ directory: path to the directory where the daily variables files ('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1') are stored sites_list: list of fluxnet sites names time frame: list of two years (e.g. [2005, 2007]) gives the time frame to plot linestyle: line style of the plot to make (can be '-', '--', '+', etc) """ plt.close('all') print "\nPlotting carbon fluxes timeseries\n" for site in sites_list: print site+'\n' filepath = os.path.join(directory,'timeseries_%s.pickle'%site) series = pickle_load(open(filepath,'rb')) # settings of the plot: figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8,10)) figs.subplots_adjust(0.1,0.07,0.98,0.95,0.,0.) variables = ['crop_no','GPP','TER','NEE'] axlabels = ['crop ID',r'GPP (g m$^{-2}$ d$^{-1}$)',r'TER (g m$^{-2}$ d$^{-1}$)',r'NEE (g m$^{-2}$ d$^{-1}$)'] ylims = [(0.,14.),(-30.,5.),(-5.,20.),(-30.,30.)] start = str(int(timeframe[0])) end = str(int(timeframe[1])) lab = '' ls = linestyle fsz = 14 # fonsize of x and y axis ticks for ax, var, axlabel, ylim in zip(axes,variables,axlabels,ylims): if (var=='TER' or var=='GPP'): lab='-' series[site][var][start:end].plot(ax=ax, lw=2, style=ls, label=lab, fontsize=fsz) if var=='GPP': series[site]['GPP_day'][start:end].plot(ax=ax, lw=2, style=ls, label='day partition', fontsize=fsz) series[site]['GPP_night'][start:end].plot(ax=ax, lw=2, style=ls, label='night partition', fontsize=fsz) if var=='TER': series[site]['TER_day'][start:end].plot(ax=ax, lw=2, style=ls, label='day partition', fontsize=fsz) series[site]['TER_night'][start:end].plot(ax=ax, lw=2, style=ls, label='night partition', fontsize=fsz) ax.axhline(y=0., c='k') ax.set_ylim(ylim) if (var=='GPP_day'): ax.legend(loc='lower left',prop={'size':12}) if (var=='TER_day'): ax.legend(loc='upper left',prop={'size':12}) ax.set_ylabel(axlabel) if var != 'NEE': ax.get_xaxis().set_visible(False) figs.suptitle(site, fontsize=14) plt.show() return None
# Any file has changed in the dir since the cache was generated or any(True for f in listdir(directory) if (getmtime(path_join(directory, f)) > cache_mtime # Ignore hidden files and not f.startswith('.'))) # The configuration is newer than the cache or getmtime(get_config_path(directory)) > cache_mtime): generate = True docstats = [] else: generate = False try: with open( cache_file_path.decode('utf-8').encode('utf-8'), 'rb') as cache_file: docstats = pickle_load(cache_file) except UnpicklingError: # Corrupt data, re-generate Messager.warning( 'Stats cache %s was corrupted; regenerating' % cache_file_path, -1) generate = True except EOFError: # Corrupt data, re-generate generate = True except OSError, e: Messager.warning( 'Failed checking file modification times for stats cache check; regenerating' ) generate = True
def __init__(self, shmem): AggregationTask.__init__(self, shmem) reader = wire.PackedMessageReader(shmem) self.state = pickle_load(reader.string())