def __init__(self, shmem):
AggregationTask.__init__(self, shmem)
reader = wire.PackedMessageReader(shmem)
# TODO: may wish to merge more than 2 at a time?
# Unpack header
self.left_inst = pickle_load(reader.string())
self.right_inst = pickle_load(reader.string())
def __init__(self, shmem):
AggregationTask.__init__(self, shmem)
reader = wire.PackedMessageReader(shmem)
# TODO: may wish to merge more than 2 at a time?
# Unpack header
self.left_inst = pickle_load(reader.string())
self.right_inst = pickle_load(reader.string())
def plot_fluxnet_LAI_CHT_AGB(directory, sites_list, timeframe, linestyle):
#===============================================================================
"""
This function plots the LAI, CHT and AGB auxilliary data from FluxNet sites
Function arguments:
------------------
directory: path to the directory where the daily variables files
('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1')
are stored
sites_list: list of fluxnet sites names
time frame: list of two years (e.g. [2005, 2007]) gives the time frame
to plot
linestyle: line style of the plot to make (can be '-', '--', '+', etc)
"""
plt.close('all')
print "\nPlotting timeseries of LAI, CHT and AGB\n"
for site in sites_list:
print site + '\n'
filepath = os.path.join(directory, 'timeseries_%s.pickle' % site)
series = pickle_load(open(filepath, 'rb'))
# settings of the plot:
figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8, 10))
figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.)
variables = ['crop_no', 'LAI', 'CHT', 'AGB']
axlabels = [
'crop ID', r'LAI (m$^{-2}$ m$^{-2}$)', 'CHT (m)',
r'AGB (kg$_{DM}$ m$^{-2}$)'
]
ylims = [(0., 14.), (0., 7.5), (0., 3.), (0., 3.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
lab = ''
ls = linestyle
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims):
if var == 'crop_no':
ls = '-'
else:
ls = linestyle
series[site][var][start:end].plot(ax=ax,
lw=2,
style=ls,
label=lab,
fontsize=fsz)
ax.axhline(y=0., c='k')
ax.set_ylim(ylim)
ax.set_ylabel(axlabel)
if var != 'AGB': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
plt.show()
return None
def test_merge(self):
klass = self._get_mean_uda()
lcol = self.col_fragments[0]
rcol = self.col_fragments[1]
left = self._update(klass, [lcol])
right = self._update(klass, [rcol])
task, mm = self._make_merge_task(left, right)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
larr = lcol.to_numpy_for_pandas()
rarr = rcol.to_numpy_for_pandas()
assert larr is not None
ex_total = (pd.Series(larr).sum() + pd.Series(rarr).sum())
assert result.total == ex_total
def init_session(remote_address, cookie_data=None):
if cookie_data is not None:
cookie = SessionCookie.load(cookie_data)
else:
cookie = None
# Default sid for the session
sid = sha224('%s-%s' % (remote_address, datetime.utcnow())).hexdigest()
if cookie is None:
cookie = SessionCookie(sid)
else:
try:
cookie.get_sid()
except KeyError:
# For some reason the cookie did not contain a SID, set to default
cookie.set_sid(sid)
# Set the session singleton (there can be only one!)
global CURRENT_SESSION
ppath = get_session_pickle_path(cookie.get_sid())
if isfile(ppath):
# Load our old session data and initialise the cookie
try:
with open(ppath, 'rb') as session_pickle:
CURRENT_SESSION = pickle_load(session_pickle)
CURRENT_SESSION.init_cookie(CURRENT_SESSION.get_sid())
except Exception, e:
# On any error, just create a new session
CURRENT_SESSION = Session(cookie)
def load_pickle(filename):
file = open(filename + '.pck', 'r')
res = pickle_load(file)
file.close()
return res
def init_session(remote_address, cookie_data=None):
if cookie_data is not None:
cookie = SessionCookie.load(cookie_data)
else:
cookie = None
# Default sid for the session
sid = sha224('%s-%s' % (remote_address, datetime.utcnow())).hexdigest()
if cookie is None:
cookie = SessionCookie(sid)
else:
try:
cookie.get_sid()
except KeyError:
# For some reason the cookie did not contain a SID, set to default
cookie.set_sid(sid)
# Set the session singleton (there can be only one!)
global CURRENT_SESSION
ppath = get_session_pickle_path(cookie.get_sid())
if isfile(ppath):
# Load our old session data and initialise the cookie
try:
with open(ppath, 'rb') as session_pickle:
CURRENT_SESSION = pickle_load(session_pickle)
CURRENT_SESSION.init_cookie(CURRENT_SESSION.get_sid())
except Exception, e:
# On any error, just create a new session
CURRENT_SESSION = Session(cookie)
def learning_curve_avg(classifiers, datasets, outdir, pickle_name='learning'):
with open(_get_learning_pickle_path(outdir, name=pickle_name),
'r') as results_file:
results = pickle_load(results_file)
for dataset in datasets:
print 'Dataset:', dataset
for classifier in classifiers:
print 'Classifier:', classifier
macro_avg = mean([
res_tup[0]
for res_tup in results[dataset][classifier].itervalues()
]) * 100
macro_tip = sorted((size, res_tup[0])
for size, res_tup in results[dataset]
[classifier].iteritems())[-1][1] * 100
amb_avg = mean([
res_tup[11]
for res_tup in results[dataset][classifier].itervalues()
])
amb_tip = sorted((size, res_tup[11]) for size, res_tup in
results[dataset][classifier].iteritems())[-1][1]
rec_avg = mean([
res_tup[13]
for res_tup in results[dataset][classifier].itervalues()
]) * 100
rec_tip = sorted((size, res_tup[13])
for size, res_tup in results[dataset]
[classifier].iteritems())[-1][1] * 100
print(
'{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f} '
'MACROAVG/MACROTIP/AMBAVG/AMBTIP/RECAVG/RECTIP').format(
macro_avg, macro_tip, amb_avg, amb_tip, rec_avg, rec_tip)
def plot_learning_curve_results(classifiers,
datasets,
outdir,
pickle_name='learning',
verbose=False):
from matplotlib import pyplot as plt
if not classifiers and not datasets:
print >> stderr, 'No classifiers or datasets specified, exiting'
return
# Load the results to be plotted
if verbose:
print >> stderr, 'Loading results...',
with open(_get_learning_pickle_path(outdir, name=pickle_name),
'r') as results_file:
results = pickle_load(results_file)
if verbose:
print >> stderr, 'Done!'
image_formats = ('svg', )
for fig_name, fig in _ambiguity_plot_gen(results, classifiers, datasets):
for image_format in image_formats:
plt.savefig(path_join(outdir, fig_name + '.' + image_format),
format=image_format,
figure=fig)
def learning_curve_avg(classifiers, datasets, outdir, pickle_name='learning'):
with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r') as results_file:
results = pickle_load(results_file)
for dataset in datasets:
print 'Dataset:', dataset
for classifier in classifiers:
print 'Classifier:', classifier
macro_avg = mean([res_tup[0] for res_tup
in results[dataset][classifier].itervalues()]) * 100
macro_tip = sorted((size, res_tup[0]) for size, res_tup
in results[dataset][classifier].iteritems())[-1][1] * 100
amb_avg = mean([res_tup[11] for res_tup
in results[dataset][classifier].itervalues()])
amb_tip = sorted((size, res_tup[11]) for size, res_tup
in results[dataset][classifier].iteritems())[-1][1]
rec_avg = mean([res_tup[13] for res_tup
in results[dataset][classifier].itervalues()]) * 100
rec_tip = sorted((size, res_tup[13]) for size, res_tup
in results[dataset][classifier].iteritems())[-1][1] * 100
print ('{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f}/{:.2f} '
'MACROAVG/MACROTIP/AMBAVG/AMBTIP/RECAVG/RECTIP').format(
macro_avg, macro_tip, amb_avg, amb_tip, rec_avg, rec_tip)
def main(args):
argp = ARGPARSER.parse_args(args[1:])
if not argp.no_cache:
# We can't do it iteratively listening to stdin, read it all
doc = Document('<classify>', [], [], '<classify>')
for _string in (l.rstrip('\n') for l in argp.input):
doc.abstract.append(_string_to_ann_sent(_string))
docs = (doc, )
else:
docs = (Document('Line: %s' % i, [], [_string_to_ann_sent(_string)],
'<stdin>') for i, _string in enumerate(
(l.rstrip('\n') for l in argp.input), start=1))
# Cache the strings for speed
if not argp.no_cache:
cache_simstring((docs, ), verbose=argp.verbose)
with open(argp.model_path, 'r') as model_file:
classifier = pickle_load(model_file)
# TODO: Faster to do it in a batch instead
for doc in docs:
for sent in doc:
for ann in sent:
print '%s\t%s' % (sent.annotation_text(ann),
str(classifier.classify(doc, sent, ann, ranked=True)))
def _load_simstring_cache():
global SIMSTRING_QUERY_CACHE
if not isfile(SIMSTRING_QUERY_CACHE_PATH):
#XXX: TODO: We need a check-sum test if we are to keep a specific db
SIMSTRING_QUERY_CACHE = {}
else:
with open(SIMSTRING_QUERY_CACHE_PATH, 'rb') as cache_file:
SIMSTRING_QUERY_CACHE = pickle_load(cache_file)
def _load_pickle(pickle_path, save=True):
if isfile(pickle_path):
try:
with open(pickle_path, 'rb') as pickle_file:
loaded_pickle = pickle_load(pickle_file)
except EOFError, e:
print >> stderr, "failed to load pickle %s: %s" % (pickle_path, e)
raise
def _load_simstring_cache():
global SIMSTRING_QUERY_CACHE
if not isfile(SIMSTRING_QUERY_CACHE_PATH):
#XXX: TODO: We need a check-sum test if we are to keep a specific db
SIMSTRING_QUERY_CACHE = {}
else:
with open(SIMSTRING_QUERY_CACHE_PATH, 'rb') as cache_file:
SIMSTRING_QUERY_CACHE = pickle_load(cache_file)
def getAnnObject2(collection,document):
'''newest version of the getAnnObject methode'''
try:
from os.path import join as path_join
from document import real_directory
real_dir = real_directory(collection)
except:
real_dir=collection
app_path = WORK_DIR + "/application/"
ann = None
full_name = collection + document
full_name = full_name.replace("/","")
if( isfile(app_path+full_name)):
temp=open (app_path+full_name , 'rb')
ann = pickle_load(temp)
temp.close()
else:
ann = TextAnnotations(real_dir+document)
ann = SimpleAnnotations(ann)
ann.folia = {}
try:
#TODO:good error message
ann.folia=get_extra_info(collection,document)
except Exception as e:
ann.folia = {}
Messager.error('Error: get extra folia info() failed: %s' % e)
#Validation:
try:
import os
import simplejson as json
import session
docdir = os.path.dirname(ann._document)
string = session.load_conf()["config"]
val = json.loads(string)["validationOn"]
#validate if config enables it and if it's not already done.
if val:
if not ann.validated:
from verify_annotations import verify_annotation
projectconf = ProjectConfiguration(docdir)
issues = []
issues = verify_annotation(ann, projectconf)
else:
issues = ann.issues
else:
ann.validated = False
issues = []
except session.NoSessionError:
issues = []
except KeyError:
issues = []
except Exception as e:
# TODO add an issue about the failure?
issues = []
ann.issues = issues
temp=open (app_path+full_name , 'wb')
pickle_dump(ann, temp)
temp.close()
return ann
def run(self):
if self.prior_state is not None:
agg_inst = self.prior_state
else:
klass = pickle_load(self.agg_class_pickled)
agg_inst = klass()
args = self._deserialize_args()
agg_inst.update(*args)
self._write_response(agg_inst)
def run(self):
if self.prior_state is not None:
agg_inst = self.prior_state
else:
klass = pickle_load(self.agg_class_pickled)
agg_inst = klass()
args = self._deserialize_args()
agg_inst.update(*args)
self._write_response(agg_inst)
def test_update(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
task, mm = self._make_update_task(klass, [col])
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
ex_total = pd.Series(col.to_numpy_for_pandas()).sum()
assert result.total == ex_total
# Test with prior state
col = self.col_fragments[1]
task, mm = self._make_update_task(klass, [col], prior_state=result)
# Executor's turn again
self.lock.release()
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
ex_total += pd.Series(col.to_numpy_for_pandas()).sum()
# pandas will yield 0 on None input strangely
assert ex_total != 0
assert result.total == ex_total
def plot_fluxnet_LAI_CHT_AGB(directory, sites_list, timeframe, linestyle):
#===============================================================================
"""
This function plots the LAI, CHT and AGB auxilliary data from FluxNet sites
Function arguments:
------------------
directory: path to the directory where the daily variables files
('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1')
are stored
sites_list: list of fluxnet sites names
time frame: list of two years (e.g. [2005, 2007]) gives the time frame
to plot
linestyle: line style of the plot to make (can be '-', '--', '+', etc)
"""
plt.close('all')
print "\nPlotting timeseries of LAI, CHT and AGB\n"
for site in sites_list:
print site+'\n'
filepath = os.path.join(directory,'timeseries_%s.pickle'%site)
series = pickle_load(open(filepath,'rb'))
# settings of the plot:
figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8,10))
figs.subplots_adjust(0.1,0.07,0.98,0.95,0.,0.)
variables = ['crop_no','LAI','CHT','AGB']
axlabels = ['crop ID',r'LAI (m$^{-2}$ m$^{-2}$)','CHT (m)',r'AGB (kg$_{DM}$ m$^{-2}$)']
ylims = [(0.,14.),(0.,7.5),(0.,3.),(0.,3.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
lab = ''
ls = linestyle
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes,variables,axlabels,ylims):
if var=='crop_no':
ls='-'
else:
ls=linestyle
series[site][var][start:end].plot(ax=ax, lw=2, style=ls,
label=lab, fontsize=fsz)
ax.axhline(y=0., c='k')
ax.set_ylim(ylim)
ax.set_ylabel(axlabel)
if var != 'AGB': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
plt.show()
return None
def _read_header(self):
reader = wire.PackedMessageReader(self.shmem)
# Unpack header
self.agg_class_pickled = reader.string()
has_prior_state = reader.uint8() != 0
if has_prior_state:
self.prior_state = pickle_load(reader.string())
else:
self.prior_state = None
def _read_header(self):
reader = wire.PackedMessageReader(self.shmem)
# Unpack header
self.agg_class_pickled = reader.string()
has_prior_state = reader.uint8() != 0
if has_prior_state:
self.prior_state = pickle_load(reader.string())
else:
self.prior_state = None
def load(filename):
"""Load a template from ``filename``, return ``Templater`` object.
This method must be used in pair with ``Templater.dump`` - it loads
the template definition from a file using cPickle, creates a
``Templater`` object with the definition and returns it.
"""
fp = open(filename)
processed_template = pickle_load(fp)
fp.close()
return processed_template
def _load_pickle(pickle_path, save=True):
if isfile(pickle_path):
with open(pickle_path, 'rb') as pickle_file:
loaded_pickle = pickle_load(pickle_file)
else:
loaded_pickle = IdMapper()
if save:
atexit_register(_save_pickle_func(loaded_pickle, pickle_path))
return loaded_pickle
def load(filename):
"""Load a template from ``filename``, return ``Templater`` object.
This method must be used in pair with ``Templater.dump`` - it loads
the template definition from a file using cPickle, creates a
``Templater`` object with the definition and returns it.
"""
fp = open(filename)
processed_template = pickle_load(fp)
fp.close()
return processed_template
def import_parser(path, auto=True, from_raw=True):
qpt = QPTokenizer()
if (auto and path.endswith('qpy')) or (not auto and not from_raw):
cfile = open(path, 'rb')
assert cfile.read(len(COMPILED_SIGNATURE)) == COMPILED_SIGNATURE, "Wrong file format"
dump = pickle_load(cfile)
return {proto.name: proto.make() for proto in dump.inner_scopes}
elif (auto and path.endswith('qp')) or (not auto and from_raw):
return qpt.parse_file(path)
else:
raise ValueError("auto parser path should end with .qp for raw parsers, "
"or .qpy for compiled ones.")
def import_parser(path, auto=True, from_raw=True):
qpt = QPTokenizer()
if (auto and path.endswith('qpy')) or (not auto and not from_raw):
cfile = open(path, 'rb')
assert cfile.read(
len(COMPILED_SIGNATURE)) == COMPILED_SIGNATURE, "Wrong file format"
dump = pickle_load(cfile)
return {proto.name: proto.make() for proto in dump.inner_scopes}
elif (auto and path.endswith('qp')) or (not auto and from_raw):
return qpt.parse_file(path)
else:
raise ValueError(
"auto parser path should end with .qp for raw parsers, "
"or .qpy for compiled ones.")
def getAnnObject(collection, document):
try:
real_dir = real_directory(collection)
except:
real_dir = collection
app_path = WORK_DIR + "/application/"
full_name = collection + document
full_name = full_name.replace("/", "")
if (os.path.isfile(app_path + full_name)):
temp = open(app_path + full_name, 'rb')
ann = pickle_load(temp)
temp.close()
else:
ann = TextAnnotations(real_dir + document)
ann = SimpleAnnotations(ann)
ann.folia = {}
try:
#TODO:good error message
ann.folia = get_extra_info(collection, document)
except Exception as e:
ann.folia = {}
Messager.error('Error: get extra folia info() failed: %s' % e)
#Validation:
try:
docdir = os.path.dirname(ann._document)
string = session.load_conf()["config"]
val = json.loads(string)["validationOn"]
#validate if config enables it and if it's not already done.
if val:
if not ann.validated:
projectconf = ProjectConfiguration(docdir)
issues = verify_annotation(ann, projectconf)
else:
issues = ann.issues
else:
ann.validated = False
issues = []
except session.NoSessionError:
issues = []
except KeyError:
issues = []
except Exception as e:
# TODO add an issue about the failure?
issues = []
Messager.error('Error: validation failed: %s' % e)
ann.issues = issues
temp = open(app_path + full_name, 'wb')
pickle_dump(ann, temp)
temp.close()
return ann
def load(cls, filename=PICKLE_CACHE_PATH):
from cPickle import UnpicklingError
from cPickle import load as pickle_load
try:
with open(filename, 'rb') as cache_file:
map_ = unordall(pickle_load(cache_file))
return cls(map_)
except UnpicklingError:
print >> sys.stderr, 'warning: failed to read cache file.'
raise
except IOError:
print >> sys.stderr, 'note: cache file not found.'
raise
except:
print >> sys.stderr, 'warning: unexpected error loading cache.'
raise
def load_cache(fn):
from cPickle import UnpicklingError
from cPickle import load as pickle_load
try:
with open(fn, 'rb') as cache_file:
data = pickle_load(cache_file)
return data
except UnpicklingError:
print >> sys.stderr, "rewritetex: warning: failed to read cache file."
raise
except IOError:
print >> sys.stderr, "rewritetex: note: cache file not found."
raise
except:
print >> sys.stderr, "rewritetex: warning: unexpected error loading cache."
raise
def load_cache(fn):
from cPickle import UnpicklingError
from cPickle import load as pickle_load
try:
with open(fn, 'rb') as cache_file:
data = pickle_load(cache_file)
return data
except UnpicklingError:
print >> sys.stderr, "rewritetex: warning: failed to read cache file."
raise
except IOError:
print >> sys.stderr, "rewritetex: note: cache file not found."
raise
except:
print >> sys.stderr, "rewritetex: warning: unexpected error loading cache."
raise
def load(cls, filename=PICKLE_CACHE_PATH):
from cPickle import UnpicklingError
from cPickle import load as pickle_load
try:
with open(filename, 'rb') as cache_file:
map_ = unordall(pickle_load(cache_file))
return cls(map_)
except UnpicklingError:
print >> sys.stderr, 'warning: failed to read cache file.'
raise
except IOError:
print >> sys.stderr, 'note: cache file not found.'
raise
except:
print >> sys.stderr, 'warning: unexpected error loading cache.'
raise
def test_finalize(self):
klass = self._get_mean_uda()
col = self.col_fragments[0]
result = self._update(klass, [col])
task, mm = self._make_finalize_task(result)
_execute_task(task, self.lock)
mm.seek(0)
reader = wire.PackedMessageReader(mm)
# success
if not reader.uint8():
raise Exception(reader.string())
result = pickle_load(reader.string())
arr = col.to_numpy_for_pandas()
ex_result = pd.Series(arr).mean()
assert result == ex_result
def plot_learning_curve_results(classifiers, datasets, outdir,
pickle_name='learning', verbose=False):
from matplotlib import pyplot as plt
if not classifiers and not datasets:
print >> stderr, 'No classifiers or datasets specified, exiting'
return
# Load the results to be plotted
if verbose:
print >> stderr, 'Loading results...',
with open(_get_learning_pickle_path(outdir, name=pickle_name), 'r'
) as results_file:
results = pickle_load(results_file)
if verbose:
print >> stderr, 'Done!'
image_formats = ('svg', )
for fig_name, fig in _ambiguity_plot_gen(results, classifiers, datasets):
for image_format in image_formats:
plt.savefig(path_join(outdir, fig_name + '.' + image_format),
format=image_format, figure=fig)
def main():
#===============================================================================
global wofostdir, sibcasadir, obsdir
#-------------------------------------------------------------------------------
# ================================= USER INPUT =================================
# read the settings from the rc file (mostly directory paths)
rcdict = rc.read('settings.rc')
sites = [s.strip(' ') for s in rcdict['sites'].split(',')]
years = [int(s.strip(' ')) for s in rcdict['years'].split(',')]
TER_method = rcdict['TER_method']
R10 = rcdict['R10']
resolution = rcdict['resolution']
# specific plotting options:
#TER_method = 'grow-only' # this is to select the corresponding WOFOST output file
#R10 = '0.08' # this is to select the corresponding WOFOST output file
#===============================================================================
#-------------------------------------------------------------------------------
# extract the needed information
# input data directory paths
rootdir = rcdict['rootdir']
sibcasadir = os.path.join(rootdir, 'intercomparison_study/SiBCASA_runs')
wofostdir = rcdict['outputdir']
obsdir = rcdict['obsdir']
figdir = os.path.join(rootdir, 'intercomparison_study/figures')
#-------------------------------------------------------------------------------
# Start a directory to store OBS, SIMW (wofost), SIMB (SiBCASA)
# recover the FluxNet observed data from pickle files
res_timeseries = dict()
res_timeseries['OBS'] = dict()
res_timeseries['SIMB'] = dict()
res_timeseries['SIMW'] = dict()
filename = os.path.join(obsdir, '%s_timeseries_OBS.pickle' % resolution)
try:
res_timeseries['OBS'] = pickle_load(open(filename, 'rb'))
except IOError:
print 'could not find the observations output file %s' % filename
res_timeseries['OBS'] = None
# recover the SiBCASA runs
filename = os.path.join(sibcasadir,
'%s_timeseries_SiBCASA.pickle' % resolution)
try:
res_timeseries['SIMB'] = pickle_load(open(filename, 'rb'))
except IOError:
print 'could not find the SiBCASA output file %s' % filename
res_timeseries['SIMB'] = None
# recover the WOFOST runs
filename = os.path.join(wofostdir, '%s_timeseries_'%resolution +\
'%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10))
try:
res_timeseries['SIMC'] = pickle_load(open(filename, 'rb'))
except IOError:
print 'could not find the WOFOST output file %s' % filename
res_timeseries['SIMC'] = None
#-------------------------------------------------------------------------------
# plot the observed and simulated timeseries with pandas library
# with pandas we plot all years one after another, and can zoom in on one
# particular year
plt.close('all')
# create figure sub-folder if it doesn't already exists
figsubdir = os.path.join(figdir,'R10=%s/TER_%s/'%(R10,TER_method)+\
'3-hourly_fluxes_perf')
if not os.path.exists(figsubdir):
print 'creating new directory %s' % figsubdir
os.makedirs(figsubdir)
#-------------------------------------------------------------------------------
# we plot the 3-hourly fluxes of simulations versus observations for the years
# and sites that perform well on the daily scale
# we plot years 2005, 2009, 2013 for site BE-Lon, which showed an extremely
# good result on the SIM vs OBS daily fluxes comparison
years = [2005, 2009, 2013]
variables = ['GPP', 'TER', 'NEE']
axlabels = [
r'GPP (g m$^{-2}$ d$^{-1}$)', r'TER (g m$^{-2}$ d$^{-1}$)',
r'NEE (g m$^{-2}$ d$^{-1}$)'
]
ylims = [(-60., 5.), (0., 20.), (-50., 15.)]
one_to_one = np.arange(-100, 100, 10)
for site in ['BE-Lon']:
if site != 'IT-BCi':
for year in years:
timeframe = [year, year]
start = str(int(timeframe[0])) + '-05-01'
end = str(int(timeframe[1])) + '-07-01'
print site
for var, axlabel, lim in zip(variables, axlabels, ylims):
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5, 5))
fig.subplots_adjust(0.15, 0.15, 0.85, 0.85, 0., 0.)
# select every 6 half-hourly flux in the observations,
# to get to 3-hourly frequency
OBS = res_timeseries['OBS'][site][var][::6][
start:end].dropna()
# convert the 3-hourly simulated fluxes from gC.m-2.s-1
# to micromol CO2 .m-2.s-1
SIM = res_timeseries['SIMC'][site][var][start:end].dropna()
SIM = SIM * 1000000. / 12.01 #micro mol CO2 per m2 per sec
# the observed GPP needs a minus sign for convention
if var == 'GPP': OBS = -OBS
# use the min and max to frame the figure
print var, min(min(OBS), min(SIM)), max(max(OBS), max(SIM))
varmin = math.floor(min(min(OBS), min(SIM)))
varmax = math.ceil(max(max(OBS), max(SIM)))
ax.scatter(OBS, SIM, marker='o')
# fit a linear regression line in OBS/SIM scatter plot
# and plot line and R2
mask = ~np.isnan(SIM)
z = np.polyfit(OBS[mask], SIM[mask], 1)
p = np.poly1d(z)
ax.plot(one_to_one, p(one_to_one), 'r-')
slope, intercept, r_value, p_value, std_err = \
linreg(OBS[mask], SIM[mask])
ax.annotate(r'r$^2$ = %.2f' % r_value**2,
xy=(0.95, 0.15),
xytext=(0.15, 0.9),
xycoords='axes fraction',
ha='center',
va='center',
color='r')
ax.plot(one_to_one, one_to_one, c='k', lw=1)
ax.set_xlabel('obs')
ax.set_ylabel('sim')
ax.set_xlim(varmin, varmax)
ax.set_ylim(varmin, varmax)
fig.suptitle(r'%s 3-hourly %s fluxes ($\mu$' %
(site, var) + r'mol m$^{-2}$ s$^{-1}$)' +
'\nfrom %s to %s\n' % (start, end))
fig.savefig(os.path.join(
figsubdir, '%s_%s_%s.png' % (site, year, var)),
dpi=300)
except NoSessionError:
pass
# New "fresh" cookie session check
init_session('127.0.0.1')
try:
session = get_session()
session['foo'] = 'bar'
except NoSessionError:
assert False
# Pickle check
init_session('127.0.0.1')
tmp_file_path = None
try:
tmp_file_fh, tmp_file_path = mkstemp()
os_close(tmp_file_fh)
session = get_session()
session['foo'] = 'bar'
with open(tmp_file_path, 'wb') as tmp_file:
pickle_dump(session, tmp_file)
del session
with open(tmp_file_path, 'rb') as tmp_file:
session = pickle_load(tmp_file)
assert session['foo'] == 'bar'
finally:
if tmp_file_path is not None:
remove(tmp_file_path)
except NoSessionError:
pass
# New "fresh" cookie session check
init_session('127.0.0.1')
try:
session = get_session()
session['foo'] = 'bar'
except NoSessionError:
assert False
# Pickle check
init_session('127.0.0.1')
tmp_file_path = None
try:
tmp_file_fh, tmp_file_path = mkstemp()
os_close(tmp_file_fh)
session = get_session()
session['foo'] = 'bar'
with open(tmp_file_path, 'wb') as tmp_file:
pickle_dump(session, tmp_file)
del session
with open(tmp_file_path, 'rb') as tmp_file:
session = pickle_load(tmp_file)
assert session['foo'] == 'bar'
finally:
if tmp_file_path is not None:
remove(tmp_file_path)
def optimize_fgap(site, crop_no, crop_name, year, NUTS_no, selec_method, ncells,
nsoils, weather, force_optimization):
#===============================================================================
# Temporarily add code directory to python path, to be able to import pcse
# modules
sys.path.insert(0, codedir)
sys.path.insert(0, os.path.join(codedir,'carbon_cycle'))
#-------------------------------------------------------------------------------
import glob
from maries_toolbox import define_opti_years,\
select_cells, select_soils
#-------------------------------------------------------------------------------
# if the optimization has already been performed and we don't want
# to redo it, we skip that region
filepath = os.path.join(optimidir,'fgap_%s_optimized.pickle'%NUTS_no)
if (os.path.exists(filepath) and force_optimization==False):
optimum = pickle_load(open(filepath,'rb'))
print "We have already calculated the optimum fgap for that "+\
"year and crop: fgap=%.2f"%optimum[2]
return optimum[2]
#-------------------------------------------------------------------------------
# we select the grid cell of the FluxNet site
gridlist = pickle_load(open(os.path.join(CGMSdir,
'gridlist_objects/shortgridlist.pickle'),'rb'))
selected_grid_cells = gridlist[NUTS_no]
#-------------------------------------------------------------------------------
# where possible, we retrieve local information about yield and sowing date
local_sowda = None
local_yield = None
for row in custom_yns:
if row[0]==site and row[1]==year and row[2]==crop_no:
local_sowda = row[3]
local_yield = row[4]
print 'We recovered local info from site %s:'%site
print 'sowing date of %s:'%crop_name, local_sowda, 'grain yield: %.3f'%local_yield
break
if local_sowda==None and local_yield==None:
print 'No local information on sowing date and yield.'
#-------------------------------------------------------------------------------
# we retrieve the EUROSTAT pre-processed yield observations:
if local_sowda==None and local_yield==None:
try:
filename1 = os.path.join(EUROSTATdir, 'preprocessed_yields.pickle')
yields_dict = pickle_load(open(filename1,'rb'))
except IOError:
print '\nYou have not preprocessed the EUROSTAT observations'
print 'Run the script 03_preprocess_obs.py first!\n'
sys.exit()
# NB: we do NOT detrend the yields anymore, since fgap is not supposed to be
# representative of multi-annual gap
obs_yields = yields_dict[crop_name][NUTS_no]
return None
#-------------------------------------------------------------------------------
# if there were no reported yield on the year X, we skip that region
if (year not in obs_yields[1]):
print 'No reported yield, we have to gap-fill later'
filename = os.path.join(optimidir,'fgap_%s_tobegapfilled.pickle'%NUTS_no)
outlist = [NUTS_no, 2, 1., selected_grid_cells]
pickle_dump(outlist, open(filename,'wb'))
return 1.
#-------------------------------------------------------------------------------
# NB: in the optimization routine, we use the observed cultivation
# fraction of the crop to calculate the soil cultivated areas, and
# thus to compute the regional yields (= weighted average of yields
# using soil cultivated areas)
# if the observed cultivated fraction is zero, we skip that region
selected_soil_types = select_soils(crop_no,[g for g,a in selected_grid_cells],
CGMSdir, method=selec_method, n=nsoils)
print 'we selected grid cell %i, top %i soil types, for optimization'%(
selected_grid_cells[0][0], nsoils)
#-------------------------------------------------------------------------------
# we set the optimization code (gives us info on how we optimize)
opti_code = 1 # 1= observations are available for optimization
# 2= no obs available
#print obs_yields[1], obs_yields[0]
# in all other cases, we optimize the yield gap factor
optimum = optimize_regional_yldgapf_dyn(NUTS_no, obs_yields,
crop_no,
selected_grid_cells,
selected_soil_types,
weather,
CGMSdir,
[year],
obs_type='yield',
plot_rmse=False)
# pickle the information per NUTS region
outlist = [NUTS_no, opti_code, optimum, selected_grid_cells]
filename = os.path.join(optimidir,'fgap_%s_optimized.pickle'%NUTS_no)
pickle_dump(outlist, open(filename,'wb'))
return optimum
#===============================================================================
# Function to optimize the regional yield gap factor using the difference
# between the regional simulated and the observed harvest or yield (ie. 1 gap to
# optimize per NUTS region). This function iterates dynamically to find the
# optimum YLDGAPF.
def optimize_regional_yldgapf_dyn(NUTS_no_, detrend, crop_no_,
selected_grid_cells_, selected_soil_types_, weather, inputdir, opti_years_,
obs_type='yield', plot_rmse=False):
#===============================================================================
import math
from operator import itemgetter as operator_itemgetter
from matplotlib import pyplot as plt
from pcse.models import Wofost71_WLP_FD
from pcse.base_classes import WeatherDataProvider
from pcse.fileinput.cabo_weather import CABOWeatherDataProvider
# aggregated yield method:
# 2- we construct a 2D array with same dimensions as TSO_regional,
# containing the observed yields
row = [] # this list will become the row of the 2D array
for y,year in enumerate(opti_years_):
index_year = np.argmin(np.absolute(detrend[1]-year))
row = row + [detrend[0][index_year]]
OBS = np.tile(row, (5,1)) # repeats the list as a row 3 times, to get a
# 2D array
# 3- we calculate all the individual yields from the selected grid cells x
# soils combinations
# NB: we explore the range of yldgapf between 0.1 and 1.
f0 = 0.
f2 = 0.5
f4 = 1.
f_step = 0.25
# Until the precision of the yield gap factor is good enough (i.e. < 0.02)
# we loop over it. We do 12 iterations in total with this method.
iter_no = 0
RMSE_stored = list()
while (f_step >= 0.02):
iter_no = iter_no + 1
# sub-method: looping over the yield gap factors
# we build a range of 3 yield gap factors to explore one low bound, one
# high bound, one in the middle
f_step = (f4 - f0)/4.
f1 = f0 + f_step
f3 = f2 + f_step
f_range = [f0, f1, f2, f3, f4]
RES = [] # list in which we will store the yields of the combinations
counter=0
for grid, arable_land in selected_grid_cells_:
frac_arable = arable_land / 625000000.
# Retrieve the weather data of one grid cell (all years are in one
# file)
if (weather == 'CGMS'):
filename = os.path.join(inputdir,'weather_objects/',
'weatherobject_g%d.pickle'%grid)
weatherdata = WeatherDataProvider()
weatherdata._load(filename)
if (weather == 'ECMWF'):
weatherdata = CABOWeatherDataProvider('%i'%grid,fpath=ECMWFdir)
# Retrieve the soil data of one grid cell (all possible soil types)
filename = os.path.join(inputdir,'soildata_objects/',
'soilobject_g%d.pickle'%grid)
soil_iterator = pickle_load(open(filename,'rb'))
for smu, stu_no, weight, soildata in selected_soil_types_[grid]:
# TSO will store all the yields of one grid cell x soil
# combination, for all years and all 3 yldgapf values
TSO = np.zeros((len(f_range), len(opti_years_)))
counter +=1
for y, year in enumerate(opti_years_):
# Retrieve yearly data
filename = os.path.join(inputdir,
'timerdata_objects/%i/c%i/'%(year,crop_no_),
'timerobject_g%d_c%d_y%d.pickle'\
%(grid,crop_no_,year))
timerdata = pickle_load(open(filename,'rb'))
filename = os.path.join(inputdir,
'cropdata_objects/%i/c%i/'%(year,crop_no_),
'cropobject_g%d_c%d_y%d.pickle'\
%(grid,crop_no_,year))
cropdata = pickle_load(open(filename,'rb'))
if str(grid).startswith('1'):
dum = str(grid)[0:2]
else:
dum = str(grid)[0]
filename = os.path.join(inputdir,
'sitedata_objects/%i/c%i/grid_%s/'
%(year,crop_no_,dum),
'siteobject_g%d_c%d_y%d_s%d.pickle'\
%(grid,crop_no_,year,stu_no))
sitedata = pickle_load(open(filename,'rb'))
for f,factor in enumerate(f_range):
cropdata['YLDGAPF']=factor
# run WOFOST
wofost_object = Wofost71_WLP_FD(sitedata, timerdata,
soildata, cropdata, weatherdata)
wofost_object.run_till_terminate()
# get the yield (in kgDM.ha-1)
TSO[f,y] = wofost_object.get_variable('TWSO')
#print grid, stu_no, year, counter, [y[0] for y in TSO], OBS[0]
RES = RES + [(grid, stu_no, weight*frac_arable, TSO)]
# 4- we aggregate the yield or harvest into the regional one with array
# operations
sum_weighted_vals = np.zeros((len(f_range), len(opti_years_)))
# empty 2D array with same dimension as TSO
sum_weights = 0.
for grid, stu_no, weight, TSO in RES:
# adding weighted 2D-arrays in the empty array sum_weighted_yields
# NB: variable 'weight' is actually the cultivated area in m2
sum_weighted_vals = sum_weighted_vals + (weight/10000.)*TSO
# computing the total sum of the cultivated area in ha
sum_weights = sum_weights + (weight/10000.)
if (obs_type == 'harvest'):
TSO_regional = sum_weighted_vals / 1000000. # sum of the individual
# harvests in 1000 tDM
elif (obs_type == 'yield'):
TSO_regional = sum_weighted_vals / sum_weights # weighted average of
# all yields in kgDM/ha
# 5- we compute the (sim-obs) differences.
DIFF = TSO_regional - OBS
if (TSO_regional[-1][0] <= 0.):
print 'WARNING: no simulated crop growth. We set the optimum fgap to 1.'
return 1.
if (TSO_regional[-1] <= OBS[-1]):
print 'WARNING: obs yield > sim yield. We set optimum to 1.'
return 1.
# 6- we calculate the RMSE (root mean squared error) of the 3 yldgapf
# The RMSE of each yldgapf is based on N obs-sim differences for the N
# years looped over
RMSE = np.zeros(len(f_range))
for f,factor in enumerate(f_range):
list_of_DIFF = []
for y, year in enumerate(opti_years_):
list_of_DIFF = list_of_DIFF + [DIFF[f,y]]
RMSE[f] = np.sqrt(np.mean( [ math.pow(j,2) for j in
list_of_DIFF ] ))
#print RMSE, f_range
# We store the value of the RMSE for plotting purposes
RMSE_stored = RMSE_stored + [(f_range[1], RMSE[1]), (f_range[3], RMSE[3])]
if (iter_no == 1):
RMSE_stored = RMSE_stored + [(f_range[0], RMSE[0]),
(f_range[2], RMSE[2]),
(f_range[4], RMSE[4])]
# 7- We update the yldgapf range to explore for the next iteration.
# For this we do a linear interpolation of RMSE between the 3 yldgapf
# explored here, and the next range to explore is the one having the
# smallest interpolated RMSE
index_new_center = RMSE.argmin()
# if the optimum is close to 1:
if index_new_center == len(f_range)-1:
f0 = f_range[index_new_center-2]
f2 = f_range[index_new_center-1]
f4 = f_range[index_new_center]
# if the optimum is close to 0:
elif index_new_center == 0:
f0 = f_range[index_new_center]
f2 = f_range[index_new_center+1]
f4 = f_range[index_new_center+2]
else:
f0 = f_range[index_new_center-1]
f2 = f_range[index_new_center]
f4 = f_range[index_new_center+1]
# when we are finished iterating on the yield gap factor range, we plot the
# RMSE as a function of the yield gap factor
if (plot_rmse == True):
RMSE_stored = sorted(RMSE_stored, key=operator_itemgetter(0))
x,y = zip(*RMSE_stored)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5,5))
fig.subplots_adjust(0.15,0.16,0.95,0.96,0.4,0.)
ax.plot(x, y, c='k', marker='o')
ax.set_xlabel('yldgapf (-)')
ax.set_ylabel('RMSE')
fig.savefig('%s_opti_fgap.png'%NUTS_no_)
#pickle_dump(RMSE_stored,open('%s_RMSE.pickle'%NUTS_no_,'wb'))
# 8- when we are finished iterating on the yield gap factor range, we return
# the optimum value. We look for the yldgapf with the lowest RMSE
index_optimum = RMSE.argmin()
optimum_yldgapf = f_range[index_optimum]
print 'optimum found: %.2f +/- %.2f'%(optimum_yldgapf, f_step)
# 10- we return the optimized YLDGAPF
return optimum_yldgapf
#===============================================================================
def str_to_bool(s):
#===============================================================================
if s.strip(' ') == 'True':
return True
elif s.strip(' ') == 'False':
return False
else:
raise ValueError
#===============================================================================
if __name__=='__main__':
main()
# Has config.py been changed?
or getmtime(get_config_py_path()) > cache_mtime
# Any file has changed in the dir since the cache was generated
or any(True for f in listdir(directory)
if (getmtime(path_join(directory, f)) > cache_mtime
# Ignore hidden files
and not f.startswith('.')))
# The configuration is newer than the cache
or getmtime(get_config_path(directory)) > cache_mtime):
generate = True
docstats = []
else:
generate = False
try:
with open(cache_file_path, 'rb') as cache_file:
docstats = pickle_load(cache_file)
except UnpicklingError:
# Corrupt data, re-generate
Messager.warning('Stats cache %s was corrupted; regenerating' % cache_file_path, -1)
generate = True
except EOFError:
# Corrupt data, re-generate
generate = True
except OSError, e:
Messager.warning('Failed checking file modification times for stats cache check; regenerating')
generate = True
# "header" and types
####stat_types = [("Entities", "int"), ("Relations", "int"), ("Events", "int")]
####if options_get_validation(directory) != 'none':
def plot_fluxnet_daily_c_fluxes(directory, sites_list, timeframe, linestyle):
#===============================================================================
"""
This function plots the average daytime Tair, average SWin, average VPD,
average Tsoil and average volumetric soil moisture content, all taken from
the formatted 2015 FluxNet DD (daily data) files.
NB: Daily averages have been computed from the half-hourly data.
Function arguments:
------------------
directory: path to the directory where the daily variables files
('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1')
are stored
sites_list: list of fluxnet sites names
time frame: list of two years (e.g. [2005, 2007]) gives the time frame
to plot
linestyle: line style of the plot to make (can be '-', '--', '+', etc)
"""
plt.close('all')
print "\nPlotting carbon fluxes timeseries\n"
for site in sites_list:
print site + '\n'
filepath = os.path.join(directory, 'timeseries_%s.pickle' % site)
series = pickle_load(open(filepath, 'rb'))
# settings of the plot:
figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8, 10))
figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.)
variables = ['crop_no', 'GPP', 'TER', 'NEE']
axlabels = [
'crop ID', r'GPP (g m$^{-2}$ d$^{-1}$)',
r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)'
]
ylims = [(0., 14.), (-30., 5.), (-5., 20.), (-30., 30.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
lab = ''
ls = linestyle
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims):
if (var == 'TER' or var == 'GPP'): lab = '-'
series[site][var][start:end].plot(ax=ax,
lw=2,
style=ls,
label=lab,
fontsize=fsz)
if var == 'GPP':
series[site]['GPP_day'][start:end].plot(ax=ax,
lw=2,
style=ls,
label='day partition',
fontsize=fsz)
series[site]['GPP_night'][start:end].plot(
ax=ax,
lw=2,
style=ls,
label='night partition',
fontsize=fsz)
if var == 'TER':
series[site]['TER_day'][start:end].plot(ax=ax,
lw=2,
style=ls,
label='day partition',
fontsize=fsz)
series[site]['TER_night'][start:end].plot(
ax=ax,
lw=2,
style=ls,
label='night partition',
fontsize=fsz)
ax.axhline(y=0., c='k')
ax.set_ylim(ylim)
if (var == 'GPP_day'):
ax.legend(loc='lower left', prop={'size': 12})
if (var == 'TER_day'):
ax.legend(loc='upper left', prop={'size': 12})
ax.set_ylabel(axlabel)
if var != 'NEE': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
plt.show()
return None
def plot_fluxnet_micromet(directory, sites_list, timeframe, linestyle):
#===============================================================================
"""
This function plots the average daytime Tair, average SWin, average VPD,
average Tsoil and average volumetric soil moisture content, all taken from
the formatted 2015 FluxNet DD (daily data) files.
NB: Daily averages have been computed from the half-hourly data.
Function arguments:
------------------
directory: path to the directory where the daily variables files
('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1')
are stored
sites_list: list of fluxnet sites names
time frame: list of two years (e.g. [2005, 2007]) gives the time frame
to plot
linestyle: line style of the plot to make (can be '-', '--', '+', etc)
"""
plt.close('all')
for site in sites_list:
print site + '\n'
filepath = os.path.join(directory, 'timeseries_%s.pickle' % site)
series = pickle_load(open(filepath, 'rb'))
# settings of the plot:
figs, axes = plt.subplots(nrows=5, ncols=1, figsize=(8, 9))
figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.)
variables = ['Ta_day', 'SWin', 'VPD', 'Ts_1', 'SWC_1']
axlabels = [
r'T$_{air}$ day (deg C)', r'SW$_{in}$ (W m$^{-2}$)', 'VPD (hPa)',
r'T$_{soil}$ (deg C)', 'SWC (%)'
]
ylims = [(-20., 40.), (0., 400.), (0., 17.), (-5., 35.), (0., 60.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
ls = linestyle
lab = 'bullshit'
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims):
if var.endswith('_1'): lab = 'layer 1'
series[site][var][start:end].plot(ax=ax,
lw=2,
style=ls,
label=lab,
fontsize=fsz)
if var == 'Ts_1':
series[site]['Ts_2'][start:end].plot(ax=ax,
lw=2,
style=ls,
label='layer 2',
fontsize=fsz)
series[site]['Ts_3'][start:end].plot(ax=ax,
lw=2,
style=ls,
label='layer 3',
fontsize=fsz)
if var == 'SWC_1':
series[site]['SWC_2'][start:end].plot(ax=ax,
lw=2,
style=ls,
label='layer 2',
fontsize=fsz)
series[site]['SWC_3'][start:end].plot(ax=ax,
lw=2,
style=ls,
label='layer 3',
fontsize=fsz)
if (var == 'Ts_1'): ax.legend(loc='upper left', prop={'size': 12})
if (var == 'SWC_1'): ax.legend(loc='lower left', prop={'size': 12})
ax.axhline(y=0., c='k')
ax.set_ylim(ylim)
ax.set_ylabel(axlabel)
if var == 'Ta_day': ax.set_yticks([-10., 0., 10., 20., 30., 40.])
if var == 'SWin': ax.set_yticks([0., 100., 200., 300., 400.])
if var == 'VPD': ax.set_yticks([0., 5., 10., 15.])
if var == 'Ts_1': ax.set_yticks([0., 10., 20., 30.])
if var != 'SWC_1': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
plt.show()
return None
def main():
#===============================================================================
global inputdir, codedir, outputdir, CGMSdir, obsdir\
#-------------------------------------------------------------------------------
import cx_Oracle
import sqlalchemy as sa
from datetime import datetime
#-------------------------------------------------------------------------------
# ================================= USER INPUT =================================
# read the settings from the rc file
rcdict = rc.read('settings.rc')
#===============================================================================
#-------------------------------------------------------------------------------
# extract the needed information from the rc file
sites = [s.strip(' ') for s in rcdict['sites'].split(',')]
crops = [s.strip(' ') for s in rcdict['crops'].split(',')]
crop_nos = [int(s.strip(' ')) for s in rcdict['crop_nos'].split(',')]
years = [int(s.strip(' ')) for s in rcdict['years'].split(',')]
obsdir = rcdict['obsdir']
inputdir = rcdict['inputdir']
CGMSdir = os.path.join(inputdir, 'CGMS')
codedir = rcdict['codedir']
#-------------------------------------------------------------------------------
# get the closest CGMS grid cell id number for each FluxNet site
# get the sites longitude and latitudes
sitdict = open_csv(os.path.join(obsdir,'regrouped_data'), 'sites_info.txt',
convert_to_float=False)
site_lons = sitdict['site_lons']
site_lats = sitdict['site_lats']
# we read the CGMS grid cells coordinates from file
CGMS_cells = open_csv(CGMSdir, 'CGMS_grid_list.csv', convert_to_float=True)
all_grids = CGMS_cells['GRID_NO']
all_lons = CGMS_cells['LONGITUDE']
all_lats = CGMS_cells['LATITUDE']
flux_gri = dict()
for i,site in enumerate(sitdict['sites']):
lon = float(site_lons[i])
lat = float(site_lats[i])
# compute the distance to site for all CGMS grid cells
dist_list = list()
for j,grid_no in enumerate(all_grids):
distance = ((all_lons[j]-lon)**2. + (all_lats[j]-lat)**2.)**(1./2.)
dist_list += [distance]
# select the closest grid cell
indx = np.argmin(np.array(dist_list))
flux_gri[site] = all_grids[indx]
print 'FluxNet site %s with lon=%5.2f, lat=%5.2f: closest grid cell is %i'%(site, lon, lat, all_grids[indx])
#-------------------------------------------------------------------------------
# create new file with grid cell number in it
filename = os.path.join(inputdir,'sites_info2.csv')
newres = open(filename,'wb')
oldres = open(os.path.join(obsdir,'regrouped_data/sites_info.txt'),'rU')
reader = oldres.readlines()
oldres.close()
for l,line in enumerate(reader):
site = line.split(',')[0].strip(' ')
if l==0: line = line.strip('\n')+', gridcells\n'
else: line = line.strip('\n') + ',%10i'%int(flux_gri[site]) + '\n'
newres.write(line)
newres.close()
print '\nWe successfully created the input file with grid cell IDs:\n%s'%filename
#-------------------------------------------------------------------------------
# retrieve the necessary input data for all sites
# settings of the connection
user = "******"
password = "******"
tns = "EURDAS.WORLD"
dsn = "oracle+cx_oracle://{user}:{pw}@{tns}".format(user=user,pw=password,tns=tns)
engine = sa.create_engine(dsn)
print engine
# test the connection:
try:
connection = cx_Oracle.connect("cgms12eu_select/[email protected]")
except cx_Oracle.DatabaseError:
print '\nBEWARE!! The Oracle database is not responding. Probably, you are'
print 'not using a computer wired within the Wageningen University network.'
print '--> Get connected with ethernet cable before trying again!'
sys.exit()
for c,crop in enumerate(crops):
crop_no = crop_nos[c]
print '\nRetrieving input data for %s (CGMS id=%i)'%(crop,crop_no)
# We add a timestamp at start of the retrieval
start_timestamp = datetime.utcnow()
# We retrieve the list of suitable soil types for the selected crop
# species
filename = os.path.join(CGMSdir, 'soildata_objects/',
'suitablesoilsobject_c%d.pickle'%(crop_no))
if os.path.exists(filename):
suitable_stu = pickle_load(open(filename,'rb'))
else:
from pcse.db.cgms11 import STU_Suitability
suitable_stu = STU_Suitability(engine, crop_no)
suitable_stu_list = []
for item in suitable_stu:
suitable_stu_list = suitable_stu_list + [item]
suitable_stu = suitable_stu_list
pickle_dump(suitable_stu,open(filename,'wb'))
print 'retrieving suitable soils for %s'%crop
# WE LOOP OVER ALL YEARS:
for y, year in enumerate(years):
print '\n######################## Year %i ##############'%year+\
'##########\n'
# if we do a serial iteration, we loop over the grid cells that
# contain arable land
for grid in flux_gri.values():
retrieve_CGMS_input(grid, year, crop_no, suitable_stu, engine)
# We add a timestamp at end of the retrieval, to time the process
end_timestamp = datetime.utcnow()
print '\nDuration of the retrieval:', end_timestamp-start_timestamp
def retrieve_CGMS_input(grid, year, crop_no, suitable_stu, engine, retrieve_weather=False):
#===============================================================================
# Temporarily add code directory to python path, to be able to import pcse
# modules
sys.path.insert(0, codedir)
#-------------------------------------------------------------------------------
from pcse.exceptions import PCSEError
from pcse.db.cgms11 import TimerDataProvider, SoilDataIterator, \
CropDataProvider, STU_Suitability, \
SiteDataProvider, WeatherObsGridDataProvider
# if the retrieval does not raise an error, the crop was cultivated that year
print ' - grid cell no %i'%grid
try:
# We retrieve the crop calendar (timerdata)
filename = os.path.join(CGMSdir,
'timerdata_objects/%i/c%i/'%(year,crop_no),
'timerobject_g%d_c%d_y%d.pickle'%(grid, crop_no, year))
if os.path.exists(filename):
pass
else:
timerdata = TimerDataProvider(engine, grid, crop_no, year)
pickle_dump(timerdata,open(filename,'wb'))
# If required by the user, we retrieve the weather data
if retrieve_weather == True:
filename = os.path.join(CGMSdir, 'weather_objects/',
'weatherobject_g%d.pickle'%(grid))
if os.path.exists(filename):
pass
else:
weatherdata = WeatherObsGridDataProvider(engine, grid)
weatherdata._dump(filename)
# We retrieve the soil data (soil_iterator)
filename = os.path.join(CGMSdir, 'soildata_objects/',
'soilobject_g%d.pickle'%(grid))
if os.path.exists(filename):
soil_iterator = pickle_load(open(filename,'rb'))
else:
soil_iterator = SoilDataIterator(engine, grid)
pickle_dump(soil_iterator,open(filename,'wb'))
# We retrieve the crop variety info (crop_data)
filename = os.path.join(CGMSdir,
'cropdata_objects/%i/c%i/'%(year,crop_no),
'cropobject_g%d_c%d_y%d.pickle'%(grid,crop_no,year))
if os.path.exists(filename):
pass
else:
cropdata = CropDataProvider(engine, grid, crop_no, year)
pickle_dump(cropdata,open(filename,'wb'))
# WE LOOP OVER ALL SOIL TYPES LOCATED IN THE GRID CELL:
for smu_no, area_smu, stu_no, percentage, soildata in soil_iterator:
# NB: we remove all unsuitable soils from the iteration
if (stu_no not in suitable_stu):
pass
else:
print ' soil type no %i'%stu_no
# We retrieve the site data (site management)
if (str(grid)).startswith('1'):
dum = str(grid)[0:2]
else:
dum = str(grid)[0]
filename = os.path.join(CGMSdir,
'sitedata_objects/%i/c%i/grid_%s/'%(year,crop_no,dum),
'siteobject_g%d_c%d_y%d_s%d.pickle'%(grid, crop_no,
year, stu_no))
if os.path.exists(filename):
pass
else:
sitedata = SiteDataProvider(engine,grid,crop_no,year,stu_no)
pickle_dump(sitedata,open(filename,'wb'))
# if an error is raised, the crop was not grown that year
except PCSEError:
print ' the crop was not grown that year in that grid cell'
except Exception as e:
print ' Unexpected error', e#sys.exc_info()[0]
return None
def main():
#===============================================================================
global wofostdir, sibcasadir, obsdir
#-------------------------------------------------------------------------------
# ================================= USER INPUT =================================
# read the settings from the rc file (mostly directory paths)
rcdict = rc.read('settings.rc')
sites = [s.strip(' ') for s in rcdict['sites'].split(',')]
years = [int(s.strip(' ')) for s in rcdict['years'].split(',')]
TER_method = rcdict['TER_method']
R10 = rcdict['R10']
resolution = rcdict['resolution']
# specific plotting options:
#TER_method = 'grow-only' # this is to select the corresponding WOFOST output file
#R10 = '0.08' # this is to select the corresponding WOFOST output file
#===============================================================================
#-------------------------------------------------------------------------------
# extract the needed information
# input data directory paths
rootdir = rcdict['rootdir']
sibcasadir = os.path.join(rootdir,'intercomparison_study/SiBCASA_runs')
wofostdir = rcdict['outputdir']
obsdir = rcdict['obsdir']
figdir = os.path.join(rootdir,'intercomparison_study/figures')
#-------------------------------------------------------------------------------
# Start a directory to store OBS, SIMW (wofost), SIMB (SiBCASA)
# recover the FluxNet observed data from pickle files
res_timeseries = dict()
res_timeseries['OBS'] = dict()
res_timeseries['SIMB'] = dict()
res_timeseries['SIMW'] = dict()
filename = os.path.join(obsdir, '%s_timeseries_OBS.pickle'%resolution)
try:
res_timeseries['OBS'] = pickle_load(open(filename,'rb'))
except IOError:
print 'could not find the observations output file %s'%filename
res_timeseries['OBS'] = None
# recover the SiBCASA runs
filename = os.path.join(sibcasadir, '%s_timeseries_SiBCASA.pickle'%resolution)
try:
res_timeseries['SIMB'] = pickle_load(open(filename,'rb'))
except IOError:
print 'could not find the SiBCASA output file %s'%filename
res_timeseries['SIMB'] = None
# recover the WOFOST runs
filename = os.path.join(wofostdir, '%s_timeseries_'%resolution +\
'%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10))
try:
res_timeseries['SIMC'] = pickle_load(open(filename,'rb'))
except IOError:
print 'could not find the WOFOST output file %s'%filename
res_timeseries['SIMC'] = None
#-------------------------------------------------------------------------------
# plot the observed and simulated timeseries with pandas library
# with pandas we plot all years one after another, and can zoom in on one
# particular year
plt.close('all')
# create figure sub-folder if it doesn't already exists
figsubdir = os.path.join(figdir,'R10=%s/TER_%s/'%(R10,TER_method)+\
'3-hourly_fluxes_perf')
if not os.path.exists(figsubdir):
print 'creating new directory %s'%figsubdir
os.makedirs(figsubdir)
#-------------------------------------------------------------------------------
# we plot the 3-hourly fluxes of simulations versus observations for the years
# and sites that perform well on the daily scale
# we plot years 2005, 2009, 2013 for site BE-Lon, which showed an extremely
# good result on the SIM vs OBS daily fluxes comparison
years = [2005,2009,2013]
variables = ['GPP','TER','NEE']
axlabels = [r'GPP (g m$^{-2}$ d$^{-1}$)',
r'TER (g m$^{-2}$ d$^{-1}$)',r'NEE (g m$^{-2}$ d$^{-1}$)']
ylims = [(-60.,5.),(0.,20.),(-50.,15.)]
one_to_one = np.arange(-100,100,10)
for site in ['BE-Lon']:
if site != 'IT-BCi':
for year in years:
timeframe = [year,year]
start = str(int(timeframe[0]))+'-05-01'
end = str(int(timeframe[1]))+'-07-01'
print site
for var, axlabel, lim in zip(variables,axlabels,ylims):
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5,5))
fig.subplots_adjust(0.15,0.15,0.85,0.85,0.,0.)
# select every 6 half-hourly flux in the observations,
# to get to 3-hourly frequency
OBS = res_timeseries['OBS'][site][var][::6][start:end].dropna()
# convert the 3-hourly simulated fluxes from gC.m-2.s-1
# to micromol CO2 .m-2.s-1
SIM = res_timeseries['SIMC'][site][var][start:end].dropna()
SIM = SIM * 1000000. / 12.01 #micro mol CO2 per m2 per sec
# the observed GPP needs a minus sign for convention
if var=='GPP': OBS=-OBS
# use the min and max to frame the figure
print var, min(min(OBS), min(SIM)), max(max(OBS), max(SIM))
varmin = math.floor(min(min(OBS), min(SIM)))
varmax = math.ceil(max(max(OBS), max(SIM)))
ax.scatter(OBS, SIM, marker='o')
# fit a linear regression line in OBS/SIM scatter plot
# and plot line and R2
mask = ~np.isnan(SIM)
z = np.polyfit(OBS[mask], SIM[mask], 1)
p = np.poly1d(z)
ax.plot(one_to_one,p(one_to_one),'r-')
slope, intercept, r_value, p_value, std_err = \
linreg(OBS[mask], SIM[mask])
ax.annotate(r'r$^2$ = %.2f'%r_value**2, xy=(0.95, 0.15),
xytext=(0.15, 0.9), xycoords='axes fraction',
ha='center', va='center', color='r')
ax.plot(one_to_one,one_to_one, c='k', lw=1)
ax.set_xlabel('obs')
ax.set_ylabel('sim')
ax.set_xlim(varmin,varmax)
ax.set_ylim(varmin,varmax)
fig.suptitle(r'%s 3-hourly %s fluxes ($\mu$'%(site, var)+
r'mol m$^{-2}$ s$^{-1}$)'+'\nfrom %s to %s\n'%(start,end))
fig.savefig(os.path.join(figsubdir,
'%s_%s_%s.png'%(site,year,var)), dpi=300)
def main():
#===============================================================================
global outputdir, obsdir
#-------------------------------------------------------------------------------
# ================================= USER INPUT =================================
# read the settings from the rc file
rcdict = rc.read('settings.rc')
#===============================================================================
#-------------------------------------------------------------------------------
# extract the needed information for that script
sites = [s.strip(' ') for s in rcdict['sites'].split(',')]
years = [s.strip(' ') for s in rcdict['years'].split(',')]
TER_method = rcdict['TER_method']
R10 = rcdict['R10']
resolution = rcdict['resolution'] # can be hourly or daily
if resolution=='daily': res='1d'
elif resolution=='3-hourly': res='3H'
# directory paths
outputdir = rcdict['outputdir']
obsdir = rcdict['obsdir']
forwardir = os.path.join(outputdir, 'forward_runs')
#-------------------------------------------------------------------------------
# load the WOFOST runs of all crops
# we store the two pandas series in one pickle file
filepath = os.path.join(forwardir,'%s_timeseries_'%resolution+\
'%s_WOFOST.pickle'%TER_method)
series = pickle_load(open(filepath,'rb'))
filepath = os.path.join(obsdir,'daily_timeseries_OBS.pickle')
obs = pickle_load(open(filepath,'rb'))
final_series = dict()
for s,site in enumerate(sites):
print site
print obs[site].keys()
final_series[site] = dict()
# read the crop rotation from FluxNet file
rotation = obs[site]['crop_no']
# slice each year's required time series, append to final series
for varname in ['GPP','TER','Raut','Rhet','NEE']:
print 'variable %s'%varname
var = []
for year in years:
# get the crop number for that year
if site != 'IT-BCi':
try:
crop_no = rotation[year:year][0]
except IndexError: # index error occurs when the year is
# not in the rotation time series
startdate = '%s-01-01 00:00:00'%year
enddate = '%s-12-31 23:59:59'%year
dtimes = pd.date_range(startdate, enddate, freq=res)
na_vals = np.array(len(dtimes)*[np.nan])
var += [pd.Series(na_vals, index=dtimes)]
print ' ',site, year, 'unknown crop cover: skip.'
continue
elif site == 'IT-BCi':
if int(year) not in np.arange(2004,2010,1):
startdate = '%s-01-01 00:00:00'%year
enddate = '%s-12-31 23:59:59'%year
dtimes = pd.date_range(startdate, enddate, freq=res)
na_vals = np.array(len(dtimes)*[np.nan])
var += [pd.Series(na_vals, index=dtimes)]
print ' ',site, year, 'unknown crop cover: skip.'
continue
else:
crop_no = 2
# try slicing and concatenating that year's timeseries from file
try:
# if the GPP = 0 (failed growing season), we set TER and
# NEE to zero as well
if np.mean(series[site]['c%i'%crop_no]['GPP'][year:year]) == 0.:
startdate = '%s-01-01 00:00:00'%year
enddate = '%s-12-31 23:59:59'%year
dtimes = pd.date_range(startdate, enddate, freq=res)
zeros = np.array(len(dtimes)*[0.])
var += [pd.Series(zeros, index=dtimes)]
else:
var += [series[site]['c%i'%crop_no][varname][year:year]]
print ' ',site, year, '%2i'%crop_no, 'slicing'
except KeyError: # key error occurs when we haven't ran a crop
# or a year with WOFOST
startdate = '%s-01-01 00:00:00'%year
enddate = '%s-12-31 23:59:59'%year
dtimes = pd.date_range(startdate, enddate, freq=res)
na_vals = np.array(len(dtimes)*[np.nan])
var += [pd.Series(na_vals, index=dtimes)]
print ' ',site, year, '%2i'%crop_no, 'skip.'
final_series[site][varname] = pd.concat(var)
#final_series[site]['GPP'].plot()
#plt.show()
# store the final WOFOST timeseries
filepath = os.path.join(outputdir,'%s_timeseries_'%resolution+\
'%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10))
pickle_dump(final_series, open(filepath,'wb'))
print 'successfully dumped %s'%filepath
def _load(self, f):
self.store = pickle_load(f)
def main():
#===============================================================================
global outputdir, obsdir
#-------------------------------------------------------------------------------
# ================================= USER INPUT =================================
# read the settings from the rc file
rcdict = rc.read('settings.rc')
#===============================================================================
#-------------------------------------------------------------------------------
# extract the needed information for that script
sites = [s.strip(' ') for s in rcdict['sites'].split(',')]
years = [s.strip(' ') for s in rcdict['years'].split(',')]
TER_method = rcdict['TER_method']
R10 = rcdict['R10']
resolution = rcdict['resolution'] # can be hourly or daily
if resolution == 'daily': res = '1d'
elif resolution == '3-hourly': res = '3H'
# directory paths
outputdir = rcdict['outputdir']
obsdir = rcdict['obsdir']
forwardir = os.path.join(outputdir, 'forward_runs')
#-------------------------------------------------------------------------------
# load the WOFOST runs of all crops
# we store the two pandas series in one pickle file
filepath = os.path.join(forwardir,'%s_timeseries_'%resolution+\
'%s_WOFOST.pickle'%TER_method)
series = pickle_load(open(filepath, 'rb'))
filepath = os.path.join(obsdir, 'daily_timeseries_OBS.pickle')
obs = pickle_load(open(filepath, 'rb'))
final_series = dict()
for s, site in enumerate(sites):
print site
print obs[site].keys()
final_series[site] = dict()
# read the crop rotation from FluxNet file
rotation = obs[site]['crop_no']
# slice each year's required time series, append to final series
for varname in ['GPP', 'TER', 'Raut', 'Rhet', 'NEE']:
print 'variable %s' % varname
var = []
for year in years:
# get the crop number for that year
if site != 'IT-BCi':
try:
crop_no = rotation[year:year][0]
except IndexError: # index error occurs when the year is
# not in the rotation time series
startdate = '%s-01-01 00:00:00' % year
enddate = '%s-12-31 23:59:59' % year
dtimes = pd.date_range(startdate, enddate, freq=res)
na_vals = np.array(len(dtimes) * [np.nan])
var += [pd.Series(na_vals, index=dtimes)]
print ' ', site, year, 'unknown crop cover: skip.'
continue
elif site == 'IT-BCi':
if int(year) not in np.arange(2004, 2010, 1):
startdate = '%s-01-01 00:00:00' % year
enddate = '%s-12-31 23:59:59' % year
dtimes = pd.date_range(startdate, enddate, freq=res)
na_vals = np.array(len(dtimes) * [np.nan])
var += [pd.Series(na_vals, index=dtimes)]
print ' ', site, year, 'unknown crop cover: skip.'
continue
else:
crop_no = 2
# try slicing and concatenating that year's timeseries from file
try:
# if the GPP = 0 (failed growing season), we set TER and
# NEE to zero as well
if np.mean(series[site]['c%i' %
crop_no]['GPP'][year:year]) == 0.:
startdate = '%s-01-01 00:00:00' % year
enddate = '%s-12-31 23:59:59' % year
dtimes = pd.date_range(startdate, enddate, freq=res)
zeros = np.array(len(dtimes) * [0.])
var += [pd.Series(zeros, index=dtimes)]
else:
var += [
series[site]['c%i' % crop_no][varname][year:year]
]
print ' ', site, year, '%2i' % crop_no, 'slicing'
except KeyError: # key error occurs when we haven't ran a crop
# or a year with WOFOST
startdate = '%s-01-01 00:00:00' % year
enddate = '%s-12-31 23:59:59' % year
dtimes = pd.date_range(startdate, enddate, freq=res)
na_vals = np.array(len(dtimes) * [np.nan])
var += [pd.Series(na_vals, index=dtimes)]
print ' ', site, year, '%2i' % crop_no, 'skip.'
final_series[site][varname] = pd.concat(var)
#final_series[site]['GPP'].plot()
#plt.show()
# store the final WOFOST timeseries
filepath = os.path.join(outputdir,'%s_timeseries_'%resolution+\
'%s_R10=%s_WOFOST_crop_rotation.pickle'%(TER_method,R10))
pickle_dump(final_series, open(filepath, 'wb'))
print 'successfully dumped %s' % filepath
def main():
#===============================================================================
global wofostdir, sibcasadir, obsdir
#-------------------------------------------------------------------------------
# ================================= USER INPUT =================================
# read the settings from the rc file (mostly directory paths)
rcdict = rc.read('settings.rc')
sites = [s.strip(' ') for s in rcdict['sites'].split(',')]
years = [int(s.strip(' ')) for s in rcdict['years'].split(',')]
TER_method = rcdict['TER_method']
R10 = rcdict['R10']
# specific plotting options:
#TER_method = 'grow-only' # this is to select the corresponding WOFOST output file
#R10 = '0.08' # this is to select the corresponding WOFOST output file
#===============================================================================
#-------------------------------------------------------------------------------
# extract the needed information
# input data directory paths
rootdir = rcdict['rootdir']
sibcasadir = os.path.join(rootdir, 'intercomparison_study/SiBCASA_runs')
wofostdir = rcdict['outputdir']
obsdir = rcdict['obsdir']
figdir = os.path.join(rootdir, 'intercomparison_study/figures')
#-------------------------------------------------------------------------------
# Start a directory to store OBS, SIMW (wofost), SIMB (SiBCASA)
# recover the FluxNet observed data from pickle files
res_timeseries = dict()
res_timeseries['OBS'] = dict()
res_timeseries['SIMB'] = dict()
res_timeseries['SIMW'] = dict()
filename = os.path.join(obsdir, 'timeseries_OBS.pickle')
try:
res_timeseries['OBS'] = pickle_load(open(filename, 'rb'))
except IOError:
print 'could not find the observations output file %s' % filename
res_timeseries['OBS'] = None
# recover the SiBCASA runs
filename = os.path.join(sibcasadir, 'timeseries_SiBCASA.pickle')
try:
res_timeseries['SIMB'] = pickle_load(open(filename, 'rb'))
except IOError:
print 'could not find the SiBCASA output file %s' % filename
res_timeseries['SIMB'] = None
# recover the WOFOST runs
filename = os.path.join(wofostdir, 'timeseries_%s_R10=%s_'%(TER_method,R10) +\
'WOFOST_crop_rotation.pickle')
try:
print 'opening the WOFOST output file %s' % filename
res_timeseries['SIMC'] = pickle_load(open(filename, 'rb'))
except IOError:
print 'could not find the WOFOST output file %s' % filename
res_timeseries['SIMC'] = None
#-------------------------------------------------------------------------------
# plot the observed and simulated timeseries with pandas library
# with pandas we plot all years one after another, and can zoom in on one
# particular year
plt.close('all')
# create figure sub-folder if it doesn't already exists
figsubdir = os.path.join(figdir, 'R10=%s/TER_%s/' % (R10, TER_method))
if not os.path.exists(figsubdir):
print 'creating new directory %s' % figsubdir
os.makedirs(figsubdir)
#-------------------------------------------------------------------------------
years = np.arange(2004, 2015, 1)
for site in sites:
for year in years:
timeframe = [year, year]
print site
figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8, 10))
figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.)
variables = ['crop_no', 'GPP', 'TER', 'NEE']
axlabels = [
'crop ID', r'GPP (g m$^{-2}$ d$^{-1}$)',
r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)'
]
ylims = [(0., 14.), (-18., 2.), (-1., 12.), (-10., 10.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
print '[%s:%s]' % (start, end)
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes, variables, axlabels,
ylims):
if (var == 'crop_no'):
try:
OBS = res_timeseries['OBS'][site][var][
start:end].dropna()
OBS[~(OBS == -9999.)].plot(
ax=ax,
lw=2,
#OBS.plot(ax=ax, lw=2,
style='-',
label='obs',
fontsize=fsz)
crop_no = OBS[0]
minobs = OBS[~(OBS == -9999.)].min()
maxobs = OBS[~(OBS == -9999.)].max()
except TypeError:
minobs = 0.
maxobs = 0.
minwof = 1.
maxwof = 1.
elif (var == 'TER'):
# observations
try:
OBS = res_timeseries['OBS'][site][var][
start:end].dropna()
OBS[~(OBS == -9999.)].plot(
ax=ax,
lw=2,
#OBS.plot(ax=ax, lw=2, c='b',
style='+',
label='obs',
fontsize=fsz)
minobs = OBS[~(OBS == -9999.)].min()
maxobs = OBS[~(OBS == -9999.)].max()
except TypeError:
minobs = 0.
maxobs = 0.
# SIBCASA sims
try:
#res_timeseries['SIMB'][site]['Raut'][start:end].plot(ax=ax,
#lw=2, c='g', style=':', label='SiBCASA Raut', fontsize=fsz)
res_timeseries['SIMB'][site][var][start:end].plot(
ax=ax,
lw=2,
c='g',
style='--',
label='SiBCASA TER',
fontsize=fsz)
except TypeError:
pass
# WOFOST sims
try:
#WOF = res_timeseries['SIMC'][site]['Raut'][start:end].dropna()
#WOF.plot(ax=ax, lw=2, c='r',
#style='_', label='WOFOST Raut', fontsize=fsz)
WOF = res_timeseries['SIMC'][site][var][
start:end].dropna()
WOF.plot(ax=ax,
lw=2,
c='r',
style='x',
label='WOFOST TER',
fontsize=fsz)
minwof = WOF.min()
maxwof = WOF.max()
except TypeError:
minwof = 0.
maxwof = 0.
WOF = 0.
else:
# observations
try:
OBS = res_timeseries['OBS'][site][var][
start:end].dropna()
OBS[~(OBS == -9999.)].plot(
ax=ax,
lw=2,
#OBS.plot(ax=ax, lw=2, c='b',
style='+',
label='obs',
fontsize=fsz)
minobs = OBS[~(OBS == -9999.)].min()
maxobs = OBS[~(OBS == -9999.)].max()
except TypeError:
minobs = 0.
maxobs = 0.
# SIBCASA sims
try:
res_timeseries['SIMB'][site][var][start:end].plot(
ax=ax,
lw=2,
c='g',
style='--',
label='SiBCASA',
fontsize=fsz)
except TypeError:
pass
# WOFOST simsA
try:
WOF = res_timeseries['SIMC'][site][var][
start:end].dropna()
#WOF[~(WOF==-9999.)].plot(ax=ax, lw=2,
WOF.plot(ax=ax,
lw=2,
c='r',
style='x',
label='WOFOST',
fontsize=fsz)
minwof = WOF.min()
maxwof = WOF.max()
except TypeError:
minwof = 0.
maxwof = 0.
WOF = 0.
ax.axhline(y=0., c='k')
minvar = math.floor(min(minobs, minwof)) - 1.
maxvar = math.ceil(max(maxobs, maxwof)) + 1.
ax.set_ylim(minvar, maxvar)
#ax.set_ylim(ylim)
if (var == 'GPP'):
ax.legend(loc='lower left', prop={'size': 12})
#if (var=='TER'): ax.legend(loc='upper left',prop={'size':10})
ax.set_ylabel(axlabel)
if var != 'NEE': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
figs.savefig(
os.path.join(
figsubdir,
'crop%i_%s_%i.png' % (crop_no, site, timeframe[0])))
plt.close('all')
#plt.show()
#-------------------------------------------------------------------------------
timeframe = [2004, 2014]
for site in sites:
print site
figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(15, 10))
figs.subplots_adjust(0.1, 0.07, 0.98, 0.95, 0., 0.)
variables = ['crop_no', 'GPP', 'TER', 'NEE']
axlabels = [
'crop ID', r'GPP (g m$^{-2}$ d$^{-1}$)',
r'TER (g m$^{-2}$ d$^{-1}$)', r'NEE (g m$^{-2}$ d$^{-1}$)'
]
ylims = [(0., 14.), (-30., 2.), (-2., 20.), (-20., 10.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
print '[%s:%s]' % (start, end)
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes, variables, axlabels, ylims):
if (var == 'crop_no'):
try:
OBS = res_timeseries['OBS'][site][var][start:end].dropna()
OBS[~(OBS == -9999.)].plot(
ax=ax,
lw=2,
#OBS.plot(ax=ax, lw=2,
style='-',
label='obs',
fontsize=fsz)
crop_no = OBS[0]
minobs = OBS[~(OBS == -9999.)].min()
maxobs = OBS[~(OBS == -9999.)].max()
except TypeError:
minobs = 0.
maxobs = 0.
minwof = 1.
maxwof = 1.
else:
# observations
try:
OBS = res_timeseries['OBS'][site][var][start:end].dropna()
OBS[~(OBS == -9999.)].plot(
ax=ax,
lw=2,
#OBS.plot(ax=ax, lw=2, c='b',
style='+',
label='obs',
fontsize=fsz)
minobs = OBS[~(OBS == -9999.)].min()
maxobs = OBS[~(OBS == -9999.)].max()
except TypeError:
minobs = 0.
maxobs = 0.
# SIBCASA sims
try:
res_timeseries['SIMB'][site][var][start:end].plot(
ax=ax,
lw=2,
c='g',
style='--',
label='SiBCASA',
fontsize=fsz)
except TypeError:
pass
# WOFOST simsA
try:
WOF = res_timeseries['SIMC'][site][var][start:end].dropna()
#WOF[~(WOF==-9999.)].plot(ax=ax, lw=2,
WOF.plot(ax=ax,
lw=2,
c='r',
style='x',
label='WOFOST',
fontsize=fsz)
minwof = WOF.min()
maxwof = WOF.max()
except TypeError:
minwof = 0.
maxwof = 0.
WOF = 0.
ax.axhline(y=0., c='k')
minvar = math.floor(min(minobs, minwof)) - 1.
maxvar = math.ceil(max(maxobs, maxwof)) + 1.
#ax.set_ylim(minvar,maxvar)
ax.set_ylim(ylim)
if (var == 'GPP'): ax.legend(loc='lower left', prop={'size': 12})
ax.set_ylabel(axlabel)
if var != 'NEE': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
figs.savefig(
os.path.join(
figsubdir, 'timeseries_crop%i_%s_%i-%i.png' %
(crop_no, site, timeframe[0], timeframe[1])))
plt.close('all')
def plot_fluxnet_micromet(directory, sites_list, timeframe, linestyle):
#===============================================================================
"""
This function plots the average daytime Tair, average SWin, average VPD,
average Tsoil and average volumetric soil moisture content, all taken from
the formatted 2015 FluxNet DD (daily data) files.
NB: Daily averages have been computed from the half-hourly data.
Function arguments:
------------------
directory: path to the directory where the daily variables files
('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1')
are stored
sites_list: list of fluxnet sites names
time frame: list of two years (e.g. [2005, 2007]) gives the time frame
to plot
linestyle: line style of the plot to make (can be '-', '--', '+', etc)
"""
plt.close('all')
for site in sites_list:
print site+'\n'
filepath = os.path.join(directory,'timeseries_%s.pickle'%site)
series = pickle_load(open(filepath,'rb'))
# settings of the plot:
figs, axes = plt.subplots(nrows=5, ncols=1, figsize=(8,9))
figs.subplots_adjust(0.1,0.07,0.98,0.95,0.,0.)
variables = ['Ta_day','SWin','VPD','Ts_1','SWC_1']
axlabels = [r'T$_{air}$ day (deg C)', r'SW$_{in}$ (W m$^{-2}$)',
'VPD (hPa)',r'T$_{soil}$ (deg C)','SWC (%)']
ylims = [(-20.,40.),(0.,400.),(0.,17.),(-5.,35.),(0.,60.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
ls = linestyle
lab = 'bullshit'
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes,variables,axlabels,ylims):
if var.endswith('_1'): lab='layer 1'
series[site][var][start:end].plot(ax=ax, lw=2, style=ls,
label=lab, fontsize=fsz)
if var=='Ts_1':
series[site]['Ts_2'][start:end].plot(ax=ax, lw=2, style=ls,
label='layer 2', fontsize=fsz)
series[site]['Ts_3'][start:end].plot(ax=ax, lw=2, style=ls,
label='layer 3', fontsize=fsz)
if var=='SWC_1':
series[site]['SWC_2'][start:end].plot(ax=ax, lw=2, style=ls,
label='layer 2', fontsize=fsz)
series[site]['SWC_3'][start:end].plot(ax=ax, lw=2, style=ls,
label='layer 3', fontsize=fsz)
if (var=='Ts_1'): ax.legend(loc='upper left',prop={'size':12})
if (var=='SWC_1'): ax.legend(loc='lower left',prop={'size':12})
ax.axhline(y=0., c='k')
ax.set_ylim(ylim)
ax.set_ylabel(axlabel)
if var=='Ta_day': ax.set_yticks([-10.,0.,10.,20.,30.,40.])
if var=='SWin': ax.set_yticks([0.,100.,200.,300.,400.])
if var=='VPD': ax.set_yticks([0.,5.,10.,15.])
if var=='Ts_1': ax.set_yticks([0.,10.,20.,30.])
if var != 'SWC_1': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
plt.show()
return None
def optimize_regional_yldgapf_dyn(NUTS_no_, detrend, crop_no_,
selected_grid_cells_, selected_soil_types_, weather, inputdir, opti_years_,
obs_type='yield', plot_rmse=False):
#===============================================================================
import math
from operator import itemgetter as operator_itemgetter
from matplotlib import pyplot as plt
from pcse.models import Wofost71_WLP_FD
from pcse.base_classes import WeatherDataProvider
from pcse.fileinput.cabo_weather import CABOWeatherDataProvider
# aggregated yield method:
# 2- we construct a 2D array with same dimensions as TSO_regional,
# containing the observed yields
row = [] # this list will become the row of the 2D array
for y,year in enumerate(opti_years_):
index_year = np.argmin(np.absolute(detrend[1]-year))
row = row + [detrend[0][index_year]]
OBS = np.tile(row, (5,1)) # repeats the list as a row 3 times, to get a
# 2D array
# 3- we calculate all the individual yields from the selected grid cells x
# soils combinations
# NB: we explore the range of yldgapf between 0.1 and 1.
f0 = 0.
f2 = 0.5
f4 = 1.
f_step = 0.25
# Until the precision of the yield gap factor is good enough (i.e. < 0.02)
# we loop over it. We do 12 iterations in total with this method.
iter_no = 0
RMSE_stored = list()
while (f_step >= 0.02):
iter_no = iter_no + 1
# sub-method: looping over the yield gap factors
# we build a range of 3 yield gap factors to explore one low bound, one
# high bound, one in the middle
f_step = (f4 - f0)/4.
f1 = f0 + f_step
f3 = f2 + f_step
f_range = [f0, f1, f2, f3, f4]
RES = [] # list in which we will store the yields of the combinations
counter=0
for grid, arable_land in selected_grid_cells_:
frac_arable = arable_land / 625000000.
# Retrieve the weather data of one grid cell (all years are in one
# file)
if (weather == 'CGMS'):
filename = os.path.join(inputdir,'weather_objects/',
'weatherobject_g%d.pickle'%grid)
weatherdata = WeatherDataProvider()
weatherdata._load(filename)
if (weather == 'ECMWF'):
weatherdata = CABOWeatherDataProvider('%i'%grid,fpath=ECMWFdir)
# Retrieve the soil data of one grid cell (all possible soil types)
filename = os.path.join(inputdir,'soildata_objects/',
'soilobject_g%d.pickle'%grid)
soil_iterator = pickle_load(open(filename,'rb'))
for smu, stu_no, weight, soildata in selected_soil_types_[grid]:
# TSO will store all the yields of one grid cell x soil
# combination, for all years and all 3 yldgapf values
TSO = np.zeros((len(f_range), len(opti_years_)))
counter +=1
for y, year in enumerate(opti_years_):
# Retrieve yearly data
filename = os.path.join(inputdir,
'timerdata_objects/%i/c%i/'%(year,crop_no_),
'timerobject_g%d_c%d_y%d.pickle'\
%(grid,crop_no_,year))
timerdata = pickle_load(open(filename,'rb'))
filename = os.path.join(inputdir,
'cropdata_objects/%i/c%i/'%(year,crop_no_),
'cropobject_g%d_c%d_y%d.pickle'\
%(grid,crop_no_,year))
cropdata = pickle_load(open(filename,'rb'))
if str(grid).startswith('1'):
dum = str(grid)[0:2]
else:
dum = str(grid)[0]
filename = os.path.join(inputdir,
'sitedata_objects/%i/c%i/grid_%s/'
%(year,crop_no_,dum),
'siteobject_g%d_c%d_y%d_s%d.pickle'\
%(grid,crop_no_,year,stu_no))
sitedata = pickle_load(open(filename,'rb'))
for f,factor in enumerate(f_range):
cropdata['YLDGAPF']=factor
# run WOFOST
wofost_object = Wofost71_WLP_FD(sitedata, timerdata,
soildata, cropdata, weatherdata)
wofost_object.run_till_terminate()
# get the yield (in kgDM.ha-1)
TSO[f,y] = wofost_object.get_variable('TWSO')
#print grid, stu_no, year, counter, [y[0] for y in TSO], OBS[0]
RES = RES + [(grid, stu_no, weight*frac_arable, TSO)]
# 4- we aggregate the yield or harvest into the regional one with array
# operations
sum_weighted_vals = np.zeros((len(f_range), len(opti_years_)))
# empty 2D array with same dimension as TSO
sum_weights = 0.
for grid, stu_no, weight, TSO in RES:
# adding weighted 2D-arrays in the empty array sum_weighted_yields
# NB: variable 'weight' is actually the cultivated area in m2
sum_weighted_vals = sum_weighted_vals + (weight/10000.)*TSO
# computing the total sum of the cultivated area in ha
sum_weights = sum_weights + (weight/10000.)
if (obs_type == 'harvest'):
TSO_regional = sum_weighted_vals / 1000000. # sum of the individual
# harvests in 1000 tDM
elif (obs_type == 'yield'):
TSO_regional = sum_weighted_vals / sum_weights # weighted average of
# all yields in kgDM/ha
# 5- we compute the (sim-obs) differences.
DIFF = TSO_regional - OBS
if (TSO_regional[-1][0] <= 0.):
print 'WARNING: no simulated crop growth. We set the optimum fgap to 1.'
return 1.
if (TSO_regional[-1] <= OBS[-1]):
print 'WARNING: obs yield > sim yield. We set optimum to 1.'
return 1.
# 6- we calculate the RMSE (root mean squared error) of the 3 yldgapf
# The RMSE of each yldgapf is based on N obs-sim differences for the N
# years looped over
RMSE = np.zeros(len(f_range))
for f,factor in enumerate(f_range):
list_of_DIFF = []
for y, year in enumerate(opti_years_):
list_of_DIFF = list_of_DIFF + [DIFF[f,y]]
RMSE[f] = np.sqrt(np.mean( [ math.pow(j,2) for j in
list_of_DIFF ] ))
#print RMSE, f_range
# We store the value of the RMSE for plotting purposes
RMSE_stored = RMSE_stored + [(f_range[1], RMSE[1]), (f_range[3], RMSE[3])]
if (iter_no == 1):
RMSE_stored = RMSE_stored + [(f_range[0], RMSE[0]),
(f_range[2], RMSE[2]),
(f_range[4], RMSE[4])]
# 7- We update the yldgapf range to explore for the next iteration.
# For this we do a linear interpolation of RMSE between the 3 yldgapf
# explored here, and the next range to explore is the one having the
# smallest interpolated RMSE
index_new_center = RMSE.argmin()
# if the optimum is close to 1:
if index_new_center == len(f_range)-1:
f0 = f_range[index_new_center-2]
f2 = f_range[index_new_center-1]
f4 = f_range[index_new_center]
# if the optimum is close to 0:
elif index_new_center == 0:
f0 = f_range[index_new_center]
f2 = f_range[index_new_center+1]
f4 = f_range[index_new_center+2]
else:
f0 = f_range[index_new_center-1]
f2 = f_range[index_new_center]
f4 = f_range[index_new_center+1]
# when we are finished iterating on the yield gap factor range, we plot the
# RMSE as a function of the yield gap factor
if (plot_rmse == True):
RMSE_stored = sorted(RMSE_stored, key=operator_itemgetter(0))
x,y = zip(*RMSE_stored)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5,5))
fig.subplots_adjust(0.15,0.16,0.95,0.96,0.4,0.)
ax.plot(x, y, c='k', marker='o')
ax.set_xlabel('yldgapf (-)')
ax.set_ylabel('RMSE')
fig.savefig('%s_opti_fgap.png'%NUTS_no_)
#pickle_dump(RMSE_stored,open('%s_RMSE.pickle'%NUTS_no_,'wb'))
# 8- when we are finished iterating on the yield gap factor range, we return
# the optimum value. We look for the yldgapf with the lowest RMSE
index_optimum = RMSE.argmin()
optimum_yldgapf = f_range[index_optimum]
print 'optimum found: %.2f +/- %.2f'%(optimum_yldgapf, f_step)
# 10- we return the optimized YLDGAPF
return optimum_yldgapf
def _load(self,f):
self.store = pickle_load(f)
def compute_timeseries_fluxes(crop_no,
grid_no,
lon,
lat,
year,
R10,
Eact0,
selec_method,
nsoils,
TER_method='grow-only',
scale='daily'):
# possible methods: 'grow-only': NEE = GPP + Rgrow + Rsoil
# 'rauto': NEE = GPP + Rgrow + Rmaint + Rsoil
#===============================================================================
import math
import pandas as pd
import datetime as dt
from maries_toolbox import open_pcse_csv_output, select_soils
print '- grid cell no %i: lon = %.2f , lat = %.2f' % (grid_no, lon, lat)
prod_figure = False
# we retrieve the tsurf and rad variables from ECMWF
filename_rad = 'rad_ecmwf_%i_lon%.2f_lat%.2f.pickle' % (year, lon, lat)
path_rad = os.path.join(ECMWFdir, filename_rad)
rad = pickle_load(open(path_rad, 'rb'))
filename_ts = 'ts_ecmwf_%i_lon%.2f_lat%.2f.pickle' % (year, lon, lat)
path_ts = os.path.join(ECMWFdir, filename_ts)
ts = pickle_load(open(path_ts, 'rb'))
# we initialize the timeseries for the grid cell
# time list for timeseries
time_cell_persec_timeseries = rad[0]
time_cell_perday_timeseries = rad[0][::8] / (3600. * 24.)
# length of all carbon fluxes timeseries
len_persec = len(rad[0])
len_perday = len(rad[0][::8])
# GPP timeseries
gpp_cell_persec_timeseries = np.array([0.] * len_persec)
gpp_cell_perday_timeseries = np.array([0.] * len_perday)
# autotrophic respiration timeseries
raut_cell_persec_timeseries = np.array([0.] * len_persec)
raut_cell_perday_timeseries = np.array([0.] * len_perday)
# heterotrophic respiration timeseries
rhet_cell_persec_timeseries = np.array([0.] * len_persec)
rhet_cell_perday_timeseries = np.array([0.] * len_perday)
# we initialize some variables
sum_stu_areas = 0. # sum of soil types areas
delta = 3600. * 3. # number of seconds in delta (here 3 hours)
if (prod_figure == True):
from matplotlib import pyplot as plt
plt.close('all')
fig1, axes = plt.subplots(nrows=3, ncols=1, figsize=(14, 10))
fig1.subplots_adjust(0.1, 0.1, 0.98, 0.9, 0.2, 0.2)
# Select soil types to loop over
soilist = select_soils(crop_no, [grid_no],
CGMSdir,
method=selec_method,
n=nsoils)
#---------------------------------------------------------------
# loop over soil types
#---------------------------------------------------------------
for smu, stu_no, stu_area, soildata in soilist[grid_no]:
# We open the WOFOST results file
filename = 'wofost_g%i_s%i.txt' % (grid_no, stu_no)
results_set = open_pcse_csv_output(forwardir, [filename])
wofost_data = results_set[0]
# We apply the short wave radiation diurnal cycle on the GPP
# and R_auto
# we create empty time series for this specific stu
gpp_cycle_timeseries = np.array([])
raut_cycle_timeseries = np.array([])
gpp_perday_timeseries = np.array([])
raut_perday_timeseries = np.array([])
# we compile the sum of the stu areas to do a weighted average of
# GPP and Rauto later on
sum_stu_areas += stu_area
#-----------------------------------------------------------
# loop over days of the year
#-----------------------------------------------------------
for DOY, timeinsec in enumerate(time_cell_persec_timeseries[::8]):
# conversion of current time in seconds into date
time = dt.date(year, 1, 1) + dt.timedelta(DOY)
#print 'date:', time
# we test to see if we are within the growing season
test_sow = (time - wofost_data[filename]['day'][0]).total_seconds()
test_rip = (time -
wofost_data[filename]['day'][-1]).total_seconds()
#print 'tests:', test_sow, test_rip
# if the day of the time series is before sowing date: plant
# fluxes are set to zero
if test_sow < 0.:
gpp_day = 0.
raut_day = 0.
# or if the day of the time series is after the harvest date:
# plant fluxes are set to zero
elif test_rip > 0.:
gpp_day = 0.
raut_day = 0.
# else we get the daily total GPP and Raut in kgCH2O/ha/day
# from wofost, and we weigh it with the stu area to later on
# calculate the weighted average GPP and Raut in the grid cell
else:
# index of the sowing date in the time_cell_timeseries:
if (test_sow == 0.): DOY_sowing = DOY
if (test_rip == 0.): DOY_harvest = DOY
#print 'DOY sowing:', DOY_sowing
# translation of cell to stu timeseries index
index_day_w = DOY - DOY_sowing
#print 'index of day in wofost record:', index_day_w
# unit conversion: from kgCH2O/ha/day to gC/m2/day
gpp_day = - wofost_data[filename]['GASS'][index_day_w] * \
(mmC / mmCH2O) * 0.1
maint_resp = wofost_data[filename]['MRES'][index_day_w] * \
(mmC / mmCH2O) * 0.1
try: # if there are any available assimilates for growth
growth_fac = (wofost_data[filename]['DMI'][index_day_w]) / \
(wofost_data[filename]['GASS'][index_day_w] -
wofost_data[filename]['MRES'][index_day_w])
growth_resp = (1. - growth_fac) * (-gpp_day - maint_resp)
except ZeroDivisionError: # otherwise there is no crop growth
growth_resp = 0.
if TER_method == 'rauto':
raut_day = growth_resp + maint_resp
elif TER_method == 'grow-only':
raut_day = growth_resp
# we select the radiation diurnal cycle for that date
# NB: the last index is ignored in the selection, so we DO have
# 8 time steps selected only (it's a 3-hourly dataset)
rad_cycle = rad[1][DOY * 8:DOY * 8 + 8]
# we apply the radiation cycle on the GPP and Rauto
# and we transform the daily integral into a rate
weights = rad_cycle / sum(rad_cycle)
# the sum of the 8 rates is equal to total/delta:
sum_gpp_rates = gpp_day / delta
sum_raut_rates = raut_day / delta
# the day's 8 values of actual gpp and raut rates per second:
gpp_cycle = weights * sum_gpp_rates
raut_cycle = weights * sum_raut_rates
# NB: we check if the applied diurnal cycle is correct
assert (sum(weights) - 1. < 0.000001), "wrong radiation kernel"
assert (len(gpp_cycle) *
int(delta) == 86400), "wrong delta in diurnal cycle"
assert ((sum(gpp_cycle)*delta-gpp_day) < 0.00001), "wrong diurnal cycle "+\
"applied on GPP: residual=%.2f "%(sum(gpp_cycle)*delta-gpp_day) +\
"on DOY %i"%DOY
assert ((sum(raut_cycle)*delta-raut_day) < 0.00001), "wrong diurnal cycle "+\
"applied on Rauto: residual=%.2f "%(sum(raut_cycle)*delta-raut_day) +\
"on DOY %i"%DOY
# if the applied diurnal cycle is ok, we append that day's cycle
# to the yearly record of the stu
gpp_cycle_timeseries = np.concatenate(
(gpp_cycle_timeseries, gpp_cycle), axis=0)
raut_cycle_timeseries = np.concatenate(
(raut_cycle_timeseries, raut_cycle), axis=0)
# we also store the carbon fluxes per day, for comparison with fluxnet
gpp_perday_timeseries = np.concatenate(
(gpp_perday_timeseries, [gpp_day]), axis=0)
raut_perday_timeseries = np.concatenate(
(raut_perday_timeseries, [raut_day]), axis=0)
#-----------------------------------------------------------
# end of day nb loop
#-----------------------------------------------------------
# plot the soil type timeseries if requested by the user
if (prod_figure == True):
for ax, var, name, lims in zip(axes.flatten(), [
gpp_perday_timeseries, raut_perday_timeseries,
gpp_perday_timeseries + raut_perday_timeseries
], ['GPP', 'Rauto', 'NPP'], [[-20., 0.], [0., 10.], [-15., 0.]]):
ax.plot(time_cell_perday_timeseries,
var,
label='stu %i' % stu_no)
#ax.set_xlim([40.,170.])
#ax.set_ylim(lims)
ax.set_ylabel(name + r' (g$_{C}$ m$^{-2}$ d$^{-1}$)',
fontsize=14)
# We compile time series of carbon fluxes in units per day and per second
# a- sum the PER SECOND timeseries
gpp_cell_persec_timeseries = gpp_cell_persec_timeseries + \
gpp_cycle_timeseries*stu_area
raut_cell_persec_timeseries = raut_cell_persec_timeseries + \
raut_cycle_timeseries*stu_area
# b- sum the PER DAY timeseries
gpp_cell_perday_timeseries = gpp_cell_perday_timeseries + \
gpp_perday_timeseries*stu_area
raut_cell_perday_timeseries = raut_cell_perday_timeseries + \
raut_perday_timeseries*stu_area
#---------------------------------------------------------------
# end of soil type loop
#---------------------------------------------------------------
# finish ploting the soil type timeseries if requested by the user
if (prod_figure == True):
plt.xlabel('time (DOY)', fontsize=14)
plt.legend(loc='upper left', ncol=2, fontsize=10)
fig1.suptitle('Daily carbon fluxes of %s for all '%crop+\
'soil types of grid cell %i in %i'%(grid_no,
year), fontsize=18)
figname = 'GPP_allsoils_%i_c%i_g%i.png'%(year,crop_no,\
grid_no)
#plt.show()
fig1.savefig(os.path.join(analysisdir, figname))
# compute the weighted average of GPP, Rauto over the grid cell
# a- PER SECOND
gpp_cell_persec_timeseries = gpp_cell_persec_timeseries / sum_stu_areas
raut_cell_persec_timeseries = raut_cell_persec_timeseries / sum_stu_areas
# b- PER DAY
gpp_cell_perday_timeseries = gpp_cell_perday_timeseries / sum_stu_areas
raut_cell_perday_timeseries = raut_cell_perday_timeseries / sum_stu_areas
# compute the heterotrophic respiration with the A-gs equation
# NB: we assume here Rhet only dependant on tsurf, not soil moisture
#fw = Cw * wsmax / (wg + wsmin)
tsurf_inter = Eact0 / (283.15 * 8.314) * (1 - 283.15 / ts[1])
# a- PER SEC:
rhet_cell_persec_timeseries = R10 * np.array(
[math.exp(t) for t in tsurf_inter])
# b- PER DAY:
for i in range(len(rhet_cell_perday_timeseries)):
rhet_cell_perday_timeseries[i] = rhet_cell_persec_timeseries[i*8] * delta +\
rhet_cell_persec_timeseries[i*8+1] * delta +\
rhet_cell_persec_timeseries[i*8+2] * delta +\
rhet_cell_persec_timeseries[i*8+3] * delta +\
rhet_cell_persec_timeseries[i*8+4] * delta +\
rhet_cell_persec_timeseries[i*8+5] * delta +\
rhet_cell_persec_timeseries[i*8+6] * delta +\
rhet_cell_persec_timeseries[i*8+7] * delta
# conversion from mgCO2 to gC
conversion_fac = (mmC / mmCO2) * 0.001
rhet_cell_persec_timeseries = rhet_cell_persec_timeseries * conversion_fac
rhet_cell_perday_timeseries = rhet_cell_perday_timeseries * conversion_fac
# calculate TER:
ter_cell_persec_timeseries = raut_cell_persec_timeseries + \
rhet_cell_persec_timeseries
ter_cell_perday_timeseries = raut_cell_perday_timeseries + \
rhet_cell_perday_timeseries
# calculate NEE:
nee_cell_persec_timeseries = gpp_cell_persec_timeseries + \
raut_cell_persec_timeseries + \
rhet_cell_persec_timeseries
nee_cell_perday_timeseries = gpp_cell_perday_timeseries + \
raut_cell_perday_timeseries + \
rhet_cell_perday_timeseries
# here we choose to return the carbon fluxes PER DAY
if scale == 'daily':
return time_cell_perday_timeseries, gpp_cell_perday_timeseries, \
raut_cell_perday_timeseries, rhet_cell_perday_timeseries, \
ter_cell_perday_timeseries, nee_cell_perday_timeseries
elif scale == '3-hourly':
return time_cell_persec_timeseries, gpp_cell_persec_timeseries, \
raut_cell_persec_timeseries, rhet_cell_persec_timeseries, \
ter_cell_persec_timeseries, nee_cell_persec_timeseries
def plot_fluxnet_daily_c_fluxes(directory, sites_list, timeframe, linestyle):
#===============================================================================
"""
This function plots the average daytime Tair, average SWin, average VPD,
average Tsoil and average volumetric soil moisture content, all taken from
the formatted 2015 FluxNet DD (daily data) files.
NB: Daily averages have been computed from the half-hourly data.
Function arguments:
------------------
directory: path to the directory where the daily variables files
('FLX_sitename_FLUXNET2015_FULLSET_DD_2004-2014_1-1')
are stored
sites_list: list of fluxnet sites names
time frame: list of two years (e.g. [2005, 2007]) gives the time frame
to plot
linestyle: line style of the plot to make (can be '-', '--', '+', etc)
"""
plt.close('all')
print "\nPlotting carbon fluxes timeseries\n"
for site in sites_list:
print site+'\n'
filepath = os.path.join(directory,'timeseries_%s.pickle'%site)
series = pickle_load(open(filepath,'rb'))
# settings of the plot:
figs, axes = plt.subplots(nrows=4, ncols=1, figsize=(8,10))
figs.subplots_adjust(0.1,0.07,0.98,0.95,0.,0.)
variables = ['crop_no','GPP','TER','NEE']
axlabels = ['crop ID',r'GPP (g m$^{-2}$ d$^{-1}$)',r'TER (g m$^{-2}$ d$^{-1}$)',r'NEE (g m$^{-2}$ d$^{-1}$)']
ylims = [(0.,14.),(-30.,5.),(-5.,20.),(-30.,30.)]
start = str(int(timeframe[0]))
end = str(int(timeframe[1]))
lab = ''
ls = linestyle
fsz = 14 # fonsize of x and y axis ticks
for ax, var, axlabel, ylim in zip(axes,variables,axlabels,ylims):
if (var=='TER' or var=='GPP'): lab='-'
series[site][var][start:end].plot(ax=ax, lw=2, style=ls,
label=lab, fontsize=fsz)
if var=='GPP':
series[site]['GPP_day'][start:end].plot(ax=ax, lw=2, style=ls,
label='day partition', fontsize=fsz)
series[site]['GPP_night'][start:end].plot(ax=ax, lw=2, style=ls,
label='night partition', fontsize=fsz)
if var=='TER':
series[site]['TER_day'][start:end].plot(ax=ax, lw=2, style=ls,
label='day partition', fontsize=fsz)
series[site]['TER_night'][start:end].plot(ax=ax, lw=2, style=ls,
label='night partition', fontsize=fsz)
ax.axhline(y=0., c='k')
ax.set_ylim(ylim)
if (var=='GPP_day'): ax.legend(loc='lower left',prop={'size':12})
if (var=='TER_day'): ax.legend(loc='upper left',prop={'size':12})
ax.set_ylabel(axlabel)
if var != 'NEE': ax.get_xaxis().set_visible(False)
figs.suptitle(site, fontsize=14)
plt.show()
return None
# Any file has changed in the dir since the cache was generated
or any(True for f in listdir(directory)
if (getmtime(path_join(directory, f)) > cache_mtime
# Ignore hidden files
and not f.startswith('.')))
# The configuration is newer than the cache
or getmtime(get_config_path(directory)) > cache_mtime):
generate = True
docstats = []
else:
generate = False
try:
with open(
cache_file_path.decode('utf-8').encode('utf-8'),
'rb') as cache_file:
docstats = pickle_load(cache_file)
except UnpicklingError:
# Corrupt data, re-generate
Messager.warning(
'Stats cache %s was corrupted; regenerating' %
cache_file_path, -1)
generate = True
except EOFError:
# Corrupt data, re-generate
generate = True
except OSError, e:
Messager.warning(
'Failed checking file modification times for stats cache check; regenerating'
)
generate = True
def __init__(self, shmem):
AggregationTask.__init__(self, shmem)
reader = wire.PackedMessageReader(shmem)
self.state = pickle_load(reader.string())
