def run_model(data_dict, NEE_model, re_configs_dict, ps_configs_dict):
if NEE_model == 'LT':
re_rslt_dict, re_params_dict = re.main(data_dict, re_configs_dict)[0:2]
ps_rslt_dict = ps.main(data_dict, ps_configs_dict, re_params_dict)[0]
data_dict['NEE_model'] = ps_rslt_dict['GPP'] + ps_rslt_dict['Re']
data_dict['NEE_filled'] = np.where(np.isnan(data_dict['NEE_series']),
data_dict['NEE_model'],
data_dict['NEE_series'])
elif NEE_model == 'ANN':
len_int = len(data_dict['NEE_series'])
input_array = np.empty([len_int, 4])
for i, var in enumerate(['TempC', 'Sws', 'Fsd', 'VPD']):
input_array[:, i] = data_dict[var]
target_array = np.empty([len_int, 1])
target_array[:, 0] = data_dict['NEE_series']
data_dict['NEE_model'] = gf.train_ANN(input_array, target_array, 100,
[4, 24, 16, 1])[:, 0]
data_dict['NEE_filled'] = np.where(np.isnan(data_dict['NEE_series']),
data_dict['NEE_model'],
data_dict['NEE_series'])
else:
raise Exception('\'' + NEE_model + '\' is not a valid model type! ' \
'Valid choices are \'ANN\' or \'LT\'')
return
def run_model(data_dict, NEE_model, re_configs_dict, ps_configs_dict):
if NEE_model == 'LT':
try:
re_rslt_dict, re_params_dict = re.main(data_dict,
re_configs_dict)[0: 2]
except Exception:
raise
try:
ps_rslt_dict = ps.main(data_dict, ps_configs_dict, re_params_dict)[0]
except Exception:
raise
data_dict['NEE_model'] = ps_rslt_dict['GPP'] + ps_rslt_dict['Re']
data_dict['NEE_filled'] = np.where(np.isnan(data_dict['NEE_series']),
data_dict['NEE_model'],
data_dict['NEE_series'])
elif NEE_model == 'ANN':
len_int = len(data_dict['NEE_series'])
input_array = np.empty([len_int, 4])
for i, var in enumerate(['TempC', 'Sws', 'Fsd', 'VPD']):
input_array[:, i] = data_dict[var]
target_array = np.empty([len_int, 1])
target_array[:, 0] = data_dict['NEE_series']
data_dict['NEE_model'] = gf.train_ANN(input_array, target_array,
100,
[4, 24, 16, 1])[:, 0]
data_dict['NEE_filled'] = np.where(np.isnan(data_dict['NEE_series']),
data_dict['NEE_model'],
data_dict['NEE_series'])
else:
raise Exception('\'' + NEE_model + '\' is not a valid model type! ' \
'Valid choices are \'ANN\' or \'LT\'')
return
this_dict = {var: data_dict[var][indices[0]: indices[1] + 1]
for var in ['Fc_series', 'Fsd', 'ustar']}
obs_year_stats['n_cases_total'][year] = indices[1] - indices[0]
obs_year_stats['n_cases_avail'][year] = len(this_dict['Fc_series']
[(this_dict['Fsd'] < 5) &
(this_dict['ustar'] > ustar_threshold) &
(~np.isnan(this_dict['Fc_series']))])
# Remove low ustar values according to threshold
data_dict['Fc_series'][data_dict['ustar'] < ustar_threshold] = np.nan
# Create a boolean array for data indicating presence of nans
nan_boolean = np.isnan(data_dict['Fc_series'])
# Calculate Re by sending data to main respiration function
re_dict, params_dict = re.main(data_dict, re_configs_dict)
data_dict['Re'] = re_dict['Re']
# Calculate sums for each year
sums_dict = {}
for year in years_input_index_dict.keys():
indices = years_input_index_dict[year]
sums_dict[year] = (data_dict['Re'][indices[0]: indices[1] + 1]
* 12 * 0.0018).sum()
#------------------------------------------------------------------------------
# Get the indices of the start and end rows of each unique date in the source
# data array
dates_input_index_dict = dtf.get_day_indices(datetime_array)
# Get the indices of the start and end rows of each window in the source
{'2011': 0.31, '2012': 0.30, '2013': 0.32, '2014': 0.32},
re_configs_dict['options']['noct_threshold'],
False)
C_dict['Fc_Sc_pt_u*_noct'] = data_dict.pop('NEE_series')
# Update variables in data dict and configs dict to allow photosynthesis to run
data_dict['PAR'] = data_dict['Fsd'] * 0.46 * 4.6
ps_configs_dict['options']['measurement_interval'] = (
re_configs_dict['options']['measurement_interval'])
# Now do gap filling
rslt_dict = {}
for var in C_dict.keys():
data_dict['NEE_series'] = cp.deepcopy(C_dict[var])
re_rslt_dict, re_params_dict = re.main(data_dict,
re_configs_dict['options'])[:2]
ps_rslt_dict = ps.main(data_dict,
ps_configs_dict['options'],
re_params_dict)[0]
this_model_array = re_rslt_dict['Re'] + ps_rslt_dict['GPP']
idx = np.isnan(data_dict['NEE_series'])
data_dict['NEE_series'][idx] = this_model_array[idx]
C_dict[var] = data_dict.pop('NEE_series')
# Do calculations: sums and means for all groups
annual_dict = {}
df = pd.DataFrame(C_dict, index = data_dict['date_time'])
years_list = list(set([this_date.year for this_date in data_dict['date_time']]))
for this_year in years_list:
annual_dict[this_year] = (df.loc[str(this_year)] * 1800 * 10**-6 * 12).sum()
#
def main(use_storage='from_config',
storage_var='from_config',
ustar_threshold='from_config',
config_file=False,
do_light_response=False):
"""
No positional arguments - prompts for a configuration file
Kwargs: use_storage - if True then algorithm looks for a variable called
Fc_storage and then sums that with Fc; if False uses
Fc alone; if set to 'from_config', looks in
config file
ustar_threshold - set to a particular value to override the ustar
threshold set in the global configs root item of
the configuration file (this is done so that can
be set in function call from another script)
"""
# Do the respiration fit
# Get master configuration file
if not config_file:
configs_master_dict = io.config_to_dict(io.file_select_dialog())
else:
configs_master_dict = io.config_to_dict(config_file)
# Build custom configuration file for this script
configs_dict = build_config_file(configs_master_dict, do_light_response)
# Override default configuration storage_var if not set to from_config
if not storage_var == 'from_config':
if isinstance(storage_var, str):
configs_dict['variables']['carbon_storage'] = storage_var
else:
raise Exception('storage_var kwarg must be a string if not '\
'set to from_config! Quitting...')
# Get data
data_dict, attr = get_data(configs_dict)
# Override default configuration file use_storage if not set to from_config
if not use_storage == 'from_config':
if isinstance(use_storage, (bool, str)):
configs_dict['global_options']['use_storage'] = use_storage
else:
raise Exception('use_storage kwarg must be a boolean if not '\
'set to from_config! Quitting...')
# Override default configuration file ustar_threshold if requested by user
if not ustar_threshold == 'from_config':
if isinstance(ustar_threshold, (int, float, dict)):
configs_dict['global_options']['ustar_threshold'] = ustar_threshold
else:
raise Exception('ustar_threshold kwarg must be set to a number or '\
'dictionary of numbers (year[int] as key, ' \
'threshold [float] as value)! Quitting...')
# Sum Fc and Sc if storage is to be included, otherwise if requested,
# remove all Fc where Sc is missing
if configs_dict['global_options']['use_storage']:
data_dict['NEE_series'] = (data_dict['NEE_series'] + data_dict['Sc'])
elif configs_dict['global_options']['unify_flux_storage_cases']:
# data_dict['NEE_series'][np.isnan(data_dict['Sc'])] = np.nan
data_dict['NEE_series'][np.isnan(data_dict['filter_var'])] = np.nan
# Remove low ustar data
data_filter.screen_low_ustar(
data_dict, configs_dict['global_options']['ustar_threshold'],
configs_dict['global_options']['noct_threshold'])
# Set up respiration configs and add measurement interval and output path
re_configs_dict = configs_master_dict['respiration_configs']['options']
re_configs_dict['measurement_interval'] = int(attr['time_step'])
re_full_path = os.path.join(configs_dict['files']['output_path'],
re_configs_dict['output_folder'])
if not os.path.isdir(re_full_path): os.makedirs(re_full_path)
re_configs_dict['output_path'] = re_full_path
# Calculate Re
re_rslt_dict, re_params_dict, re_error_dict = re.main(
cp.copy(data_dict), re_configs_dict)
# Add original time series
re_rslt_dict['NEE_series'] = data_dict['NEE_series']
# Do the light response parameters (or not)
if do_light_response:
# Set up light response configs and add measurement interval and output path
li_configs_dict = configs_master_dict['photosynthesis_configs'][
'options']
li_configs_dict['measurement_interval'] = int(attr['time_step'])
li_full_path = os.path.join(configs_dict['files']['output_path'],
li_configs_dict['output_folder'])
if not os.path.isdir(li_full_path): os.makedirs(li_full_path)
li_configs_dict['output_path'] = li_full_path
# Convert insolation to PPFD
data_dict['PAR'] = data_dict['Fsd'] * 0.46 * 4.6
# Call light response function
li_rslt_dict, li_params_dict, li_error_dict = ps.main(
data_dict, li_configs_dict, re_params_dict)
# Add original time series
li_rslt_dict['NEE_series'] = data_dict['NEE_series']
# Write data to file
var_order_list = [
'date', 'Eo', 'Eo_error_code', 'rb', 'alpha', 'beta', 'k',
'light_response_error_code'
]
if li_configs_dict['use_nocturnal_rb']:
var_order_list.insert(4, 'rb_error_code')
io.array_dict_to_csv(li_params_dict,
os.path.join(li_full_path, 'params.csv'),
var_order_list)
io.array_dict_to_csv(li_rslt_dict,
os.path.join(li_full_path, 'Re_GPP.csv'),
['date_time', 'Re', 'GPP'])
io.text_dict_to_text_file(
li_error_dict, os.path.join(li_full_path, 'error_codes.txt'))
return li_rslt_dict, li_params_dict
else:
# Write data to file
io.array_dict_to_csv(
re_params_dict, os.path.join(re_full_path, 'params.csv'),
['date', 'Eo', 'Eo_error_code', 'rb', 'rb_error_code'])
io.array_dict_to_csv(re_rslt_dict,
os.path.join(re_full_path,
'Re.csv'), ['date_time', 'Re'])
io.text_dict_to_text_file(
re_error_dict, os.path.join(re_full_path, 'error_codes.txt'))
return re_rslt_dict, re_params_dict
# Do the respiration fit
# Get configurations
configs_dict = io.config_to_dict(io.file_select_dialog())
# Get data
data_dict, attr = get_data(configs_dict)
# Set output path for results
full_path = os.path.join(configs_dict['files']['output_path'],
configs_dict['respiration_configs']['output_folder'])
if not os.path.isdir(full_path): os.makedirs(full_path)
# Drop additional config items and add measurement interval to configs
configs_dict = configs_dict['respiration_configs']
configs_dict['measurement_interval'] = int(attr['time_step'])
configs_dict['output_path'] = full_path
# Remove low ustar values according to threshold
data_dict['NEE_series'][data_dict['ustar'] <
configs_dict['ustar_threshold']] = np.nan
# Calculate Re by sending data to main respiration function
re_dict, params_dict, error_dict = re.main(cp.copy(data_dict), configs_dict)
# Write data to file
io.array_dict_to_csv(params_dict,
os.path.join(full_path, 'params.csv'),
['date', 'Eo', 'Eo_error_code', 'rb', 'rb_error_code'])
io.array_dict_to_csv(re_dict, os.path.join(full_path, 'Re.csv'), ['date_time',
'Re'])
# Do respiration
re_configs_dict = io.build_algorithm_configs_dict(config_file_location =
config_file,
algorithm =
'respiration_configs')
re_data_dict = get_data(re_configs_dict, False)
filt_re_data_dict = cp.deepcopy(re_data_dict)
filt_re_data_dict['NEE_series'] = filt_re_data_dict['NEE_series'] + filt_re_data_dict['Sc']
filt.screen_low_ustar(filt_re_data_dict, re_configs_dict['options']['ustar_threshold'],
True)
re_rslt_dict, re_params_dict, re_error_dict = re.main(filt_re_data_dict,
re_configs_dict['options'])
###############################################################################
# Do photosynthesis
ps_configs_dict = io.build_algorithm_configs_dict(config_file_location =
config_file,
algorithm =
'photosynthesis_configs')
ps_data_dict = get_data(ps_configs_dict, False)
filt_ps_data_dict = cp.deepcopy(ps_data_dict)
ps_data_dict['PAR'] = ps_data_dict['Fsd'] * 0.46 * 4.6
filt_ps_data_dict = cp.deepcopy(ps_data_dict)
filt_ps_data_dict['NEE_series'] = filt_ps_data_dict['NEE_series'] + filt_ps_data_dict['Sc']