def rpLL_createdict_outputs(cf, erll, target, called_by, flag_code):
level = cf["level"]
eo = erll["outputs"]
# loop over the outputs listed in the control file
section = "EcosystemRespiration"
outputs = cf[section][target][called_by].keys()
for output in outputs:
# create the dictionary keys for this series
eo[output] = {}
# get the target
sl = [section, target, called_by, output]
eo[output]["target"] = pfp_utils.get_keyvaluefromcf(cf, sl, "target", default=target)
eo[output]["source"] = pfp_utils.get_keyvaluefromcf(cf, sl, "source", default="Fc")
# add the flag_code
eo[output]["flag_code"] = flag_code
# list of drivers
opt = pfp_utils.get_keyvaluefromcf(cf, sl, "drivers", default="Ta")
eo[output]["drivers"] = pfp_cfg.cfg_string_to_list(opt)
opt = pfp_utils.get_keyvaluefromcf(cf, sl, "step_size_days", default=5)
eo[output]["step_size_days"] = int(opt)
opt = pfp_utils.get_keyvaluefromcf(cf, sl, "window_size_days", default=15)
eo[output]["window_size_days"] = int(opt)
opt = pfp_utils.get_keyvaluefromcf(cf, sl, "output_plots", default="False")
eo[output]["output_plots"] = (opt == "True")
opt = pfp_utils.get_keyvaluefromcf(cf, sl, "fsd_threshold", default=10)
eo[output]["fsd_threshold"] = int(opt)
# fit statistics for plotting later on
eo[output]["results"] = {"startdate":[],"enddate":[],"No. points":[],"r":[],
"Bias":[],"RMSE":[],"Frac Bias":[],"NMSE":[],
"Avg (obs)":[],"Avg (LL)":[],
"Var (obs)":[],"Var (LL)":[],"Var ratio":[],
"m_ols":[],"b_ols":[]}
return
def change_variable_attributes(cfg, ds):
"""
Purpose:
Clean up the variable attributes.
Usage:
Author: PRI
Date: November 2018
"""
# rename existing long_name to description, introduce a
# consistent long_name attribute and introduce the group_name
# attribute
vattr_list = list(cfg["variable_attributes"].keys())
series_list = list(ds.series.keys())
descr = "description_" + ds.globalattributes["nc_level"]
for label in series_list:
variable = pfp_utils.GetVariable(ds, label)
variable["Attr"][descr] = copy.deepcopy(variable["Attr"]["long_name"])
for item in vattr_list:
if label[:len(item)] == item:
for key in list(cfg["variable_attributes"][item].keys()):
variable["Attr"][key] = cfg["variable_attributes"][item][
key]
pfp_utils.CreateVariable(ds, variable)
# parse variable attributes to new format, remove deprecated variable attributes
# and fix valid_range == "-1e+35,1e+35"
tmp = cfg["variable_attributes"]["deprecated"]
deprecated = pfp_cfg.cfg_string_to_list(tmp)
series_list = list(ds.series.keys())
for label in series_list:
variable = pfp_utils.GetVariable(ds, label)
# parse variable attributes to new format
variable["Attr"] = parse_variable_attributes(variable["Attr"])
# remove deprecated variable attributes
for vattr in deprecated:
if vattr in list(variable["Attr"].keys()):
del variable["Attr"][vattr]
# fix valid_range == "-1e+35,1e+35"
if "valid_range" in variable["Attr"]:
valid_range = variable["Attr"]["valid_range"]
if valid_range == "-1e+35,1e+35":
d = numpy.ma.min(variable["Data"])
mn = pfp_utils.round2significant(d, 4, direction='down')
d = numpy.ma.max(variable["Data"])
mx = pfp_utils.round2significant(d, 4, direction='up')
variable["Attr"]["valid_range"] = repr(mn) + "," + repr(mx)
pfp_utils.CreateVariable(ds, variable)
return
def do_levels_batch(cf_batch):
batch_log_path = pfp_log.get_batch_log_path("logfiles")
batch_log_file_name = os.path.join(batch_log_path, "batch.log")
logger = pfp_log.init_logger("pfp_log", batch_log_file_name, to_file=True, to_screen=True)
start = datetime.datetime.now()
msg = " Started batch processing at " + start.strftime("%Y%m%d%H%M")
logger.info(msg)
if "Options" in cf_batch:
if "levels" in cf_batch["Options"]:
levels = pfp_cfg.cfg_string_to_list(cf_batch["Options"]["levels"])
else:
msg = "No 'levels' entry found in [Options] section"
logger.error(msg)
sys.exit()
else:
msg = "No [Options] section in control file"
logger.error(msg)
sys.exit()
processing_levels = ["l1", "l2", "l3",
"ecostress", "fluxnet", "reddyproc",
"concatenate", "climatology",
"cpd1", "cpd2", "mpt",
"l4", "l5", "l6"]
for level in levels:
if level.lower() not in processing_levels:
msg = "Unrecognised level " + level
logger.warning(msg)
continue
if level.lower() == "l1":
# L1 processing
do_L1_batch(cf_batch["Levels"][level])
elif level.lower() == "l2":
# L2 processing
do_L2_batch(cf_batch["Levels"][level])
elif level.lower() == "l3":
# L3 processing
do_L3_batch(cf_batch["Levels"][level])
elif level.lower() == "ecostress":
# convert netCDF files to ECOSTRESS CSV files
do_ecostress_batch(cf_batch["Levels"][level])
elif level.lower() == "fluxnet":
# convert netCDF files to FluxNet CSV files
do_fluxnet_batch(cf_batch["Levels"][level])
elif level.lower() == "reddyproc":
# convert netCDF files to REddyProc CSV files
do_reddyproc_batch(cf_batch["Levels"][level])
elif level.lower() == "concatenate":
# concatenate netCDF files
do_concatenate_batch(cf_batch["Levels"][level])
elif level.lower() == "climatology":
# climatology
do_climatology_batch(cf_batch["Levels"][level])
elif level.lower() == "cpd1":
# ustar threshold from change point detection
do_cpd1_batch(cf_batch["Levels"][level])
elif level.lower() == "cpd2":
# ustar threshold from change point detection
do_cpd2_batch(cf_batch["Levels"][level])
elif level.lower() == "mpt":
# ustar threshold from change point detection
do_mpt_batch(cf_batch["Levels"][level])
elif level.lower() == "l4":
# L4 processing
do_L4_batch(cf_batch["Levels"][level])
elif level.lower() == "l5":
# L5 processing
do_L5_batch(cf_batch["Levels"][level])
elif level.lower() == "l6":
# L6 processing
do_L6_batch(cf_batch["Levels"][level])
logger = pfp_log.change_logger_filename("pfp_log", "batch")
end = datetime.datetime.now()
msg = " Finished batch processing at " + end.strftime("%Y%m%d%H%M")
logger.info(msg)
return
def do_levels_batch(cf_batch):
if "Options" in cf_batch:
if "levels" in cf_batch["Options"]:
levels = pfp_cfg.cfg_string_to_list(cf_batch["Options"]["levels"])
else:
msg = "No 'levels' entry found in [Options] section"
logger.error(msg)
sys.exit()
else:
msg = "No [Options] section in control file"
logger.error(msg)
sys.exit()
processing_levels = ["l1", "l2", "l3",
"ecostress", "fluxnet", "reddyproc",
"concatenate", "climatology",
"cpd", "mpt",
"l4", "l5", "l6"]
for level in levels:
if level.lower() not in processing_levels:
msg = "Unrecognised level " + level
logger.warning(msg)
continue
if level.lower() == "l1":
# L1 processing
do_L1_batch(cf_batch["Levels"][level])
elif level.lower() == "l2":
# L2 processing
do_L2_batch(cf_batch["Levels"][level])
elif level.lower() == "l3":
# L3 processing
do_L3_batch(cf_batch["Levels"][level])
elif level.lower() == "ecostress":
# convert netCDF files to ECOSTRESS CSV files
do_ecostress_batch(cf_batch["Levels"][level])
elif level.lower() == "fluxnet":
# convert netCDF files to FluxNet CSV files
do_fluxnet_batch(cf_batch["Levels"][level])
elif level.lower() == "reddyproc":
# convert netCDF files to REddyProc CSV files
do_reddyproc_batch(cf_batch["Levels"][level])
elif level.lower() == "concatenate":
# concatenate netCDF files
do_concatenate_batch(cf_batch["Levels"][level])
elif level.lower() == "climatology":
# climatology
do_climatology_batch(cf_batch["Levels"][level])
elif level.lower() == "cpd":
# ustar threshold from change point detection
do_cpd_batch(cf_batch["Levels"][level])
elif level.lower() == "mpt":
# ustar threshold from change point detection
do_mpt_batch(cf_batch["Levels"][level])
elif level.lower() == "l4":
# L4 processing
do_L4_batch(cf_batch["Levels"][level])
elif level.lower() == "l5":
# L5 processing
do_L5_batch(cf_batch["Levels"][level])
elif level.lower() == "l6":
# L6 processing
do_L6_batch(cf_batch["Levels"][level])
return