def rpLL_createdict_outputs(cf, erll, target, called_by, flag_code): level = cf["level"] eo = erll["outputs"] # loop over the outputs listed in the control file section = "EcosystemRespiration" outputs = cf[section][target][called_by].keys() for output in outputs: # create the dictionary keys for this series eo[output] = {} # get the target sl = [section, target, called_by, output] eo[output]["target"] = pfp_utils.get_keyvaluefromcf(cf, sl, "target", default=target) eo[output]["source"] = pfp_utils.get_keyvaluefromcf(cf, sl, "source", default="Fc") # add the flag_code eo[output]["flag_code"] = flag_code # list of drivers opt = pfp_utils.get_keyvaluefromcf(cf, sl, "drivers", default="Ta") eo[output]["drivers"] = pfp_cfg.cfg_string_to_list(opt) opt = pfp_utils.get_keyvaluefromcf(cf, sl, "step_size_days", default=5) eo[output]["step_size_days"] = int(opt) opt = pfp_utils.get_keyvaluefromcf(cf, sl, "window_size_days", default=15) eo[output]["window_size_days"] = int(opt) opt = pfp_utils.get_keyvaluefromcf(cf, sl, "output_plots", default="False") eo[output]["output_plots"] = (opt == "True") opt = pfp_utils.get_keyvaluefromcf(cf, sl, "fsd_threshold", default=10) eo[output]["fsd_threshold"] = int(opt) # fit statistics for plotting later on eo[output]["results"] = {"startdate":[],"enddate":[],"No. points":[],"r":[], "Bias":[],"RMSE":[],"Frac Bias":[],"NMSE":[], "Avg (obs)":[],"Avg (LL)":[], "Var (obs)":[],"Var (LL)":[],"Var ratio":[], "m_ols":[],"b_ols":[]} return
def change_variable_attributes(cfg, ds): """ Purpose: Clean up the variable attributes. Usage: Author: PRI Date: November 2018 """ # rename existing long_name to description, introduce a # consistent long_name attribute and introduce the group_name # attribute vattr_list = list(cfg["variable_attributes"].keys()) series_list = list(ds.series.keys()) descr = "description_" + ds.globalattributes["nc_level"] for label in series_list: variable = pfp_utils.GetVariable(ds, label) variable["Attr"][descr] = copy.deepcopy(variable["Attr"]["long_name"]) for item in vattr_list: if label[:len(item)] == item: for key in list(cfg["variable_attributes"][item].keys()): variable["Attr"][key] = cfg["variable_attributes"][item][ key] pfp_utils.CreateVariable(ds, variable) # parse variable attributes to new format, remove deprecated variable attributes # and fix valid_range == "-1e+35,1e+35" tmp = cfg["variable_attributes"]["deprecated"] deprecated = pfp_cfg.cfg_string_to_list(tmp) series_list = list(ds.series.keys()) for label in series_list: variable = pfp_utils.GetVariable(ds, label) # parse variable attributes to new format variable["Attr"] = parse_variable_attributes(variable["Attr"]) # remove deprecated variable attributes for vattr in deprecated: if vattr in list(variable["Attr"].keys()): del variable["Attr"][vattr] # fix valid_range == "-1e+35,1e+35" if "valid_range" in variable["Attr"]: valid_range = variable["Attr"]["valid_range"] if valid_range == "-1e+35,1e+35": d = numpy.ma.min(variable["Data"]) mn = pfp_utils.round2significant(d, 4, direction='down') d = numpy.ma.max(variable["Data"]) mx = pfp_utils.round2significant(d, 4, direction='up') variable["Attr"]["valid_range"] = repr(mn) + "," + repr(mx) pfp_utils.CreateVariable(ds, variable) return
def do_levels_batch(cf_batch): batch_log_path = pfp_log.get_batch_log_path("logfiles") batch_log_file_name = os.path.join(batch_log_path, "batch.log") logger = pfp_log.init_logger("pfp_log", batch_log_file_name, to_file=True, to_screen=True) start = datetime.datetime.now() msg = " Started batch processing at " + start.strftime("%Y%m%d%H%M") logger.info(msg) if "Options" in cf_batch: if "levels" in cf_batch["Options"]: levels = pfp_cfg.cfg_string_to_list(cf_batch["Options"]["levels"]) else: msg = "No 'levels' entry found in [Options] section" logger.error(msg) sys.exit() else: msg = "No [Options] section in control file" logger.error(msg) sys.exit() processing_levels = ["l1", "l2", "l3", "ecostress", "fluxnet", "reddyproc", "concatenate", "climatology", "cpd1", "cpd2", "mpt", "l4", "l5", "l6"] for level in levels: if level.lower() not in processing_levels: msg = "Unrecognised level " + level logger.warning(msg) continue if level.lower() == "l1": # L1 processing do_L1_batch(cf_batch["Levels"][level]) elif level.lower() == "l2": # L2 processing do_L2_batch(cf_batch["Levels"][level]) elif level.lower() == "l3": # L3 processing do_L3_batch(cf_batch["Levels"][level]) elif level.lower() == "ecostress": # convert netCDF files to ECOSTRESS CSV files do_ecostress_batch(cf_batch["Levels"][level]) elif level.lower() == "fluxnet": # convert netCDF files to FluxNet CSV files do_fluxnet_batch(cf_batch["Levels"][level]) elif level.lower() == "reddyproc": # convert netCDF files to REddyProc CSV files do_reddyproc_batch(cf_batch["Levels"][level]) elif level.lower() == "concatenate": # concatenate netCDF files do_concatenate_batch(cf_batch["Levels"][level]) elif level.lower() == "climatology": # climatology do_climatology_batch(cf_batch["Levels"][level]) elif level.lower() == "cpd1": # ustar threshold from change point detection do_cpd1_batch(cf_batch["Levels"][level]) elif level.lower() == "cpd2": # ustar threshold from change point detection do_cpd2_batch(cf_batch["Levels"][level]) elif level.lower() == "mpt": # ustar threshold from change point detection do_mpt_batch(cf_batch["Levels"][level]) elif level.lower() == "l4": # L4 processing do_L4_batch(cf_batch["Levels"][level]) elif level.lower() == "l5": # L5 processing do_L5_batch(cf_batch["Levels"][level]) elif level.lower() == "l6": # L6 processing do_L6_batch(cf_batch["Levels"][level]) logger = pfp_log.change_logger_filename("pfp_log", "batch") end = datetime.datetime.now() msg = " Finished batch processing at " + end.strftime("%Y%m%d%H%M") logger.info(msg) return
def do_levels_batch(cf_batch): if "Options" in cf_batch: if "levels" in cf_batch["Options"]: levels = pfp_cfg.cfg_string_to_list(cf_batch["Options"]["levels"]) else: msg = "No 'levels' entry found in [Options] section" logger.error(msg) sys.exit() else: msg = "No [Options] section in control file" logger.error(msg) sys.exit() processing_levels = ["l1", "l2", "l3", "ecostress", "fluxnet", "reddyproc", "concatenate", "climatology", "cpd", "mpt", "l4", "l5", "l6"] for level in levels: if level.lower() not in processing_levels: msg = "Unrecognised level " + level logger.warning(msg) continue if level.lower() == "l1": # L1 processing do_L1_batch(cf_batch["Levels"][level]) elif level.lower() == "l2": # L2 processing do_L2_batch(cf_batch["Levels"][level]) elif level.lower() == "l3": # L3 processing do_L3_batch(cf_batch["Levels"][level]) elif level.lower() == "ecostress": # convert netCDF files to ECOSTRESS CSV files do_ecostress_batch(cf_batch["Levels"][level]) elif level.lower() == "fluxnet": # convert netCDF files to FluxNet CSV files do_fluxnet_batch(cf_batch["Levels"][level]) elif level.lower() == "reddyproc": # convert netCDF files to REddyProc CSV files do_reddyproc_batch(cf_batch["Levels"][level]) elif level.lower() == "concatenate": # concatenate netCDF files do_concatenate_batch(cf_batch["Levels"][level]) elif level.lower() == "climatology": # climatology do_climatology_batch(cf_batch["Levels"][level]) elif level.lower() == "cpd": # ustar threshold from change point detection do_cpd_batch(cf_batch["Levels"][level]) elif level.lower() == "mpt": # ustar threshold from change point detection do_mpt_batch(cf_batch["Levels"][level]) elif level.lower() == "l4": # L4 processing do_L4_batch(cf_batch["Levels"][level]) elif level.lower() == "l5": # L5 processing do_L5_batch(cf_batch["Levels"][level]) elif level.lower() == "l6": # L6 processing do_L6_batch(cf_batch["Levels"][level]) return