def mgCO2pm3_to_umolpmol(ds, MF_out, CO2_in, Ta_in, ps_in): """ Purpose: Calculate CO2 mole fraction in uml/mol from mass density in mgCO2/m3. Usage: pfp_func_units.mgCO2pm3_to_umolpmol(ds, MF_out, CO2_in, Ta_in, ps_in) Author: PRI Date: August 2019 """ nRecs = int(ds.globalattributes["nc_nrecs"]) zeros = numpy.zeros(nRecs, dtype=numpy.int32) ones = numpy.ones(nRecs, dtype=numpy.int32) for item in [CO2_in, Ta_in, ps_in]: if item not in ds.series.keys(): msg = " Requested series " + item + " not found, " + MF_out + " not calculated" logger.error(msg) return 0 CO2 = pfp_utils.GetVariable(ds, CO2_in) CO2 = pfp_utils.convert_units_func(ds, CO2, "mg/m^3") Ta = pfp_utils.GetVariable(ds, Ta_in) Ta = pfp_utils.convert_units_func(ds, Ta, "degC") ps = pfp_utils.GetVariable(ds, ps_in) ps = pfp_utils.convert_units_func(ds, ps, "kPa") MF = pfp_utils.GetVariable(ds, MF_out) MF["Data"] = pfp_mf.co2_ppmfrommgCO2pm3(CO2["Data"], Ta["Data"], ps["Data"]) MF["Flag"] = numpy.where( numpy.ma.getmaskarray(MF["Data"]) == True, ones, zeros) MF["Attr"]["units"] = "umol/mol" pfp_utils.CreateVariable(ds, MF) return 1
def do_EPQCFlagCheck(cf, ds, section, series, code=9): """ Purpose: Mask data according to the value of an EddyPro QC flag. Usage: Author: PRI Date: August 2017 """ if 'EPQCFlagCheck' not in cf[section][series].keys(): return nRecs = int(ds.globalattributes["nc_nrecs"]) flag = numpy.zeros(nRecs, dtype=numpy.int32) source_list = ast.literal_eval( cf[section][series]['EPQCFlagCheck']["Source"]) reject_list = ast.literal_eval( cf[section][series]['EPQCFlagCheck']["Reject"]) variable = pfp_utils.GetVariable(ds, series) for source in source_list: epflag = pfp_utils.GetVariable(ds, source) for value in reject_list: bool_array = numpy.isclose(epflag["Data"], float(value)) idx = numpy.where(bool_array == True)[0] flag[idx] = numpy.int32(1) idx = numpy.where(flag == 1)[0] variable["Data"][idx] = numpy.float(c.missing_value) variable["Flag"][idx] = numpy.int32(9) pfp_utils.CreateVariable(ds, variable) return
def percent_to_mmolpmol(ds, MF_out, RH_in, Ta_in, ps_in): """ Purpose: Calculate H2O mole fraction from relative humidity (RH). """ nRecs = int(ds.globalattributes["nc_nrecs"]) zeros = numpy.zeros(nRecs, dtype=numpy.int32) ones = numpy.ones(nRecs, dtype=numpy.int32) for item in [RH_in, Ta_in, ps_in]: if item not in list(ds.series.keys()): msg = " Requested series " + item + " not found, " + MF_out + " not calculated" logger.error(msg) return 0 # get the relative humidity and check the units RH = pfp_utils.GetVariable(ds, RH_in) RH = pfp_utils.convert_units_func(ds, RH, "percent") # get the temperature and check the units Ta = pfp_utils.GetVariable(ds, Ta_in) Ta = pfp_utils.convert_units_func(ds, Ta, "degC") # get the absoulte humidity AH_data = pfp_mf.absolutehumidityfromRH(Ta["Data"], RH["Data"]) # get the atmospheric pressure and check the units ps = pfp_utils.GetVariable(ds, ps_in) ps = pfp_utils.convert_units_func(ds, ps, "kPa") # get the output variable (created in pfp_ts.DoFunctions()) MF = pfp_utils.GetVariable(ds, MF_out) # do the business MF["Data"] = pfp_mf.h2o_mmolpmolfromgpm3(AH_data, Ta["Data"], ps["Data"]) MF["Flag"] = numpy.where( numpy.ma.getmaskarray(MF["Data"]) == True, ones, zeros) MF["Attr"]["units"] = "mmol/mol" # put the output variable back into the data structure pfp_utils.CreateVariable(ds, MF) return 1
def gH2Opm3_to_percent(ds, RH_out, AH_in, Ta_in): """ Purpose: Function to convert absolute humidity in units of g/m^3 to relative humidity in percent. Usage: pfp_func_units.gH2Opm3_to_percent(ds, RH_out, AH_in, Ta_in) Author: PRI Date: September 2020 """ nRecs = int(ds.globalattributes["nc_nrecs"]) zeros = numpy.zeros(nRecs, dtype=numpy.int32) ones = numpy.ones(nRecs, dtype=numpy.int32) for item in [AH_in, Ta_in]: if item not in ds.series.keys(): msg = " Requested series " + item + " not found, " + RH_out + " not calculated" logger.error(msg) return 0 AH = pfp_utils.GetVariable(ds, AH_in) Ta = pfp_utils.GetVariable(ds, Ta_in) RH = pfp_utils.GetVariable(ds, RH_out) RH["Data"] = pfp_mf.relativehumidityfromabsolutehumidity( AH["Data"], Ta["Data"]) RH["Flag"] = numpy.where( numpy.ma.getmaskarray(RH["Data"]) == True, ones, zeros) RH["Attr"]["units"] = "percent" pfp_utils.CreateVariable(ds, RH) return 1
def gfMDS_mask_long_gaps(ds, mds_label, l5_info, called_by): """ Purpose: Mask gaps that are longer than a specified maximum length. Usage: Side effects: Author: PRI Date: June 2019 """ if "MaxShortGapRecords" not in l5_info[called_by]["info"]: return max_short_gap_days = l5_info[called_by]["info"]["MaxShortGapDays"] msg = " Masking gaps longer than " + str(max_short_gap_days) + " days" logger.info(msg) label = l5_info[called_by]["outputs"][mds_label]["target"] target = pfp_utils.GetVariable(ds, label) variable = pfp_utils.GetVariable(ds, mds_label) mask = numpy.ma.getmaskarray(target["Data"]) # start and stop indices of contiguous blocks max_short_gap_records = l5_info[called_by]["info"]["MaxShortGapRecords"] gap_start_end = pfp_utils.contiguous_regions(mask) for start, stop in gap_start_end: gap_length = stop - start if gap_length > max_short_gap_records: variable["Data"][start: stop] = target["Data"][start: stop] variable["Flag"][start: stop] = target["Flag"][start: stop] # put data_int back into the data structure pfp_utils.CreateVariable(ds, variable) return
def gH2Opm3_to_mmolpmol(ds, MF_out, AH_in, Ta_in, ps_in): """ Purpose: Calculate H2O mole fraction in mml/mol from absolute humidity in g/m^3. Usage: pfp_func_units.gH2Opm3_to_mmolpmol(ds, MF_out, AH_in, Ta_in, ps_in) Author: PRI Date: August 2019 """ nRecs = int(ds.globalattributes["nc_nrecs"]) zeros = numpy.zeros(nRecs, dtype=numpy.int32) ones = numpy.ones(nRecs, dtype=numpy.int32) for item in [AH_in, Ta_in, ps_in]: if item not in ds.series.keys(): msg = " Requested series " + item + " not found, " + MF_out + " not calculated" logger.error(msg) return 0 AH = pfp_utils.GetVariable(ds, AH_in) AH = pfp_utils.convert_units_func(ds, AH, "g/m^3") Ta = pfp_utils.GetVariable(ds, Ta_in) Ta = pfp_utils.convert_units_func(ds, Ta, "degC") ps = pfp_utils.GetVariable(ds, ps_in) ps = pfp_utils.convert_units_func(ds, ps, "kPa") MF = pfp_utils.GetVariable(ds, MF_out) MF["Data"] = pfp_mf.h2o_mmolpmolfromgpm3(AH["Data"], Ta["Data"], ps["Data"]) MF["Flag"] = numpy.where( numpy.ma.getmaskarray(MF["Data"]) == True, ones, zeros) MF["Attr"]["units"] = "mmol/mol" pfp_utils.CreateVariable(ds, MF) return 1
def make_data_array(cf, ds, current_year): ldt = pfp_utils.GetVariable(ds, "DateTime") nrecs = int(ds.globalattributes["nc_nrecs"]) ts = int(ds.globalattributes["time_step"]) start = datetime.datetime(current_year, 1, 1, 0, 0, 0) + datetime.timedelta(minutes=ts) end = datetime.datetime(current_year + 1, 1, 1, 0, 0, 0) cdt = numpy.array([ dt for dt in pfp_utils.perdelta(start, end, datetime.timedelta( minutes=ts)) ]) mt = numpy.ones(len(cdt)) * float(-9999) mt_list = [cdt] + [mt for n in list(cf["Variables"].keys())] data = numpy.stack(mt_list, axis=-1) si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0) ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs) dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei) idx1, idx2 = pfp_utils.FindMatchingIndices(cdt, dt["Data"]) for n, cf_label in enumerate(list(cf["Variables"].keys())): label = cf["Variables"][cf_label]["name"] var = pfp_utils.GetVariable(ds, label, start=si, end=ei) data[idx1, n + 1] = var["Data"] # convert datetime to ISO dates data[:, 0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt]) return data
def gfMDS_get_mds_output(ds, mds_label, out_file_path, l5_info, called_by): """ Purpose: Reads the CSV file output by the MDS C code and puts the contents into the data structure. Usage: gfMDS_get_mds_output(ds, out_file_path, first_date, last_date, include_qc=False) where ds is a data structure out_file_path is the full path to the MDS output file Side effects: New series are created in the data structure to hold the MDS data. Author: PRI Date: May 2018 """ ldt = pfp_utils.GetVariable(ds, "DateTime") first_date = ldt["Data"][0] last_date = ldt["Data"][-1] data_mds = numpy.genfromtxt(out_file_path, delimiter=",", names=True, autostrip=True, dtype=None) dt_mds = numpy.array([dateutil.parser.parse(str(dt)) for dt in data_mds["TIMESTAMP"]]) si_mds = pfp_utils.GetDateIndex(dt_mds, first_date) ei_mds = pfp_utils.GetDateIndex(dt_mds, last_date) # get a list of the names in the data array mds_output_names = list(data_mds.dtype.names) # strip out the timestamp and the original data for item in ["TIMESTAMP", l5_info[called_by]["outputs"][mds_label]["target_mds"]]: if item in mds_output_names: mds_output_names.remove(item) # and now loop over the MDS output series for mds_output_name in mds_output_names: if mds_output_name == "FILLED": # get the gap filled target and write it to the data structure var_in = pfp_utils.GetVariable(ds, l5_info[called_by]["outputs"][mds_label]["target"]) data = data_mds[mds_output_name][si_mds:ei_mds+1] idx = numpy.where((numpy.ma.getmaskarray(var_in["Data"]) == True) & (abs(data - c.missing_value) > c.eps))[0] flag = numpy.array(var_in["Flag"]) flag[idx] = numpy.int32(40) attr = copy.deepcopy(var_in["Attr"]) attr["long_name"] = attr["long_name"]+", gap filled using MDS" var_out = {"Label":mds_label, "Data":data, "Flag":flag, "Attr":attr} pfp_utils.CreateVariable(ds, var_out) elif mds_output_name == "TIMEWINDOW": # make the series name for the data structure mds_qc_label = "MDS"+"_"+l5_info[called_by]["outputs"][mds_label]["target"]+"_"+mds_output_name data = data_mds[mds_output_name][si_mds:ei_mds+1] flag = numpy.zeros(len(data)) attr = {"long_name":"TIMEWINDOW from MDS gap filling for "+l5_info[called_by]["outputs"][mds_label]["target"]} var_out = {"Label":mds_qc_label, "Data":data, "Flag":flag, "Attr":attr} pfp_utils.CreateVariable(ds, var_out) else: # make the series name for the data structure mds_qc_label = "MDS"+"_"+l5_info[called_by]["outputs"][mds_label]["target"]+"_"+mds_output_name data = data_mds[mds_output_name][si_mds:ei_mds+1] flag = numpy.zeros(len(data)) attr = {"long_name":"QC field from MDS gap filling for "+l5_info[called_by]["outputs"][mds_label]["target"]} var_out = {"Label":mds_qc_label, "Data":data, "Flag":flag, "Attr":attr} pfp_utils.CreateVariable(ds, var_out) return
def consistent_Fc_storage(cfg, ds, site): """ Purpose: Make the various incarnations of single point Fc storage consistent. Author: PRI Date: November 2019 """ ## save Fc_single if it exists - debug only #labels = ds.series.keys() #if "Fc_single" in labels: #variable = pfp_utils.GetVariable(ds, "Fc_single") #variable["Label"] = "Fc_sinorg" #pfp_utils.CreateVariable(ds, variable) #pfp_utils.DeleteVariable(ds, "Fc_single") # do nothing if Fc_single exists labels = list(ds.series.keys()) if "Fc_single" in labels: pass # Fc_single may be called Fc_storage elif "Fc_storage" in labels: level = ds.globalattributes["nc_level"] descr = "description_" + level variable = pfp_utils.GetVariable(ds, "Fc_storage") if "using single point CO2 measurement" in variable["Attr"][descr]: variable["Label"] = "Fc_single" pfp_utils.CreateVariable(ds, variable) pfp_utils.DeleteVariable(ds, "Fc_storage") else: # neither Fc_single nor Fc_storage exist, try to calculate # check to see if the measurement height is defined zms = None CO2 = pfp_utils.GetVariable(ds, "CO2") if "height" in CO2["Attr"]: zms = pfp_utils.get_number_from_heightstring(CO2["Attr"]["height"]) if zms is None: xls_name = cfg["Files"]["site_information"] site_information = xl_read_site_information(xls_name, site) if len(site_information) != 0: s = site_information["IRGA"]["Height"] zms = pfp_utils.get_number_from_heightstring(s) else: while zms is None: file_name = cfg["Files"]["in_filename"] prompt = "Enter CO2 measuement height in metres" text, ok = QtWidgets.QInputDialog.getText( None, file_name, prompt, QtWidgets.QLineEdit.Normal, "") zms = pfp_utils.get_number_from_heightstring(text) # update the CO2 variable attribute CO2["Attr"]["height"] = zms pfp_utils.CreateVariable(ds, CO2) # calculate single point Fc storage term cf = {"Options": {"zms": zms}} pfp_ts.CalculateFcStorageSinglePoint(cf, ds) # convert Fc_single from mg/m2/s to umol/m2/s pfp_utils.CheckUnits(ds, "Fc_single", "umol/m2/s", convert_units=True) return
def mmolpm3_to_gH2Opm3(ds, AH_out, H2O_in): """ Purpose: Function to convert mmol/m^3 (molar density) to g/m^3 (mass density). Usage: pfp_func_units.mmolpm3_to_gpm3(ds, AH_out, H2O_in) Author: PRI Date: August 2020 """ for item in [H2O_in]: if item not in list(ds.series.keys()): msg = " Requested series " + item + " not found, " + AH_out + " not calculated" logger.error(msg) return 0 var_in = pfp_utils.GetVariable(ds, H2O_in) got_variance = False if var_in["Label"][-3:] == "_Vr" and var_in["Attr"][ "units"] == "mmol^2/m^6": got_variance = True var_in["Data"] = numpy.ma.sqrt(var_in["Data"]) var_in["Attr"]["units"] = "mmol/m^3" var_out = pfp_utils.convert_units_func(ds, var_in, "g/m^3", mode="quiet") var_out["Label"] = AH_out if got_variance: var_out["Data"] = var_out["Data"] * var_out["Data"] var_out["Attr"]["units"] = "g^2/m^6" pfp_utils.CreateVariable(ds, var_out) return 1
def mgCO2pm3_to_mmolpm3(ds, CO2_out, CO2_in): """ Purpose: Calculate CO2 molar density in mmol/m3 from CO2 concentration in mg/m3. Usage: pfp_func_units.mgCO2pm3_to_mmolpm3(ds, CO2_out, CO2_in) Author: PRI Date: September 2020 """ for item in [CO2_in]: if item not in ds.series.keys(): msg = " Requested series " + item + " not found, " + CO2_out + " not calculated" logger.error(msg) return 0 var_in = pfp_utils.GetVariable(ds, CO2_in) got_variance = False if var_in["Label"][-3:] == "_Vr" and var_in["Attr"]["units"] in [ "mg^2/m^6", "mgCO2^2/m^6" ]: got_variance = True var_in["Data"] = numpy.ma.sqrt(var_in["Data"]) var_in["Attr"]["units"] = "mg/m^3" var_out = pfp_utils.convert_units_func(ds, var_in, "mmol/m^3", mode="quiet") var_out["Label"] = CO2_out if got_variance: var_out["Data"] = var_out["Data"] * var_out["Data"] var_out["Attr"]["units"] = "mmol^2/m^6" pfp_utils.CreateVariable(ds, var_out) return 1
def gH2Opm3_to_mmolpm3(ds, H2O_out, AH_in): """ Purpose: Calculate H2O molar density in mmol/m^3 from absolute humidity in g/m^3. Usage: pfp_func_units.gH2Opm3_to_mmolpm3(ds, MD_out, AH_in) Author: PRI Date: September 2020 """ for item in [AH_in]: if item not in ds.series.keys(): msg = " Requested series " + item + " not found, " + H2O_out + " not calculated" logger.error(msg) return 0 var_in = pfp_utils.GetVariable(ds, AH_in) got_variance = False if var_in["Label"][-3:] == "_Vr" and var_in["Attr"]["units"] in [ "g^2/m^6", "gH2O^2/m^6" ]: got_variance = True var_in["Data"] = numpy.ma.sqrt(var_in["Data"]) var_in["Attr"]["units"] = "g/m^3" var_out = pfp_utils.convert_units_func(ds, var_in, "mmol/m^3", mode="quiet") var_out["Label"] = H2O_out if got_variance: var_out["Data"] = var_out["Data"] * var_out["Data"] var_out["Attr"]["units"] = "mmol^2/m^6" pfp_utils.CreateVariable(ds, var_out) return 1
def Variance_from_standard_deviation(ds, Vr_out, Sd_in): """ Purpose: Function to convert standard deviation to variance. Usage: pfp_func_statistics.Variance_from_standard_deviation(ds, Vr_out, Sd_in) Author: PRI Date: October 2020 """ sd_units = {"mg/m3": "mg^2/m^6", "mmol/m^3": "mmol^2/m^6", "g/m^3": "g^2/m^6", "degC": "degC^2", "K": "K^2", "m/s": "m^2/s^2"} sd = pfp_utils.GetVariable(ds, Sd_in) if sd["Attr"]["units"] not in list(sd_units.keys()): msg = " Unrecognised units (" + sd["Attr"]["units"] + ") for variable " + Sd_in logger.error(msg) msg = " Variance not calculated from standard deviation" logger.error(msg) return 0 vr = copy.deepcopy(sd) vr["Label"] = Vr_out vr["Data"] = sd["Data"]*sd["Data"] vr["Attr"]["units"] = sd_units[sd["Attr"]["units"]] if "statistic_type" in vr["Attr"]: vr["Attr"]["statistic_type"] = "variance" pfp_utils.CreateVariable(ds, vr) return 1
def rpLL_createdict(cf, ds, l6_info, output, called_by, flag_code): """ Purpose: Creates a dictionary in ds to hold information about estimating ecosystem respiration using the Lasslop method. Usage: Side effects: Author: PRI Date April 2016 """ nrecs = int(ds.globalattributes["nc_nrecs"]) # create the Lasslop settings directory if called_by not in l6_info.keys(): l6_info[called_by] = {"outputs": {}, "info": {}, "gui": {}} # get the info section rpLL_createdict_info(cf, ds, l6_info[called_by], called_by) if ds.returncodes["value"] != 0: return # get the outputs section rpLL_createdict_outputs(cf, l6_info[called_by], output, called_by, flag_code) # create an empty series in ds if the output series doesn't exist yet Fc = pfp_utils.GetVariable(ds, l6_info[called_by]["info"]["source"]) model_outputs = cf["EcosystemRespiration"][output][called_by].keys() for model_output in model_outputs: if model_output not in ds.series.keys(): # create an empty variable variable = pfp_utils.CreateEmptyVariable(model_output, nrecs) variable["Attr"]["long_name"] = "Ecosystem respiration" variable["Attr"]["drivers"] = l6_info[called_by]["outputs"][model_output]["drivers"] variable["Attr"]["description_l6"] = "Modeled by Lasslop et al. (2010)" variable["Attr"]["target"] = l6_info[called_by]["info"]["target"] variable["Attr"]["source"] = l6_info[called_by]["info"]["source"] variable["Attr"]["units"] = Fc["Attr"]["units"] pfp_utils.CreateVariable(ds, variable) return
def Standard_deviation_from_variance(ds, Sd_out, Vr_in): """ Purpose: Function to convert variance to standard deviation. Usage: pfp_func_statistics.Standard_deviation_from_variance(ds, Sd_out, Vr_in) Author: PRI Date: October 2020 """ vr_units = {"mg^2/m^6": "mg/m3", "mmol^2/m^6": "mmol/m^3", "g^2/m^6": "g/m^3", "degC^2": "degC", "K^2": "K", "m^2/s^2": "m/s"} vr = pfp_utils.GetVariable(ds, Vr_in) if vr["Attr"]["units"] not in list(vr_units.keys()): msg = " Unrecognised units (" + vr["Attr"]["units"] + ") for variable " + Vr_in logger.error(msg) msg = " Standard deviation not calculated from variance" logger.error(msg) return 0 sd = copy.deepcopy(vr) sd["Label"] = Sd_out sd["Data"] = numpy.ma.sqrt(vr["Data"]) sd["Attr"]["units"] = vr_units[vr["Attr"]["units"]] if "statistic_type" in sd["Attr"]: sd["Attr"]["statistic_type"] = "standard deviation" pfp_utils.CreateVariable(ds, sd) return 1
def change_variable_attributes(cfg, ds): """ Purpose: Clean up the variable attributes. Usage: Author: PRI Date: November 2018 """ # rename existing long_name to description, introduce a # consistent long_name attribute and introduce the group_name # attribute vattr_list = list(cfg["variable_attributes"].keys()) series_list = list(ds.series.keys()) descr = "description_" + ds.globalattributes["nc_level"] for label in series_list: variable = pfp_utils.GetVariable(ds, label) variable["Attr"][descr] = copy.deepcopy(variable["Attr"]["long_name"]) for item in vattr_list: if label[:len(item)] == item: for key in list(cfg["variable_attributes"][item].keys()): variable["Attr"][key] = cfg["variable_attributes"][item][ key] pfp_utils.CreateVariable(ds, variable) # parse variable attributes to new format, remove deprecated variable attributes # and fix valid_range == "-1e+35,1e+35" tmp = cfg["variable_attributes"]["deprecated"] deprecated = pfp_cfg.cfg_string_to_list(tmp) series_list = list(ds.series.keys()) for label in series_list: variable = pfp_utils.GetVariable(ds, label) # parse variable attributes to new format variable["Attr"] = parse_variable_attributes(variable["Attr"]) # remove deprecated variable attributes for vattr in deprecated: if vattr in list(variable["Attr"].keys()): del variable["Attr"][vattr] # fix valid_range == "-1e+35,1e+35" if "valid_range" in variable["Attr"]: valid_range = variable["Attr"]["valid_range"] if valid_range == "-1e+35,1e+35": d = numpy.ma.min(variable["Data"]) mn = pfp_utils.round2significant(d, 4, direction='down') d = numpy.ma.max(variable["Data"]) mx = pfp_utils.round2significant(d, 4, direction='up') variable["Attr"]["valid_range"] = repr(mn) + "," + repr(mx) pfp_utils.CreateVariable(ds, variable) return
def gfMDS_make_data_array(ds, current_year, info): """ Purpose: Create a data array for the MDS gap filling routine. The array constructed here will be written to a CSV file that is read by the MDS C code. Usage: Side Effects: The constructed data arrays are full years. That is they run from YYYY-01-01 00:30 to YYYY+1-01-01 00:00. Missing data is represented as -9999. Author: PRI Date: May 2018 """ ldt = pfp_utils.GetVariable(ds, "DateTime") nrecs = int(ds.globalattributes["nc_nrecs"]) ts = int(ds.globalattributes["time_step"]) start = datetime.datetime(current_year, 1, 1, 0, 30, 0) end = datetime.datetime(current_year + 1, 1, 1, 0, 0, 0) cdt = numpy.array([ dt for dt in pfp_utils.perdelta(start, end, datetime.timedelta( minutes=ts)) ]) mt = numpy.ones(len(cdt)) * float(-9999) # need entry for the timestamp and the target ... array_list = [cdt, mt] # ... and entries for the drivers for driver in info["drivers"]: array_list.append(mt) # now we can create the data array data = numpy.stack(array_list, axis=-1) si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0) ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs) dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei) idx1, _ = pfp_utils.FindMatchingIndices(cdt, dt["Data"]) pfp_label_list = [info["target"]] + info["drivers"] mds_label_list = [info["target_mds"]] + info["drivers_mds"] header = "TIMESTAMP" fmt = "%12i" for n, label in enumerate(pfp_label_list): var = pfp_utils.GetVariable(ds, label, start=si, end=ei) data[idx1, n + 1] = var["Data"] header = header + "," + mds_label_list[n] fmt = fmt + "," + "%f" # convert datetime to ISO dates data[:, 0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt]) return data, header, fmt
def make_data_array(ds, current_year): ldt = pfp_utils.GetVariable(ds, "DateTime") nrecs = ds.globalattributes["nc_nrecs"] ts = int(ds.globalattributes["time_step"]) start = datetime.datetime(current_year,1,1,0,30,0) end = datetime.datetime(current_year+1,1,1,0,0,0) cdt = numpy.array([dt for dt in pfp_utils.perdelta(start, end, datetime.timedelta(minutes=ts))]) mt = numpy.ones(len(cdt))*float(-9999) data = numpy.stack([cdt, mt, mt, mt, mt, mt, mt, mt], axis=-1) si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0) ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs) dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei) idx1, idx2 = pfp_utils.FindMatchingIndices(cdt, dt["Data"]) for n, label in enumerate(["Fc", "VPD", "ustar", "Ta", "Fsd", "Fh", "Fe"]): var = pfp_utils.GetVariable(ds, label, start=si, end=ei) data[idx1,n+1] = var["Data"] # convert datetime to ISO dates data[:,0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt]) return data
def kgpm3_to_gpm3(ds, AH_out, AH_in): """ Purpose: Function to convert absolute humidity from kg/m^3 to g/m^3. Usage: pfp_func_units.kgpm3_to_gpm3(ds, AH_out, AH_in) Author: PRI Date: August 2020 """ var_in = pfp_utils.GetVariable(ds, AH_in) var_out = pfp_utils.convert_units_func(ds, var_in, "g/m^3", mode="quiet") var_out["Label"] = AH_out pfp_utils.CreateVariable(ds, var_out) return 1
def ConverthPa2kPa(ds, ps_out, ps_in): """ Purpose: Function to convert pressure from hPa (mb) to kPa. Usage: pfp_func.ConverthPa2kPa(ds, ps_in, ps_out) Author: PRI Date: February 2018 """ var_in = pfp_utils.GetVariable(ds, ps_in) var_out = pfp_utils.convert_units_func(ds, var_in, "kPa", mode="quiet") var_out["Label"] = ps_out pfp_utils.CreateVariable(ds, var_out) return 1
def ConvertPercent2m3pm3(ds, Sws_out, Sws_in): """ Purpose: Function to convert Sws in units of "percent" (1 to 100) to "frac" (0 to 1). Usage: pfp_func.ConvertPercent2m3pm3(ds, Sws_out, Sws_in) Author: PRI Date: April 2020 """ var_in = pfp_utils.GetVariable(ds, Sws_in) var_out = pfp_utils.convert_units_func(ds, var_in, "m3/m3", mode="quiet") var_out["Label"] = Sws_out pfp_utils.CreateVariable(ds, var_out) return
def ConvertRHtoPercent(ds, RH_out, RH_in): """ Purpose: Function to convert RH in units of "frac" (0 to 1) to "percent" (1 to 100). Usage: pfp_func.ConvertRHtoPercent(ds, RH_out, RH_in) Author: PRI Date: August 2019 """ var_in = pfp_utils.GetVariable(ds, RH_in) var_out = pfp_utils.convert_units_func(ds, var_in, "%", mode="quiet") var_out["Label"] = RH_out pfp_utils.CreateVariable(ds, var_out) return
def compare_eddypro(): epname = pfp_io.get_filename_dialog(title='Choose an EddyPro full output file') ofname = pfp_io.get_filename_dialog(title='Choose an L3 output file') ds_ep = pfp_io.read_eddypro_full(epname) ds_of = pfp_io.nc_read_series(ofname) dt_ep = ds_ep.series['DateTime']['Data'] dt_of = ds_of.series['DateTime']['Data'] start_datetime = max([dt_ep[0],dt_of[0]]) end_datetime = min([dt_ep[-1],dt_of[-1]]) si_of = pfp_utils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact') ei_of = pfp_utils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact') si_ep = pfp_utils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact') ei_ep = pfp_utils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact') us_of = pfp_utils.GetVariable(ds_of,'ustar',start=si_of,end=ei_of) us_ep = pfp_utils.GetVariable(ds_ep,'ustar',start=si_ep,end=ei_ep) Fh_of = pfp_utils.GetVariable(ds_of,'Fh',start=si_of,end=ei_of) Fh_ep = pfp_utils.GetVariable(ds_ep,'Fh',start=si_ep,end=ei_ep) Fe_of = pfp_utils.GetVariable(ds_of,'Fe',start=si_of,end=ei_of) Fe_ep = pfp_utils.GetVariable(ds_ep,'Fe',start=si_ep,end=ei_ep) Fc_of = pfp_utils.GetVariable(ds_of,'Fc',start=si_of,end=ei_of) Fc_ep = pfp_utils.GetVariable(ds_ep,'Fc',start=si_ep,end=ei_ep) # copy the range check values from the OFQC attributes to the EP attributes for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]): for item in ["rangecheck_upper", "rangecheck_lower"]: if item in of["Attr"]: ep["Attr"][item] = of["Attr"][item] # apply QC to the EddyPro data pfp_ck.ApplyRangeCheckToVariable(us_ep) pfp_ck.ApplyRangeCheckToVariable(Fc_ep) pfp_ck.ApplyRangeCheckToVariable(Fe_ep) pfp_ck.ApplyRangeCheckToVariable(Fh_ep) # plot the comparison plt.ion() fig = plt.figure(1,figsize=(8,8)) pfp_plot.xyplot(us_ep["Data"],us_of["Data"],sub=[2,2,1],regr=2,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)') pfp_plot.xyplot(Fh_ep["Data"],Fh_of["Data"],sub=[2,2,2],regr=2,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)') pfp_plot.xyplot(Fe_ep["Data"],Fe_of["Data"],sub=[2,2,3],regr=2,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)') pfp_plot.xyplot(Fc_ep["Data"],Fc_of["Data"],sub=[2,2,4],regr=2,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)') plt.tight_layout() plt.draw() plt.ioff()
def percent_to_gH2Opm3(ds, AH_out, RH_in, Ta_in): """ Purpose: Function to calculate absolute humidity given relative humidity and air temperature. Absolute humidity is not calculated if any of the input series are missing or if the specified output series already exists in the data structure. The calculated absolute humidity is created as a new series in the data structure. Usage: pfp_func_units.percent_to_gpm3(ds,"AH_HMP_2m","RH_HMP_2m","Ta_HMP_2m") Author: PRI Date: September 2015 """ nRecs = int(ds.globalattributes["nc_nrecs"]) zeros = numpy.zeros(nRecs, dtype=numpy.int32) ones = numpy.ones(nRecs, dtype=numpy.int32) for item in [RH_in, Ta_in]: if item not in ds.series.keys(): msg = " Requested series " + item + " not found, " + AH_out + " not calculated" logger.error(msg) return 0 # get the relative humidity and check the units RH = pfp_utils.GetVariable(ds, RH_in) RH = pfp_utils.convert_units_func(ds, RH, "percent") # get the temperature and check the units Ta = pfp_utils.GetVariable(ds, Ta_in) Ta = pfp_utils.convert_units_func(ds, Ta, "degC") # get the absolute humidity AH = pfp_utils.GetVariable(ds, AH_out) AH["Data"] = pfp_mf.absolutehumidityfromRH(Ta["Data"], RH["Data"]) AH["Flag"] = numpy.where( numpy.ma.getmaskarray(AH["Data"]) == True, ones, zeros) AH["Attr"]["units"] = "g/m^3" pfp_utils.CreateVariable(ds, AH) return 1
def mmolpmol_to_gH2Opm3(ds, AH_out, MF_in, Ta_in, ps_in): """ Purpose: Function to calculate absolute humidity given the water vapour mole fraction, air temperature and pressure. Absolute humidity is not calculated if any of the input series are missing or if the specified output series already exists in the data structure. The calculated absolute humidity is created as a new series in the data structure. Usage: pfp_func_units.mmolpmol_to_gpm3(ds,"AH_IRGA_Av","H2O_IRGA_Av","Ta_HMP_2m","ps") Author: PRI Date: September 2015 """ nRecs = int(ds.globalattributes["nc_nrecs"]) zeros = numpy.zeros(nRecs, dtype=numpy.int32) ones = numpy.ones(nRecs, dtype=numpy.int32) for item in [MF_in, Ta_in, ps_in]: if item not in list(ds.series.keys()): msg = " Requested series " + item + " not found, " + AH_out + " not calculated" logger.error(msg) return 0 MF = pfp_utils.GetVariable(ds, MF_in) MF = pfp_utils.convert_units_func(ds, MF, "mmol/mol") Ta = pfp_utils.GetVariable(ds, Ta_in) Ta = pfp_utils.convert_units_func(ds, Ta, "degC") ps = pfp_utils.GetVariable(ds, ps_in) ps = pfp_utils.convert_units_func(ds, ps, "kPa") AH = pfp_utils.GetVariable(ds, AH_out) AH["Data"] = pfp_mf.h2o_gpm3frommmolpmol(MF["Data"], Ta["Data"], ps["Data"]) AH["Flag"] = numpy.where( numpy.ma.getmaskarray(AH["Data"]) == True, ones, zeros) AH["Attr"]["units"] = "g/m^3" pfp_utils.CreateVariable(ds, AH) return 1
def rpLL_createdict(cf, ds, l6_info, output, called_by, flag_code): """ Purpose: Creates a dictionary in ds to hold information about estimating ecosystem respiration using the Lasslop method. Usage: Side effects: Author: PRI Date April 2016 """ nrecs = int(ds.globalattributes["nc_nrecs"]) # make the L6 "description" attrubute for the target variable descr_level = "description_" + ds.globalattributes["nc_level"] # create the Lasslop settings directory if called_by not in list(l6_info.keys()): l6_info[called_by] = {"outputs": {}, "info": {}, "gui": {}} # get the info section rpLL_createdict_info(cf, ds, l6_info[called_by], called_by) if ds.returncodes["value"] != 0: return # get the outputs section rpLL_createdict_outputs(cf, l6_info[called_by], output, called_by, flag_code) # create an empty series in ds if the output series doesn't exist yet Fco2 = pfp_utils.GetVariable(ds, l6_info[called_by]["info"]["source"]) model_outputs = list(cf["EcosystemRespiration"][output][called_by].keys()) for model_output in model_outputs: if model_output not in list(ds.series.keys()): l6_info["RemoveIntermediateSeries"]["not_output"].append( model_output) # create an empty variable variable = pfp_utils.CreateEmptyVariable(model_output, nrecs) variable["Attr"]["long_name"] = "Ecosystem respiration" variable["Attr"]["drivers"] = l6_info[called_by]["outputs"][ model_output]["drivers"] variable["Attr"][descr_level] = "Modeled by Lasslop et al. (2010)" variable["Attr"]["target"] = l6_info[called_by]["info"]["target"] variable["Attr"]["source"] = l6_info[called_by]["info"]["source"] variable["Attr"]["units"] = Fco2["Attr"]["units"] pfp_utils.CreateVariable(ds, variable) # intermediate series to be deleted for item in [ "alpha_LL", "beta_LL", "E0_LL", "k_LL", "rb_LL", "NEE_LL_all", "GPP_LL_all" ]: l6_info["RemoveIntermediateSeries"]["not_output"].append(item) return
def rpLT_createdict(cf, ds, l6_info, output, called_by, flag_code): """ Purpose: Creates a dictionary in ds to hold information about estimating ecosystem respiration using the Lloyd-Taylor method. Usage: Author: PRI Date October 2015 """ nrecs = int(ds.globalattributes["nc_nrecs"]) # make the L6 "description" attrubute for the target variable descr_level = "description_" + ds.globalattributes["nc_level"] # create the LT settings directory if called_by not in l6_info.keys(): l6_info[called_by] = {"outputs": {}, "info": {"source": "Fc", "target": "ER"}, "gui": {}} # get the info section rpLT_createdict_info(cf, ds, l6_info[called_by], called_by) if ds.returncodes["value"] != 0: return # get the outputs section rpLT_createdict_outputs(cf, l6_info[called_by], output, called_by, flag_code) # create an empty series in ds if the output series doesn't exist yet Fc = pfp_utils.GetVariable(ds, l6_info[called_by]["info"]["source"]) model_outputs = cf["EcosystemRespiration"][output][called_by].keys() for model_output in model_outputs: if model_output not in ds.series.keys(): l6_info["RemoveIntermediateSeries"]["not_output"].append(model_output) # create an empty variable variable = pfp_utils.CreateEmptyVariable(model_output, nrecs) variable["Attr"]["long_name"] = "Ecosystem respiration" variable["Attr"]["drivers"] = l6_info[called_by]["outputs"][model_output]["drivers"] variable["Attr"][descr_level] = "Modeled by Lloyd-Taylor" variable["Attr"]["target"] = l6_info[called_by]["info"]["target"] variable["Attr"]["source"] = l6_info[called_by]["info"]["source"] variable["Attr"]["units"] = Fc["Attr"]["units"] pfp_utils.CreateVariable(ds, variable) return
def ConvertK2C(ds, T_out, T_in): """ Purpose: Function to convert temperature from K to C. Usage: pfp_func.ConvertK2C(ds, T_out, T_in) Author: PRI Date: February 2018 """ if T_in not in ds.series.keys(): msg = " ConvertK2C: variable " + T_in + " not found, skipping ..." logger.warning(msg) return 0 if "<" in T_out or ">" in T_out: logger.warning(" ***") msg = " *** " + T_in + ": illegal name (" + T_out + ") in function, skipping ..." logger.warning(msg) logger.warning(" ***") return 0 var_in = pfp_utils.GetVariable(ds, T_in) var_out = pfp_utils.convert_units_func(ds, var_in, "C", mode="quiet") var_out["Label"] = T_out pfp_utils.CreateVariable(ds, var_out) return 1
def run_mpt_code(ds, nc_file_name): ldt = pfp_utils.GetVariable(ds, "DateTime") out_file_paths = {} header = "TIMESTAMP,NEE,VPD,USTAR,TA,SW_IN,H,LE" fmt = "%12i,%f,%f,%f,%f,%f,%f,%f" first_year = ldt["Data"][0].year last_year = ldt["Data"][-1].year log_file_path = os.path.join("mpt", "log", "mpt.log") mptlogfile = open(log_file_path, "wb") in_base_path = os.path.join("mpt", "input", "") out_base_path = os.path.join("mpt", "output", "") for current_year in range(first_year, last_year + 1): msg = " MPT: processing year " + str(current_year) logger.info(msg) in_name = nc_file_name.replace(".nc", "_" + str(current_year) + "_MPT.csv") in_full_path = os.path.join(in_base_path, in_name) out_full_path = in_full_path.replace("input", "output").replace( ".csv", "_ut.txt") data = make_data_array(ds, current_year) numpy.savetxt(in_full_path, data, header=header, delimiter=",", comments="", fmt=fmt) ustar_mp_exe = os.path.join(".", "mpt", "bin", "ustar_mp") cmd = [ ustar_mp_exe, "-input_path=" + in_full_path, "-output_path=" + out_base_path ] subprocess.call(cmd, stdout=mptlogfile) if os.path.isfile(out_full_path): out_file_paths[current_year] = out_full_path mptlogfile.close() return out_file_paths
def gfMDS_plot(pd, ds, mds_label, l5_info, called_by): """ Purpose: Plot the drivers, target and gap filled variable. Usage: Author: PRI Date: Back in the day """ ts = int(ds.globalattributes["time_step"]) drivers = l5_info[called_by]["outputs"][mds_label]["drivers"] target = l5_info[called_by]["outputs"][mds_label]["target"] Hdh = pfp_utils.GetVariable(ds, "Hdh") obs = pfp_utils.GetVariable(ds, target) mds = pfp_utils.GetVariable(ds, mds_label) if pd["show_plots"]: plt.ion() else: plt.ioff() if plt.fignum_exists(1): fig = plt.figure(1) plt.clf() else: fig = plt.figure(1, figsize=(13, 8)) fig.canvas.set_window_title(target) plt.figtext(0.5, 0.95, pd["title"], ha='center', size=16) # diurnal plot # XY plot of the diurnal variation rect1 = [0.10, pd["margin_bottom"], pd["xy_width"], pd["xy_height"]] ax1 = plt.axes(rect1) # get the diurnal stats of the observations mask = numpy.ma.mask_or(obs["Data"].mask, mds["Data"].mask) obs_mor = numpy.ma.array(obs["Data"], mask=mask) _, Hr1, Av1, _, _, _ = gf_getdiurnalstats(Hdh["Data"], obs_mor, ts) ax1.plot(Hr1, Av1, 'b-', label="Obs") # get the diurnal stats of all SOLO predictions _, Hr2, Av2, _, _, _ = gf_getdiurnalstats(Hdh["Data"], mds["Data"], ts) ax1.plot(Hr2, Av2, 'r-', label="MDS") plt.xlim(0, 24) plt.xticks([0, 6, 12, 18, 24]) ax1.set_ylabel(target) ax1.set_xlabel('Hour') ax1.legend(loc='upper right', frameon=False, prop={'size':8}) # histogram of window size time_window = pfp_utils.GetVariable(ds, "MDS_"+target+"_TIMEWINDOW") idx = numpy.where(mds["Flag"] == 40)[0] if len(idx) != 0: tw_hist_data = time_window["Data"][idx] rect2 = [0.40,pd["margin_bottom"],pd["xy_width"],pd["xy_height"]] ax2 = plt.axes(rect2) ax2.hist(tw_hist_data) ax2.set_ylabel("Occurrence") ax2.set_xlabel("MDS window length") # write statistics to the plot numpoints = numpy.ma.count(obs["Data"]) numfilled = numpy.ma.count(mds["Data"])-numpy.ma.count(obs["Data"]) plt.figtext(0.65,0.225,'No. points') plt.figtext(0.75,0.225,str(numpoints)) plt.figtext(0.65,0.200,'No. filled') plt.figtext(0.75,0.200,str(numfilled)) avg_obs = numpy.ma.mean(obs["Data"]) avg_mds = numpy.ma.mean(mds["Data"]) plt.figtext(0.65,0.175,'Avg (obs)') plt.figtext(0.75,0.175,'%.4g'%(avg_obs)) plt.figtext(0.65,0.150,'Avg (MDS)') plt.figtext(0.75,0.150,'%.4g'%(avg_mds)) var_obs = numpy.ma.var(obs["Data"]) var_mds = numpy.ma.var(mds["Data"]) plt.figtext(0.65,0.125,'Var (obs)') plt.figtext(0.75,0.125,'%.4g'%(var_obs)) plt.figtext(0.65,0.100,'Var (MDS)') plt.figtext(0.75,0.100,'%.4g'%(var_mds)) # time series of drivers and target ts_axes = [] rect = [pd["margin_left"],pd["ts_bottom"],pd["ts_width"],pd["ts_height"]] ts_axes.append(plt.axes(rect)) ts_axes[0].plot(obs["DateTime"], obs["Data"], 'b.', mds["DateTime"], mds["Data"], 'r-') ts_axes[0].set_xlim(obs["DateTime"][0], obs["DateTime"][-1]) TextStr = target+'_obs ('+obs['Attr']['units']+')' ts_axes[0].text(0.05,0.85,TextStr,color='b',horizontalalignment='left',transform=ts_axes[0].transAxes) TextStr = target+'('+mds['Attr']['units']+')' ts_axes[0].text(0.85,0.85,TextStr,color='r',horizontalalignment='right',transform=ts_axes[0].transAxes) for i, driver in enumerate(drivers): this_bottom = pd["ts_bottom"] + (i+1)*pd["ts_height"] rect = [pd["margin_left"], this_bottom, pd["ts_width"], pd["ts_height"]] ts_axes.append(plt.axes(rect, sharex=ts_axes[0])) drv = pfp_utils.GetVariable(ds, driver) drv_notgf = numpy.ma.masked_where(drv["Flag"] != 0, drv["Data"]) drv_gf = numpy.ma.masked_where(drv["Flag"] == 0, drv["Data"]) ts_axes[i+1].plot(drv["DateTime"], drv_notgf, 'b-') ts_axes[i+1].plot(drv["DateTime"], drv_gf, 'r-', linewidth=2) plt.setp(ts_axes[i+1].get_xticklabels(), visible=False) TextStr = driver+'('+drv['Attr']['units']+')' ts_axes[i+1].text(0.05,0.85,TextStr,color='b',horizontalalignment='left',transform=ts_axes[i+1].transAxes) # save a hard copy sdt = obs["DateTime"][0].strftime("%Y%m%d") edt = obs["DateTime"][-1].strftime("%Y%m%d") figure_name = pd["site_name"].replace(" ","") + "_MDS_" + pd["label"] figure_name = figure_name + "_" + sdt + "_" + edt + ".png" figure_path = os.path.join(l5_info[called_by]["info"]["plot_path"], figure_name) fig.savefig(figure_path, format='png') if pd["show_plots"]: plt.draw() #plt.pause(1) mypause(1) plt.ioff() else: plt.close(fig) plt.ion() return