def calculate_ws_and_wd_from_u_and_v(ds):
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
U = pfp_utils.GetVariable(ds, "U")
V = pfp_utils.GetVariable(ds, "V")
Ws = pfp_utils.CreateEmptyVariable("Ws", nrecs,
datetime=U["DateTime"])
Wd = pfp_utils.CreateEmptyVariable("Wd", nrecs,
datetime=U["DateTime"])
# get the wind speed and direction from the components
Wd["Data"] = float(270) - (numpy.degrees(numpy.ma.arctan2(V["Data"], U["Data"])))
Wd["Data"] = numpy.ma.mod(Wd["Data"], 360)
Ws["Data"] = numpy.ma.sqrt(U["Data"]*U["Data"] + V["Data"]*V["Data"])
# mask wind direction when the wind speed is less than 0.01
Wd["Data"] = numpy.ma.masked_where(Ws["Data"] < 0.01, Wd["Data"])
# now set the QC flag
Ws["Flag"] = numpy.where(numpy.ma.getmaskarray(Ws["Data"]) == True, ones, zeros)
Wd["Flag"] = numpy.where(numpy.ma.getmaskarray(Wd["Data"]) == True, ones, zeros)
# update the variable attributes
Ws["Attr"] = {"standard_name": "wind_speed", "long_name": "Wind speed",
"units": "m/s", "statistic_type": "average"}
Wd["Attr"] = {"standard_name": "wind_from_direction", "long_name": "Wind direction",
"units": "degrees", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, Ws)
pfp_utils.CreateVariable(ds, Wd)
return
def percent_to_mmolpmol(ds, MF_out, RH_in, Ta_in, ps_in):
"""
Purpose:
Calculate H2O mole fraction from relative humidity (RH).
"""
nRecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [RH_in, Ta_in, ps_in]:
if item not in list(ds.root["Variables"].keys()):
msg = " Requested series " + item + " not found, " + MF_out + " not calculated"
logger.error(msg)
return 0
# get the relative humidity and check the units
RH = pfp_utils.GetVariable(ds, RH_in)
RH = pfp_utils.convert_units_func(ds, RH, "percent")
# get the temperature and check the units
Ta = pfp_utils.GetVariable(ds, Ta_in)
Ta = pfp_utils.convert_units_func(ds, Ta, "degC")
# get the absoulte humidity
AH_data = pfp_mf.absolutehumidityfromrelativehumidity(
Ta["Data"], RH["Data"])
# get the atmospheric pressure and check the units
ps = pfp_utils.GetVariable(ds, ps_in)
ps = pfp_utils.convert_units_func(ds, ps, "kPa")
# get the output variable (created in pfp_ts.DoFunctions())
MF = pfp_utils.GetVariable(ds, MF_out)
# do the business
MF["Data"] = pfp_mf.h2o_mmolpmolfromgpm3(AH_data, Ta["Data"], ps["Data"])
MF["Flag"] = numpy.where(
numpy.ma.getmaskarray(MF["Data"]) == True, ones, zeros)
MF["Attr"]["units"] = "mmol/mol"
# put the output variable back into the data structure
pfp_utils.CreateVariable(ds, MF)
return 1
def gH2Opm3_to_percent(ds, RH_out, AH_in, Ta_in):
"""
Purpose:
Function to convert absolute humidity in units of g/m^3 to relative humidity in percent.
Usage:
pfp_func_units.gH2Opm3_to_percent(ds, RH_out, AH_in, Ta_in)
Author: PRI
Date: September 2020
"""
nRecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [AH_in, Ta_in]:
if item not in ds.root["Variables"].keys():
msg = " Requested series " + item + " not found, " + RH_out + " not calculated"
logger.error(msg)
return 0
AH = pfp_utils.GetVariable(ds, AH_in)
Ta = pfp_utils.GetVariable(ds, Ta_in)
RH = pfp_utils.GetVariable(ds, RH_out)
RH["Data"] = pfp_mf.relativehumidityfromabsolutehumidity(
AH["Data"], Ta["Data"])
RH["Flag"] = numpy.where(
numpy.ma.getmaskarray(RH["Data"]) == True, ones, zeros)
RH["Attr"]["units"] = "percent"
pfp_utils.CreateVariable(ds, RH)
return 1
def mgCO2pm3_to_umolpmol(ds, MF_out, CO2_in, Ta_in, ps_in):
"""
Purpose:
Calculate CO2 mole fraction in uml/mol from mass density in mgCO2/m3.
Usage:
pfp_func_units.mgCO2pm3_to_umolpmol(ds, MF_out, CO2_in, Ta_in, ps_in)
Author: PRI
Date: August 2019
"""
nRecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [CO2_in, Ta_in, ps_in]:
if item not in ds.root["Variables"].keys():
msg = " Requested series " + item + " not found, " + MF_out + " not calculated"
logger.error(msg)
return 0
CO2 = pfp_utils.GetVariable(ds, CO2_in)
CO2 = pfp_utils.convert_units_func(ds, CO2, "mg/m^3")
Ta = pfp_utils.GetVariable(ds, Ta_in)
Ta = pfp_utils.convert_units_func(ds, Ta, "degC")
ps = pfp_utils.GetVariable(ds, ps_in)
ps = pfp_utils.convert_units_func(ds, ps, "kPa")
MF = pfp_utils.GetVariable(ds, MF_out)
MF["Data"] = pfp_mf.co2_ppmfrommgCO2pm3(CO2["Data"], Ta["Data"],
ps["Data"])
MF["Flag"] = numpy.where(
numpy.ma.getmaskarray(MF["Data"]) == True, ones, zeros)
MF["Attr"]["units"] = "umol/mol"
pfp_utils.CreateVariable(ds, MF)
return 1
def gfMDS_mask_long_gaps(ds, mds_label, l5_info, called_by):
"""
Purpose:
Mask gaps that are longer than a specified maximum length.
Usage:
Side effects:
Author: PRI
Date: June 2019
"""
if "MaxShortGapRecords" not in l5_info[called_by]["info"]:
return
max_short_gap_days = l5_info[called_by]["info"]["MaxShortGapDays"]
msg = " Masking gaps longer than " + str(max_short_gap_days) + " days"
logger.info(msg)
label = l5_info[called_by]["outputs"][mds_label]["target"]
target = pfp_utils.GetVariable(ds, label)
variable = pfp_utils.GetVariable(ds, mds_label)
mask = numpy.ma.getmaskarray(target["Data"])
# start and stop indices of contiguous blocks
max_short_gap_records = l5_info[called_by]["info"]["MaxShortGapRecords"]
gap_start_end = pfp_utils.contiguous_regions(mask)
for start, stop in gap_start_end:
gap_length = stop - start
if gap_length > max_short_gap_records:
variable["Data"][start:stop] = target["Data"][start:stop]
variable["Flag"][start:stop] = target["Flag"][start:stop]
# put data_int back into the data structure
pfp_utils.CreateVariable(ds, variable)
return
def remove_duplicates(ds):
"""
Remove duplicate timestamps, similar to Peter's solution in pfp_ts.py
MergeDataStructures at L1
"""
# get the datetime
dtn = pfp_utils.GetVariable(ds, "DateTime")
# remove duplicate timestamps
dtn_unique, index_unique = numpy.unique(dtn["Data"], return_index=True)
# restore the original order of the unique timestamps
dtn_sorted = dtn_unique[numpy.argsort(index_unique)]
# check to see if there were duplicates
if len(dtn_sorted) < len(dtn["Data"]):
n = len(dtn["Data"]) - len(dtn_sorted)
msg = str(n) + " duplicate time stamps were removed for isd site "
logger.warning(msg)
nrecs = len(dtn_sorted)
labels = list(ds.series.keys())
#if "DateTime" in labels:
# labels.remove("DateTime")
for label in labels:
var1 = pfp_utils.CreateEmptyVariable(label, nrecs)
varn = pfp_utils.GetVariable(ds, label)
var1["Data"] = varn["Data"][index_unique]
var1["Flag"] = varn["Flag"][index_unique]
var1["Attr"] = varn["Attr"]
pfp_utils.CreateVariable(ds, var1)
return ds
def gH2Opm3_to_mmolpmol(ds, MF_out, AH_in, Ta_in, ps_in):
"""
Purpose:
Calculate H2O mole fraction in mml/mol from absolute humidity in g/m^3.
Usage:
pfp_func_units.gH2Opm3_to_mmolpmol(ds, MF_out, AH_in, Ta_in, ps_in)
Author: PRI
Date: August 2019
"""
nRecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [AH_in, Ta_in, ps_in]:
if item not in ds.root["Variables"].keys():
msg = " Requested series " + item + " not found, " + MF_out + " not calculated"
logger.error(msg)
return 0
AH = pfp_utils.GetVariable(ds, AH_in)
AH = pfp_utils.convert_units_func(ds, AH, "g/m^3")
Ta = pfp_utils.GetVariable(ds, Ta_in)
Ta = pfp_utils.convert_units_func(ds, Ta, "degC")
ps = pfp_utils.GetVariable(ds, ps_in)
ps = pfp_utils.convert_units_func(ds, ps, "kPa")
MF = pfp_utils.GetVariable(ds, MF_out)
MF["Data"] = pfp_mf.h2o_mmolpmolfromgpm3(AH["Data"], Ta["Data"],
ps["Data"])
MF["Flag"] = numpy.where(
numpy.ma.getmaskarray(MF["Data"]) == True, ones, zeros)
MF["Attr"]["units"] = "mmol/mol"
pfp_utils.CreateVariable(ds, MF)
return 1
def make_data_array(cf, ds, current_year):
ldt = pfp_utils.GetVariable(ds, "DateTime")
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
ts = int(float(ds.root["Attributes"]["time_step"]))
start = datetime.datetime(current_year, 1, 1, 0, 0,
0) + datetime.timedelta(minutes=ts)
end = datetime.datetime(current_year + 1, 1, 1, 0, 0, 0)
cdt = numpy.array([
dt
for dt in pfp_utils.perdelta(start, end, datetime.timedelta(
minutes=ts))
])
mt = numpy.ones(len(cdt)) * float(-9999)
mt_list = [cdt] + [mt for n in list(cf["Variables"].keys())]
data = numpy.stack(mt_list, axis=-1)
si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0)
ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs)
dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei)
idx1, idx2 = pfp_utils.FindMatchingIndices(cdt, dt["Data"])
for n, cf_label in enumerate(list(cf["Variables"].keys())):
label = cf["Variables"][cf_label]["name"]
var = pfp_utils.GetVariable(ds, label, start=si, end=ei)
data[idx1, n + 1] = var["Data"]
# convert datetime to ISO dates
data[:, 0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt])
return data
def calculate_available_energy(ds):
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
Fh = pfp_utils.GetVariable(ds, "Fh")
Fe = pfp_utils.GetVariable(ds, "Fe")
Fa = pfp_utils.CreateEmptyVariable("Fa", nrecs,
datetime=Fh["DateTime"])
Fa["Data"] = Fh["Data"] + Fe["Data"]
Fa["Flag"] = numpy.where(numpy.ma.getmaskarray(Fa["Data"]) == True, ones, zeros)
Fa["Attr"] = {"long_name": "Available energy",
"units": "W/m^2", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, Fa)
return
def calculate_net_radiation(ds):
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
Fnsw = pfp_utils.GetVariable(ds, "Fnsw")
Fnlw = pfp_utils.GetVariable(ds, "Fnlw")
Fn = pfp_utils.CreateEmptyVariable("Fn", nrecs,
datetime=Fnsw["DateTime"])
Fn["Data"] = Fnsw["Data"] + Fnlw["Data"]
Fn["Flag"] = numpy.where(numpy.ma.getmaskarray(Fn["Data"]) == True, ones, zeros)
Fn["Attr"] = {"standard_name": "surface_net_downwawrd_radiative_flux",
"long_name": "Net radiation",
"units": "W/m^2", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, Fn)
return
def calculate_upwelling_longwave_radiation(ds):
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
Fld = pfp_utils.GetVariable(ds, "Fld")
Fnlw = pfp_utils.GetVariable(ds, "Fnlw")
Flu = pfp_utils.CreateEmptyVariable("Flu", nrecs,
datetime=Fld["DateTime"])
Flu["Data"] = Fld["Data"] - Fnlw["Data"]
Flu["Flag"] = numpy.where(numpy.ma.getmaskarray(Fld["Data"]) == True, ones, zeros)
Flu["Attr"] = {"standard_name": "surface_upwelling_longwave_flux_in_air",
"long_name": "Up-welling shortwave radiation",
"units": "W/m^2", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, Flu)
return
def Variance_from_standard_deviation(ds, Vr_out, Sd_in):
"""
Purpose:
Function to convert standard deviation to variance.
Usage:
pfp_func_statistics.Variance_from_standard_deviation(ds, Vr_out, Sd_in)
Author: PRI
Date: October 2020
"""
sd_units = {
"mg/m3": "mg^2/m^6",
"mmol/m^3": "mmol^2/m^6",
"g/m^3": "g^2/m^6",
"degC": "degC^2",
"K": "K^2",
"m/s": "m^2/s^2"
}
sd = pfp_utils.GetVariable(ds, Sd_in)
if sd["Attr"]["units"] not in list(sd_units.keys()):
msg = " Unrecognised units (" + sd["Attr"][
"units"] + ") for variable " + Sd_in
logger.error(msg)
msg = " Variance not calculated from standard deviation"
logger.error(msg)
return 0
vr = copy.deepcopy(sd)
vr["Label"] = Vr_out
vr["Data"] = sd["Data"] * sd["Data"]
vr["Attr"]["units"] = sd_units[sd["Attr"]["units"]]
if "statistic_type" in vr["Attr"]:
vr["Attr"]["statistic_type"] = "variance"
pfp_utils.CreateVariable(ds, vr)
return 1
def Standard_deviation_from_variance(ds, Sd_out, Vr_in):
"""
Purpose:
Function to convert variance to standard deviation.
Usage:
pfp_func_statistics.Standard_deviation_from_variance(ds, Sd_out, Vr_in)
Author: PRI
Date: October 2020
"""
vr_units = {
"mg^2/m^6": "mg/m3",
"mmol^2/m^6": "mmol/m^3",
"g^2/m^6": "g/m^3",
"degC^2": "degC",
"K^2": "K",
"m^2/s^2": "m/s"
}
vr = pfp_utils.GetVariable(ds, Vr_in)
if vr["Attr"]["units"] not in list(vr_units.keys()):
msg = " Unrecognised units (" + vr["Attr"][
"units"] + ") for variable " + Vr_in
logger.error(msg)
msg = " Standard deviation not calculated from variance"
logger.error(msg)
return 0
sd = copy.deepcopy(vr)
sd["Label"] = Sd_out
sd["Data"] = numpy.ma.sqrt(vr["Data"])
sd["Attr"]["units"] = vr_units[vr["Attr"]["units"]]
if "statistic_type" in sd["Attr"]:
sd["Attr"]["statistic_type"] = "standard_deviation"
pfp_utils.CreateVariable(ds, sd)
return 1
def mmolpm3_to_gH2Opm3(ds, AH_out, H2O_in):
"""
Purpose:
Function to convert mmol/m^3 (molar density) to g/m^3 (mass density).
Usage:
pfp_func_units.mmolpm3_to_gpm3(ds, AH_out, H2O_in)
Author: PRI
Date: August 2020
"""
for item in [H2O_in]:
if item not in list(ds.root["Variables"].keys()):
msg = " Requested series " + item + " not found, " + AH_out + " not calculated"
logger.error(msg)
return 0
var_in = pfp_utils.GetVariable(ds, H2O_in)
got_variance = False
if var_in["Label"][-3:] == "_Vr" and var_in["Attr"][
"units"] == "mmol^2/m^6":
got_variance = True
var_in["Data"] = numpy.ma.sqrt(var_in["Data"])
var_in["Attr"]["units"] = "mmol/m^3"
var_out = pfp_utils.convert_units_func(ds, var_in, "g/m^3", mode="quiet")
var_out["Label"] = AH_out
if got_variance:
var_out["Data"] = var_out["Data"] * var_out["Data"]
var_out["Attr"]["units"] = "g^2/m^6"
pfp_utils.CreateVariable(ds, var_out)
return 1
def calculate_absolute_humidity(ds):
# from relative humidity
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
RH = pfp_utils.GetVariable(ds, "RH")
Ta = pfp_utils.GetVariable(ds, "Ta")
AH = pfp_utils.CreateEmptyVariable("AH", nrecs,
datetime=RH["DateTime"])
AH["Data"] = pfp_mf.absolutehumidityfromrelativehumidity(Ta["Data"], RH["Data"])
AH["Flag"] = numpy.where(numpy.ma.getmaskarray(AH["Data"]) == True, ones, zeros)
AH["Attr"] = {"standard_name": "mass_concentration_of_water_vapor_in_air",
"long_name": "Absolute humidity",
"units": "g/m^3", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, AH)
return
def calculate_relative_humidity(ds):
# from dew point temperature
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
Td = pfp_utils.GetVariable(ds, "Td")
Ta = pfp_utils.GetVariable(ds, "Ta")
RH = pfp_utils.CreateEmptyVariable("RH", nrecs,
datetime=Td["DateTime"])
RH["Data"] = pfp_mf.relativehumidityfromdewpoint(Td["Data"], Ta["Data"])
RH["Flag"] = numpy.where(numpy.ma.getmaskarray(RH["Data"]) == True, ones, zeros)
RH["Attr"] = {"standard_name": "relative_humidity",
"long_name": "Relative humidity",
"units": "percent", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, RH)
return
def calculate_ground_heat_flux(ds):
# as residual from net rad - avail energy
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
Fn = pfp_utils.GetVariable(ds, "Fn")
Fa = pfp_utils.GetVariable(ds, "Fa")
Fg = pfp_utils.CreateEmptyVariable("Fg", nrecs,
datetime=Fn["DateTime"])
Fg["Data"] = Fn["Data"] - Fa["Data"]
Fg["Flag"] = numpy.where(numpy.ma.getmaskarray(Fg["Data"]) == True, ones, zeros)
Fg["Attr"] = {"standard_name": "downward_heat_flux_in_soil",
"long_name": "Ground heat flux",
"units": "W/m^2", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, Fg)
return
def mgCO2pm3_to_mmolpm3(ds, CO2_out, CO2_in):
"""
Purpose:
Calculate CO2 molar density in mmol/m3 from CO2 concentration in mg/m3.
Usage:
pfp_func_units.mgCO2pm3_to_mmolpm3(ds, CO2_out, CO2_in)
Author: PRI
Date: September 2020
"""
for item in [CO2_in]:
if item not in ds.root["Variables"].keys():
msg = " Requested series " + item + " not found, " + CO2_out + " not calculated"
logger.error(msg)
return 0
var_in = pfp_utils.GetVariable(ds, CO2_in)
got_variance = False
if var_in["Label"][-3:] == "_Vr" and var_in["Attr"]["units"] in [
"mg^2/m^6", "mgCO2^2/m^6"
]:
got_variance = True
var_in["Data"] = numpy.ma.sqrt(var_in["Data"])
var_in["Attr"]["units"] = "mg/m^3"
var_out = pfp_utils.convert_units_func(ds,
var_in,
"mmol/m^3",
mode="quiet")
var_out["Label"] = CO2_out
if got_variance:
var_out["Data"] = var_out["Data"] * var_out["Data"]
var_out["Attr"]["units"] = "mmol^2/m^6"
pfp_utils.CreateVariable(ds, var_out)
return 1
def gH2Opm3_to_mmolpm3(ds, H2O_out, AH_in):
"""
Purpose:
Calculate H2O molar density in mmol/m^3 from absolute humidity in g/m^3.
Usage:
pfp_func_units.gH2Opm3_to_mmolpm3(ds, MD_out, AH_in)
Author: PRI
Date: September 2020
"""
for item in [AH_in]:
if item not in ds.root["Variables"].keys():
msg = " Requested series " + item + " not found, " + H2O_out + " not calculated"
logger.error(msg)
return 0
var_in = pfp_utils.GetVariable(ds, AH_in)
got_variance = False
if var_in["Label"][-3:] == "_Vr" and var_in["Attr"]["units"] in [
"g^2/m^6", "gH2O^2/m^6"
]:
got_variance = True
var_in["Data"] = numpy.ma.sqrt(var_in["Data"])
var_in["Attr"]["units"] = "g/m^3"
var_out = pfp_utils.convert_units_func(ds,
var_in,
"mmol/m^3",
mode="quiet")
var_out["Label"] = H2O_out
if got_variance:
var_out["Data"] = var_out["Data"] * var_out["Data"]
var_out["Attr"]["units"] = "mmol^2/m^6"
pfp_utils.CreateVariable(ds, var_out)
return 1
def calculate_specific_humidity(ds):
# from relative humidity
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
Ta = pfp_utils.GetVariable(ds,"Ta")
ps = pfp_utils.GetVariable(ds,"ps")
RH = pfp_utils.GetVariable(ds,"RH")
SH = pfp_utils.CreateEmptyVariable("SH", nrecs,
datetime=RH["DateTime"])
SH["Data"] = pfp_mf.specifichumidityfromrelativehumidity(RH["Data"], Ta["Data"], ps["Data"])
SH["Flag"] = numpy.where(numpy.ma.getmaskarray(SH["Data"]) == True, ones, zeros)
SH["Attr"] = {"standard_name": "specific_humidity",
"long_name": "Specific humidity",
"units": "kg/kg", "statistic_type": "average"}
pfp_utils.CreateVariable(ds, SH)
def gfMDS_make_data_array(ds, current_year, info):
"""
Purpose:
Create a data array for the MDS gap filling routine. The array constructed
here will be written to a CSV file that is read by the MDS C code.
Usage:
Side Effects:
The constructed data arrays are full years. That is they run from YYYY-01-01 00:30
to YYYY+1-01-01 00:00. Missing data is represented as -9999.
Author: PRI
Date: May 2018
"""
ldt = pfp_utils.GetVariable(ds, "DateTime")
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
ts = int(float(ds.root["Attributes"]["time_step"]))
start = datetime.datetime(current_year, 1, 1, 0, 0,
0) + datetime.timedelta(minutes=ts)
end = datetime.datetime(current_year + 1, 1, 1, 0, 0, 0)
cdt = numpy.array([
dt
for dt in pfp_utils.perdelta(start, end, datetime.timedelta(
minutes=ts))
])
mt = numpy.ones(len(cdt)) * float(-9999)
# need entry for the timestamp and the target ...
array_list = [cdt, mt]
# ... and entries for the drivers
for driver in info["drivers"]:
array_list.append(mt)
# now we can create the data array
data = numpy.stack(array_list, axis=-1)
si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0)
ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs)
dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei)
idx1, _ = pfp_utils.FindMatchingIndices(cdt, dt["Data"])
pfp_label_list = [info["target"]] + info["drivers"]
mds_label_list = [info["target_mds"]] + info["drivers_mds"]
header = "TIMESTAMP"
fmt = "%12i"
for n, label in enumerate(pfp_label_list):
var = pfp_utils.GetVariable(ds, label, start=si, end=ei)
data[idx1, n + 1] = var["Data"]
header = header + "," + mds_label_list[n]
fmt = fmt + "," + "%f"
# convert datetime to ISO dates
data[:, 0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt])
return data, header, fmt
def calculate_ground_heat_flux(ds):
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
for i in range(3):
for j in range(3):
Fn = pfp_utils.GetVariable(ds, "Fn" + "_" + str(i) + str(j))
Fa = pfp_utils.GetVariable(ds, "Fa" + "_" + str(i) + str(j))
Fg = pfp_utils.CreateEmptyVariable("Fg" + "_" + str(i) + str(j),
nrecs,
datetime=Fn["DateTime"])
Fg["Data"] = Fn["Data"] - Fa["Data"]
Fg["Flag"] = numpy.where(
numpy.ma.getmaskarray(Fg["Data"]) == True, ones, zeros)
Fg["Attr"] = {
"standard_name": "downward_heat_flux_in_soil",
"long_name": "Ground heat flux",
"units": "W/m^2",
"statistic_type": "average"
}
pfp_utils.CreateVariable(ds, Fg)
return
def calculate_upwelling_shortwave_radiation(ds):
nrecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nrecs)
ones = numpy.ones(nrecs)
for i in range(3):
for j in range(3):
Fsd = pfp_utils.GetVariable(ds, "Fsd" + "_" + str(i) + str(j))
Fnsw = pfp_utils.GetVariable(ds, "Fnsw" + "_" + str(i) + str(j))
Fsu = pfp_utils.CreateEmptyVariable("Fsu" + "_" + str(i) + str(j),
nrecs,
datetime=Fsd["DateTime"])
Fsu["Data"] = Fsd["Data"] - Fnsw["Data"]
Fsu["Flag"] = numpy.where(
numpy.ma.getmaskarray(Fsd["Data"]) == True, ones, zeros)
Fsu["Attr"] = {
"standard_name": "surface_upwelling_shortwave_flux_in_air",
"long_name": "Up-welling shortwave radiation",
"units": "W/m^2",
"statistic_type": "average"
}
pfp_utils.CreateVariable(ds, Fsu)
return
def percent_to_m3pm3(ds, Sws_out, Sws_in):
"""
Purpose:
Function to convert Sws in units of "percent" (1 to 100) to "frac" (0 to 1).
Usage:
pfp_func_units.percent_to_m3pm3(ds, Sws_out, Sws_in)
Author: PRI
Date: April 2020
"""
var_in = pfp_utils.GetVariable(ds, Sws_in)
var_out = pfp_utils.convert_units_func(ds, var_in, "m^3/m^3", mode="quiet")
var_out["Label"] = Sws_out
pfp_utils.CreateVariable(ds, var_out)
return 1
def fraction_to_percent(ds, RH_out, RH_in):
"""
Purpose:
Function to convert RH in units of "frac" (0 to 1) to "percent" (1 to 100).
Usage:
pfp_func_units.fraction_to_percent(ds, RH_out, RH_in)
Author: PRI
Date: August 2019
"""
var_in = pfp_utils.GetVariable(ds, RH_in)
var_out = pfp_utils.convert_units_func(ds, var_in, "percent", mode="quiet")
var_out["Label"] = RH_out
pfp_utils.CreateVariable(ds, var_out)
return 1
def Pa_to_kPa(ds, ps_out, ps_in):
"""
Purpose:
Function to convert pressure from Pa to kPa.
Usage:
pfp_func_units.Pa_to_kPa(ds, ps_out, ps_in)
Author: PRI
Date: February 2018
"""
var_in = pfp_utils.GetVariable(ds, ps_in)
var_out = pfp_utils.convert_units_func(ds, var_in, "kPa", mode="quiet")
var_out["Label"] = ps_out
pfp_utils.CreateVariable(ds, var_out)
return 1
def kgpm3_to_gpm3(ds, AH_out, AH_in):
"""
Purpose:
Function to convert absolute humidity from kg/m^3 to g/m^3.
Usage:
pfp_func_units.kgpm3_to_gpm3(ds, AH_out, AH_in)
Author: PRI
Date: August 2020
"""
var_in = pfp_utils.GetVariable(ds, AH_in)
var_out = pfp_utils.convert_units_func(ds, var_in, "g/m^3", mode="quiet")
var_out["Label"] = AH_out
pfp_utils.CreateVariable(ds, var_out)
return 1
def percent_to_gH2Opm3(ds, AH_out, RH_in, Ta_in):
"""
Purpose:
Function to calculate absolute humidity given relative humidity and
air temperature. Absolute humidity is not calculated if any of the
input series are missing or if the specified output series already
exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
pfp_func_units.percent_to_gpm3(ds,"AH_HMP_2m","RH_HMP_2m","Ta_HMP_2m")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [RH_in, Ta_in]:
if item not in ds.root["Variables"].keys():
msg = " Requested series " + item + " not found, " + AH_out + " not calculated"
logger.error(msg)
return 0
# get the relative humidity and check the units
RH = pfp_utils.GetVariable(ds, RH_in)
RH = pfp_utils.convert_units_func(ds, RH, "percent")
# get the temperature and check the units
Ta = pfp_utils.GetVariable(ds, Ta_in)
Ta = pfp_utils.convert_units_func(ds, Ta, "degC")
# get the absolute humidity
AH = pfp_utils.GetVariable(ds, AH_out)
AH["Data"] = pfp_mf.absolutehumidityfromrelativehumidity(
Ta["Data"], RH["Data"])
AH["Flag"] = numpy.where(
numpy.ma.getmaskarray(AH["Data"]) == True, ones, zeros)
AH["Attr"]["units"] = "g/m^3"
pfp_utils.CreateVariable(ds, AH)
return 1
def mmolpmol_to_gH2Opm3(ds, AH_out, MF_in, Ta_in, ps_in):
"""
Purpose:
Function to calculate absolute humidity given the water vapour mole
fraction, air temperature and pressure. Absolute humidity is not calculated
if any of the input series are missing or if the specified output series
already exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
pfp_func_units.mmolpmol_to_gpm3(ds,"AH_IRGA_Av","H2O_IRGA_Av","Ta_HMP_2m","ps")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.root["Attributes"]["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [MF_in, Ta_in, ps_in]:
if item not in list(ds.root["Variables"].keys()):
msg = " Requested series " + item + " not found, " + AH_out + " not calculated"
logger.error(msg)
return 0
MF = pfp_utils.GetVariable(ds, MF_in)
MF = pfp_utils.convert_units_func(ds, MF, "mmol/mol")
Ta = pfp_utils.GetVariable(ds, Ta_in)
Ta = pfp_utils.convert_units_func(ds, Ta, "degC")
ps = pfp_utils.GetVariable(ds, ps_in)
ps = pfp_utils.convert_units_func(ds, ps, "kPa")
AH = pfp_utils.GetVariable(ds, AH_out)
AH["Data"] = pfp_mf.h2o_gpm3frommmolpmol(MF["Data"], Ta["Data"],
ps["Data"])
AH["Flag"] = numpy.where(
numpy.ma.getmaskarray(AH["Data"]) == True, ones, zeros)
AH["Attr"]["units"] = "g/m^3"
pfp_utils.CreateVariable(ds, AH)
return 1
def K_to_C(ds, T_out, T_in):
"""
Purpose:
Function to convert temperature from K to C.
Usage:
pfp_func_units.K_to_C(ds, T_out, T_in)
Author: PRI
Date: February 2018
"""
if T_in not in list(ds.root["Variables"].keys()):
msg = " Convert_K_to_C: variable " + T_in + " not found, skipping ..."
logger.warning(msg)
return 0
if "<" in T_out or ">" in T_out:
logger.warning(" ***")
msg = " *** " + T_in + ": illegal name (" + T_out + ") in function, skipping ..."
logger.warning(msg)
logger.warning(" ***")
return 0
var_in = pfp_utils.GetVariable(ds, T_in)
var_out = pfp_utils.convert_units_func(ds, var_in, "degC", mode="quiet")
var_out["Label"] = T_out
pfp_utils.CreateVariable(ds, var_out)
return 1
