def MRfromRH(ds, MR_out, RH_in, Ta_in, ps_in):
"""
Purpose:
Calculate H2O mixing ratio from RH.
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [RH_in, Ta_in, ps_in]:
if item not in ds.series.keys():
msg = " MRfromRH: Requested series " + item + " not found, " + MR_out + " not calculated"
logger.error(msg)
return 0
if MR_out in ds.series.keys():
msg = " MRfromRH: Output series " + MR_out + " already exists, skipping ..."
logger.error(msg)
return 0
RH_data, RH_flag, RH_attr = pfp_utils.GetSeriesasMA(ds, RH_in)
Ta_data, Ta_flag, Ta_attr = pfp_utils.GetSeriesasMA(ds, Ta_in)
Ah_data = pfp_mf.absolutehumidityfromRH(Ta_data, RH_data)
ps_data, ps_flag, ps_attr = pfp_utils.GetSeriesasMA(ds, ps_in)
MR_data = pfp_mf.h2o_mmolpmolfromgpm3(Ah_data, Ta_data, ps_data)
MR_attr = pfp_utils.MakeAttributeDictionary(
long_name="H2O mixing ratio calculated from " + RH_in + ", " + Ta_in +
" and " + ps_in,
height=RH_attr["height"],
units="mmol/mol")
flag = numpy.where(numpy.ma.getmaskarray(MR_data) == True, ones, zeros)
pfp_utils.CreateSeries(ds, MR_out, MR_data, flag, MR_attr)
return 1
def AhfromRH(ds, Ah_out, RH_in, Ta_in):
"""
Purpose:
Function to calculate absolute humidity given relative humidity and
air temperature. Absolute humidity is not calculated if any of the
input series are missing or if the specified output series already
exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
pfp_func.AhfromRH(ds,"Ah_HMP_2m","RH_HMP_2m","Ta_HMP_2m")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs,dtype=numpy.int32)
ones = numpy.ones(nRecs,dtype=numpy.int32)
for item in [RH_in,Ta_in]:
if item not in ds.series.keys():
msg = " AhfromRH: Requested series "+item+" not found, "+Ah_out+" not calculated"
logger.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromRH: Output series "+Ah_out+" already exists, skipping ..."
logger.error(msg)
return 0
RH_data,RH_flag,RH_attr = pfp_utils.GetSeriesasMA(ds,RH_in)
Ta_data,Ta_flag,Ta_attr = pfp_utils.GetSeriesasMA(ds,Ta_in)
Ah_data = pfp_mf.absolutehumidityfromRH(Ta_data,RH_data)
Ah_attr = pfp_utils.MakeAttributeDictionary(long_name="Absolute humidity calculated from "+RH_in+" and "+Ta_in,
height=RH_attr["height"],
units="g/m3")
flag = numpy.where(numpy.ma.getmaskarray(Ah_data)==True,ones,zeros)
pfp_utils.CreateSeries(ds,Ah_out,Ah_data,flag,Ah_attr)
return 1
def AhfromMR(ds, Ah_out, MR_in, Ta_in, ps_in):
"""
Purpose:
Function to calculate absolute humidity given the water vapour mixing
ratio, air temperature and pressure. Absolute humidity is not calculated
if any of the input series are missing or if the specified output series
already exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
pfp_func.AhfromMR(ds,"Ah_IRGA_Av","H2O_IRGA_Av","Ta_HMP_2m","ps")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [MR_in, Ta_in, ps_in]:
if item not in ds.series.keys():
msg = " AhfromMR: Requested series " + item + " not found, " + Ah_out + " not calculated"
logger.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromMR: Output series " + Ah_out + " already exists, skipping ..."
logger.error(msg)
return 0
MR_data, MR_flag, MR_attr = pfp_utils.GetSeriesasMA(ds, MR_in)
Ta_data, Ta_flag, Ta_attr = pfp_utils.GetSeriesasMA(ds, Ta_in)
ps_data, ps_flag, ps_attr = pfp_utils.GetSeriesasMA(ds, ps_in)
Ah_data = pfp_mf.h2o_gpm3frommmolpmol(MR_data, Ta_data, ps_data)
long_name = "Absolute humidity calculated from " + MR_in + ", " + Ta_in + " and " + ps_in
Ah_attr = pfp_utils.MakeAttributeDictionary(long_name=long_name,
height=MR_attr["height"],
units="g/m3")
flag = numpy.where(numpy.ma.getmaskarray(Ah_data) == True, ones, zeros)
pfp_utils.CreateSeries(ds, Ah_out, Ah_data, flag, Ah_attr)
return 1
# remove any daylight saving adjustments (towers run on standard time)
dt_loc = [x - x.dst() for x in dt_loc]
# strip the time zone from the local datetime series
dt_loc = [x.replace(tzinfo=None) for x in dt_loc]
ds.series["DateTime"]["Data"] = dt_loc
# update global attributes
ds.globalattributes["nc_nrecs"] = len(dt_loc)
ds.globalattributes["start_datetime"] = str(dt_loc[0])
ds.globalattributes["end_datetime"] = str(dt_loc[-1])
# put the QC'd, smoothed and interpolated EVI into the data structure
flag = numpy.zeros(len(dt_loc), dtype=numpy.int32)
attr = pfp_utils.MakeAttributeDictionary(
long_name="MODIS EVI, smoothed and interpolated",
units="none",
horiz_resolution="250m",
cutout_size=str(3),
evi_min=str(evi_min),
evi_max=str(evi_max),
sg_num_points=str(sgnp),
sg_order=str(sgo))
pfp_utils.CreateSeries(ds, "EVI", evi_ts["smoothed"], flag, attr)
attr = pfp_utils.MakeAttributeDictionary(long_name="MODIS EVI, interpolated",
units="none",
horiz_resolution="250m",
cutout_size=str(3),
evi_min=str(evi_min),
evi_max=str(evi_max))
pfp_utils.CreateSeries(ds, "EVI_notsmoothed", evi_ts["mean"], flag, attr)
# now write the data structure to a netCDF file
out_name = os.path.join(cf["Files"]["base_path"], site, "Data", "MODIS",
def ApplyTurbulenceFilter(cf, ds, ustar_threshold=None):
"""
Purpose:
Usage:
Author:
Date:
"""
opt = ApplyTurbulenceFilter_checks(cf, ds)
if not opt["OK"]: return
# local point to datetime series
ldt = ds.series["DateTime"]["Data"]
# time step
ts = int(ds.globalattributes["time_step"])
# dictionary of utar thresold values
if ustar_threshold == None:
ustar_dict = pfp_rp.get_ustar_thresholds(cf, ldt)
else:
ustar_dict = pfp_rp.get_ustar_thresholds_annual(ldt, ustar_threshold)
# initialise a dictionary for the indicator series
indicators = {}
# get data for the indicator series
ustar, ustar_flag, ustar_attr = pfp_utils.GetSeriesasMA(ds, "ustar")
Fsd, f, a = pfp_utils.GetSeriesasMA(ds, "Fsd")
if "solar_altitude" not in ds.series.keys():
pfp_ts.get_synthetic_fsd(ds)
Fsd_syn, f, a = pfp_utils.GetSeriesasMA(ds, "Fsd_syn")
sa, f, a = pfp_utils.GetSeriesasMA(ds, "solar_altitude")
# get the day/night indicator series
# indicators["day"] = 1 ==> day time, indicators["day"] = 0 ==> night time
indicators["day"] = pfp_rp.get_day_indicator(cf, Fsd, Fsd_syn, sa)
ind_day = indicators["day"]["values"]
# get the turbulence indicator series
if opt["turbulence_filter"].lower() == "ustar":
# indicators["turbulence"] = 1 ==> turbulent, indicators["turbulence"] = 0 ==> not turbulent
indicators["turbulence"] = pfp_rp.get_turbulence_indicator_ustar(
ldt, ustar, ustar_dict, ts)
elif opt["turbulence_filter"].lower() == "ustar_evg":
# ustar >= threshold ==> ind_ustar = 1, ustar < threshold == ind_ustar = 0
indicators["ustar"] = pfp_rp.get_turbulence_indicator_ustar(
ldt, ustar, ustar_dict, ts)
ind_ustar = indicators["ustar"]["values"]
# ustar >= threshold during day AND ustar has been >= threshold since sunset ==> indicators["turbulence"] = 1
# indicators["turbulence"] = 0 during night once ustar has dropped below threshold even if it
# increases above the threshold later in the night
indicators["turbulence"] = pfp_rp.get_turbulence_indicator_ustar_evg(
ldt, ind_day, ind_ustar, ustar, ustar_dict, ts)
elif opt["turbulence_filter"].lower() == "l":
#indicators["turbulence] = get_turbulence_indicator_l(ldt,L,z,d,zmdonL_threshold)
indicators["turbulence"] = numpy.ones(len(ldt))
msg = " Use of L as turbulence indicator not implemented, no filter applied"
logger.warning(msg)
else:
msg = " Unrecognised turbulence filter option ("
msg = msg + opt["turbulence_filter"] + "), no filter applied"
logger.error(msg)
return
# initialise the final indicator series as the turbulence indicator
# subsequent filters will modify the final indicator series
# we must use copy.deepcopy() otherwise the "values" array will only
# be copied by reference not value. Damn Python's default of copy by reference!
indicators["final"] = copy.deepcopy(indicators["turbulence"])
# check to see if the user wants to accept all day time observations
# regardless of ustar value
if opt["accept_day_times"].lower() == "yes":
# if yes, then we force the final indicator to be 1
# if ustar is below the threshold during the day.
idx = numpy.where(indicators["day"]["values"] == 1)[0]
indicators["final"]["values"][idx] = numpy.int(1)
indicators["final"]["attr"].update(indicators["day"]["attr"])
# get the evening indicator series
indicators["evening"] = pfp_rp.get_evening_indicator(
cf, Fsd, Fsd_syn, sa, ts)
indicators["dayevening"] = {
"values": indicators["day"]["values"] + indicators["evening"]["values"]
}
indicators["dayevening"]["attr"] = indicators["day"]["attr"].copy()
indicators["dayevening"]["attr"].update(indicators["evening"]["attr"])
if opt["use_evening_filter"].lower() == "yes":
idx = numpy.where(indicators["dayevening"]["values"] == 0)[0]
indicators["final"]["values"][idx] = numpy.int(0)
indicators["final"]["attr"].update(indicators["dayevening"]["attr"])
# save the indicator series
ind_flag = numpy.zeros(len(ldt))
long_name = "Turbulence indicator, 1 for turbulent, 0 for non-turbulent"
ind_attr = pfp_utils.MakeAttributeDictionary(long_name=long_name,
units="None")
pfp_utils.CreateSeries(ds, "turbulence_indicator",
indicators["turbulence"]["values"], ind_flag,
ind_attr)
long_name = "Day indicator, 1 for day time, 0 for night time"
ind_attr = pfp_utils.MakeAttributeDictionary(long_name=long_name,
units="None")
pfp_utils.CreateSeries(ds, "day_indicator", indicators["day"]["values"],
ind_flag, ind_attr)
long_name = "Evening indicator, 1 for evening, 0 for not evening"
ind_attr = pfp_utils.MakeAttributeDictionary(long_name=long_name,
units="None")
pfp_utils.CreateSeries(ds, "evening_indicator",
indicators["evening"]["values"], ind_flag, ind_attr)
long_name = "Day/evening indicator, 1 for day/evening, 0 for not day/evening"
ind_attr = pfp_utils.MakeAttributeDictionary(long_name=long_name,
units="None")
pfp_utils.CreateSeries(ds, "dayevening_indicator",
indicators["dayevening"]["values"], ind_flag,
ind_attr)
long_name = "Final indicator, 1 for use data, 0 for don't use data"
ind_attr = pfp_utils.MakeAttributeDictionary(long_name=long_name,
units="None")
pfp_utils.CreateSeries(ds, "final_indicator",
indicators["final"]["values"], ind_flag, ind_attr)
# loop over the series to be filtered
for series in opt["filter_list"]:
msg = " Applying " + opt["turbulence_filter"] + " filter to " + series
logger.info(msg)
# get the data
data, flag, attr = pfp_utils.GetSeriesasMA(ds, series)
# continue to next series if this series has been filtered before
if "turbulence_filter" in attr:
msg = " Series " + series + " has already been filtered, skipping ..."
logger.warning(msg)
continue
# save the non-filtered data
pfp_utils.CreateSeries(ds, series + "_nofilter", data, flag, attr)
# now apply the filter
data_filtered = numpy.ma.masked_where(
indicators["final"]["values"] == 0, data, copy=True)
flag_filtered = numpy.copy(flag)
idx = numpy.where(indicators["final"]["values"] == 0)[0]
flag_filtered[idx] = numpy.int32(61)
# update the series attributes
for item in indicators["final"]["attr"].keys():
attr[item] = indicators["final"]["attr"][item]
# and write the filtered data to the data structure
pfp_utils.CreateSeries(ds, series, data_filtered, flag_filtered, attr)
# and write a copy of the filtered datas to the data structure so it
# will still exist once the gap filling has been done
pfp_utils.CreateSeries(ds, series + "_filtered", data_filtered,
flag_filtered, attr)
return
# get the Akima interpolator function
int_fn = scipy.interpolate.Akima1DInterpolator(era5_time_1hr, coef_1hr)
# get the coefficient at the tower time step
coef_tts = int_fn(era5_time_tts)
# ==== old = UnivariateSpline ====
# get the spline interpolation function
#s = InterpolatedUnivariateSpline(era5_time_1hr, coef_1hr, k=1)
# get the coefficient at the tower time step
#coef_tts = s(era5_time_tts)
# get the downwelling solar radiation at the tower time step
Fsd_era5_tts = coef_tts * numpy.sin(numpy.deg2rad(alt_solar_tts))
flag = numpy.zeros(len(Fsd_era5_tts), dtype=numpy.int32)
attr = pfp_utils.MakeAttributeDictionary(
long_name="Downwelling short wave radiation", units="W/m2")
pfp_utils.CreateSeries(ds_era5, "Fsd", Fsd_era5_tts, flag, attr)
# === NET-SHORTWAVE Fn_sw === #
# Interpolate the 1 hourly accumulated net shortwave to the tower time step
# NOTE: ERA-5 variables are dimensioned [time,latitude,longitude]
Fn_sw_3d = era5_file.variables["ssr"][:, :, :]
Fn_sw_accum = Fn_sw_3d[:, site_lat_index, site_lon_index]
# Net shortwave in ERA-5 is a cummulative value that is reset to 0 at 0300 and 1500 UTC.
# Here we convert the cummulative values to 3 hourly values.
#Fn_sw_era5_1hr = numpy.ediff1d(Fn_sw_accum,to_begin=0)
# deal with the reset times at 0300 and 1500
#idx = numpy.where((hour_utc==3)|(hour_utc==15))[0]
#Fn_sw_era5_1hr[idx] = Fn_sw_accum[idx]
# get the average value over the 1 hourly period
#Fn_sw_era5_1hr = Fn_sw_era5_1hr/(era5_timestep*60)