def make_data_array(cf, ds, current_year):
ldt = pfp_utils.GetVariable(ds, "DateTime")
nrecs = int(ds.globalattributes["nc_nrecs"])
ts = int(ds.globalattributes["time_step"])
start = datetime.datetime(current_year, 1, 1, 0, 0,
0) + datetime.timedelta(minutes=ts)
end = datetime.datetime(current_year + 1, 1, 1, 0, 0, 0)
cdt = numpy.array([
dt
for dt in pfp_utils.perdelta(start, end, datetime.timedelta(
minutes=ts))
])
mt = numpy.ones(len(cdt)) * float(-9999)
mt_list = [cdt] + [mt for n in list(cf["Variables"].keys())]
data = numpy.stack(mt_list, axis=-1)
si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0)
ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs)
dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei)
idx1, idx2 = pfp_utils.FindMatchingIndices(cdt, dt["Data"])
for n, cf_label in enumerate(list(cf["Variables"].keys())):
label = cf["Variables"][cf_label]["name"]
var = pfp_utils.GetVariable(ds, label, start=si, end=ei)
data[idx1, n + 1] = var["Data"]
# convert datetime to ISO dates
data[:, 0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt])
return data
def gfMDS_make_data_array(ds, current_year, info):
"""
Purpose:
Create a data array for the MDS gap filling routine. The array constructed
here will be written to a CSV file that is read by the MDS C code.
Usage:
Side Effects:
The constructed data arrays are full years. That is they run from YYYY-01-01 00:30
to YYYY+1-01-01 00:00. Missing data is represented as -9999.
Author: PRI
Date: May 2018
"""
ldt = pfp_utils.GetVariable(ds, "DateTime")
nrecs = int(ds.globalattributes["nc_nrecs"])
ts = int(ds.globalattributes["time_step"])
start = datetime.datetime(current_year, 1, 1, 0, 30, 0)
end = datetime.datetime(current_year + 1, 1, 1, 0, 0, 0)
cdt = numpy.array([
dt
for dt in pfp_utils.perdelta(start, end, datetime.timedelta(
minutes=ts))
])
mt = numpy.ones(len(cdt)) * float(-9999)
# need entry for the timestamp and the target ...
array_list = [cdt, mt]
# ... and entries for the drivers
for driver in info["drivers"]:
array_list.append(mt)
# now we can create the data array
data = numpy.stack(array_list, axis=-1)
si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0)
ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs)
dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei)
idx1, _ = pfp_utils.FindMatchingIndices(cdt, dt["Data"])
pfp_label_list = [info["target"]] + info["drivers"]
mds_label_list = [info["target_mds"]] + info["drivers_mds"]
header = "TIMESTAMP"
fmt = "%12i"
for n, label in enumerate(pfp_label_list):
var = pfp_utils.GetVariable(ds, label, start=si, end=ei)
data[idx1, n + 1] = var["Data"]
header = header + "," + mds_label_list[n]
fmt = fmt + "," + "%f"
# convert datetime to ISO dates
data[:, 0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt])
return data, header, fmt
def make_data_array(ds, current_year):
ldt = pfp_utils.GetVariable(ds, "DateTime")
nrecs = ds.globalattributes["nc_nrecs"]
ts = int(ds.globalattributes["time_step"])
start = datetime.datetime(current_year,1,1,0,30,0)
end = datetime.datetime(current_year+1,1,1,0,0,0)
cdt = numpy.array([dt for dt in pfp_utils.perdelta(start, end, datetime.timedelta(minutes=ts))])
mt = numpy.ones(len(cdt))*float(-9999)
data = numpy.stack([cdt, mt, mt, mt, mt, mt, mt, mt], axis=-1)
si = pfp_utils.GetDateIndex(ldt["Data"], start, default=0)
ei = pfp_utils.GetDateIndex(ldt["Data"], end, default=nrecs)
dt = pfp_utils.GetVariable(ds, "DateTime", start=si, end=ei)
idx1, idx2 = pfp_utils.FindMatchingIndices(cdt, dt["Data"])
for n, label in enumerate(["Fc", "VPD", "ustar", "Ta", "Fsd", "Fh", "Fe"]):
var = pfp_utils.GetVariable(ds, label, start=si, end=ei)
data[idx1,n+1] = var["Data"]
# convert datetime to ISO dates
data[:,0] = numpy.array([int(xdt.strftime("%Y%m%d%H%M")) for xdt in cdt])
return data
axs[0].plot(evi["DateTime"], evi["min"], 'r+')
axs[0].plot(evi["DateTime"], evi["max"], 'gx')
axs[1].plot(evi_qc["DateTime"], evi_qc["mean"], 'b.')
axs[1].plot(evi_qc["DateTime"], evi_qc["min"], 'r+')
axs[1].plot(evi_qc["DateTime"], evi_qc["max"], 'gx')
axs[0].set_ylabel("EVI")
axs[1].set_ylabel("EVI QC")
fig.show()
# interpolate onto a regular 8 day time step
evi_8day = {}
start = evi_qc["DateTime"][0]
end = evi_qc["DateTime"][-1]
tdts = datetime.timedelta(days=8)
evi_8day["DateTime"] = [
result for result in pfp_utils.perdelta(start, end, tdts)
]
evi_8day["time"] = netCDF4.date2num(evi_8day["DateTime"], evi_time_units)
# get rid of any masked elements
m1 = numpy.ma.getmaskarray(evi_qc["time"])
m2 = numpy.ma.getmaskarray(evi_qc["mean"])
mask = numpy.ma.mask_or(m1, m2)
time = numpy.ma.compressed(numpy.ma.masked_where(mask, evi_qc["time"]))
mean = numpy.ma.compressed(numpy.ma.masked_where(mask, evi_qc["mean"]))
# now we can do the interpolation and smoothing
f = scipy.interpolate.Akima1DInterpolator(time, mean)
evi_8day["mean"] = f(evi_8day["time"])
# run a Savitzky-Golay filter through the 8 day values
evi_8day["smoothed"] = scipy.signal.savgol_filter(evi_8day["mean"],
sgnp,
sgo,
# now we get the datetime series at the tower time step
tdts = datetime.timedelta(minutes=site_timestep)
# get the start and end datetimes rounded to the nearest time steps
# that lie between the first and last times
start_date = pfp_utils.rounddttots(dt_era5_utc_cor[0],
ts=site_timestep)
if start_date < dt_era5_utc_cor[0]: start_date = start_date + tdts
end_date = pfp_utils.rounddttots(dt_era5_utc_cor[-1], ts=site_timestep)
if end_date > dt_era5_utc_cor[-1]: end_date = end_date - tdts
msg = "Data: " + start_date.strftime("%Y-%m-%d %H:%M") + " UTC to "
msg = msg + end_date.strftime("%Y-%m-%d %H:%M") + " UTC"
logger.info(msg)
#print site_name,end_date,dt_era5_utc_cor[-1]
# UTC datetime series at the tower time step
dt_era5_utc_tts = [
x for x in pfp_utils.perdelta(start_date, end_date, tdts)
]
# UTC netCDF time series at tower time step for interpolation
tmp = [x.replace(tzinfo=None) for x in dt_era5_utc_tts]
era5_time_tts = netCDF4.date2num(tmp, time_units)
# local datetime series at tower time step
dt_era5_loc_tts = [x.astimezone(site_tz) for x in dt_era5_utc_tts]
# NOTE: will have to disable daylight saving at some stage, towers stay on Standard Time
# PRI hopes that the following line will do this ...
dt_era5_loc_tts = [x - x.dst() for x in dt_era5_loc_tts]
# make the datetime series timezone naive and put it in data structure
dt_era5_loc_tts = [x.replace(tzinfo=None) for x in dt_era5_loc_tts]
ds_era5.series["DateTime"]["Data"] = dt_era5_loc_tts
ds_era5.series["DateTime"]["Flag"] = numpy.zeros(len(dt_era5_loc_tts))
ds_era5.globalattributes["nc_nrecs"] = len(dt_era5_loc_tts)
ds_era5.globalattributes["start_datetime"] = str(dt_era5_loc_tts[0])