def interpolate_wswd(ds_in, x1, ds_out, x2):
"""
Purpose:
Interpolate wind speed and direction by converting them to U and V
components first, doing the interpolation and then converting back
to wind speed and direction.
Usage:
Side effects
Author: PRI
Date: December 2017
"""
# get the number of records in the output data
nrecs = ds_out.globalattributes["nc_nrecs"]
f0 = numpy.zeros(nrecs, dtype=numpy.int32)
f1 = numpy.ones(nrecs, dtype=numpy.int32)
# local pointer to the output datetime
ldt = qcutils.GetVariable(ds_out, "DateTime")
# create empty variables for the output dara
Ws_out = qcutils.create_empty_variable("Ws", nrecs, datetime=ldt["Data"])
Wd_out = qcutils.create_empty_variable("Wd", nrecs, datetime=ldt["Data"])
U_out = qcutils.create_empty_variable("u", nrecs, datetime=ldt["Data"])
V_out = qcutils.create_empty_variable("v", nrecs, datetime=ldt["Data"])
# get the input wind speed and direction
Ws_in = qcutils.GetVariable(ds_in, "Ws")
Wd_in = qcutils.GetVariable(ds_in, "Wd")
# covert to U and V components
U_in, V_in = qcutils.convert_WSWDtoUV(Ws_in, Wd_in)
# interpolate the components to the output time stamp
U_out["Data"] = interpolate_1d(x1, U_in["Data"], x2)
V_out["Data"] = interpolate_1d(x1, V_in["Data"], x2)
# add the QC flag and update variable attributes
U_out["Flag"] = numpy.ma.where(numpy.ma.getmaskarray(U_out["Data"])==True, f1, f0)
V_out["Flag"] = numpy.ma.where(numpy.ma.getmaskarray(V_out["Data"])==True, f1, f0)
U_out["Attr"]["long_name"] = "U component of wind velocity, postive east"
U_out["Attr"]["units"] = "m/s"
V_out["Attr"]["long_name"] = "V component of wind velocity, postive north"
V_out["Attr"]["units"] = "m/s"
# write the U and V components to the output data structure
qcutils.CreateVariable(ds_out, U_out)
qcutils.CreateVariable(ds_out, V_out)
# convert the interpolated components to wind speed and direction
Ws_out, Wd_out = qcutils.convert_UVtoWSWD(U_out, V_out)
# add the Qc flag and update the variable attributes
Ws_out["Flag"] = numpy.ma.where(numpy.ma.getmaskarray(Ws_out["Data"])==True, f1, f0)
Wd_out["Flag"] = numpy.ma.where(numpy.ma.getmaskarray(Wd_out["Data"])==True, f1, f0)
Ws_out["Attr"] = copy.deepcopy(Ws_in["Attr"])
Wd_out["Attr"] = copy.deepcopy(Wd_in["Attr"])
# write the wind speed and direction into the output data structure
qcutils.CreateVariable(ds_out, Ws_out)
qcutils.CreateVariable(ds_out, Wd_out)
return
def interpolate_precip(ds_in, x1, ds_out, x2):
"""
Purpose:
Transfer precipitation data from one time stamp to another.
Interpolating precipitation does not make sense since interpolation
will change the total amount of the precipitation and because
precipitation is very intermittent, we do not know how it will
behave at the missing times. Instead of interpolation, we assign
all of the precipitation total at the input time stamp to the
matching output time stamp.
Assumptions:
That the input time stamps are a subset of the output time stamps.
Usage:
Side effects:
Author: PRI
Date: December 2017
"""
nrecs = ds_out.globalattributes["nc_nrecs"]
# local pointer to the output datetime
ldt = qcutils.GetVariable(ds_out, "DateTime")
# just assign any precipitation to the ISD time stamp.
precip_out = qcutils.create_empty_variable("Precip", nrecs, datetime=ldt["Data"])
# zero the data array
precip_out["Data"] = precip_out["Data"] * float(0)
precip_in = qcutils.GetVariable(ds_in, "Precip")
# get indices of elements where the times match
idx = numpy.searchsorted(x2, numpy.intersect1d(x2, x1))
precip_out["Data"][idx] = precip_in["Data"]
precip_out["Flag"] = numpy.zeros(nrecs, dtype=numpy.int32)
precip_out["Attr"] = copy.deepcopy(precip_in["Attr"])
qcutils.CreateVariable(ds_out, precip_out)
return
def interpolate_ds(ds_in, ts):
"""
Purpose:
Interpolate the contents of a data structure onto a different time step.
Assumptions:
Usage:
Author: PRI
Date: June 2017
"""
logger.info("Interpolating data")
# instance the output data structure
ds_out = qcio.DataStructure()
# copy the global attributes
ds_out.globalattributes = copy.deepcopy(ds_in.globalattributes)
# add the time step
ds_out.globalattributes["time_step"] = str(ts)
# generate a regular time series at the required time step
dt = ds_in.series["DateTime"]["Data"]
dt0 = qcutils.rounddttots(dt[0], ts=ts)
if dt0 < dt[0]:
dt0 = dt0 + datetime.timedelta(minutes=ts)
dt1 = qcutils.rounddttots(dt[-1], ts=ts)
if dt1 > dt[-1]:
dt1 = dt1 - datetime.timedelta(minutes=ts)
idt = [result for result in qcutils.perdelta(dt0, dt1, datetime.timedelta(minutes=ts))]
x1 = numpy.array([toTimestamp(dt[i]) for i in range(len(dt))])
x2 = numpy.array([toTimestamp(idt[i]) for i in range(len(idt))])
# loop over the series in the data structure and interpolate
flag = numpy.zeros(len(idt), dtype=numpy.int32)
attr = {"long_name":"Datetime", "units":"none"}
ldt_var = {"Label":"DateTime", "Data":idt, "Flag":flag, "Attr":attr}
qcutils.CreateVariable(ds_out, ldt_var)
qcutils.get_nctime_from_datetime(ds_out)
nrecs = len(idt)
ds_out.globalattributes["nc_nrecs"] = nrecs
# first, we do the air temperature, dew point temperature and surface pressure
f0 = numpy.zeros(nrecs, dtype=numpy.int32)
f1 = numpy.ones(nrecs, dtype=numpy.int32)
for label in ["Ta", "Td", "ps", "RH", "Ah", "q"]:
var_out = qcutils.create_empty_variable(label, nrecs, datetime=idt)
var_in = qcutils.GetVariable(ds_in, label)
var_out["Data"] = interpolate_1d(x1, var_in["Data"], x2)
var_out["Flag"] = numpy.where(numpy.ma.getmaskarray(var_out["Data"])==True, f1, f0)
var_out["Attr"] = copy.deepcopy(var_in["Attr"])
qcutils.CreateVariable(ds_out, var_out)
# now clamp the dew point so that TD <= TA
Ta = qcutils.GetVariable(ds_out, "Ta")
Td = qcutils.GetVariable(ds_out, "Td")
Td["Data"] = numpy.ma.where(Td["Data"]<=Ta["Data"], x=Td["Data"], y=Ta["Data"])
qcutils.CreateVariable(ds_out, Td)
# now we do wind speed and direction by converting to U and V components
interpolate_wswd(ds_in, x1, ds_out, x2)
# and lastly, do precipitation
interpolate_precip(ds_in, x1, ds_out, x2)
return ds_out
ds_all.globalattributes["latitude_"+isd_site_id] = ds_out[i].globalattributes["latitude"]
ds_all.globalattributes["longitude_"+isd_site_id] = ds_out[i].globalattributes["longitude"]
ds_all.globalattributes["altitude_"+isd_site_id] = ds_out[i].globalattributes["altitude"]
# now copy the variables
# first, we get the indices of matching datetimes
ldt_one = ds_out[i].series["DateTime"]["Data"]
idx = numpy.searchsorted(ldt_all, numpy.intersect1d(ldt_all, ldt_one))
# then we get a list of the variables to copy
labels = [label for label in ds_out[i].series.keys() if label not in ["DateTime"]]
# and then we loop over the variables to be copied
for label in labels:
# read the data out of the ISD site data structure
var_out = qcutils.GetVariable(ds_out[i], label)
# create an empty output variable with a number, unique to each ISD station,
# appended to the label
var_all = qcutils.create_empty_variable(label+"_"+str(i), nrecs)
# copy the variable attributes
var_all["Attr"] = copy.deepcopy(var_out["Attr"])
# add the ISD site ID
var_all["Attr"]["isd_site_id"] = isd_site_id
# copy the data and flag onto the matching times
var_all["Data"][idx] = var_out["Data"]
var_all["Flag"][idx] = var_out["Flag"]
# put the data, flag and attributes into the all-in-one data structure
qcutils.CreateVariable(ds_all, var_all)
# write the netCDF file with the combined data for this year
if len(fluxnet_id) == 0:
nc_dir_path = os.path.join(out_base_path,site,"Data","ISD")
nc_file_name = site+"_ISD_"+str(year)+".nc"
else:
nc_dir_path = os.path.join(out_base_path,fluxnet_id,"Data","ISD")
# now copy the variables
# first, we get the indices of matching datetimes
ldt_one = ds_out[i].series["DateTime"]["Data"]
idx = numpy.searchsorted(ldt_all,
numpy.intersect1d(ldt_all, ldt_one))
# then we get a list of the variables to copy
series_list = ds_out[i].series.keys()
# and remove the datetime
if "DateTime" in series_list:
series_list.remove("DateTime")
# and then we loop over the variables to be copied
for label in series_list:
# append a number, unique to each ISD station, to the variable label
all_label = label + "_" + str(i)
# create empty data and flag arrays
variable = qcutils.create_empty_variable(all_label, nrecs)
qcutils.CreateSeries(ds_all,
all_label,
variable["Data"],
Flag=variable["Flag"],
Attr=variable["Attr"])
# read the data out of the ISD site data structure
data, flag, attr = qcutils.GetSeriesasMA(ds_out[i], label)
# add the ISD site ID
attr["isd_site_id"] = isd_site_id
# put the data, flag and attributes into the all-in-one data structure
ds_all.series[all_label]["Data"][idx] = data
ds_all.series[all_label]["Flag"][idx] = flag
ds_all.series[all_label]["Attr"] = copy.deepcopy(attr)
# write the netCDF file with the combined data for this year
if len(fluxnet_id) == 0: