def get_absolutehumidity(ds_60minutes):
for i in range(0, 3):
for j in range(0, 3):
Ta_label = "Ta_" + str(i) + str(j)
RH_label = "RH_" + str(i) + str(j)
Ah_label = "Ah_" + str(i) + str(j)
Ta, f, a = qcutils.GetSeriesasMA(ds_60minutes, Ta_label)
RH, f, a = qcutils.GetSeriesasMA(ds_60minutes, RH_label)
Ah = mf.absolutehumidityfromRH(Ta, RH)
attr = qcutils.MakeAttributeDictionary(
long_name='Absolute humidity',
units='g/m3',
standard_name='not defined')
qcutils.CreateSeries(ds_60minutes, Ah_label, Ah, Flag=f, Attr=attr)
return
def get_groundheatflux(ds_60minutes):
for i in range(0,3):
for j in range(0,3):
label_Fg = "Fg_"+str(i)+str(j)
label_Fn = "Fn_"+str(i)+str(j)
label_Fh = "Fh_"+str(i)+str(j)
label_Fe = "Fe_"+str(i)+str(j)
Fn,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fn)
Fh,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fh)
Fe,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fe)
Fg = Fn - Fh - Fe
attr = qcutils.MakeAttributeDictionary(long_name='Calculated ground heat flux',
standard_name='downward_heat_flux_in_soil',
units='W/m2')
qcutils.CreateSeries(ds_60minutes,label_Fg,Fg,Flag=f,Attr=attr)
return
def get_relativehumidity(ds_60minutes):
for i in range(0, 3):
for j in range(0, 3):
q_label = "q_" + str(i) + str(j)
Ta_label = "Ta_" + str(i) + str(j)
ps_label = "ps_" + str(i) + str(j)
RH_label = "RH_" + str(i) + str(j)
q, f, a = qcutils.GetSeriesasMA(ds_60minutes, q_label)
Ta, f, a = qcutils.GetSeriesasMA(ds_60minutes, Ta_label)
ps, f, a = qcutils.GetSeriesasMA(ds_60minutes, ps_label)
RH = mf.RHfromspecifichumidity(q, Ta, ps)
attr = qcutils.MakeAttributeDictionary(
long_name='Relative humidity',
units='%',
standard_name='not defined')
qcutils.CreateSeries(ds_60minutes, RH_label, RH, Flag=f, Attr=attr)
return
def AhfromMR(ds, Ah_out, MR_in, Ta_in, ps_in):
"""
Purpose:
Function to calculate absolute humidity given the water vapour mixing
ratio, air temperature and pressure. Absolute humidity is not calculated
if any of the input series are missing or if the specified output series
already exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
qcfunc.AhfromMR(ds,"Ah_IRGA_Av","H2O_IRGA_Av","Ta_HMP_2m","ps")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [MR_in, Ta_in, ps_in]:
if item not in ds.series.keys():
msg = " AhfromMR: Requested series " + item + " not found, " + Ah_out + " not calculated"
logger.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromMR: Output series " + Ah_out + " already exists, skipping ..."
logger.error(msg)
return 0
MR_data, MR_flag, MR_attr = qcutils.GetSeriesasMA(ds, MR_in)
Ta_data, Ta_flag, Ta_attr = qcutils.GetSeriesasMA(ds, Ta_in)
ps_data, ps_flag, ps_attr = qcutils.GetSeriesasMA(ds, ps_in)
Ah_data = mf.h2o_gpm3frommmolpmol(MR_data, Ta_data, ps_data)
long_name = "Absolute humidity calculated from " + MR_in + ", " + Ta_in + " and " + ps_in
Ah_attr = qcutils.MakeAttributeDictionary(long_name=long_name,
height=MR_attr["height"],
units="g/m3")
flag = numpy.where(numpy.ma.getmaskarray(Ah_data) == True, ones, zeros)
qcutils.CreateSeries(ds, Ah_out, Ah_data, flag, Ah_attr)
return 1
def AhfromRH(ds, Ah_out, RH_in, Ta_in):
"""
Purpose:
Function to calculate absolute humidity given relative humidity and
air temperature. Absolute humidity is not calculated if any of the
input series are missing or if the specified output series already
exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
qcfunc.AhfromRH(ds,"Ah_HMP_2m","RH_HMP_2m","Ta_HMP_2m")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [RH_in, Ta_in]:
if item not in ds.series.keys():
msg = " AhfromRH: Requested series " + item + " not found, " + Ah_out + " not calculated"
logger.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromRH: Output series " + Ah_out + " already exists, skipping ..."
logger.error(msg)
return 0
RH_data, RH_flag, RH_attr = qcutils.GetSeriesasMA(ds, RH_in)
Ta_data, Ta_flag, Ta_attr = qcutils.GetSeriesasMA(ds, Ta_in)
Ah_data = mf.absolutehumidityfromRH(Ta_data, RH_data)
Ah_attr = qcutils.MakeAttributeDictionary(
long_name="Absolute humidity calculated from " + RH_in + " and " +
Ta_in,
height=RH_attr["height"],
units="g/m3")
flag = numpy.where(numpy.ma.getmaskarray(Ah_data) == True, ones, zeros)
qcutils.CreateSeries(ds, Ah_out, Ah_data, flag, Ah_attr)
return 1
def AhfromRH(ds, Ah_out, RH_in, Ta_in):
"""
Purpose:
Function to calculate absolute humidity given relative humidity and
air temperature. Absolute humidity is not calculated if any of the
input series are missing or if the specified output series already
exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
qcfunc.AhfromRH(ds,"Ah_HMP_2m","RH_HMP_2m","Ta_HMP_2m")
Author: PRI
Date: September 2015
"""
for item in [RH_in, Ta_in]:
if item not in ds.series.keys():
msg = " AhfromRH: Requested series " + item + " not found, " + Ah_out + " not calculated"
log.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromRH: Output series " + Ah_out + " already exists, skipping ..."
log.error(msg)
return 0
RH_data, RH_flag, RH_attr = qcutils.GetSeriesasMA(ds, RH_in)
Ta_data, Ta_flag, Ta_attr = qcutils.GetSeriesasMA(ds, Ta_in)
Ah_data = mf.absolutehumidityfromRH(Ta_data, RH_data)
Ah_attr = qcutils.MakeAttributeDictionary(
long_name="Absolute humidity calculated from " + RH_in + " and " +
Ta_in,
height=RH_attr["height"],
units="g/m3")
qcutils.CreateSeries(ds,
Ah_out,
Ah_data,
FList=[RH_in, Ta_in],
Attr=Ah_attr)
return 1
# update global attributes
ds.globalattributes["nc_nrecs"] = len(dt_loc)
ds.globalattributes["start_datetime"] = str(dt_loc[0])
ds.globalattributes["end_datetime"] = str(dt_loc[-1])
# get the Excel datetime
qcutils.get_xldatefromdatetime(ds)
# get the year, month, day, hour, minute and second
qcutils.get_ymdhmsfromdatetime(ds)
# put the QC'd, smoothed and interpolated EVI into the data structure
flag = numpy.zeros(len(dt_loc), dtype=numpy.int32)
attr = qcutils.MakeAttributeDictionary(
long_name="MODIS EVI, smoothed and interpolated",
units="none",
horiz_resolution="250m",
cutout_size=str(site_cutout),
evi_quality_threshold=str(evi_quality_threshold),
evi_sd_threshold=str(evi_sd_threshold),
evi_interpolate=str(evi_interpolate),
evi_smooth_filter=str(evi_smooth_filter),
sg_num_points=str(sg_num_points),
sg_order=str(sg_num_points))
qcutils.CreateSeries(ds, "EVI", evi_interp2_smooth, Flag=flag, Attr=attr)
attr = qcutils.MakeAttributeDictionary(
long_name="MODIS EVI, interpolated",
units="none",
horiz_resolution="250m",
cutout_size=str(site_cutout),
evi_quality_threshold=str(evi_quality_threshold),
evi_sd_threshold=str(evi_sd_threshold),
evi_interpolate=str(evi_interpolate))
def ApplyTurbulenceFilter(cf,ds,ustar_threshold=None):
"""
Purpose:
Usage:
Author:
Date:
"""
opt = ApplyTurbulenceFilter_checks(cf,ds)
if not opt["OK"]: return
# local point to datetime series
ldt = ds.series["DateTime"]["Data"]
# time step
ts = int(ds.globalattributes["time_step"])
# dictionary of utar thresold values
if ustar_threshold==None:
ustar_dict = qcrp.get_ustar_thresholds(cf,ldt)
else:
ustar_dict = qcrp.get_ustar_thresholds_annual(ldt,ustar_threshold)
# initialise a dictionary for the indicator series
indicators = {}
# get data for the indicator series
ustar,ustar_flag,ustar_attr = qcutils.GetSeriesasMA(ds,"ustar")
Fsd,f,a = qcutils.GetSeriesasMA(ds,"Fsd")
if "solar_altitude" not in ds.series.keys(): qcts.get_synthetic_fsd(ds)
Fsd_syn,f,a = qcutils.GetSeriesasMA(ds,"Fsd_syn")
sa,f,a = qcutils.GetSeriesasMA(ds,"solar_altitude")
# get the day/night indicator series
# indicators["day"] = 1 ==> day time, indicators["day"] = 0 ==> night time
indicators["day"] = qcrp.get_day_indicator(cf,Fsd,Fsd_syn,sa)
ind_day = indicators["day"]["values"]
# get the turbulence indicator series
if opt["turbulence_filter"].lower()=="ustar":
# indicators["turbulence"] = 1 ==> turbulent, indicators["turbulence"] = 0 ==> not turbulent
indicators["turbulence"] = qcrp.get_turbulence_indicator_ustar(ldt,ustar,ustar_dict,ts)
elif opt["turbulence_filter"].lower()=="ustar_evg":
# ustar >= threshold ==> ind_ustar = 1, ustar < threshold == ind_ustar = 0
indicators["ustar"] = qcrp.get_turbulence_indicator_ustar(ldt,ustar,ustar_dict,ts)
ind_ustar = indicators["ustar"]["values"]
# ustar >= threshold during day AND ustar has been >= threshold since sunset ==> indicators["turbulence"] = 1
# indicators["turbulence"] = 0 during night once ustar has dropped below threshold even if it
# increases above the threshold later in the night
indicators["turbulence"] = qcrp.get_turbulence_indicator_ustar_evg(ldt,ind_day,ind_ustar,ustar,ustar_dict,ts)
elif opt["turbulence_filter"].lower()=="l":
#indicators["turbulence] = get_turbulence_indicator_l(ldt,L,z,d,zmdonL_threshold)
indicators["turbulence"] = numpy.ones(len(ldt))
msg = " Use of L as turbulence indicator not implemented, no filter applied"
log.warning(msg)
else:
msg = " Unrecognised turbulence filter option ("
msg = msg+opt["turbulence_filter"]+"), no filter applied"
log.error(msg)
return
# initialise the final indicator series as the turbulence indicator
# subsequent filters will modify the final indicator series
# we must use copy.deepcopy() otherwise the "values" array will only
# be copied by reference not value. Damn Python's default of copy by reference!
indicators["final"] = copy.deepcopy(indicators["turbulence"])
# check to see if the user wants to accept all day time observations
# regardless of ustar value
if opt["accept_day_times"].lower()=="yes":
# if yes, then we force the final indicator to be 1
# if ustar is below the threshold during the day.
idx = numpy.where(indicators["day"]["values"]==1)[0]
indicators["final"]["values"][idx] = numpy.int(1)
indicators["final"]["attr"].update(indicators["day"]["attr"])
# get the evening indicator series
indicators["evening"] = qcrp.get_evening_indicator(cf,Fsd,Fsd_syn,sa,ts)
indicators["dayevening"] = {"values":indicators["day"]["values"]+indicators["evening"]["values"]}
indicators["dayevening"]["attr"] = indicators["day"]["attr"].copy()
indicators["dayevening"]["attr"].update(indicators["evening"]["attr"])
if opt["use_evening_filter"].lower()=="yes":
idx = numpy.where(indicators["dayevening"]["values"]==0)[0]
indicators["final"]["values"][idx] = numpy.int(0)
indicators["final"]["attr"].update(indicators["dayevening"]["attr"])
# save the indicator series
ind_flag = numpy.zeros(len(ldt))
long_name = "Turbulence indicator, 1 for turbulent, 0 for non-turbulent"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"turbulence_indicator",indicators["turbulence"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Day indicator, 1 for day time, 0 for night time"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"day_indicator",indicators["day"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Evening indicator, 1 for evening, 0 for not evening"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"evening_indicator",indicators["evening"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Day/evening indicator, 1 for day/evening, 0 for not day/evening"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"dayevening_indicator",indicators["dayevening"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Final indicator, 1 for use data, 0 for don't use data"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"final_indicator",indicators["final"]["values"],Flag=ind_flag,Attr=ind_attr)
# loop over the series to be filtered
for series in opt["filter_list"]:
msg = " Applying "+opt["turbulence_filter"]+" filter to "+series
log.info(msg)
# get the data
data,flag,attr = qcutils.GetSeriesasMA(ds,series)
# continue to next series if this series has been filtered before
if "turbulence_filter" in attr:
msg = " Series "+series+" has already been filtered, skipping ..."
log.warning(msg)
continue
# save the non-filtered data
qcutils.CreateSeries(ds,series+"_nofilter",data,Flag=flag,Attr=attr)
# now apply the filter
data_filtered = numpy.ma.masked_where(indicators["final"]["values"]==0,data,copy=True)
flag_filtered = numpy.copy(flag)
idx = numpy.where(indicators["final"]["values"]==0)[0]
flag_filtered[idx] = numpy.int32(61)
# update the series attributes
for item in indicators["final"]["attr"].keys():
attr[item] = indicators["final"]["attr"][item]
# and write the filtered data to the data structure
qcutils.CreateSeries(ds,series,data_filtered,Flag=flag_filtered,Attr=attr)
# and write a copy of the filtered datas to the data structure so it
# will still exist once the gap filling has been done
qcutils.CreateSeries(ds,series+"_filtered",data_filtered,Flag=flag_filtered,Attr=attr)
return
Fsd_erai_3hr = Fsd_erai_3hr / (erai_timestep * 60)
# normalise the ERA-I downwelling shortwave by the solar altitude
# clamp solar altitude to a minimum value to avoid numerical problems
# when alt_solar is close to 0
alt_solar_limit = float(site_sa_limit) * numpy.ones(len(alt_solar_3hr))
sa = numpy.where(alt_solar_3hr <= float(site_sa_limit),
alt_solar_limit, alt_solar_3hr)
coef_3hr = Fsd_erai_3hr / numpy.sin(numpy.deg2rad(sa))
# get the spline interpolation function
s = InterpolatedUnivariateSpline(erai_time_3hr, coef_3hr, k=1)
# get the coefficient at the tower time step
coef_tts = s(erai_time_tts)
# get the downwelling solar radiation at the tower time step
Fsd_erai_tts = coef_tts * numpy.sin(numpy.deg2rad(alt_solar_tts))
flag = numpy.zeros(len(Fsd_erai_tts), dtype=numpy.int32)
attr = qcutils.MakeAttributeDictionary(
long_name="Downwelling short wave radiation", units="W/m2")
qcutils.CreateSeries(ds_erai,
"Fsd",
Fsd_erai_tts,
Flag=flag,
Attr=attr)
# Interpolate the 3 hourly accumulated net shortwave to the tower time step
# NOTE: ERA-I variables are dimensioned [time,latitude,longitude]
Fn_sw_3d = erai_file.variables["ssr"][:, :, :]
Fn_sw_accum = Fn_sw_3d[:, site_lat_index, site_lon_index]
# Net shortwave in ERA-I is a cummulative value that is reset to 0 at 0300 and 1500 UTC.
# Here we convert the cummulative values to 3 hourly values.
Fn_sw_erai_3hr = numpy.ediff1d(Fn_sw_accum, to_begin=0)
# deal with the reset times at 0300 and 1500
idx = numpy.where((hour_utc == 3) | (hour_utc == 15))[0]
def get_accessdata(cf,ds_60minutes,f,info):
# latitude and longitude, chose central pixel of 3x3 grid
ds_60minutes.globalattributes["latitude"] = f.variables["lat"][1]
ds_60minutes.globalattributes["longitude"] = f.variables["lon"][1]
# list of variables to process
var_list = list(cf["Variables"].keys())
# get a series of Python datetimes and put this into the data structure
valid_date = f.variables["valid_date"][:]
nRecs = len(valid_date)
valid_time = f.variables["valid_time"][:]
dl = [datetime.datetime.strptime(str(int(valid_date[i])*10000+int(valid_time[i])),"%Y%m%d%H%M") for i in range(0,nRecs)]
dt_utc_all = numpy.array(dl)
time_step = numpy.array([(dt_utc_all[i]-dt_utc_all[i-1]).total_seconds() for i in range(1,len(dt_utc_all))])
time_step = numpy.append(time_step,3600)
idxne0 = numpy.where(time_step!=0)[0]
idxeq0 = numpy.where(time_step==0)[0]
idx_clipped = numpy.where((idxeq0>0)&(idxeq0<nRecs))[0]
idxeq0 = idxeq0[idx_clipped]
dt_utc = dt_utc_all[idxne0]
dt_utc = [x.replace(tzinfo=pytz.utc) for x in dt_utc]
dt_loc = [x.astimezone(info["site_tz"]) for x in dt_utc]
dt_loc = [x-x.dst() for x in dt_loc]
dt_loc = [x.replace(tzinfo=None) for x in dt_loc]
flag = numpy.zeros(len(dt_loc),dtype=numpy.int32)
ds_60minutes.series["DateTime"] = {}
ds_60minutes.series["DateTime"]["Data"] = dt_loc
ds_60minutes.series["DateTime"]["Flag"] = flag
ds_60minutes.series["DateTime_UTC"] = {}
ds_60minutes.series["DateTime_UTC"]["Data"] = dt_utc
ds_60minutes.series["DateTime_UTC"]["Flag"] = flag
nRecs = len(ds_60minutes.series["DateTime"]["Data"])
ds_60minutes.globalattributes["nc_nrecs"] = nRecs
# we're done with valid_date and valid_time, drop them from the variable list
for item in ["valid_date","valid_time","lat","lon"]:
if item in var_list: var_list.remove(item)
# create the QC flag with all zeros
nRecs = ds_60minutes.globalattributes["nc_nrecs"]
flag_60minutes = numpy.zeros(nRecs,dtype=numpy.int32)
# get the UTC hour
hr_utc = [x.hour for x in dt_utc]
attr = qcutils.MakeAttributeDictionary(long_name='UTC hour')
qcutils.CreateSeries(ds_60minutes,'Hr_UTC',hr_utc,Flag=flag_60minutes,Attr=attr)
# now loop over the variables listed in the control file
for label in var_list:
# get the name of the variable in the ACCESS file
access_name = qcutils.get_keyvaluefromcf(cf,["Variables",label],"access_name",default=label)
# warn the user if the variable not found
if access_name not in list(f.variables.keys()):
msg = "Requested variable "+access_name
msg = msg+" not found in ACCESS data"
logging.error(msg)
continue
# get the variable attibutes
attr = get_variableattributes(f,access_name)
# loop over the 3x3 matrix of ACCESS grid data supplied
for i in range(0,3):
for j in range(0,3):
label_ij = label+'_'+str(i)+str(j)
if len(f.variables[access_name].shape)==3:
series = f.variables[access_name][:,i,j]
elif len(f.variables[access_name].shape)==4:
series = f.variables[access_name][:,0,i,j]
else:
msg = "Unrecognised variable ("+label
msg = msg+") dimension in ACCESS file"
logging.error(msg)
series = series[idxne0]
qcutils.CreateSeries(ds_60minutes,label_ij,series,
Flag=flag_60minutes,Attr=attr)
return
def interpolate_to_30minutes(ds_60minutes):
ds_30minutes = qcio.DataStructure()
# copy the global attributes
for this_attr in ds_60minutes.globalattributes.keys():
ds_30minutes.globalattributes[
this_attr] = ds_60minutes.globalattributes[this_attr]
# update the global attribute "time_step"
ds_30minutes.globalattributes["time_step"] = 30
# generate the 30 minute datetime series
dt_loc_60minutes = ds_60minutes.series["DateTime"]["Data"]
dt_loc_30minutes = [
x for x in perdelta(dt_loc_60minutes[0], dt_loc_60minutes[-1],
datetime.timedelta(minutes=30))
]
nRecs_30minutes = len(dt_loc_30minutes)
dt_utc_60minutes = ds_60minutes.series["DateTime_UTC"]["Data"]
dt_utc_30minutes = [
x for x in perdelta(dt_utc_60minutes[0], dt_utc_60minutes[-1],
datetime.timedelta(minutes=30))
]
# update the global attribute "nc_nrecs"
ds_30minutes.globalattributes['nc_nrecs'] = nRecs_30minutes
ds_30minutes.series["DateTime"] = {}
ds_30minutes.series["DateTime"]["Data"] = dt_loc_30minutes
flag = numpy.zeros(len(dt_loc_30minutes), dtype=numpy.int32)
ds_30minutes.series["DateTime"]["Flag"] = flag
ds_30minutes.series["DateTime_UTC"] = {}
ds_30minutes.series["DateTime_UTC"]["Data"] = dt_utc_30minutes
flag = numpy.zeros(len(dt_utc_30minutes), dtype=numpy.int32)
ds_30minutes.series["DateTime_UTC"]["Flag"] = flag
# get the year, month etc from the datetime
qcutils.get_xldatefromdatetime(ds_30minutes)
qcutils.get_ymdhmsfromdatetime(ds_30minutes)
# interpolate to 30 minutes
nRecs_60 = len(ds_60minutes.series["DateTime"]["Data"])
nRecs_30 = len(ds_30minutes.series["DateTime"]["Data"])
x_60minutes = numpy.arange(0, nRecs_60, 1)
x_30minutes = numpy.arange(0, nRecs_60 - 0.5, 0.5)
varlist_60 = ds_60minutes.series.keys()
# strip out the date and time variables already done
for item in [
"DateTime", "DateTime_UTC", "xlDateTime", "Year", "Month", "Day",
"Hour", "Minute", "Second", "Hdh", "Hr_UTC"
]:
if item in varlist_60: varlist_60.remove(item)
# now do the interpolation (its OK to interpolate accumulated precipitation)
for label in varlist_60:
series_60minutes, flag, attr = qcutils.GetSeries(ds_60minutes, label)
ci_60minutes = numpy.zeros(len(series_60minutes))
idx = numpy.where(
abs(series_60minutes - float(c.missing_value)) < c.eps)[0]
ci_60minutes[idx] = float(1)
int_fn = interp1d(x_60minutes, series_60minutes)
series_30minutes = int_fn(x_30minutes)
int_fn = interp1d(x_60minutes, ci_60minutes)
ci_30minutes = int_fn(x_30minutes)
idx = numpy.where(abs(ci_30minutes - float(0)) > c.eps)[0]
series_30minutes[idx] = numpy.float64(c.missing_value)
flag_30minutes = numpy.zeros(nRecs_30, dtype=numpy.int32)
flag_30minutes[idx] = numpy.int32(1)
qcutils.CreateSeries(ds_30minutes,
label,
series_30minutes,
Flag=flag_30minutes,
Attr=attr)
# get the UTC hour
hr_utc = [float(x.hour) + float(x.minute) / 60 for x in dt_utc_30minutes]
attr = qcutils.MakeAttributeDictionary(long_name='UTC hour')
flag_30minutes = numpy.zeros(nRecs_30, dtype=numpy.int32)
qcutils.CreateSeries(ds_30minutes,
'Hr_UTC',
hr_utc,
Flag=flag_30minutes,
Attr=attr)
return ds_30minutes
# get the 60 minute data structure
ds_aws_60minute = qcio.DataStructure()
# get the global attributes
for item in list(ds_aws_30minute.globalattributes.keys()):
ds_aws_60minute.globalattributes[
item] = ds_aws_30minute.globalattributes[item]
# overwrite with 60 minute values as appropriate
ds_aws_60minute.globalattributes["nc_nrecs"] = str(nRecs_60minute)
ds_aws_60minute.globalattributes["time_step"] = str(60)
# put the Python datetime into the data structure
ds_aws_60minute.series["DateTime"] = {}
ds_aws_60minute.series["DateTime"]["Data"] = dt_aws_60minute
ds_aws_60minute.series["DateTime"]["Flag"] = numpy.zeros(nRecs_60minute,
dtype=numpy.int32)
ds_aws_60minute.series["DateTime"][
"Attr"] = qcutils.MakeAttributeDictionary(
long_name="DateTime in local time zone", units="None")
# add the Excel datetime, year, month etc
qcutils.get_xldatefromdatetime(ds_aws_60minute)
qcutils.get_ymdhmsfromdatetime(ds_aws_60minute)
# loop over the series and take the average (every thing but Precip) or sum (Precip)
for item in series_list:
if "Precip" in item:
data_30minute, flag_30minute, attr = qcutils.GetSeriesasMA(
ds_aws_30minute, item, si=si_wholehour, ei=ei_wholehour)
data_2d = numpy.reshape(data_30minute, (nRecs_30minute / 2, 2))
flag_2d = numpy.reshape(flag_30minute, (nRecs_30minute / 2, 2))
data_60minute = numpy.ma.sum(data_2d, axis=1)
flag_60minute = numpy.ma.max(flag_2d, axis=1)
qcutils.CreateSeries(ds_aws_60minute, item, data_60minute,
flag_60minute, attr)
elif "Wd" in item:
# now pull the data out and put it in separate data structures, one per station, all
# of which are held in a data structure dictionary
ds_dict = {}
for bom_id in data_dict.keys():
log.info("Processing BoM station: "+str(bom_id))
# create a data structure
ds=qcio.DataStructure()
# put the year, month, day, hour and minute into the data structure
nRecs = data_dict[bom_id].shape[0]
ds.globalattributes["nc_nrecs"] = nRecs
ds.globalattributes["time_step"] = 30
ds.globalattributes["latitude"] = bom_sites_info[site_name][str(bom_id)]["latitude"]
ds.globalattributes["longitude"] = bom_sites_info[site_name][str(bom_id)]["longitude"]
flag = numpy.zeros(nRecs,dtype=numpy.int32)
Seconds = numpy.zeros(nRecs,dtype=numpy.float64)
qcutils.CreateSeries(ds,'Year',data_dict[bom_id][:,1],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Year',units='none'))
qcutils.CreateSeries(ds,'Month',data_dict[bom_id][:,2],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Month',units='none'))
qcutils.CreateSeries(ds,'Day',data_dict[bom_id][:,3],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Day',units='none'))
qcutils.CreateSeries(ds,'Hour',data_dict[bom_id][:,4],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Hour',units='none'))
qcutils.CreateSeries(ds,'Minute',data_dict[bom_id][:,5],Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Minute',units='none'))
qcutils.CreateSeries(ds,'Second',Seconds,Flag=flag,Attr=qcutils.MakeAttributeDictionary(long_name='Second',units='none'))
# now get the Python datetime
qcutils.get_datetimefromymdhms(ds)
# now put the data into the data structure
attr=qcutils.MakeAttributeDictionary(long_name='Precipitation since 0900',units='mm',
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
qcutils.CreateSeries(ds,'Precip',data_dict[bom_id][:,6],Flag=flag,Attr=attr)
attr=qcutils.MakeAttributeDictionary(long_name='Air temperature',units='C',
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
ds_30.globalattributes["site_name"] = cf["Sites"][site]["site_name"]
time_units = getattr(bios_ncfile.variables["time"],"units")
qcutils.get_datetimefromnctime(ds_30,time,time_units)
qcutils.round_datetime(ds_30,mode="nearest_timestep")
if qcutils.CheckTimeStep(ds_30): qcutils.FixTimeStep(ds_30)
ldt_30 = ds_30.series["DateTime"]["Data"]
si = qcutils.GetDateIndex(ldt_30,start_date,default=0,ts=ts,match="startnexthour")
ei = qcutils.GetDateIndex(ldt_30,end_date,default=len(ldt_30),ts=ts,match="endprevioushour")
ds_30.series["DateTime"]["Data"] = ds_30.series["DateTime"]["Data"][si:ei+1]
ds_30.series["DateTime"]["Flag"] = ds_30.series["DateTime"]["Flag"][si:ei+1]
ldt_30 = ds_30.series["DateTime"]["Data"]
nRecs = ds_30.globalattributes["nc_nrecs"] = len(ldt_30)
flag = numpy.zeros(nRecs)
qcutils.get_ymdhmsfromdatetime(ds_30)
xl_date_loc = qcutils.get_xldatefromdatetime(ds_30)
attr = qcutils.MakeAttributeDictionary(long_name="Date/time (local) in Excel format",units="days since 1899-12-31 00:00:00")
qcutils.CreateSeries(ds_30,"xlDateTime",xl_date_loc,flag,attr)
# get the data
for label in var_list:
bios_name = cf["Variables"][label]["bios_name"]
if len(bios_ncfile.variables[bios_name].shape)==1:
#print label+" has 1 dimension"
data = bios_ncfile.variables[bios_name][:][si:ei+1]
elif len(bios_ncfile.variables[bios_name].shape)==2:
#print label+" has 2 dimensions"
data = bios_ncfile.variables[bios_name][:,0][si:ei+1]
elif len(bios_ncfile.variables[bios_name].shape)==3:
#print label+" has 3 dimensions"
data = bios_ncfile.variables[bios_name][:,0,0][si:ei+1]
attr = {}
for this_attr in bios_ncfile.variables[bios_name].ncattrs():
lon = nc_file.variables["longitude"][:]
modis_time = nc_file.variables["time"][:]
# get a Python datetime series from the netCDF time
modis_time_units = getattr(nc_file.variables["time"], "units")
modis_dt = netCDF4.num2date(modis_time, modis_time_units)
print modis_dt[0], modis_dt[-1]
# get the index of points within the latitude and longitude bounds
lat_index = numpy.where((lat >= lat_bound_lower)
& (lat <= lat_bound_upper))[0]
lon_index = numpy.where((lon >= lon_bound_lower)
& (lon <= lon_bound_upper))[0]
# loop over MODIS products here
# get the EVI
evi_attr = qcutils.MakeAttributeDictionary(
long_name="Enhanced Vegetation Index from MODIS",
MODIS_product="MOD13Q1",
source="TERN-AusCover",
dap_url=dap_url,
evi_name=evi_name)
evi = nc_file.variables["evi"][:, lat_index, lon_index]
evi_flag = numpy.zeros(len(evi), dtype=numpy.int32)
# get the quality flags
quality = nc_file.variables["quality"][:, lat_index, lon_index]
ok_mask = numpy.ones_like(evi)
ok_list = [
2048, 2049, 2052, 2053, 2112, 2113, 2116, 2117, 2560, 2561, 2564, 2565,
2624, 2625, 2628, 2629
]
for item in ok_list:
index = numpy.ma.where(quality == item)[0]
ok_mask[index] = 0
evi_masked = numpy.ma.masked_where(ok_mask != 0, evi)
qcutils.CreateSeries(ds_60minutes,
label,
ps,
Flag=flag_60minutes,
Attr=attr)
# wind speed from components
for i in range(0, 3):
for j in range(0, 3):
u_label = "u_" + str(i) + str(j)
v_label = "v_" + str(i) + str(j)
Ws_label = "Ws_" + str(i) + str(j)
u, f, a = qcutils.GetSeriesasMA(ds_60minutes, u_label)
v, f, a = qcutils.GetSeriesasMA(ds_60minutes, v_label)
Ws = numpy.sqrt(u * u + v * v)
attr = qcutils.MakeAttributeDictionary(long_name="Wind speed",
units="m/s",
height="10m")
qcutils.CreateSeries(ds_60minutes, Ws_label, Ws, Flag=f, Attr=attr)
# wind direction from components
for i in range(0, 3):
for j in range(0, 3):
u_label = "u_" + str(i) + str(j)
v_label = "v_" + str(i) + str(j)
Wd_label = "Wd_" + str(i) + str(j)
u, f, a = qcutils.GetSeriesasMA(ds_60minutes, u_label)
v, f, a = qcutils.GetSeriesasMA(ds_60minutes, v_label)
Wd = float(270) - numpy.ma.arctan2(v, u) * float(180) / numpy.pi
index = numpy.ma.where(Wd > 360)[0]
if len(index) > 0: Wd[index] = Wd[index] - float(360)
attr = qcutils.MakeAttributeDictionary(long_name="Wind direction",
units="degrees",
