def get_instantaneous_precip30(ds_30minutes):
hr_utc,f,a = qcutils.GetSeries(ds_30minutes,'Hr_UTC')
for i in range(0,3):
for j in range(0,3):
label = "Precip_"+str(i)+str(j)
# get the accumulated precipitation
accum,flag,attr = qcutils.GetSeries(ds_30minutes,label)
# get the 30 minute precipitation
precip = numpy.ediff1d(accum,to_begin=0)
# now we deal with the reset of accumulated precipitation at 00, 06, 12 and 18 UTC
# indices of analysis times 00, 06, 12, and 18
idx1 = numpy.where(numpy.mod(hr_utc,6)==0)[0]
# set 30 minute precipitation at these times to half of the analysis value
precip[idx1] = accum[idx1]/float(2)
# now get the indices of the 30 minute period immediately the analysis time
# these values will have been interpolated between the last forecast value
# and the analysis value, they need to be set to half of the analysis value
idx2 = idx1-1
# remove negative indices
idx2 = idx2[idx2>=0]
# set these 30 minute times to half the analysis value
precip[idx2] = accum[idx2+1]/float(2)
# set precipitations less than 0.01 mm to 0
idx3 = numpy.ma.where(precip<0.01)[0]
precip[idx3] = float(0)
# set instantaneous precipitation to missing when accumlated precipitation was missing
idx = numpy.where(flag!=0)[0]
precip[idx] = float(c.missing_value)
# set some variable attributes
attr["long_name"] = "Precipitation total over time step"
attr["units"] = "mm/30 minutes"
qcutils.CreateSeries(ds_30minutes,label,precip,Flag=flag,Attr=attr)
return
def get_instantaneous_precip60(ds_60minutes):
hr_utc, f, a = qcutils.GetSeries(ds_60minutes, 'Hr_UTC')
for i in range(0, 3):
for j in range(0, 3):
label = "Precip_" + str(i) + str(j)
# get the accumulated precipitation
accum, flag, attr = qcutils.GetSeries(ds_60minutes, label)
# get the 30 minute precipitation
precip = numpy.ediff1d(accum, to_begin=0)
# now we deal with the reset of accumulated precipitation at 00, 06, 12 and 18 UTC
# indices of analysis times 00, 06, 12, and 18
idx1 = numpy.where(numpy.mod(hr_utc, 6) == 0)[0]
# set 30 minute precipitation at these times to the analysis value
precip[idx1] = accum[idx1]
# set accumulated precipitations less than 0.001 mm to 0
idx2 = numpy.ma.where(precip < 0.01)[0]
precip[idx2] = float(0)
# set instantaneous precipitation to missing when accumlated precipitation was missing
idx = numpy.where(flag != 0)[0]
precip[idx] = float(c.missing_value)
# set some variable attributes
attr["long_name"] = "Precipitation total over time step"
attr["units"] = "mm/60 minutes"
qcutils.CreateSeries(ds_60minutes,
label,
precip,
Flag=flag,
Attr=attr)
def interpolate_to_30minutes(ds_60minutes):
ds_30minutes = qcio.DataStructure()
# copy the global attributes
for this_attr in list(ds_60minutes.globalattributes.keys()):
ds_30minutes.globalattributes[this_attr] = ds_60minutes.globalattributes[this_attr]
# update the global attribute "time_step"
ds_30minutes.globalattributes["time_step"] = 30
# generate the 30 minute datetime series
dt_loc_60minutes = ds_60minutes.series["DateTime"]["Data"]
dt_loc_30minutes = [x for x in perdelta(dt_loc_60minutes[0],dt_loc_60minutes[-1],datetime.timedelta(minutes=30))]
nRecs_30minutes = len(dt_loc_30minutes)
dt_utc_60minutes = ds_60minutes.series["DateTime_UTC"]["Data"]
dt_utc_30minutes = [x for x in perdelta(dt_utc_60minutes[0],dt_utc_60minutes[-1],datetime.timedelta(minutes=30))]
# update the global attribute "nc_nrecs"
ds_30minutes.globalattributes['nc_nrecs'] = nRecs_30minutes
ds_30minutes.series["DateTime"] = {}
ds_30minutes.series["DateTime"]["Data"] = dt_loc_30minutes
flag = numpy.zeros(len(dt_loc_30minutes),dtype=numpy.int32)
ds_30minutes.series["DateTime"]["Flag"] = flag
ds_30minutes.series["DateTime_UTC"] = {}
ds_30minutes.series["DateTime_UTC"]["Data"] = dt_utc_30minutes
flag = numpy.zeros(len(dt_utc_30minutes),dtype=numpy.int32)
ds_30minutes.series["DateTime_UTC"]["Flag"] = flag
# get the year, month etc from the datetime
qcutils.get_xldatefromdatetime(ds_30minutes)
qcutils.get_ymdhmsfromdatetime(ds_30minutes)
# interpolate to 30 minutes
nRecs_60 = len(ds_60minutes.series["DateTime"]["Data"])
nRecs_30 = len(ds_30minutes.series["DateTime"]["Data"])
x_60minutes = numpy.arange(0,nRecs_60,1)
x_30minutes = numpy.arange(0,nRecs_60-0.5,0.5)
varlist_60 = list(ds_60minutes.series.keys())
# strip out the date and time variables already done
for item in ["DateTime","DateTime_UTC","xlDateTime","Year","Month","Day","Hour","Minute","Second","Hdh","Hr_UTC"]:
if item in varlist_60: varlist_60.remove(item)
# now do the interpolation (its OK to interpolate accumulated precipitation)
for label in varlist_60:
series_60minutes,flag,attr = qcutils.GetSeries(ds_60minutes,label)
ci_60minutes = numpy.zeros(len(series_60minutes))
idx = numpy.where(abs(series_60minutes-float(c.missing_value))<c.eps)[0]
ci_60minutes[idx] = float(1)
int_fn = interp1d(x_60minutes,series_60minutes)
series_30minutes = int_fn(x_30minutes)
int_fn = interp1d(x_60minutes,ci_60minutes)
ci_30minutes = int_fn(x_30minutes)
idx = numpy.where(abs(ci_30minutes-float(0))>c.eps)[0]
series_30minutes[idx] = numpy.float64(c.missing_value)
flag_30minutes = numpy.zeros(nRecs_30, dtype=numpy.int32)
flag_30minutes[idx] = numpy.int32(1)
qcutils.CreateSeries(ds_30minutes,label,series_30minutes,Flag=flag_30minutes,Attr=attr)
# get the UTC hour
hr_utc = [float(x.hour)+float(x.minute)/60 for x in dt_utc_30minutes]
attr = qcutils.MakeAttributeDictionary(long_name='UTC hour')
flag_30minutes = numpy.zeros(nRecs_30, dtype=numpy.int32)
qcutils.CreateSeries(ds_30minutes,'Hr_UTC',hr_utc,Flag=flag_30minutes,Attr=attr)
return ds_30minutes
def get_data_dict(ds, configs_dict):
data = {}
# NOTE: series are ndarrays not masked arrays
Fc, Fc_flag, a = qcutils.GetSeries(ds, "Fc")
target = configs_dict["target"]
ER, ER_flag, a = qcutils.GetSeries(ds, target)
Fsd, Fsd_flag, a = qcutils.GetSeries(ds, "Fsd")
T_label = configs_dict["drivers"]
T, T_flag, a = qcutils.GetSeries(ds, T_label)
VPD, VPD_flag, a = qcutils.GetSeries(ds, "VPD")
ustar, ustar_flag, a = qcutils.GetSeries(ds, "ustar")
# replace c.missing_value with numpy.nan
Fc = numpy.where((Fc_flag != 0) | (Fc == c.missing_value), numpy.nan, Fc)
ustar = numpy.where((ustar_flag != 0) | (ustar == c.missing_value),
numpy.nan, ustar)
ER = numpy.where((ER_flag != 0) | (ER == c.missing_value), numpy.nan, ER)
#Fsd = numpy.where((Fsd_flag!=0)|(Fsd==c.missing_value),
#numpy.nan,Fsd)
#T = numpy.where((T_flag!=0)|(T==c.missing_value),
#numpy.nan,T)
#VPD = numpy.where((VPD_flag!=0)|(VPD==c.missing_value),
#numpy.nan,VPD)
# put the data in the dictionary
#data["NEE"] = Fc
data["NEE"] = ER
data["PAR"] = Fsd * 0.46 * 4.6
data["TempC"] = T
data["VPD"] = VPD
data["ustar"] = ustar
data["date_time"] = numpy.array(ds.series["DateTime"]["Data"])
return data
def do_dependencycheck(cf, ds, section, series, code=23, mode="quiet"):
"""
Purpose:
Usage:
Author: PRI
Date: Back in the day
"""
if len(section) == 0 and len(series) == 0: return
if len(section) == 0:
section = qcutils.get_cfsection(cf, series=series, mode='quiet')
if "DependencyCheck" not in cf[section][series].keys(): return
if "Source" not in cf[section][series]["DependencyCheck"]:
msg = " DependencyCheck: keyword Source not found for series " + series + ", skipping ..."
logger.error(msg)
return
if mode == "verbose":
msg = " Doing DependencyCheck for " + series
logger.info(msg)
# get the precursor source list from the control file
source_list = ast.literal_eval(
cf[section][series]["DependencyCheck"]["Source"])
# check to see if the "ignore_missing" flag is set
opt = qcutils.get_keyvaluefromcf(cf, [section, series, "DependencyCheck"],
"ignore_missing",
default="no")
ignore_missing = False
if opt.lower() in ["yes", "y", "true", "t"]:
ignore_missing = True
# get the data
dependent_data, dependent_flag, dependent_attr = qcutils.GetSeries(
ds, series)
# loop over the precursor source list
for item in source_list:
# check the precursor is in the data structure
if item not in ds.series.keys():
msg = " DependencyCheck: " + series + " precursor series " + item + " not found, skipping ..."
logger.warning(msg)
continue
# get the precursor data
precursor_data, precursor_flag, precursor_attr = qcutils.GetSeries(
ds, item)
# check if the user wants to ignore missing precursor data
if ignore_missing:
# they do, so make an array of missing values
nRecs = int(ds.globalattributes["nc_nrecs"])
missing_array = numpy.ones(nRecs) * float(c.missing_value)
# and find the indicies of elements equal to the missing value
bool_array = numpy.isclose(precursor_data, missing_array)
idx = numpy.where(bool_array == True)[0]
# and set these flags to 0 so missing data is ignored
precursor_flag[idx] = numpy.int32(0)
# mask the dependent data where the precursor flag shows data not OK
dependent_data = numpy.ma.masked_where(
numpy.mod(precursor_flag, 10) != 0, dependent_data)
# get an index where the precursor flag shows data not OK
idx = numpy.ma.where(numpy.mod(precursor_flag, 10) != 0)[0]
# set the dependent QC flag
dependent_flag[idx] = numpy.int32(code)
# put the data back into the data structure
dependent_attr["DependencyCheck_source"] = str(source_list)
qcutils.CreateSeries(ds, series, dependent_data, dependent_flag,
dependent_attr)
# our work here is done
return
def get_seriesstats(cf,ds):
# open an Excel file for the flag statistics
level = ds.globalattributes['nc_level']
out_filename = get_outfilename_from_cf(cf)
xl_filename = out_filename.replace('.nc','_FlagStats.xls')
log.info(' Writing flag stats to Excel file '+xl_filename)
xlFile = xlwt.Workbook()
xlFlagSheet = xlFile.add_sheet('Flag')
# get the flag statistics
xlRow = 0
xlCol = 0
xlFlagSheet.write(xlRow,xlCol,'0:')
xlFlagSheet.write(xlRow,xlCol+1,ds.globalattributes['Flag0'])
xlFlagSheet.write(xlRow,xlCol+2,'1:')
xlFlagSheet.write(xlRow,xlCol+3,ds.globalattributes['Flag1'])
xlFlagSheet.write(xlRow,xlCol+4,'2:')
xlFlagSheet.write(xlRow,xlCol+5,ds.globalattributes['Flag2'])
xlFlagSheet.write(xlRow,xlCol+6,'3:')
xlFlagSheet.write(xlRow,xlCol+7,ds.globalattributes['Flag3'])
xlFlagSheet.write(xlRow,xlCol+8,'4:')
xlFlagSheet.write(xlRow,xlCol+9,ds.globalattributes['Flag4'])
xlFlagSheet.write(xlRow,xlCol+10,'5:')
xlFlagSheet.write(xlRow,xlCol+11,ds.globalattributes['Flag5'])
xlFlagSheet.write(xlRow,xlCol+12,'6:')
xlFlagSheet.write(xlRow,xlCol+13,ds.globalattributes['Flag6'])
xlFlagSheet.write(xlRow,xlCol+14,'7:')
xlFlagSheet.write(xlRow,xlCol+15,ds.globalattributes['Flag7'])
xlRow = xlRow + 1
xlFlagSheet.write(xlRow,xlCol,'10:')
xlFlagSheet.write(xlRow,xlCol+1,ds.globalattributes['Flag10'])
xlFlagSheet.write(xlRow,xlCol+2,'11:')
xlFlagSheet.write(xlRow,xlCol+3,ds.globalattributes['Flag11'])
xlFlagSheet.write(xlRow,xlCol+4,'12:')
xlFlagSheet.write(xlRow,xlCol+5,ds.globalattributes['Flag12'])
xlFlagSheet.write(xlRow,xlCol+6,'13:')
xlFlagSheet.write(xlRow,xlCol+7,ds.globalattributes['Flag13'])
xlFlagSheet.write(xlRow,xlCol+8,'14:')
xlFlagSheet.write(xlRow,xlCol+9,ds.globalattributes['Flag14'])
xlFlagSheet.write(xlRow,xlCol+10,'15:')
xlFlagSheet.write(xlRow,xlCol+11,ds.globalattributes['Flag15'])
xlFlagSheet.write(xlRow,xlCol+12,'16:')
xlFlagSheet.write(xlRow,xlCol+13,ds.globalattributes['Flag16'])
xlFlagSheet.write(xlRow,xlCol+14,'17:')
xlFlagSheet.write(xlRow,xlCol+15,ds.globalattributes['Flag17'])
xlFlagSheet.write(xlRow,xlCol+16,'18:')
xlFlagSheet.write(xlRow,xlCol+17,ds.globalattributes['Flag18'])
xlFlagSheet.write(xlRow,xlCol+18,'19:')
xlFlagSheet.write(xlRow,xlCol+19,ds.globalattributes['Flag19'])
xlRow = xlRow + 1
xlFlagSheet.write(xlRow,xlCol,'30:')
xlFlagSheet.write(xlRow,xlCol+1,ds.globalattributes['Flag30'])
xlFlagSheet.write(xlRow,xlCol+2,'31:')
xlFlagSheet.write(xlRow,xlCol+3,ds.globalattributes['Flag31'])
xlFlagSheet.write(xlRow,xlCol+4,'32:')
xlFlagSheet.write(xlRow,xlCol+5,ds.globalattributes['Flag32'])
xlFlagSheet.write(xlRow,xlCol+6,'33:')
xlFlagSheet.write(xlRow,xlCol+7,ds.globalattributes['Flag33'])
xlFlagSheet.write(xlRow,xlCol+8,'34:')
xlFlagSheet.write(xlRow,xlCol+9,ds.globalattributes['Flag34'])
xlFlagSheet.write(xlRow,xlCol+10,'35:')
xlFlagSheet.write(xlRow,xlCol+11,ds.globalattributes['Flag35'])
xlFlagSheet.write(xlRow,xlCol+12,'36:')
xlFlagSheet.write(xlRow,xlCol+13,ds.globalattributes['Flag36'])
xlFlagSheet.write(xlRow,xlCol+14,'37:')
xlFlagSheet.write(xlRow,xlCol+15,ds.globalattributes['Flag37'])
xlFlagSheet.write(xlRow,xlCol+16,'38:')
xlFlagSheet.write(xlRow,xlCol+17,ds.globalattributes['Flag38'])
xlFlagSheet.write(xlRow,xlCol+18,'39:')
xlFlagSheet.write(xlRow,xlCol+19,ds.globalattributes['Flag39'])
bins = numpy.arange(-0.5,23.5)
xlRow = 5
xlCol = 1
for Value in bins[:len(bins)-1]:
xlFlagSheet.write(xlRow,xlCol,int(Value+0.5))
xlCol = xlCol + 1
xlRow = xlRow + 1
xlCol = 0
dsVarNames = ds.series.keys()
dsVarNames.sort(key=unicode.lower)
for ThisOne in dsVarNames:
data,flag = qcutils.GetSeries(ds, ThisOne)
hist, bin_edges = numpy.histogram(flag, bins=bins)
xlFlagSheet.write(xlRow,xlCol,ThisOne)
xlCol = xlCol + 1
for Value in hist:
xlFlagSheet.write(xlRow,xlCol,float(Value))
xlCol = xlCol + 1
xlCol = 0
xlRow = xlRow + 1
xlFile.save(xl_filename)
def CPD_run(cf):
# Set input file and output path and create directories for plots and results
path_out = cf['Files']['file_path']
file_in = os.path.join(cf['Files']['file_path'],
cf['Files']['in_filename'])
#
if "out_filename" in cf['Files']:
file_out = os.path.join(cf['Files']['file_path'],
cf['Files']['out_filename'])
else:
file_out = os.path.join(
cf['Files']['file_path'],
cf['Files']['in_filename'].replace(".nc", "_CPD.xls"))
plot_path = "plots/"
if "plot_path" in cf["Files"]:
plot_path = os.path.join(cf["Files"]["plot_path"], "CPD/")
if not os.path.isdir(plot_path): os.makedirs(plot_path)
results_path = path_out
if not os.path.isdir(results_path): os.makedirs(results_path)
# get a dictionary of the variable names
var_list = cf["Variables"].keys()
names = {}
for item in var_list:
if "AltVarName" in cf["Variables"][item].keys():
names[item] = cf["Variables"][item]["AltVarName"]
else:
names[item] = item
# add the xlDateTime
names["xlDateTime"] = "xlDateTime"
names["Year"] = "Year"
# read the netcdf file
logger.info(' Reading netCDF file ' + file_in)
ds = qcio.nc_read_series(file_in)
dates_list = ds.series["DateTime"]["Data"]
nrecs = int(ds.globalattributes["nc_nrecs"])
# now get the data
d = {}
f = {}
for item in names.keys():
data, flag, attr = qcutils.GetSeries(ds, names[item])
d[item] = np.where(data == c.missing_value, np.nan, data)
f[item] = flag
# set all data to NaNs if any flag not 0 or 10
for item in f.keys():
for f_OK in [0, 10]:
idx = np.where(f[item] != 0)[0]
if len(idx) != 0:
for itemd in d.keys():
d[itemd][idx] = np.nan
df = pd.DataFrame(d, index=dates_list)
# replace missing values with NaN
df.replace(c.missing_value, np.nan)
# Build dictionary of additional configs
d = {}
d['radiation_threshold'] = int(cf['Options']['Fsd_threshold'])
d['num_bootstraps'] = int(cf['Options']['Num_bootstraps'])
d['flux_period'] = int(ds.globalattributes["time_step"])
d['site_name'] = ds.globalattributes["site_name"]
d["call_mode"] = qcutils.get_keyvaluefromcf(cf, ["Options"],
"call_mode",
default="interactive",
mode="quiet")
d["show_plots"] = qcutils.get_keyvaluefromcf(cf, ["Options"],
"show_plots",
default=True,
mode="quiet")
d['plot_tclass'] = False
if cf['Options']['Plot_TClass'] == 'True': d['plot_tclass'] = True
if cf['Options']['Output_plots'] == 'True':
d['plot_path'] = plot_path
if cf['Options']['Output_results'] == 'True':
d['results_path'] = results_path
d["file_out"] = file_out
return df, d
qcutils.get_ymdhmsfromdatetime(ds_30minutes)
# interpolate to 30 minutes
nRecs_60 = len(ds_60minutes.series["DateTime"]["Data"])
nRecs_30 = len(ds_30minutes.series["DateTime"]["Data"])
x_60minutes = numpy.arange(0, nRecs_60, 1)
x_30minutes = numpy.arange(0, nRecs_60 - 0.5, 0.5)
varlist_60 = ds_60minutes.series.keys()
# strip out the date and time variables already done
for item in [
"DateTime", "DateTime_UTC", "xlDateTime", "Year", "Month",
"Day", "Hour", "Minute", "Second", "Hdh"
]:
if item in varlist_60: varlist_60.remove(item)
# now do the interpolation (its OK to interpolate accumulated precipitation)
for label in varlist_60:
series_60minutes, flag, attr = qcutils.GetSeries(
ds_60minutes, label)
ci_60minutes = numpy.zeros(len(series_60minutes))
index = numpy.where(
abs(series_60minutes - float(c.missing_value)) < c.eps)[0]
ci_60minutes[index] = float(1)
int_fn = interp1d(x_60minutes, series_60minutes)
series_30minutes = int_fn(x_30minutes)
int_fn = interp1d(x_60minutes, ci_60minutes)
ci_30minutes = int_fn(x_30minutes)
index = numpy.where(abs(ci_30minutes - float(0)) > c.eps)[0]
series_30minutes[index] = numpy.float64(c.missing_value)
qcutils.CreateSeries(ds_30minutes,
label,
series_30minutes,
Flag=flag_30minutes,
Attr=attr)