def DateTimeFromDoY(ds,Year_in,DoY_in,Hdh_in):
year,f,a = qcutils.GetSeriesasMA(ds,Year_in)
doy,f,a = qcutils.GetSeriesasMA(ds,DoY_in)
hdh,f,a = qcutils.GetSeriesasMA(ds,Hdh_in)
idx = numpy.ma.where((numpy.ma.getmaskarray(year)==False)&
(numpy.ma.getmaskarray(doy)==False)&
(numpy.ma.getmaskarray(hdh)==False))[0]
year = year[idx]
doy = doy[idx]
hdh = hdh[idx]
hour = numpy.array(hdh,dtype=numpy.integer)
minute = numpy.array((hdh-hour)*60,dtype=numpy.integer)
dt = [datetime.datetime(int(y),1,1,h,m)+datetime.timedelta(int(d)-1) for y,d,h,m in zip(year,doy,hour,minute)]
nRecs = len(dt)
ds.series["DateTime"] = {}
ds.series["DateTime"]["Data"] = dt
ds.series["DateTime"]["Flag"] = numpy.zeros(len(dt),dtype=numpy.int32)
ds.series["DateTime"]["Attr"] = {}
ds.series["DateTime"]["Attr"]["long_name"] = "Datetime in local timezone"
ds.series["DateTime"]["Attr"]["units"] = "None"
# now remove any "data"" from empty lines
series_list = ds.series.keys()
if "DateTime" in series_list: series_list.remove("DateTime")
for item in series_list:
ds.series[item]["Data"] = ds.series[item]["Data"][idx]
ds.series[item]["Flag"] = ds.series[item]["Flag"][idx]
ds.globalattributes["nc_nrecs"] = nRecs
return 1
def CoordinateFluxGaps(cf, ds, Fc_in='Fc', Fe_in='Fe', Fh_in='Fh'):
if not qcutils.cfoptionskey(cf, Key='CoordinateFluxGaps'): return
if qcutils.cfkeycheck(cf, Base='FunctionArgs', ThisOne='gapsvars'):
vars = ast.literal_eval(cf['FunctionArgs']['gapsvars'])
Fc_in = vars[0]
Fe_in = vars[1]
Fh_in = vars[2]
Fc, f = qcutils.GetSeriesasMA(ds, Fc_in)
Fe, f = qcutils.GetSeriesasMA(ds, Fe_in)
Fh, f = qcutils.GetSeriesasMA(ds, Fh_in)
index = numpy.ma.where((Fc.mask == True) | (Fe.mask == True)
| (Fh.mask == True))[0]
# the following for ... in loop is not necessary
for i in range(len(index)):
j = index[i]
if Fc.mask[j] == False:
Fc.mask[j] = True
Fc[j] = numpy.float64(-9999)
ds.series[Fc_in]['Flag'][j] = numpy.int32(19)
if Fe.mask[j] == False:
Fe.mask[j] = True
Fe[j] = numpy.float64(-9999)
ds.series[Fe_in]['Flag'][j] = numpy.int32(19)
if Fh.mask[j] == False:
Fh.mask[j] = True
Fh[j] = numpy.float64(-9999)
ds.series[Fh_in]['Flag'][j] = numpy.int32(19)
ds.series[Fc_in]['Data'] = numpy.ma.filled(Fc, float(-9999))
ds.series[Fe_in]['Data'] = numpy.ma.filled(Fe, float(-9999))
ds.series[Fh_in]['Data'] = numpy.ma.filled(Fh, float(-9999))
log.info(' Finished gap co-ordination')
def AhfromRH(ds,Ah_out,RH_in,Ta_in):
"""
Purpose:
Function to calculate absolute humidity given relative humidity and
air temperature. Absolute humidity is not calculated if any of the
input series are missing or if the specified output series already
exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
qcfunc.AhfromRH(ds,"Ah_HMP_2m","RH_HMP_2m","Ta_HMP_2m")
Author: PRI
Date: September 2015
"""
for item in [RH_in,Ta_in]:
if item not in ds.series.keys():
msg = " AhfromRH: Requested series "+item+" not found, "+Ah_out+" not calculated"
log.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromRH: Output series "+Ah_out+" already exists, skipping ..."
log.error(msg)
return 0
RH_data,RH_flag,RH_attr = qcutils.GetSeriesasMA(ds,RH_in)
Ta_data,Ta_flag,Ta_attr = qcutils.GetSeriesasMA(ds,Ta_in)
Ah_data = mf.absolutehumidityfromRH(Ta_data,RH_data)
Ah_attr = qcutils.MakeAttributeDictionary(long_name="Absolute humidity calculated from "+RH_in+" and "+Ta_in,
height=RH_attr["height"],
units="g/m3")
qcutils.CreateSeries(ds,Ah_out,Ah_data,FList=[RH_in,Ta_in],Attr=Ah_attr)
return 1
def do_dependencycheck(cf,ds,section='',series='',code=23,mode="quiet"):
if len(section)==0 and len(series)==0: return
if len(section)==0: section = qcutils.get_cfsection(cf,series=series,mode='quiet')
if "DependencyCheck" not in cf[section][series].keys(): return
if "Source" not in cf[section][series]["DependencyCheck"]:
msg = " DependencyCheck: keyword Source not found for series "+series+", skipping ..."
log.error(msg)
return
if mode=="verbose":
msg = " Doing DependencyCheck for "+series
log.info(msg)
# get the precursor source list from the control file
source_list = ast.literal_eval(cf[section][series]["DependencyCheck"]["Source"])
# get the data
dependent_data,dependent_flag,dependent_attr = qcutils.GetSeriesasMA(ds,series)
# loop over the precursor source list
for item in source_list:
# check the precursor is in the data structure
if item not in ds.series.keys():
msg = " DependencyCheck: "+series+" precursor series "+item+" not found, skipping ..."
continue
# get the precursor data
precursor_data,precursor_flag,precursor_attr = qcutils.GetSeriesasMA(ds,item)
# mask the dependent data where the precurso is masked
dependent_data = numpy.ma.masked_where(numpy.ma.getmaskarray(precursor_data)==True,dependent_data)
# get an index of masked precursor data
index = numpy.ma.where(numpy.ma.getmaskarray(precursor_data)==True)[0]
# set the dependent QC flag
dependent_flag[index] = numpy.int32(code)
# put the data back into the data structure
dependent_attr["DependencyCheck_source"] = str(source_list)
qcutils.CreateSeries(ds,series,dependent_data,Flag=dependent_flag,Attr=dependent_attr)
if 'do_dependencychecks' not in ds.globalattributes['Functions']:
ds.globalattributes['Functions'] = ds.globalattributes['Functions']+',do_dependencychecks'
def AhfromMR(ds,Ah_out,MR_in,Ta_in,ps_in):
"""
Purpose:
Function to calculate absolute humidity given the water vapour mixing
ratio, air temperature and pressure. Absolute humidity is not calculated
if any of the input series are missing or if the specified output series
already exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
qcfunc.AhfromMR(ds,"Ah_IRGA_Av","H2O_IRGA_Av","Ta_HMP_2m","ps")
Author: PRI
Date: September 2015
"""
for item in [MR_in,Ta_in,ps_in]:
if item not in ds.series.keys():
msg = " AhfromMR: Requested series "+item+" not found, "+Ah_out+" not calculated"
log.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromMR: Output series "+Ah_out+" already exists, skipping ..."
log.error(msg)
return 0
MR_data,MR_flag,MR_attr = qcutils.GetSeriesasMA(ds,MR_in)
Ta_data,Ta_flag,Ta_attr = qcutils.GetSeriesasMA(ds,Ta_in)
ps_data,ps_flag,ps_attr = qcutils.GetSeriesasMA(ds,ps_in)
Ah_data = mf.h2o_gpm3frommmolpmol(MR_data,Ta_data,ps_data)
long_name = "Absolute humidity calculated from "+MR_in+", "+Ta_in+" and "+ps_in
Ah_attr = qcutils.MakeAttributeDictionary(long_name=long_name,
height=MR_attr["height"],
units="g/m3")
qcutils.CreateSeries(ds,Ah_out,Ah_data,FList=[MR_in,Ta_in,ps_in],Attr=Ah_attr)
return 1
def MRfromRH(ds, MR_out, RH_in, Ta_in, ps_in):
"""
Purpose:
Calculate H2O mixing ratio from RH.
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [RH_in, Ta_in, ps_in]:
if item not in ds.series.keys():
msg = " MRfromRH: Requested series " + item + " not found, " + MR_out + " not calculated"
logger.error(msg)
return 0
if MR_out in ds.series.keys():
msg = " MRfromRH: Output series " + MR_out + " already exists, skipping ..."
logger.error(msg)
return 0
RH_data, RH_flag, RH_attr = qcutils.GetSeriesasMA(ds, RH_in)
Ta_data, Ta_flag, Ta_attr = qcutils.GetSeriesasMA(ds, Ta_in)
Ah_data = mf.absolutehumidityfromRH(Ta_data, RH_data)
ps_data, ps_flag, ps_attr = qcutils.GetSeriesasMA(ds, ps_in)
MR_data = mf.h2o_mmolpmolfromgpm3(Ah_data, Ta_data, ps_data)
MR_attr = qcutils.MakeAttributeDictionary(
long_name="H2O mixing ratio calculated from " + RH_in + ", " + Ta_in +
" and " + ps_in,
height=RH_attr["height"],
units="mmol/mol")
flag = numpy.where(numpy.ma.getmaskarray(MR_data) == True, ones, zeros)
qcutils.CreateSeries(ds, MR_out, MR_data, flag, MR_attr)
return 1
def get_radiation(ds_60minutes):
for i in range(0,3):
for j in range(0,3):
label_Fn = "Fn_"+str(i)+str(j)
label_Fsd = "Fsd_"+str(i)+str(j)
label_Fld = "Fld_"+str(i)+str(j)
label_Fsu = "Fsu_"+str(i)+str(j)
label_Flu = "Flu_"+str(i)+str(j)
label_Fn_sw = "Fn_sw_"+str(i)+str(j)
label_Fn_lw = "Fn_lw_"+str(i)+str(j)
Fsd,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fsd)
Fld,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fld)
Fn_sw,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fn_sw)
Fn_lw,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fn_lw)
Fsu = Fsd - Fn_sw
Flu = Fld - Fn_lw
Fn = (Fsd-Fsu)+(Fld-Flu)
attr = qcutils.MakeAttributeDictionary(long_name='Up-welling long wave',
standard_name='surface_upwelling_longwave_flux_in_air',
units='W/m2')
qcutils.CreateSeries(ds_60minutes,label_Flu,Flu,Flag=f,Attr=attr)
attr = qcutils.MakeAttributeDictionary(long_name='Up-welling short wave',
standard_name='surface_upwelling_shortwave_flux_in_air',
units='W/m2')
qcutils.CreateSeries(ds_60minutes,label_Fsu,Fsu,Flag=f,Attr=attr)
attr = qcutils.MakeAttributeDictionary(long_name='Calculated net radiation',
standard_name='surface_net_allwave_radiation',
units='W/m2')
qcutils.CreateSeries(ds_60minutes,label_Fn,Fn,Flag=f,Attr=attr)
return
def get_windspeedanddirection(ds_60minutes):
for i in range(0,3):
for j in range(0,3):
u_label = "u_"+str(i)+str(j)
v_label = "v_"+str(i)+str(j)
Ws_label = "Ws_"+str(i)+str(j)
u,f,a = qcutils.GetSeriesasMA(ds_60minutes,u_label)
v,f,a = qcutils.GetSeriesasMA(ds_60minutes,v_label)
Ws = numpy.sqrt(u*u+v*v)
attr = qcutils.MakeAttributeDictionary(long_name="Wind speed",
units="m/s",height="10m")
qcutils.CreateSeries(ds_60minutes,Ws_label,Ws,Flag=f,Attr=attr)
# wind direction from components
for i in range(0,3):
for j in range(0,3):
u_label = "u_"+str(i)+str(j)
v_label = "v_"+str(i)+str(j)
Wd_label = "Wd_"+str(i)+str(j)
u,f,a = qcutils.GetSeriesasMA(ds_60minutes,u_label)
v,f,a = qcutils.GetSeriesasMA(ds_60minutes,v_label)
Wd = float(270) - numpy.ma.arctan2(v,u)*float(180)/numpy.pi
index = numpy.ma.where(Wd>360)[0]
if len(index)>0: Wd[index] = Wd[index] - float(360)
attr = qcutils.MakeAttributeDictionary(long_name="Wind direction",
units="degrees",height="10m")
qcutils.CreateSeries(ds_60minutes,Wd_label,Wd,Flag=f,Attr=attr)
return
def get_absolutehumidity(ds_60minutes):
for i in range(0,3):
for j in range(0,3):
Ta_label = "Ta_"+str(i)+str(j)
RH_label = "RH_"+str(i)+str(j)
Ah_label = "Ah_"+str(i)+str(j)
Ta,f,a = qcutils.GetSeriesasMA(ds_60minutes,Ta_label)
RH,f,a = qcutils.GetSeriesasMA(ds_60minutes,RH_label)
Ah = mf.absolutehumidityfromRH(Ta, RH)
attr = qcutils.MakeAttributeDictionary(long_name='Absolute humidity',
units='g/m3',standard_name='not defined')
qcutils.CreateSeries(ds_60minutes,Ah_label,Ah,Flag=f,Attr=attr)
return
def get_availableenergy(ds_60miutes):
for i in range(0,3):
for j in range(0,3):
label_Fg = "Fg_"+str(i)+str(j)
label_Fn = "Fn_"+str(i)+str(j)
label_Fa = "Fa_"+str(i)+str(j)
Fn,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fn)
Fg,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fg)
Fa = Fn - Fg
attr = qcutils.MakeAttributeDictionary(long_name='Calculated available energy',
standard_name='not defined',units='W/m2')
qcutils.CreateSeries(ds_60minutes,label_Fa,Fa,Flag=f,Attr=attr)
return
def get_relativehumidity(ds_60minutes):
for i in range(0,3):
for j in range(0,3):
q_label = "q_"+str(i)+str(j)
Ta_label = "Ta_"+str(i)+str(j)
ps_label = "ps_"+str(i)+str(j)
RH_label = "RH_"+str(i)+str(j)
q,f,a = qcutils.GetSeriesasMA(ds_60minutes,q_label)
Ta,f,a = qcutils.GetSeriesasMA(ds_60minutes,Ta_label)
ps,f,a = qcutils.GetSeriesasMA(ds_60minutes,ps_label)
RH = mf.RHfromspecifichumidity(q, Ta, ps)
attr = qcutils.MakeAttributeDictionary(long_name='Relative humidity',
units='%',standard_name='not defined')
qcutils.CreateSeries(ds_60minutes,RH_label,RH,Flag=f,Attr=attr)
return
def get_groundheatflux(ds_60minutes):
for i in range(0,3):
for j in range(0,3):
label_Fg = "Fg_"+str(i)+str(j)
label_Fn = "Fn_"+str(i)+str(j)
label_Fh = "Fh_"+str(i)+str(j)
label_Fe = "Fe_"+str(i)+str(j)
Fn,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fn)
Fh,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fh)
Fe,f,a = qcutils.GetSeriesasMA(ds_60minutes,label_Fe)
Fg = Fn - Fh - Fe
attr = qcutils.MakeAttributeDictionary(long_name='Calculated ground heat flux',
standard_name='downward_heat_flux_in_soil',
units='W/m2')
qcutils.CreateSeries(ds_60minutes,label_Fg,Fg,Flag=f,Attr=attr)
return
def changeunits_soilmoisture(ds_60minutes):
attr = qcutils.GetAttributeDictionary(ds_60minutes,"Sws_00")
for i in range(0,3):
for j in range(0,3):
label = "Sws_"+str(i)+str(j)
Sws,f,a = qcutils.GetSeriesasMA(ds_60minutes,label)
Sws = Sws/float(100)
attr["units"] = "frac"
qcutils.CreateSeries(ds_60minutes,label,Sws,Flag=f,Attr=attr)
return
def changeunits_pressure(ds_60minutes):
attr = qcutils.GetAttributeDictionary(ds_60minutes,"ps_00")
if attr["units"] == "Pa":
for i in range(0,3):
for j in range(0,3):
label = "ps_"+str(i)+str(j)
ps,f,a = qcutils.GetSeriesasMA(ds_60minutes,label)
ps = ps/float(1000)
attr["units"] = "kPa"
qcutils.CreateSeries(ds_60minutes,label,ps,Flag=f,Attr=attr)
return
def changeunits_soiltemperature(ds_60minutes):
attr = qcutils.GetAttributeDictionary(ds_60minutes,"Ts_00")
if attr["units"] == "K":
for i in range(0,3):
for j in range(0,3):
label = "Ts_"+str(i)+str(j)
Ts,f,a = qcutils.GetSeriesasMA(ds_60minutes,label)
Ts = Ts - c.C2K
attr["units"] = "C"
qcutils.CreateSeries(ds_60minutes,label,Ts,Flag=f,Attr=attr)
return
def CoordinateFluxGaps(cf,ds,Fc_in='Fc',Fe_in='Fe',Fh_in='Fh'):
if qcutils.cfkeycheck(cf,Base='FunctionArgs',ThisOne='gapsvars'):
vars = ast.literal_eval(cf['FunctionArgs']['gapsvars'])
Fc_in = vars[0]
Fe_in = vars[1]
Fh_in = vars[2]
Fc,flagC,a = qcutils.GetSeriesasMA(ds,Fc_in)
Fe,flagE,a = qcutils.GetSeriesasMA(ds,Fe_in)
Fh,flagH,a = qcutils.GetSeriesasMA(ds,Fh_in)
index = numpy.ma.where((numpy.mod(flagC,10)!=0) | (numpy.mod(flagE,10)!=0) | (numpy.mod(flagH,10)!=0))[0]
rC_i = numpy.ma.where((numpy.mod(flagC,10)==0) & ((numpy.mod(flagE,10)!=0) | (numpy.mod(flagH,10)!=0)))[0]
rE_i = numpy.ma.where((numpy.mod(flagE,10)==0) & ((numpy.mod(flagC,10)!=0) | (numpy.mod(flagH,10)!=0)))[0]
rH_i = numpy.ma.where((numpy.mod(flagH,10)==0) & ((numpy.mod(flagC,10)!=0) | (numpy.mod(flagE,10)!=0)))[0]
ds.series[Fc_in]['Flag'][rC_i] = numpy.int32(19)
ds.series[Fe_in]['Flag'][rE_i] = numpy.int32(19)
ds.series[Fh_in]['Flag'][rH_i] = numpy.int32(19)
flux_series = [Fc_in,Fe_in,Fh_in]
for ThisOne in flux_series:
index = numpy.where(ds.series[ThisOne]['Flag'] == 19)[0]
ds.series[ThisOne]['Data'][index] = numpy.float64(c.missing_value)
log.info(' Finished gap co-ordination')
def do_lowercheck(cf, ds, section, series, code=2):
"""
Purpose:
Usage:
Author: PRI
Date: February 2017
"""
# check to see if LowerCheck requested for this variable
if "LowerCheck" not in cf[section][series]:
return
# Check to see if limits have been specified
if len(cf[section][series]["LowerCheck"].keys()) == 0:
msg = "do_lowercheck: no date ranges specified"
logger.info(msg)
return
ldt = ds.series["DateTime"]["Data"]
ts = ds.globalattributes["time_step"]
data, flag, attr = qcutils.GetSeriesasMA(ds, series)
lc_list = list(cf[section][series]["LowerCheck"].keys())
for n, item in enumerate(lc_list):
# this should be a list and we should probably check for compliance
lwr_info = cf[section][series]["LowerCheck"][item]
attr["lowercheck_" + str(n)] = str(lwr_info)
start_date = dateutil.parser.parse(lwr_info[0])
su = float(lwr_info[1])
end_date = dateutil.parser.parse(lwr_info[2])
eu = float(lwr_info[3])
# get the start and end indices
si = qcutils.GetDateIndex(ldt,
start_date,
ts=ts,
default=0,
match="exact")
ei = qcutils.GetDateIndex(ldt,
end_date,
ts=ts,
default=len(ldt) - 1,
match="exact")
# get the segment of data between this start and end date
seg_data = data[si:ei + 1]
seg_flag = flag[si:ei + 1]
x = numpy.arange(si, ei + 1, 1)
lower = numpy.interp(x, [si, ei], [su, eu])
index = numpy.ma.where((seg_data < lower))[0]
seg_data[index] = numpy.ma.masked
seg_flag[index] = numpy.int32(code)
data[si:ei + 1] = seg_data
flag[si:ei + 1] = seg_flag
# now put the data back into the data structure
qcutils.CreateSeries(ds, series, data, Flag=flag, Attr=attr)
return
def plotxy(cf, nFig, plt_cf, dsa, dsb, si, ei):
SiteName = dsa.globalattributes['site_name']
PlotDescription = cf['Plots'][str(nFig)]['Title']
fig = plt.figure(int(nFig))
fig.clf()
plt.figtext(0.5,
0.95,
SiteName + ': ' + PlotDescription,
ha='center',
size=16)
XSeries = ast.literal_eval(plt_cf['XSeries'])
YSeries = ast.literal_eval(plt_cf['YSeries'])
log.info(' Plotting xy: ' + str(XSeries) + ' v ' + str(YSeries))
if dsa == dsb:
for xname, yname in zip(XSeries, YSeries):
xa, flag = qcutils.GetSeriesasMA(dsa, xname, si=si, ei=ei)
ya, flag = qcutils.GetSeriesasMA(dsa, yname, si=si, ei=ei)
xyplot(xa, ya, sub=[1, 1, 1], regr=1, xlabel=xname, ylabel=yname)
else:
for xname, yname in zip(XSeries, YSeries):
xa, flag = qcutils.GetSeriesasMA(dsa, xname, si=si, ei=ei)
ya, flag = qcutils.GetSeriesasMA(dsa, yname, si=si, ei=ei)
xb, flag = qcutils.GetSeriesasMA(dsb, xname, si=si, ei=ei)
yb, flag = qcutils.GetSeriesasMA(dsb, yname, si=si, ei=ei)
xyplot(xa, ya, sub=[1, 2, 1], xlabel=xname, ylabel=yname)
xyplot(xb, yb, sub=[1, 2, 2], regr=1, xlabel=xname, ylabel=yname)
fig.show()
def AhfromRH(ds, Ah_out, RH_in, Ta_in):
"""
Purpose:
Function to calculate absolute humidity given relative humidity and
air temperature. Absolute humidity is not calculated if any of the
input series are missing or if the specified output series already
exists in the data structure.
The calculated absolute humidity is created as a new series in the
data structure.
Usage:
qcfunc.AhfromRH(ds,"Ah_HMP_2m","RH_HMP_2m","Ta_HMP_2m")
Author: PRI
Date: September 2015
"""
nRecs = int(ds.globalattributes["nc_nrecs"])
zeros = numpy.zeros(nRecs, dtype=numpy.int32)
ones = numpy.ones(nRecs, dtype=numpy.int32)
for item in [RH_in, Ta_in]:
if item not in ds.series.keys():
msg = " AhfromRH: Requested series " + item + " not found, " + Ah_out + " not calculated"
logger.error(msg)
return 0
if Ah_out in ds.series.keys():
msg = " AhfromRH: Output series " + Ah_out + " already exists, skipping ..."
logger.error(msg)
return 0
RH_data, RH_flag, RH_attr = qcutils.GetSeriesasMA(ds, RH_in)
Ta_data, Ta_flag, Ta_attr = qcutils.GetSeriesasMA(ds, Ta_in)
Ah_data = mf.absolutehumidityfromRH(Ta_data, RH_data)
Ah_attr = qcutils.MakeAttributeDictionary(
long_name="Absolute humidity calculated from " + RH_in + " and " +
Ta_in,
height=RH_attr["height"],
units="g/m3")
flag = numpy.where(numpy.ma.getmaskarray(Ah_data) == True, ones, zeros)
qcutils.CreateSeries(ds, Ah_out, Ah_data, flag, Ah_attr)
return 1
def ImportSeries(cf, ds):
# check to see if there is an Imports section
if "Imports" not in cf.keys(): return
# number of records
nRecs = int(ds.globalattributes["nc_nrecs"])
# get the start and end datetime
ldt = ds.series["DateTime"]["Data"]
start_date = ldt[0]
end_date = ldt[-1]
# loop over the series in the Imports section
for label in cf["Imports"].keys():
import_filename = qcutils.get_keyvaluefromcf(cf, ["Imports", label],
"file_name",
default="")
if import_filename == "":
msg = " ImportSeries: import filename not found in control file, skipping ..."
logger.warning(msg)
continue
var_name = qcutils.get_keyvaluefromcf(cf, ["Imports", label],
"var_name",
default="")
if var_name == "":
msg = " ImportSeries: variable name not found in control file, skipping ..."
logger.warning(msg)
continue
ds_import = qcio.nc_read_series(import_filename)
ts_import = ds_import.globalattributes["time_step"]
ldt_import = ds_import.series["DateTime"]["Data"]
si = qcutils.GetDateIndex(ldt_import,
str(start_date),
ts=ts_import,
default=0,
match="exact")
ei = qcutils.GetDateIndex(ldt_import,
str(end_date),
ts=ts_import,
default=len(ldt_import) - 1,
match="exact")
data = numpy.ma.ones(nRecs) * float(c.missing_value)
flag = numpy.ma.ones(nRecs)
data_import, flag_import, attr_import = qcutils.GetSeriesasMA(
ds_import, var_name, si=si, ei=ei)
ldt_import = ldt_import[si:ei + 1]
index = qcutils.FindIndicesOfBInA(ldt_import, ldt)
data[index] = data_import
flag[index] = flag_import
qcutils.CreateSeries(ds, label, data, flag, attr_import)
def do_rangecheck(cf, ds, section, series, code=2):
"""
Purpose:
Applies a range check to data series listed in the control file. Data values that
are less than the lower limit or greater than the upper limit are replaced with
c.missing_value and the corresponding QC flag element is set to 2.
Usage:
Author: PRI
Date: Back in the day
"""
# check that RangeCheck has been requested for this series
if 'RangeCheck' not in cf[section][series].keys():
return
# check that the upper and lower limits have been given
if ("Lower" not in cf[section][series]["RangeCheck"].keys()
or "Upper" not in cf[section][series]["RangeCheck"].keys()):
msg = "RangeCheck: key not found in control file for " + series + ", skipping ..."
logger.warning(msg)
return
# get the upper and lower limits
upr = numpy.array(eval(cf[section][series]['RangeCheck']['Upper']))
valid_upper = numpy.min(upr)
upr = upr[ds.series['Month']['Data'] - 1]
lwr = numpy.array(eval(cf[section][series]['RangeCheck']['Lower']))
valid_lower = numpy.min(lwr)
lwr = lwr[ds.series['Month']['Data'] - 1]
# get the data, flag and attributes
data, flag, attr = qcutils.GetSeriesasMA(ds, series)
# convert the data from a masked array to an ndarray so the range check works
data = numpy.ma.filled(data, fill_value=c.missing_value)
# get the indices of elements outside this range
idx = numpy.where((data < lwr) | (data > upr))[0]
# set elements outside range to missing and set the QC flag
data[idx] = numpy.float64(c.missing_value)
flag[idx] = numpy.int32(code)
# update the variable attributes
attr["rangecheck_lower"] = cf[section][series]["RangeCheck"]["Lower"]
attr["rangecheck_upper"] = cf[section][series]["RangeCheck"]["Upper"]
attr["valid_range"] = str(valid_lower) + "," + str(valid_upper)
# and now put the data back into the data structure
qcutils.CreateSeries(ds, series, data, Flag=flag, Attr=attr)
# now we can return
return
def plotxy(cf, nFig, plt_cf, dsa, dsb, si, ei):
SiteName = dsa.globalattributes['site_name']
PlotDescription = cf['Plots'][str(nFig)]['Title']
fig = plt.figure(numpy.int32(nFig))
fig.clf()
plt.figtext(0.5,
0.95,
SiteName + ': ' + PlotDescription,
ha='center',
size=16)
XSeries = ast.literal_eval(plt_cf['XSeries'])
YSeries = ast.literal_eval(plt_cf['YSeries'])
log.info(' Plotting xy: ' + str(XSeries) + ' v ' + str(YSeries))
if dsa == dsb:
for xname, yname in zip(XSeries, YSeries):
xa, flag, attr = qcutils.GetSeriesasMA(dsa, xname, si=si, ei=ei)
ya, flag, attr = qcutils.GetSeriesasMA(dsa, yname, si=si, ei=ei)
xyplot(xa, ya, sub=[1, 1, 1], regr=1, xlabel=xname, ylabel=yname)
else:
for xname, yname in zip(XSeries, YSeries):
xa, flag, attr = qcutils.GetSeriesasMA(dsa, xname, si=si, ei=ei)
ya, flag, attr = qcutils.GetSeriesasMA(dsa, yname, si=si, ei=ei)
xb, flag, attr = qcutils.GetSeriesasMA(dsb, xname, si=si, ei=ei)
yb, flag, attr = qcutils.GetSeriesasMA(dsb, yname, si=si, ei=ei)
xyplot(xa, ya, sub=[1, 2, 1], xlabel=xname, ylabel=yname)
xyplot(xb, yb, sub=[1, 2, 2], regr=1, xlabel=xname, ylabel=yname)
STList = []
ETList = []
if ei == -1:
L1XArray = numpy.array(dsa.series['DateTime']['Data'][si:ei])
else:
L1XArray = numpy.array(dsa.series['DateTime']['Data'][si:ei + 1])
for fmt in ['%Y', '_', '%m', '_', '%d', '_', '%H', '%M']:
STList.append(L1XArray[0].strftime(fmt))
if ei == -1:
ETList.append(dsa.series['DateTime']['Data'][-1].strftime(fmt))
else:
ETList.append(L1XArray[-1].strftime(fmt))
if qcutils.cfkeycheck(
cf, Base='Output',
ThisOne='PNGFile') and cf['Output']['PNGFile'] == 'True':
log.info(' Generating a PNG file of the plot')
PNGFileName = cf['Files']['PNG'][
'PNGFilePath'] + 'Fig' + nFig + '_' + ''.join(
STList) + '-' + ''.join(ETList) + '.png'
plt.savefig(PNGFileName)
fig.show()
def read_isd_file(isd_file_path):
"""
Purpose:
Reads an ISD CSV file (gz or uncompressed) and returns the data in a data structure.
Assumptions:
Usage:
Author: PRI
Date: June 2017
"""
isd_file_name = os.path.split(isd_file_path)[1]
msg = "Reading ISD file " + isd_file_name
logger.info(msg)
isd_site_id = isd_file_name.split("-")
isd_site_id = isd_site_id[0] + "-" + isd_site_id[1]
# read the file
if os.path.splitext(isd_file_path)[1] == ".gz":
with gzip.open(isd_file_path, 'rb') as fp:
content = fp.readlines()
else:
with open(isd_file_path) as fp:
content = fp.readlines()
# get a data structure
ds = qcio.DataStructure()
# get the site latitude, longitude and altitude
ds.globalattributes["altitude"] = float(content[0][46:51])
ds.globalattributes["latitude"] = float(content[0][28:34]) / float(1000)
ds.globalattributes["longitude"] = float(content[0][34:41]) / float(1000)
ds.globalattributes["isd_site_id"] = isd_site_id
# initialise the data structure
ds.series["DateTime"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Datetime",
"units": "none"
}
}
ds.series["Wd"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Wind direction",
"units": "degrees"
}
}
ds.series["Ws"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Wind speed",
"units": "m/s"
}
}
ds.series["Ta"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Air temperature",
"units": "C"
}
}
ds.series["Td"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Dew point temperature",
"units": "C"
}
}
ds.series["ps"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Surface pressure",
"units": "kPa"
}
}
ds.series["Precip"] = {
"Data": [],
"Flag": [],
"Attr": {
"long_name": "Precipitation",
"units": "mm"
}
}
# define the codes for good data in the ISD file
OK_obs_code = [
"AUTO ", "CRN05", "CRN15", "FM-12", "FM-15", "FM-16", "SY-MT"
]
# iterate over the lines in the file and decode the data
for i in range(len(content) - 1):
#for i in range(10):
# filter out anything other than hourly data
if content[i][41:46] not in OK_obs_code: continue
YY = int(content[i][15:19])
MM = int(content[i][19:21])
DD = int(content[i][21:23])
HH = int(content[i][23:25])
mm = int(content[i][25:27])
dt = datetime.datetime(YY, MM, DD, HH, mm, 0)
ds.series["DateTime"]["Data"].append(pytz.utc.localize(dt))
# wind direction, degT
try:
ds.series["Wd"]["Data"].append(float(content[i][60:63]))
except:
ds.series["Wd"]["Data"].append(float(999))
# wind speed, m/s
try:
ds.series["Ws"]["Data"].append(
float(content[i][65:69]) / float(10))
except:
ds.series["Ws"]["Data"].append(float(999.9))
# air temperature, C
try:
ds.series["Ta"]["Data"].append(
float(content[i][87:92]) / float(10))
except:
ds.series["Ta"]["Data"].append(float(999.9))
# dew point temperature, C
try:
ds.series["Td"]["Data"].append(
float(content[i][93:98]) / float(10))
except:
ds.series["Td"]["Data"].append(float(999.9))
# sea level pressure, hPa
try:
ds.series["ps"]["Data"].append(
float(content[i][99:104]) / float(10))
except:
ds.series["ps"]["Data"].append(float(9999.9))
# precipitation, mm
if content[i][108:111] == "AA1":
try:
ds.series["Precip"]["Data"].append(
float(content[i][113:117]) / float(10))
except:
ds.series["Precip"]["Data"].append(float(999.9))
else:
ds.series["Precip"]["Data"].append(float(999.9))
# add the time zone to the DateTime ataributes
ds.series["DateTime"]["Attr"]["time_zone"] = "UTC"
# convert from lists to masked arrays
f0 = numpy.zeros(len(ds.series["DateTime"]["Data"]))
f1 = numpy.ones(len(ds.series["DateTime"]["Data"]))
ds.series["DateTime"]["Data"] = numpy.array(ds.series["DateTime"]["Data"])
ds.series["DateTime"]["Flag"] = f0
ds.globalattributes["nc_nrecs"] = len(f0)
dt_delta = qcutils.get_timestep(ds)
ts = scipy.stats.mode(dt_delta)[0] / 60
ds.globalattributes["time_step"] = ts[0]
ds.series["Wd"]["Data"] = numpy.ma.masked_equal(ds.series["Wd"]["Data"],
999)
ds.series["Wd"]["Flag"] = numpy.where(
numpy.ma.getmaskarray(ds.series["Wd"]["Data"]) == True, f1, f0)
ds.series["Ws"]["Data"] = numpy.ma.masked_equal(ds.series["Ws"]["Data"],
999.9)
ds.series["Ws"]["Flag"] = numpy.where(
numpy.ma.getmaskarray(ds.series["Ws"]["Data"]) == True, f1, f0)
ds.series["Ta"]["Data"] = numpy.ma.masked_equal(ds.series["Ta"]["Data"],
999.9)
ds.series["Ta"]["Flag"] = numpy.where(
numpy.ma.getmaskarray(ds.series["Ta"]["Data"]) == True, f1, f0)
ds.series["Td"]["Data"] = numpy.ma.masked_equal(ds.series["Td"]["Data"],
999.9)
ds.series["Td"]["Flag"] = numpy.where(
numpy.ma.getmaskarray(ds.series["Td"]["Data"]) == True, f1, f0)
# hPa to kPa
ds.series["ps"]["Data"] = numpy.ma.masked_equal(ds.series["ps"]["Data"],
9999.9) / float(10)
ds.series["ps"]["Flag"] = numpy.where(
numpy.ma.getmaskarray(ds.series["ps"]["Data"]) == True, f1, f0)
# convert sea level pressure to station pressure
site_altitude = float(ds.globalattributes["altitude"])
cfac = numpy.ma.exp(
(-1 * site_altitude) / ((ds.series["Ta"]["Data"] + 273.15) * 29.263))
ds.series["ps"]["Data"] = ds.series["ps"]["Data"] * cfac
# do precipitation and apply crude limits
ds.series["Precip"]["Data"] = numpy.ma.masked_equal(
ds.series["Precip"]["Data"], 999.9)
condition = (ds.series["Precip"]["Data"] <
0) | (ds.series["Precip"]["Data"] > 100)
ds.series["Precip"]["Data"] = numpy.ma.masked_where(
condition, ds.series["Precip"]["Data"])
ds.series["Precip"]["Flag"] = numpy.where(
numpy.ma.getmaskarray(ds.series["Precip"]["Data"]) == True, f1, f0)
# get the humidities from Td
Ta, flag, attr = qcutils.GetSeriesasMA(ds, "Ta")
Td, flag, attr = qcutils.GetSeriesasMA(ds, "Td")
ps, flag, attr = qcutils.GetSeriesasMA(ds, "ps")
RH = mf.RHfromdewpoint(Td, Ta)
flag = numpy.where(numpy.ma.getmaskarray(RH) == True, f1, f0)
attr = {"long_name": "Relative humidity", "units": "%"}
qcutils.CreateSeries(ds, "RH", RH, Flag=flag, Attr=attr)
Ah = mf.absolutehumidityfromRH(Ta, RH)
flag = numpy.where(numpy.ma.getmaskarray(Ah) == True, f1, f0)
attr = {"long_name": "Absolute humidity", "units": "g/m3"}
qcutils.CreateSeries(ds, "Ah", Ah, Flag=flag, Attr=attr)
q = mf.specifichumidityfromRH(RH, Ta, ps)
flag = numpy.where(numpy.ma.getmaskarray(q) == True, f1, f0)
attr = {"long_name": "Specific humidity", "units": "kg/kg"}
qcutils.CreateSeries(ds, "q", q, Flag=flag, Attr=attr)
# return the data
return ds
def ApplyTurbulenceFilter(cf,ds,ustar_threshold=None):
"""
Purpose:
Usage:
Author:
Date:
"""
opt = ApplyTurbulenceFilter_checks(cf,ds)
if not opt["OK"]: return
# local point to datetime series
ldt = ds.series["DateTime"]["Data"]
# time step
ts = int(ds.globalattributes["time_step"])
# dictionary of utar thresold values
if ustar_threshold==None:
ustar_dict = qcrp.get_ustar_thresholds(cf,ldt)
else:
ustar_dict = qcrp.get_ustar_thresholds_annual(ldt,ustar_threshold)
# initialise a dictionary for the indicator series
indicators = {}
# get data for the indicator series
ustar,ustar_flag,ustar_attr = qcutils.GetSeriesasMA(ds,"ustar")
Fsd,f,a = qcutils.GetSeriesasMA(ds,"Fsd")
if "solar_altitude" not in ds.series.keys(): qcts.get_synthetic_fsd(ds)
Fsd_syn,f,a = qcutils.GetSeriesasMA(ds,"Fsd_syn")
sa,f,a = qcutils.GetSeriesasMA(ds,"solar_altitude")
# get the day/night indicator series
# indicators["day"] = 1 ==> day time, indicators["day"] = 0 ==> night time
indicators["day"] = qcrp.get_day_indicator(cf,Fsd,Fsd_syn,sa)
ind_day = indicators["day"]["values"]
# get the turbulence indicator series
if opt["turbulence_filter"].lower()=="ustar":
# indicators["turbulence"] = 1 ==> turbulent, indicators["turbulence"] = 0 ==> not turbulent
indicators["turbulence"] = qcrp.get_turbulence_indicator_ustar(ldt,ustar,ustar_dict,ts)
elif opt["turbulence_filter"].lower()=="ustar_evg":
# ustar >= threshold ==> ind_ustar = 1, ustar < threshold == ind_ustar = 0
indicators["ustar"] = qcrp.get_turbulence_indicator_ustar(ldt,ustar,ustar_dict,ts)
ind_ustar = indicators["ustar"]["values"]
# ustar >= threshold during day AND ustar has been >= threshold since sunset ==> indicators["turbulence"] = 1
# indicators["turbulence"] = 0 during night once ustar has dropped below threshold even if it
# increases above the threshold later in the night
indicators["turbulence"] = qcrp.get_turbulence_indicator_ustar_evg(ldt,ind_day,ind_ustar,ustar,ustar_dict,ts)
elif opt["turbulence_filter"].lower()=="l":
#indicators["turbulence] = get_turbulence_indicator_l(ldt,L,z,d,zmdonL_threshold)
indicators["turbulence"] = numpy.ones(len(ldt))
msg = " Use of L as turbulence indicator not implemented, no filter applied"
log.warning(msg)
else:
msg = " Unrecognised turbulence filter option ("
msg = msg+opt["turbulence_filter"]+"), no filter applied"
log.error(msg)
return
# initialise the final indicator series as the turbulence indicator
# subsequent filters will modify the final indicator series
# we must use copy.deepcopy() otherwise the "values" array will only
# be copied by reference not value. Damn Python's default of copy by reference!
indicators["final"] = copy.deepcopy(indicators["turbulence"])
# check to see if the user wants to accept all day time observations
# regardless of ustar value
if opt["accept_day_times"].lower()=="yes":
# if yes, then we force the final indicator to be 1
# if ustar is below the threshold during the day.
idx = numpy.where(indicators["day"]["values"]==1)[0]
indicators["final"]["values"][idx] = numpy.int(1)
indicators["final"]["attr"].update(indicators["day"]["attr"])
# get the evening indicator series
indicators["evening"] = qcrp.get_evening_indicator(cf,Fsd,Fsd_syn,sa,ts)
indicators["dayevening"] = {"values":indicators["day"]["values"]+indicators["evening"]["values"]}
indicators["dayevening"]["attr"] = indicators["day"]["attr"].copy()
indicators["dayevening"]["attr"].update(indicators["evening"]["attr"])
if opt["use_evening_filter"].lower()=="yes":
idx = numpy.where(indicators["dayevening"]["values"]==0)[0]
indicators["final"]["values"][idx] = numpy.int(0)
indicators["final"]["attr"].update(indicators["dayevening"]["attr"])
# save the indicator series
ind_flag = numpy.zeros(len(ldt))
long_name = "Turbulence indicator, 1 for turbulent, 0 for non-turbulent"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"turbulence_indicator",indicators["turbulence"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Day indicator, 1 for day time, 0 for night time"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"day_indicator",indicators["day"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Evening indicator, 1 for evening, 0 for not evening"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"evening_indicator",indicators["evening"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Day/evening indicator, 1 for day/evening, 0 for not day/evening"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"dayevening_indicator",indicators["dayevening"]["values"],Flag=ind_flag,Attr=ind_attr)
long_name = "Final indicator, 1 for use data, 0 for don't use data"
ind_attr = qcutils.MakeAttributeDictionary(long_name=long_name,units="None")
qcutils.CreateSeries(ds,"final_indicator",indicators["final"]["values"],Flag=ind_flag,Attr=ind_attr)
# loop over the series to be filtered
for series in opt["filter_list"]:
msg = " Applying "+opt["turbulence_filter"]+" filter to "+series
log.info(msg)
# get the data
data,flag,attr = qcutils.GetSeriesasMA(ds,series)
# continue to next series if this series has been filtered before
if "turbulence_filter" in attr:
msg = " Series "+series+" has already been filtered, skipping ..."
log.warning(msg)
continue
# save the non-filtered data
qcutils.CreateSeries(ds,series+"_nofilter",data,Flag=flag,Attr=attr)
# now apply the filter
data_filtered = numpy.ma.masked_where(indicators["final"]["values"]==0,data,copy=True)
flag_filtered = numpy.copy(flag)
idx = numpy.where(indicators["final"]["values"]==0)[0]
flag_filtered[idx] = numpy.int32(61)
# update the series attributes
for item in indicators["final"]["attr"].keys():
attr[item] = indicators["final"]["attr"][item]
# and write the filtered data to the data structure
qcutils.CreateSeries(ds,series,data_filtered,Flag=flag_filtered,Attr=attr)
# and write a copy of the filtered datas to the data structure so it
# will still exist once the gap filling has been done
qcutils.CreateSeries(ds,series+"_filtered",data_filtered,Flag=flag_filtered,Attr=attr)
return
ds_aws_60minute.globalattributes["time_step"] = str(60)
# put the Python datetime into the data structure
ds_aws_60minute.series["DateTime"] = {}
ds_aws_60minute.series["DateTime"]["Data"] = dt_aws_60minute
ds_aws_60minute.series["DateTime"]["Flag"] = numpy.zeros(nRecs_60minute,
dtype=numpy.int32)
ds_aws_60minute.series["DateTime"][
"Attr"] = qcutils.MakeAttributeDictionary(
long_name="DateTime in local time zone", units="None")
# add the Excel datetime, year, month etc
qcutils.get_xldatefromdatetime(ds_aws_60minute)
qcutils.get_ymdhmsfromdatetime(ds_aws_60minute)
# loop over the series and take the average (every thing but Precip) or sum (Precip)
for item in series_list:
if "Precip" in item:
data_30minute, flag_30minute, attr = qcutils.GetSeriesasMA(
ds_aws_30minute, item, si=si_wholehour, ei=ei_wholehour)
data_2d = numpy.reshape(data_30minute, (nRecs_30minute / 2, 2))
flag_2d = numpy.reshape(flag_30minute, (nRecs_30minute / 2, 2))
data_60minute = numpy.ma.sum(data_2d, axis=1)
flag_60minute = numpy.ma.max(flag_2d, axis=1)
qcutils.CreateSeries(ds_aws_60minute, item, data_60minute,
flag_60minute, attr)
elif "Wd" in item:
Ws_30minute, flag_30minute, attr = qcutils.GetSeriesasMA(
ds_aws_30minute, item, si=si_wholehour, ei=ei_wholehour)
Wd_30minute, flag_30minute, attr = qcutils.GetSeriesasMA(
ds_aws_30minute, item, si=si_wholehour, ei=ei_wholehour)
U_30minute, V_30minute = qcutils.convert_WsWdtoUV(
Ws_30minute, Wd_30minute)
U_2d = numpy.reshape(U_30minute, (nRecs_30minute / 2, 2))
V_2d = numpy.reshape(V_30minute, (nRecs_30minute / 2, 2))
def climatology(cf):
nc_filename = qcio.get_infilenamefromcf(cf)
if not qcutils.file_exists(nc_filename): return
xl_filename = nc_filename.replace(".nc", "_Climatology.xls")
xlFile = xlwt.Workbook()
ds = qcio.nc_read_series(nc_filename)
# calculate Fa if it is not in the data structure
if "Fa" not in ds.series.keys():
if "Fn" in ds.series.keys() and "Fg" in ds.series.keys():
qcts.CalculateAvailableEnergy(ds,
Fa_out='Fa',
Fn_in='Fn',
Fg_in='Fg')
else:
log.warning(" Climatology: Fn or Fg not in data struicture")
# get the time step
ts = int(ds.globalattributes['time_step'])
# get the site name
SiteName = ds.globalattributes['site_name']
# get the datetime series
dt = ds.series['DateTime']['Data']
Hdh = ds.series['Hdh']['Data']
Month = ds.series['Month']['Data']
# get the initial start and end dates
StartDate = str(dt[0])
EndDate = str(dt[-1])
# find the start index of the first whole day (time=00:30)
si = qcutils.GetDateIndex(dt,
StartDate,
ts=ts,
default=0,
match='startnextday')
# find the end index of the last whole day (time=00:00)
ei = qcutils.GetDateIndex(dt,
EndDate,
ts=ts,
default=-1,
match='endpreviousday')
# get local views of the datetime series
ldt = dt[si:ei + 1]
Hdh = Hdh[si:ei + 1]
Month = Month[si:ei + 1]
# get the number of time steps in a day and the number of days in the data
ntsInDay = int(24.0 * 60.0 / float(ts))
nDays = int(len(ldt)) / ntsInDay
for ThisOne in cf['Variables'].keys():
if "AltVarName" in cf['Variables'][ThisOne].keys():
ThisOne = cf['Variables'][ThisOne]["AltVarName"]
if ThisOne in ds.series.keys():
log.info(" Doing climatology for " + ThisOne)
data, f, a = qcutils.GetSeriesasMA(ds, ThisOne, si=si, ei=ei)
if numpy.ma.count(data) == 0:
log.warning(" No data for " + ThisOne + ", skipping ...")
continue
fmt_str = get_formatstring(cf, ThisOne, fmt_def='')
xlSheet = xlFile.add_sheet(ThisOne)
Av_all = do_diurnalstats(Month,
Hdh,
data,
xlSheet,
format_string=fmt_str,
ts=ts)
# now do it for each day
# we want to preserve any data that has been truncated by the use of the "startnextday"
# and "endpreviousday" match options used above. Here we revisit the start and end indices
# and adjust these backwards and forwards respectively if data has been truncated.
nDays_daily = nDays
ei_daily = ei
si_daily = si
sdate = ldt[0]
edate = ldt[-1]
# is there data after the current end date?
if dt[-1] > ldt[-1]:
# if so, push the end index back by 1 day so it is included
ei_daily = ei + ntsInDay
nDays_daily = nDays_daily + 1
edate = ldt[-1] + datetime.timedelta(days=1)
# is there data before the current start date?
if dt[0] < ldt[0]:
# if so, push the start index back by 1 day so it is included
si_daily = si - ntsInDay
nDays_daily = nDays_daily + 1
sdate = ldt[0] - datetime.timedelta(days=1)
# get the data and use the "pad" option to add missing data if required to
# complete the extra days
data, f, a = qcutils.GetSeriesasMA(ds,
ThisOne,
si=si_daily,
ei=ei_daily,
mode="pad")
data_daily = data.reshape(nDays_daily, ntsInDay)
xlSheet = xlFile.add_sheet(ThisOne + '(day)')
write_data_1columnpertimestep(xlSheet,
data_daily,
ts,
startdate=sdate,
format_string=fmt_str)
data_daily_i = do_2dinterpolation(data_daily)
xlSheet = xlFile.add_sheet(ThisOne + 'i(day)')
write_data_1columnpertimestep(xlSheet,
data_daily_i,
ts,
startdate=sdate,
format_string=fmt_str)
elif ThisOne == "EF":
log.info(" Doing evaporative fraction")
EF = numpy.ma.zeros([48, 12]) + float(c.missing_value)
Hdh, f, a = qcutils.GetSeriesasMA(ds, 'Hdh', si=si, ei=ei)
Fa, f, a = qcutils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei)
Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei)
for m in range(1, 13):
mi = numpy.where(Month == m)[0]
Fa_Num, Hr, Fa_Av, Sd, Mx, Mn = get_diurnalstats(
Hdh[mi], Fa[mi], ts)
Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats(
Hdh[mi], Fe[mi], ts)
index = numpy.ma.where((Fa_Num > 4) & (Fe_Num > 4))
EF[:, m - 1][index] = Fe_Av[index] / Fa_Av[index]
# reject EF values greater than upper limit or less than lower limit
upr, lwr = get_rangecheck_limit(cf, 'EF')
EF = numpy.ma.filled(
numpy.ma.masked_where((EF > upr) | (EF < lwr), EF),
float(c.missing_value))
# write the EF to the Excel file
xlSheet = xlFile.add_sheet('EF')
write_data_1columnpermonth(xlSheet, EF, ts, format_string='0.00')
# do the 2D interpolation to fill missing EF values
EFi = do_2dinterpolation(EF)
xlSheet = xlFile.add_sheet('EFi')
write_data_1columnpermonth(xlSheet, EFi, ts, format_string='0.00')
# now do EF for each day
Fa, f, a = qcutils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei)
Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei)
EF = Fe / Fa
EF = numpy.ma.filled(
numpy.ma.masked_where((EF > upr) | (EF < lwr), EF),
float(c.missing_value))
EF_daily = EF.reshape(nDays, ntsInDay)
xlSheet = xlFile.add_sheet('EF(day)')
write_data_1columnpertimestep(xlSheet,
EF_daily,
ts,
startdate=ldt[0],
format_string='0.00')
EFi = do_2dinterpolation(EF_daily)
xlSheet = xlFile.add_sheet('EFi(day)')
write_data_1columnpertimestep(xlSheet,
EFi,
ts,
startdate=ldt[0],
format_string='0.00')
elif ThisOne == "BR":
log.info(" Doing Bowen ratio")
BR = numpy.ma.zeros([48, 12]) + float(c.missing_value)
Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei)
Fh, f, a = qcutils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei)
for m in range(1, 13):
mi = numpy.where(Month == m)[0]
Fh_Num, Hr, Fh_Av, Sd, Mx, Mn = get_diurnalstats(
Hdh[mi], Fh[mi], ts)
Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats(
Hdh[mi], Fe[mi], ts)
index = numpy.ma.where((Fh_Num > 4) & (Fe_Num > 4))
BR[:, m - 1][index] = Fh_Av[index] / Fe_Av[index]
# reject BR values greater than upper limit or less than lower limit
upr, lwr = get_rangecheck_limit(cf, 'BR')
BR = numpy.ma.filled(
numpy.ma.masked_where((BR > upr) | (BR < lwr), BR),
float(c.missing_value))
# write the BR to the Excel file
xlSheet = xlFile.add_sheet('BR')
write_data_1columnpermonth(xlSheet, BR, ts, format_string='0.00')
# do the 2D interpolation to fill missing EF values
BRi = do_2dinterpolation(BR)
xlSheet = xlFile.add_sheet('BRi')
write_data_1columnpermonth(xlSheet, BRi, ts, format_string='0.00')
# now do BR for each day ...
Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei)
Fh, f, a = qcutils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei)
BR = Fh / Fe
BR = numpy.ma.filled(
numpy.ma.masked_where((BR > upr) | (BR < lwr), BR),
float(c.missing_value))
BR_daily = BR.reshape(nDays, ntsInDay)
xlSheet = xlFile.add_sheet('BR(day)')
write_data_1columnpertimestep(xlSheet,
BR_daily,
ts,
startdate=ldt[0],
format_string='0.00')
BRi = do_2dinterpolation(BR_daily)
xlSheet = xlFile.add_sheet('BRi(day)')
write_data_1columnpertimestep(xlSheet,
BRi,
ts,
startdate=ldt[0],
format_string='0.00')
elif ThisOne == "WUE":
log.info(" Doing ecosystem WUE")
WUE = numpy.ma.zeros([48, 12]) + float(c.missing_value)
Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei)
Fc, f, a = qcutils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei)
for m in range(1, 13):
mi = numpy.where(Month == m)[0]
Fc_Num, Hr, Fc_Av, Sd, Mx, Mn = get_diurnalstats(
Hdh[mi], Fc[mi], ts)
Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats(
Hdh[mi], Fe[mi], ts)
index = numpy.ma.where((Fc_Num > 4) & (Fe_Num > 4))
WUE[:, m - 1][index] = Fc_Av[index] / Fe_Av[index]
# reject WUE values greater than upper limit or less than lower limit
upr, lwr = get_rangecheck_limit(cf, 'WUE')
WUE = numpy.ma.filled(
numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE),
float(c.missing_value))
# write the WUE to the Excel file
xlSheet = xlFile.add_sheet('WUE')
write_data_1columnpermonth(xlSheet,
WUE,
ts,
format_string='0.00000')
# do the 2D interpolation to fill missing EF values
WUEi = do_2dinterpolation(WUE)
xlSheet = xlFile.add_sheet('WUEi')
write_data_1columnpermonth(xlSheet,
WUEi,
ts,
format_string='0.00000')
# now do WUE for each day ...
Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei)
Fc, f, a = qcutils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei)
WUE = Fc / Fe
WUE = numpy.ma.filled(
numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE),
float(c.missing_value))
WUE_daily = WUE.reshape(nDays, ntsInDay)
xlSheet = xlFile.add_sheet('WUE(day)')
write_data_1columnpertimestep(xlSheet,
WUE_daily,
ts,
startdate=ldt[0],
format_string='0.00000')
WUEi = do_2dinterpolation(WUE_daily)
xlSheet = xlFile.add_sheet('WUEi(day)')
write_data_1columnpertimestep(xlSheet,
WUEi,
ts,
startdate=ldt[0],
format_string='0.00000')
else:
log.warning(" qcclim.climatology: requested variable " + ThisOne +
" not in data structure")
continue
log.info(" Saving Excel file " + xl_filename)
xlFile.save(xl_filename)
def rpLT_createdict(cf, ds, series):
"""
Purpose:
Creates a dictionary in ds to hold information about estimating ecosystem
respiration using the Lloyd-Taylor method.
Usage:
Author: PRI
Date October 2015
"""
# get the section of the control file containing the series
section = qcutils.get_cfsection(cf, series=series, mode="quiet")
# return without doing anything if the series isn't in a control file section
if len(section) == 0:
logger.error("ERUsingLloydTaylor: Series " + series +
" not found in control file, skipping ...")
return
# check that none of the drivers have missing data
driver_list = ast.literal_eval(
cf[section][series]["ERUsingLloydTaylor"]["drivers"])
target = cf[section][series]["ERUsingLloydTaylor"]["target"]
for label in driver_list:
data, flag, attr = qcutils.GetSeriesasMA(ds, label)
if numpy.ma.count_masked(data) != 0:
logger.error("ERUsingLloydTaylor: driver " + label +
" contains missing data, skipping target " + target)
return
# create the dictionary keys for this series
rpLT_info = {}
# site name
rpLT_info["site_name"] = ds.globalattributes["site_name"]
# source series for ER
opt = qcutils.get_keyvaluefromcf(cf,
[section, series, "ERUsingLloydTaylor"],
"source",
default="Fc")
rpLT_info["source"] = opt
# target series name
rpLT_info["target"] = cf[section][series]["ERUsingLloydTaylor"]["target"]
# list of drivers
rpLT_info["drivers"] = ast.literal_eval(
cf[section][series]["ERUsingLloydTaylor"]["drivers"])
# name of SOLO output series in ds
rpLT_info["output"] = cf[section][series]["ERUsingLloydTaylor"]["output"]
# results of best fit for plotting later on
rpLT_info["results"] = {
"startdate": [],
"enddate": [],
"No. points": [],
"r": [],
"Bias": [],
"RMSE": [],
"Frac Bias": [],
"NMSE": [],
"Avg (obs)": [],
"Avg (LT)": [],
"Var (obs)": [],
"Var (LT)": [],
"Var ratio": [],
"m_ols": [],
"b_ols": []
}
# create the configuration dictionary
rpLT_info["configs_dict"] = get_configs_dict(cf, ds)
# create an empty series in ds if the output series doesn't exist yet
if rpLT_info["output"] not in ds.series.keys():
data, flag, attr = qcutils.MakeEmptySeries(ds, rpLT_info["output"])
qcutils.CreateSeries(ds, rpLT_info["output"], data, flag, attr)
# create the merge directory in the data structure
if "merge" not in dir(ds): ds.merge = {}
if "standard" not in ds.merge.keys(): ds.merge["standard"] = {}
# create the dictionary keys for this series
ds.merge["standard"][series] = {}
# output series name
ds.merge["standard"][series]["output"] = series
# source
ds.merge["standard"][series]["source"] = ast.literal_eval(
cf[section][series]["MergeSeries"]["Source"])
# create an empty series in ds if the output series doesn't exist yet
if ds.merge["standard"][series]["output"] not in ds.series.keys():
data, flag, attr = qcutils.MakeEmptySeries(
ds, ds.merge["standard"][series]["output"])
qcutils.CreateSeries(ds, ds.merge["standard"][series]["output"], data,
flag, attr)
return rpLT_info
def rpLT_plot(pd,
ds,
series,
driverlist,
targetlabel,
outputlabel,
LT_info,
si=0,
ei=-1):
""" Plot the results of the Lloyd-Taylor run. """
# get the time step
ts = int(ds.globalattributes['time_step'])
# get a local copy of the datetime series
if ei == -1:
dt = ds.series['DateTime']['Data'][si:]
else:
dt = ds.series['DateTime']['Data'][si:ei + 1]
xdt = numpy.array(dt)
Hdh, f, a = qcutils.GetSeriesasMA(ds, 'Hdh', si=si, ei=ei)
# get the observed and modelled values
obs, f, a = qcutils.GetSeriesasMA(ds, targetlabel, si=si, ei=ei)
mod, f, a = qcutils.GetSeriesasMA(ds, outputlabel, si=si, ei=ei)
# make the figure
if LT_info["show_plots"]:
plt.ion()
else:
plt.ioff()
fig = plt.figure(pd["fig_num"], figsize=(13, 8))
fig.clf()
fig.canvas.set_window_title(targetlabel + " (LT): " + pd["startdate"] +
" to " + pd["enddate"])
plt.figtext(0.5, 0.95, pd["title"], ha='center', size=16)
# XY plot of the diurnal variation
rect1 = [0.10, pd["margin_bottom"], pd["xy_width"], pd["xy_height"]]
ax1 = plt.axes(rect1)
# get the diurnal stats of the observations
mask = numpy.ma.mask_or(obs.mask, mod.mask)
obs_mor = numpy.ma.array(obs, mask=mask)
dstats = qcutils.get_diurnalstats(dt, obs_mor, LT_info)
ax1.plot(dstats["Hr"], dstats["Av"], 'b-', label="Obs")
# get the diurnal stats of all SOLO predictions
dstats = qcutils.get_diurnalstats(dt, mod, LT_info)
ax1.plot(dstats["Hr"], dstats["Av"], 'r-', label="LT(all)")
mod_mor = numpy.ma.masked_where(numpy.ma.getmaskarray(obs) == True,
mod,
copy=True)
dstats = qcutils.get_diurnalstats(dt, mod_mor, LT_info)
ax1.plot(dstats["Hr"], dstats["Av"], 'g-', label="LT(obs)")
plt.xlim(0, 24)
plt.xticks([0, 6, 12, 18, 24])
ax1.set_ylabel(targetlabel)
ax1.set_xlabel('Hour')
ax1.legend(loc='upper right', frameon=False, prop={'size': 8})
# XY plot of the 30 minute data
rect2 = [0.40, pd["margin_bottom"], pd["xy_width"], pd["xy_height"]]
ax2 = plt.axes(rect2)
ax2.plot(mod, obs, 'b.')
ax2.set_ylabel(targetlabel + '_obs')
ax2.set_xlabel(targetlabel + '_LT')
# plot the best fit line
coefs = numpy.ma.polyfit(numpy.ma.copy(mod), numpy.ma.copy(obs), 1)
xfit = numpy.ma.array([numpy.ma.minimum(mod), numpy.ma.maximum(mod)])
yfit = numpy.polyval(coefs, xfit)
r = numpy.ma.corrcoef(mod, obs)
ax2.plot(xfit, yfit, 'r--', linewidth=3)
eqnstr = 'y = %.3fx + %.3f, r = %.3f' % (coefs[0], coefs[1], r[0][1])
ax2.text(0.5,
0.875,
eqnstr,
fontsize=8,
horizontalalignment='center',
transform=ax2.transAxes)
# write the fit statistics to the plot
numpoints = numpy.ma.count(obs)
numfilled = numpy.ma.count(mod) - numpy.ma.count(obs)
diff = mod - obs
bias = numpy.ma.average(diff)
LT_info["er"][series]["results"]["Bias"].append(bias)
rmse = numpy.ma.sqrt(numpy.ma.mean((obs - mod) * (obs - mod)))
plt.figtext(0.725, 0.225, 'No. points')
plt.figtext(0.825, 0.225, str(numpoints))
LT_info["er"][series]["results"]["No. points"].append(numpoints)
plt.figtext(0.725, 0.200, 'No. filled')
plt.figtext(0.825, 0.200, str(numfilled))
plt.figtext(0.725, 0.175, 'Slope')
plt.figtext(0.825, 0.175, str(qcutils.round2sig(coefs[0], sig=4)))
LT_info["er"][series]["results"]["m_ols"].append(coefs[0])
plt.figtext(0.725, 0.150, 'Offset')
plt.figtext(0.825, 0.150, str(qcutils.round2sig(coefs[1], sig=4)))
LT_info["er"][series]["results"]["b_ols"].append(coefs[1])
plt.figtext(0.725, 0.125, 'r')
plt.figtext(0.825, 0.125, str(qcutils.round2sig(r[0][1], sig=4)))
LT_info["er"][series]["results"]["r"].append(r[0][1])
plt.figtext(0.725, 0.100, 'RMSE')
plt.figtext(0.825, 0.100, str(qcutils.round2sig(rmse, sig=4)))
LT_info["er"][series]["results"]["RMSE"].append(rmse)
var_obs = numpy.ma.var(obs)
LT_info["er"][series]["results"]["Var (obs)"].append(var_obs)
var_mod = numpy.ma.var(mod)
LT_info["er"][series]["results"]["Var (LT)"].append(var_mod)
LT_info["er"][series]["results"]["Var ratio"].append(var_obs / var_mod)
LT_info["er"][series]["results"]["Avg (obs)"].append(numpy.ma.average(obs))
LT_info["er"][series]["results"]["Avg (LT)"].append(numpy.ma.average(mod))
# time series of drivers and target
ts_axes = []
rect = [
pd["margin_left"], pd["ts_bottom"], pd["ts_width"], pd["ts_height"]
]
ts_axes.append(plt.axes(rect))
#ts_axes[0].plot(xdt,obs,'b.',xdt,mod,'r-')
ts_axes[0].scatter(xdt, obs, c=Hdh)
ts_axes[0].plot(xdt, mod, 'r-')
plt.axhline(0)
ts_axes[0].set_xlim(xdt[0], xdt[-1])
TextStr = targetlabel + '_obs (' + ds.series[targetlabel]['Attr'][
'units'] + ')'
ts_axes[0].text(0.05,
0.85,
TextStr,
color='b',
horizontalalignment='left',
transform=ts_axes[0].transAxes)
TextStr = outputlabel + '(' + ds.series[outputlabel]['Attr']['units'] + ')'
ts_axes[0].text(0.85,
0.85,
TextStr,
color='r',
horizontalalignment='right',
transform=ts_axes[0].transAxes)
for ThisOne, i in zip(driverlist, range(1, pd["nDrivers"] + 1)):
this_bottom = pd["ts_bottom"] + i * pd["ts_height"]
rect = [
pd["margin_left"], this_bottom, pd["ts_width"], pd["ts_height"]
]
ts_axes.append(plt.axes(rect, sharex=ts_axes[0]))
data, flag, attr = qcutils.GetSeriesasMA(ds, ThisOne, si=si, ei=ei)
data_notgf = numpy.ma.masked_where(flag != 0, data)
data_gf = numpy.ma.masked_where(flag == 0, data)
ts_axes[i].plot(xdt, data_notgf, 'b-')
ts_axes[i].plot(xdt, data_gf, 'r-')
plt.setp(ts_axes[i].get_xticklabels(), visible=False)
TextStr = ThisOne + '(' + ds.series[ThisOne]['Attr']['units'] + ')'
ts_axes[i].text(0.05,
0.85,
TextStr,
color='b',
horizontalalignment='left',
transform=ts_axes[i].transAxes)
# save a hard copy of the plot
sdt = xdt[0].strftime("%Y%m%d")
edt = xdt[-1].strftime("%Y%m%d")
plot_path = LT_info["plot_path"] + "L6/"
if not os.path.exists(plot_path): os.makedirs(plot_path)
figname = plot_path + pd["site_name"].replace(" ",
"") + "_LT_" + pd["label"]
figname = figname + "_" + sdt + "_" + edt + '.png'
fig.savefig(figname, format='png')
# draw the plot on the screen
if LT_info["show_plots"]:
plt.draw()
plt.pause(1)
plt.ioff()
else:
plt.close(fig)
plt.ion()
# and remove the datetime
if "DateTime" in series_list:
series_list.remove("DateTime")
# and then we loop over the variables to be copied
for label in series_list:
# append a number, unique to each ISD station, to the variable label
all_label = label + "_" + str(i)
# create empty data and flag arrays
variable = qcutils.create_empty_variable(all_label, nrecs)
qcutils.CreateSeries(ds_all,
all_label,
variable["Data"],
Flag=variable["Flag"],
Attr=variable["Attr"])
# read the data out of the ISD site data structure
data, flag, attr = qcutils.GetSeriesasMA(ds_out[i], label)
# add the ISD site ID
attr["isd_site_id"] = isd_site_id
# put the data, flag and attributes into the all-in-one data structure
ds_all.series[all_label]["Data"][idx] = data
ds_all.series[all_label]["Flag"][idx] = flag
ds_all.series[all_label]["Attr"] = copy.deepcopy(attr)
# write the netCDF file with the combined data for this year
if len(fluxnet_id) == 0:
nc_dir_path = os.path.join(out_base_path, site, "Data", "ISD")
nc_file_name = site + "_ISD_" + str(year) + ".nc"
else:
nc_dir_path = os.path.join(out_base_path, fluxnet_id, "Data",
"ISD")
nc_file_name = fluxnet_id + "_ISD_" + str(year) + ".nc"
if not os.path.exists(nc_dir_path):
def interpolate_ds(ds_in, ts, k=3):
"""
Purpose:
Interpolate the contents of a data structure onto a different time step.
Assumptions:
Usage:
Author: PRI
Date: June 2017
"""
# instance the output data structure
ds_out = qcio.DataStructure()
# copy the global attributes
for key in ds_in.globalattributes.keys():
ds_out.globalattributes[key] = ds_in.globalattributes[key]
# add the time step
ds_out.globalattributes["time_step"] = str(ts)
# generate a regular time series at the required time step
dt = ds_in.series["DateTime"]["Data"]
dt0 = dt[0] - datetime.timedelta(minutes=30)
start = datetime.datetime(dt0.year, dt0.month, dt0.day, dt0.hour, 0, 0)
dt1 = dt[-1] + datetime.timedelta(minutes=30)
end = datetime.datetime(dt1.year, dt1.month, dt1.day, dt1.hour, 0, 0)
idt = [
result
for result in perdelta(start, end, datetime.timedelta(minutes=ts))
]
x1 = numpy.array([toTimestamp(dt[i]) for i in range(len(dt))])
x2 = numpy.array([toTimestamp(idt[i]) for i in range(len(idt))])
# loop over the series in the data structure and interpolate
ds_out.series["DateTime"] = {}
ds_out.series["DateTime"]["Data"] = idt
ds_out.series["DateTime"]["Flag"] = numpy.zeros(len(idt))
ds_out.series["DateTime"]["Attr"] = {
"long_name": "Datetime",
"units": "none"
}
ds_out.globalattributes["nc_nrecs"] = len(idt)
series_list = list(ds_in.series.keys())
if "DateTime" in series_list:
series_list.remove("DateTime")
for label in series_list:
#print label
data_in, flag_in, attr_in = qcutils.GetSeriesasMA(ds_in, label)
# check if we are dealing with precipitation
if "Precip" in label:
# precipitation shouldn't be interpolated, just assign any precipitation
# to the ISD time stamp.
data_out = numpy.ma.zeros(len(idt), dtype=numpy.float64)
idx = numpy.searchsorted(x2, numpy.intersect1d(x2, x1))
data_out[idx] = data_in
else:
# interpolate everything else
data_out = interpolate_1d(x1, data_in, x2)
flag_out = numpy.zeros(len(idt))
attr_out = attr_in
qcutils.CreateSeries(ds_out,
label,
data_out,
Flag=flag_out,
Attr=attr_out)
return ds_out
