def GapFillUsingInterpolation(cf, ds):
"""
Purpose:
Gap fill variables in the data structure using interpolation.
All variables in the [Variables], [Drivers] and [Fluxes] section
are processed.
Usage:
qcgf.GapFillUsingInterpolation(cf,ds)
where cf is a control file object
ds is a data structure
Author: PRI
Date: September 2016
"""
label_list = qcutils.get_label_list_from_cf(cf)
maxlen = int(
qcutils.get_keyvaluefromcf(cf, ["Options"],
"MaxGapInterpolate",
default=2))
if maxlen == 0:
msg = " Gap fill by interpolation disabled in control file"
logger.info(msg)
return
for label in label_list:
section = qcutils.get_cfsection(cf, series=label)
if "MaxGapInterpolate" in cf[section][label]:
maxlen = int(
qcutils.get_keyvaluefromcf(cf, [section, label],
"MaxGapInterpolate",
default=2))
if maxlen == 0:
msg = " Gap fill by interpolation disabled for " + label
logger.info(msg)
continue
qcts.InterpolateOverMissing(ds, series=label, maxlen=2)
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
qcutils.CreateSeries(ds,'Wg',data_dict[bom_id][:,12],Flag=flag,Attr=attr)
data_dict[bom_id][:,13] = data_dict[bom_id][:,13]/float(10)
attr=qcutils.MakeAttributeDictionary(long_name='Air Pressure',units='kPa',
bom_id=str(bom_id),bom_name=bom_sites_info[site_name][str(bom_id)]["site_name"],
bom_dist=bom_sites_info[site_name][str(bom_id)]["distance"])
qcutils.CreateSeries(ds,'ps',data_dict[bom_id][:,13],Flag=flag,Attr=attr)
# fix any time stamp issues
if qcutils.CheckTimeStep(ds):
qcutils.FixTimeStep(ds)
# update the Year, Month, Day etc from the Python datetime
qcutils.get_ymdhmsfromdatetime(ds)
# now interpolate
for label in ["Precip","Ta","Td","RH","Ws","Wd","Wg","ps"]:
qcts.InterpolateOverMissing(ds,series=label,maxlen=2)
# put this stations data into the data structure dictionary
ds_dict[bom_id] = ds
# get the earliest start datetime and the latest end datetime
log.info("Finding the start and end dates")
bom_id_list = ds_dict.keys()
ds0 = ds_dict[bom_id_list[0]]
ldt = ds0.series["DateTime"]["Data"]
#print bom_id_list[0],":",ldt[0],ldt[-1]
start_date = ldt[0]
end_date = ldt[-1]
bom_id_list.remove(bom_id_list[0])
for bom_id in bom_id_list:
dsn = ds_dict[bom_id]
ldtn = dsn.series["DateTime"]["Data"]
def l4qc(cf, ds3, InLevel, x):
ds4 = copy.deepcopy(ds3)
ds4.globalattributes['nc_level'] = 'L4'
if (qcutils.cfkeycheck(cf, Base='Functions', ThisOne='L4_offline') and
cf['Functions']['L4_offline'] == 'True') and qcutils.cfkeycheck(
cf, Base='Functions', ThisOne='L4_keys'):
try:
ds4.globalattributes['L4Functions'] = ds4.globalattributes[
'L4Functions'] + ', ' + cf['Functions']['L4_keys']
except:
ds4.globalattributes['L4Functions'] = cf['Functions']['L4_keys']
x = x + 1
# determine HMP Ah if not output by datalogger
if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='CalculateAh'
) and cf['Functions']['CalculateAh'] == 'True':
try:
ds4.globalattributes['L4Functions'] = ds4.globalattributes[
'L4Functions'] + ', CalculateAh'
except:
ds4.globalattributes['L4Functions'] = 'CalculateAh'
log.info(' Adding HMP Ah to database')
qcts.CalculateAhHMP(cf, ds4)
# add relevant meteorological values to L4 data
if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='CalculateMetVars'
) and cf['Functions']['CalculateMetVars'] == 'True':
try:
ds4.globalattributes['L4Functions'] = ds4.globalattributes[
'L4Functions'] + ', CalculateMetVars'
except:
ds4.globalattributes['L4Functions'] = 'CalculateMetVars'
log.info(' Adding standard met variables to database')
qcts.CalculateMeteorologicalVariables(ds4)
# merge CSAT and wind sentry wind speed
if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='MergeSeriesWS'
) and cf['Functions']['MergeSeriesWS'] == 'True':
try:
ds4.globalattributes['L4Functions'] = ds4.globalattributes[
'L4Functions'] + ', MergeSeriesWS'
except:
ds4.globalattributes['L4Functions'] = 'MergeSeriesWS'
qcts.MergeSeries(cf, ds4, 'Ws', [0, 10])
# linear interpolation to fill missing values over gaps of 1 hour
if qcutils.cfkeycheck(
cf, Base='Functions', ThisOne='InterpolateOverMissing'
) and cf['Functions']['InterpolateOverMissing'] == 'True':
try:
ds4.globalattributes['L4Functions'] = ds4.globalattributes[
'L4Functions'] + ', InterpolateOverMissing'
except:
ds4.globalattributes['L4Functions'] = 'InterpolateOverMissing'
log.info(
' Gap filling by linear interpolation to fill missing values over gaps of 1 hour'
)
for ThisOne in cf['InterpolateVars'].keys():
qcts.InterpolateOverMissing(cf, ds4, series=ThisOne, maxlen=2)
x = x + 1
# re-apply the quality control checks (range, diurnal and rules)
if x > 0:
log.info(' Doing QC checks on L4 data')
qcck.do_qcchecks(cf, ds4)
try:
ds4.globalattributes['L4Functions'] = ds4.globalattributes[
'L4Functions'] + ', do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours)'
except:
ds4.globalattributes[
'L4Functions'] = 'do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours)'
# interpolate over any ramaining gaps up to 3 hours in length
if qcutils.cfkeycheck(
cf, Base='Functions', ThisOne='InterpolateOverMissing'
) and cf['Functions']['InterpolateOverMissing'] == 'True':
for ThisOne in cf['InterpolateVars'].keys():
qcts.InterpolateOverMissing(cf, ds4, series=ThisOne, maxlen=6)
ds4.globalattributes['Functions'] = ds4.globalattributes[
'Functions'] + ', ' + ds4.globalattributes['L4Functions']
return ds4, x