def CoordinateAh7500AndFcGaps(cf,ds,Fcvar='Fc'):
'''Cleans up Ah_7500_Av based upon Fc gaps to for QA check on Ah_7500_Av v Ah_HMP.'''
if not qcutils.cfoptionskeylogical(cf,Key='CoordinateAh7500&FcGaps'): return
log.info(' Doing the Ah_7500 check')
if qcutils.cfkeycheck(cf,Base='FunctionArgs',ThisOne='AhcheckFc'):
Fclist = ast.literal_eval(cf['FunctionArgs']['AhcheckFc'])
Fcvar = Fclist[0]
# index1 Index of bad Ah_7500_Av observations
index1 = numpy.where((ds.series['Ah_7500_Av']['Flag']!=0) & (ds.series['Ah_7500_Av']['Flag']!=10))
# index2 Index of bad Fc observations
index2 = numpy.where((ds.series[Fcvar]['Flag']!=0) & (ds.series[Fcvar]['Flag']!=10))
ds.series['Ah_7500_Av']['Data'][index2] = numpy.float64(c.missing_value)
ds.series['Ah_7500_Av']['Flag'][index2] = ds.series[Fcvar]['Flag'][index2]
ds.series['Ah_7500_Av']['Flag'][index1] = ds.series['Ah_7500_Av']['Flag'][index1]
if 'CoordinateAh7500AndFcGaps' not in ds.globalattributes['Functions']:
ds.globalattributes['Functions'] = ds.globalattributes['Functions']+',CoordinateAh7500AndFcGaps'
def CoordinateFluxGaps(cf,ds,Fc_in='Fc',Fe_in='Fe',Fh_in='Fh'):
if not qcutils.cfoptionskeylogical(cf,Key='CoordinateFluxGaps'): return
if qcutils.cfkeycheck(cf,Base='FunctionArgs',ThisOne='gapsvars'):
vars = ast.literal_eval(cf['FunctionArgs']['gapsvars'])
Fc_in = vars[0]
Fe_in = vars[1]
Fh_in = vars[2]
Fc,f,a = qcutils.GetSeriesasMA(ds,Fc_in)
Fe,f,a = qcutils.GetSeriesasMA(ds,Fe_in)
Fh,f,a = qcutils.GetSeriesasMA(ds,Fh_in)
# April 2015 PRI - changed numpy.ma.where to numpy.where
index = numpy.where((numpy.ma.getmaskarray(Fc)==True)|
(numpy.ma.getmaskarray(Fe)==True)|
(numpy.ma.getmaskarray(Fh)==True))[0]
#index = numpy.ma.where((numpy.ma.getmaskarray(Fc)==True)|
#(numpy.ma.getmaskarray(Fe)==True)|
#(numpy.ma.getmaskarray(Fh)==True))[0]
# the following for ... in loop is not necessary
for i in range(len(index)):
j = index[i]
if Fc.mask[j]==False:
Fc.mask[j]=True
Fc[j] = numpy.float64(c.missing_value)
ds.series[Fc_in]['Flag'][j] = numpy.int32(19)
if Fe.mask[j]==False:
Fe.mask[j]=True
Fe[j] = numpy.float64(c.missing_value)
ds.series[Fe_in]['Flag'][j] = numpy.int32(19)
if Fh.mask[j]==False:
Fh.mask[j]=True
Fh[j] = numpy.float64(c.missing_value)
ds.series[Fh_in]['Flag'][j] = numpy.int32(19)
ds.series[Fc_in]['Data']=numpy.ma.filled(Fc,float(c.missing_value))
ds.series[Fe_in]['Data']=numpy.ma.filled(Fe,float(c.missing_value))
ds.series[Fh_in]['Data']=numpy.ma.filled(Fh,float(c.missing_value))
log.info(' Finished gap co-ordination')
def l3qc(cf,ds2):
"""
Corrections
Generates L3 from L2 data
Functions performed:
qcts.AddMetVars (optional)
qcts.CorrectSWC (optional*)
qcck.do_linear (all sites)
qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x
qcts.TaFromTv (optional)
qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x
qcts.CoordRotation2D (all sites)
qcts.MassmanApprox (optional*)y
qcts.Massman (optional*)y
qcts.CalculateFluxes (used if Massman not optioned)x
qcts.CalculateFluxesRM (used if Massman optioned)y
qcts.FhvtoFh (all sites)
qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
qcts.CalculateNetRadiation (optional)
qcutils.GetMergeList + qcts.MergeSeries Fsd (optional)
qcutils.GetMergeList + qcts.MergeSeries Fn (optional*)
qcts.InterpolateOverMissing (optional)
AverageSeriesByElements (optional)
qcts.CorrectFgForStorage (all sites)
qcts.Average3SeriesByElements (optional)
qcts.CalculateAvailableEnergy (optional)
qcck.do_qcchecks (all sites)
qcck.gaps (optional)
*: requires ancillary measurements for paratmerisation
+: each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov
x: required together in option set
y: required together in option set
"""
# make a copy of the L2 data
ds3 = copy.deepcopy(ds2)
# set some attributes for this level
qcutils.UpdateGlobalAttributes(cf,ds3,"L3")
# initialise the global attribute to document the functions used
ds3.globalattributes['Functions'] = ''
# put the control file name into the global attributes
ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
# check to see if we have any imports
qcgf.ImportSeries(cf,ds3)
# correct measured soil water content using empirical relationship to collected samples
qcts.CorrectSWC(cf,ds3)
# apply linear corrections to the data
qcck.do_linear(cf,ds3)
# merge whatever humidities are available
qcts.MergeHumidities(cf,ds3,convert_units=True)
# get the air temperature from the CSAT virtual temperature
qcts.TaFromTv(cf,ds3)
# merge the HMP and corrected CSAT data
qcts.MergeSeries(cf,ds3,'Ta',[0,10],convert_units=True)
qcutils.CheckUnits(ds3,"Ta","C",convert_units=True)
# calculate humidities (absolute, specific and relative) from whatever is available
qcts.CalculateHumidities(ds3)
# merge the 7500 CO2 concentration
qcts.MergeSeries(cf,ds3,'Cc',[0,10],convert_units=True)
# PRI - disable CO2 units conversion from whatever to mg/m3
# - this step is, as far as I can see, redundant, see qcts.Fc_WPL()
#qcutils.CheckUnits(ds3,"Cc","mg/m3",convert_units=True)
# add relevant meteorological values to L3 data
qcts.CalculateMeteorologicalVariables(ds3)
# check to see if the user wants to use the fluxes in the L2 file
if not qcutils.cfoptionskeylogical(cf,Key="UseL2Fluxes",default=False):
# check the covariancve units and change if necessary
qcts.CheckCovarianceUnits(ds3)
# do the 2D coordinate rotation
qcts.CoordRotation2D(cf,ds3)
# do the Massman frequency attenuation correction
qcts.MassmanStandard(cf,ds3)
# calculate the fluxes
qcts.CalculateFluxes(cf,ds3)
# approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
qcts.FhvtoFh(cf,ds3)
# correct the H2O & CO2 flux due to effects of flux on density measurements
qcts.Fe_WPL(cf,ds3)
qcts.Fc_WPL(cf,ds3)
# convert CO2 units if required
qcutils.ConvertCO2Units(cf,ds3,Cc='Cc')
# calculate Fc storage term - single height only at present
qcts.CalculateFcStorage(cf,ds3)
# convert Fc and Fc_storage units if required
qcutils.ConvertFcUnits(cf,ds3,Fc='Fc',Fc_storage='Fc_storage')
# correct Fc for storage term - only recommended if storage calculated from profile available
qcts.CorrectFcForStorage(cf,ds3)
# merge the incoming shortwave radiation
qcts.MergeSeries(cf,ds3,'Fsd',[0,10])
# calculate the net radiation from the Kipp and Zonen CNR1
qcts.CalculateNetRadiation(cf,ds3,Fn_out='Fn_KZ',Fsd_in='Fsd',Fsu_in='Fsu',Fld_in='Fld',Flu_in='Flu')
qcts.MergeSeries(cf,ds3,'Fn',[0,10])
# combine wind speed from the Wind Sentry and the CSAT
qcts.MergeSeries(cf,ds3,'Ws',[0,10])
# combine wind direction from the Wind Sentry and the CSAT
qcts.MergeSeries(cf,ds3,'Wd',[0,10])
# correct soil heat flux for storage
# ... either average the raw ground heat flux, soil temperature and moisture
# and then do the correction (OzFlux "standard")
qcts.AverageSeriesByElements(cf,ds3,'Ts')
qcts.AverageSeriesByElements(cf,ds3,'Sws')
if qcutils.cfoptionskeylogical(cf,Key='CorrectIndividualFg'):
# ... or correct the individual ground heat flux measurements (James' method)
qcts.CorrectIndividualFgForStorage(cf,ds3)
qcts.AverageSeriesByElements(cf,ds3,'Fg')
else:
qcts.AverageSeriesByElements(cf,ds3,'Fg')
qcts.CorrectFgForStorage(cf,ds3,Fg_out='Fg',Fg_in='Fg',Ts_in='Ts',Sws_in='Sws')
# calculate the available energy
qcts.CalculateAvailableEnergy(ds3,Fa_out='Fa',Fn_in='Fn',Fg_in='Fg')
# create new series using MergeSeries or AverageSeries
qcck.CreateNewSeries(cf,ds3)
# create a series of daily averaged soil moisture interpolated back to the time step
#qcts.DailyAverageSws_Interpolated(cf,ds3,Sws_out='Sws_daily',Sws_in='Sws')
# re-apply the quality control checks (range, diurnal and rules)
qcck.do_qcchecks(cf,ds3)
# coordinate gaps in the three main fluxes
qcck.CoordinateFluxGaps(cf,ds3)
# coordinate gaps in Ah_7500_Av with Fc
qcck.CoordinateAh7500AndFcGaps(cf,ds3)
# get the statistics for the QC flags and write these to an Excel spreadsheet
qcio.get_seriesstats(cf,ds3)
# write the percentage of good data as a variable attribute
qcutils.get_coverage_individual(ds3)
# write the percentage of good data for groups
qcutils.get_coverage_groups(ds3)
return ds3
def l3qc(cf, ds2):
"""
Corrections
Generates L3 from L2 data
Functions performed:
qcts.AddMetVars (optional)
qcts.CorrectSWC (optional*)
qcck.do_linear (all sites)
qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x
qcts.TaFromTv (optional)
qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x
qcts.CoordRotation2D (all sites)
qcts.MassmanApprox (optional*)y
qcts.Massman (optional*)y
qcts.CalculateFluxes (used if Massman not optioned)x
qcts.CalculateFluxesRM (used if Massman optioned)y
qcts.FhvtoFh (all sites)
qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
qcts.CalculateNetRadiation (optional)
qcutils.GetMergeList + qcts.MergeSeries Fsd (optional)
qcutils.GetMergeList + qcts.MergeSeries Fn (optional*)
qcts.InterpolateOverMissing (optional)
AverageSeriesByElements (optional)
qcts.CorrectFgForStorage (all sites)
qcts.Average3SeriesByElements (optional)
qcts.CalculateAvailableEnergy (optional)
qcck.do_qcchecks (all sites)
qcck.gaps (optional)
*: requires ancillary measurements for paratmerisation
+: each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov
x: required together in option set
y: required together in option set
"""
# make a copy of the L2 data
ds3 = copy.deepcopy(ds2)
# set some attributes for this level
qcutils.UpdateGlobalAttributes(cf, ds3, "L3")
# initialise the global attribute to document the functions used
ds3.globalattributes['Functions'] = ''
# put the control file name into the global attributes
ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
# check to see if we have any imports
qcgf.ImportSeries(cf, ds3)
# correct measured soil water content using empirical relationship to collected samples
qcts.CorrectSWC(cf, ds3)
# apply linear corrections to the data
qcck.do_linear(cf, ds3)
# merge whatever humidities are available
qcts.MergeHumidities(cf, ds3, convert_units=True)
# get the air temperature from the CSAT virtual temperature
qcts.TaFromTv(cf, ds3)
# merge the HMP and corrected CSAT data
qcts.MergeSeries(cf, ds3, 'Ta', [0, 10], convert_units=True)
qcutils.CheckUnits(ds3, "Ta", "C", convert_units=True)
# calculate humidities (absolute, specific and relative) from whatever is available
qcts.CalculateHumidities(ds3)
# merge the 7500 CO2 concentration
qcts.MergeSeries(cf, ds3, 'Cc', [0, 10], convert_units=True)
qcutils.CheckUnits(ds3, "Cc", "mg/m3", convert_units=True)
# add relevant meteorological values to L3 data
qcts.CalculateMeteorologicalVariables(ds3)
# check to see if the user wants to use the fluxes in the L2 file
if not qcutils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False):
# check the covariancve units and change if necessary
qcts.CheckCovarianceUnits(ds3)
# do the 2D coordinate rotation
qcts.CoordRotation2D(cf, ds3)
# do the Massman frequency attenuation correction
qcts.MassmanStandard(cf, ds3)
# calculate the fluxes
qcts.CalculateFluxes(cf, ds3)
# approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
qcts.FhvtoFh(cf, ds3)
# correct the H2O & CO2 flux due to effects of flux on density measurements
qcts.Fe_WPL(cf, ds3)
qcts.Fc_WPL(cf, ds3)
# convert CO2 units if required
qcutils.ConvertCO2Units(cf, ds3, Cc='Cc')
# calculate Fc storage term - single height only at present
qcts.CalculateFcStorage(cf, ds3)
# convert Fc and Fc_storage units if required
qcutils.ConvertFcUnits(cf, ds3, Fc='Fc', Fc_storage='Fc_storage')
# correct Fc for storage term - only recommended if storage calculated from profile available
qcts.CorrectFcForStorage(cf, ds3)
# merge the incoming shortwave radiation
qcts.MergeSeries(cf, ds3, 'Fsd', [0, 10])
# calculate the net radiation from the Kipp and Zonen CNR1
qcts.CalculateNetRadiation(cf,
ds3,
Fn_out='Fn_KZ',
Fsd_in='Fsd',
Fsu_in='Fsu',
Fld_in='Fld',
Flu_in='Flu')
qcts.MergeSeries(cf, ds3, 'Fn', [0, 10])
# combine wind speed from the Wind Sentry and the CSAT
qcts.MergeSeries(cf, ds3, 'Ws', [0, 10])
# combine wind direction from the Wind Sentry and the CSAT
qcts.MergeSeries(cf, ds3, 'Wd', [0, 10])
# correct soil heat flux for storage
# ... either average the raw ground heat flux, soil temperature and moisture
# and then do the correction (OzFlux "standard")
qcts.AverageSeriesByElements(cf, ds3, 'Ts')
qcts.AverageSeriesByElements(cf, ds3, 'Sws')
if qcutils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'):
# ... or correct the individual ground heat flux measurements (James' method)
qcts.CorrectIndividualFgForStorage(cf, ds3)
qcts.AverageSeriesByElements(cf, ds3, 'Fg')
else:
qcts.AverageSeriesByElements(cf, ds3, 'Fg')
qcts.CorrectFgForStorage(cf,
ds3,
Fg_out='Fg',
Fg_in='Fg',
Ts_in='Ts',
Sws_in='Sws')
# calculate the available energy
qcts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg')
# create new series using MergeSeries or AverageSeries
qcck.CreateNewSeries(cf, ds3)
# create a series of daily averaged soil moisture interpolated back to the time step
#qcts.DailyAverageSws_Interpolated(cf,ds3,Sws_out='Sws_daily',Sws_in='Sws')
# re-apply the quality control checks (range, diurnal and rules)
qcck.do_qcchecks(cf, ds3)
# coordinate gaps in the three main fluxes
qcck.CoordinateFluxGaps(cf, ds3)
# coordinate gaps in Ah_7500_Av with Fc
qcck.CoordinateAh7500AndFcGaps(cf, ds3)
# get the statistics for the QC flags and write these to an Excel spreadsheet
qcio.get_seriesstats(cf, ds3)
# write the percentage of good data as a variable attribute
qcutils.get_coverage_individual(ds3)
# write the percentage of good data for groups
qcutils.get_coverage_groups(ds3)
return ds3
def l3qc(cf, ds2):
"""
"""
# make a copy of the L2 data
ds3 = copy.deepcopy(ds2)
# set some attributes for this level
qcutils.UpdateGlobalAttributes(cf, ds3, "L3")
# put the control file name into the global attributes
ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
# check to see if we have any imports
qcgf.ImportSeries(cf, ds3)
# apply linear corrections to the data
qcck.do_linear(cf, ds3)
# ************************
# *** Merge humidities ***
# ************************
# merge whatever humidities are available
qcts.MergeHumidities(cf, ds3, convert_units=True)
# **************************
# *** Merge temperatures ***
# **************************
# get the air temperature from the CSAT virtual temperature
qcts.TaFromTv(cf, ds3)
# merge the HMP and corrected CSAT data
qcts.MergeSeries(cf, ds3, "Ta", convert_units=True)
qcutils.CheckUnits(ds3, "Ta", "C", convert_units=True)
# ***************************
# *** Calcuate humidities ***
# ***************************
# calculate humidities (absolute, specific and relative) from whatever is available
qcts.CalculateHumidities(ds3)
# ********************************
# *** Merge CO2 concentrations ***
# ********************************
# merge the 7500 CO2 concentration
# PRI 09/08/2017 possibly the ugliest thing I have done yet
# This needs to be abstracted to a general alias checking routine at the
# start of the L3 processing so that possible aliases are mapped to a single
# set of variable names.
if "CO2" in cf["Variables"]:
CO2 = "CO2"
elif "Cc" in cf["Variables"]:
CO2 = "Cc"
else:
msg = "Label for CO2 ('CO2','Cc') not found in control file"
logger.error(msg)
return
qcts.MergeSeries(cf, ds3, CO2, convert_units=True)
# ******************************************
# *** Calculate meteorological variables ***
# ******************************************
# Update meteorological variables
qcts.CalculateMeteorologicalVariables(ds3)
# *************************************************
# *** Calculate fluxes from covariances section ***
# *************************************************
# check to see if the user wants to use the fluxes in the L2 file
if not qcutils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False):
# check the covariance units and change if necessary
qcts.CheckCovarianceUnits(ds3)
# do the 2D coordinate rotation
qcts.CoordRotation2D(cf, ds3)
# do the Massman frequency attenuation correction
qcts.MassmanStandard(cf, ds3)
# calculate the fluxes
qcts.CalculateFluxes(cf, ds3)
# approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
qcts.FhvtoFh(cf, ds3)
# correct the H2O & CO2 flux due to effects of flux on density measurements
qcts.Fe_WPL(cf, ds3)
qcts.Fc_WPL(cf, ds3)
# **************************************
# *** Calculate Monin-Obukhov length ***
# **************************************
qcts.CalculateMoninObukhovLength(ds3)
# **************************
# *** CO2 and Fc section ***
# **************************
# convert CO2 units if required
qcutils.ConvertCO2Units(cf, ds3, CO2=CO2)
# calculate Fc storage term - single height only at present
qcts.CalculateFcStorageSinglePoint(cf, ds3, Fc_out='Fc_single', CO2_in=CO2)
# convert Fc and Fc_storage units if required
qcutils.ConvertFcUnits(cf, ds3)
# merge Fc and Fc_storage series if required
merge_list = [
label for label in cf["Variables"].keys() if label[0:2] == "Fc"
and "MergeSeries" in cf["Variables"][label].keys()
]
for label in merge_list:
qcts.MergeSeries(cf, ds3, label, save_originals=True)
# correct Fc for storage term - only recommended if storage calculated from profile available
qcts.CorrectFcForStorage(cf, ds3)
# *************************
# *** Radiation section ***
# *************************
# merge the incoming shortwave radiation
qcts.MergeSeries(cf, ds3, 'Fsd')
# calculate the net radiation from the Kipp and Zonen CNR1
qcts.CalculateNetRadiation(cf,
ds3,
Fn_out='Fn_KZ',
Fsd_in='Fsd',
Fsu_in='Fsu',
Fld_in='Fld',
Flu_in='Flu')
qcts.MergeSeries(cf, ds3, 'Fn')
# ****************************************
# *** Wind speed and direction section ***
# ****************************************
# combine wind speed from the Wind Sentry and the SONIC
qcts.MergeSeries(cf, ds3, 'Ws')
# combine wind direction from the Wind Sentry and the SONIC
qcts.MergeSeries(cf, ds3, 'Wd')
# ********************
# *** Soil section ***
# ********************
# correct soil heat flux for storage
# ... either average the raw ground heat flux, soil temperature and moisture
# and then do the correction (OzFlux "standard")
qcts.AverageSeriesByElements(cf, ds3, 'Ts')
qcts.AverageSeriesByElements(cf, ds3, 'Sws')
if qcutils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'):
# ... or correct the individual ground heat flux measurements (James' method)
qcts.CorrectIndividualFgForStorage(cf, ds3)
qcts.AverageSeriesByElements(cf, ds3, 'Fg')
else:
qcts.AverageSeriesByElements(cf, ds3, 'Fg')
qcts.CorrectFgForStorage(cf,
ds3,
Fg_out='Fg',
Fg_in='Fg',
Ts_in='Ts',
Sws_in='Sws')
# calculate the available energy
qcts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg')
# create new series using MergeSeries or AverageSeries
qcck.CreateNewSeries(cf, ds3)
# re-apply the quality control checks (range, diurnal and rules)
qcck.do_qcchecks(cf, ds3)
# coordinate gaps in the three main fluxes
qcck.CoordinateFluxGaps(cf, ds3)
# coordinate gaps in Ah_7500_Av with Fc
qcck.CoordinateAh7500AndFcGaps(cf, ds3)
# check missing data and QC flags are consistent
qcutils.CheckQCFlags(ds3)
# get the statistics for the QC flags and write these to an Excel spreadsheet
qcio.get_seriesstats(cf, ds3)
# write the percentage of good data as a variable attribute
qcutils.get_coverage_individual(ds3)
# write the percentage of good data for groups
qcutils.get_coverage_groups(ds3)
return ds3