Exemplo n.º 1
0
def l2qc(cf, ds1):
    """
        Perform initial QA/QC on flux data
        Generates L2 from L1 data
        * check parameters specified in control file
        
        Functions performed:
            qcck.do_rangecheck*
            qcck.do_CSATcheck
            qcck.do_7500check
            qcck.do_diurnalcheck*
            qcck.do_excludedates*
            qcck.do_excludehours*
            qcts.albedo
        """
    # make a copy of the L1 data
    ds2 = copy.deepcopy(ds1)
    ds2.globalattributes['nc_level'] = 'L2'
    ds2.globalattributes['EPDversion'] = sys.version
    ds2.globalattributes['QC_version_history'] = cfg.__doc__
    ds2.globalattributes[
        'L2Functions'] = 'do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours), CSATcheck, 7500check, albedo'
    ds2.globalattributes[
        'Functions'] = 'do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours), CSATcheck, 7500check, albedo'
    # put the control file name into the global attributes
    ds2.globalattributes['controlfile_name'] = cf['controlfile_name']
    # apply the quality control checks (range, diurnal, exclude dates and exclude hours
    qcck.do_qcchecks(cf, ds2)
    # do the CSAT diagnostic check
    qcck.do_CSATcheck(cf, ds2)
    # do the LI-7500 diagnostic check
    qcck.do_7500check(cf, ds2)
    # constrain albedo estimates to full sun angles
    qcts.albedo(cf, ds2)
    log.info(' Finished the albedo constraints'
             )  # apply linear corrections to the data
    log.info(' Applying linear corrections ...')
    qcck.do_linear(cf, ds2)
    # write series statistics to file
    qcio.get_seriesstats(cf, ds2)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds2)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds2)
    return ds2
Exemplo n.º 2
0
def l2qc(cf,ds1):
    """
        Perform initial QA/QC on flux data
        Generates L2 from L1 data
        * check parameters specified in control file
        
        Functions performed:
            qcck.do_rangecheck*
            qcck.do_CSATcheck
            qcck.do_7500check
            qcck.do_diurnalcheck*
            qcck.do_excludedates*
            qcck.do_excludehours*
            qcts.albedo
        """
    # make a copy of the L1 data
    ds2 = copy.deepcopy(ds1)
    ds2.globalattributes['nc_level'] = 'L2'
    ds2.globalattributes['EPDversion'] = sys.version
    ds2.globalattributes['QC_version_history'] = cfg.__doc__
    ds2.globalattributes['L2Functions'] = 'do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours), CSATcheck, 7500check, albedo'
    ds2.globalattributes['Functions'] = 'do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours), CSATcheck, 7500check, albedo'
    # put the control file name into the global attributes
    ds2.globalattributes['controlfile_name'] = cf['controlfile_name']
    # apply the quality control checks (range, diurnal, exclude dates and exclude hours
    qcck.do_qcchecks(cf,ds2)
    # do the CSAT diagnostic check
    qcck.do_CSATcheck(cf,ds2)
    # do the LI-7500 diagnostic check
    qcck.do_7500check(cf,ds2)
    # constrain albedo estimates to full sun angles
    qcts.albedo(cf,ds2)
    log.info(' Finished the albedo constraints')    # apply linear corrections to the data
    log.info(' Applying linear corrections ...')
    qcck.do_linear(cf,ds2)
    # write series statistics to file
    qcio.get_seriesstats(cf,ds2)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds2)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds2)
    return ds2
Exemplo n.º 3
0
def l2qc(cf, ds1):
    """
        Perform initial QA/QC on flux data
        Generates L2 from L1 data
        * check parameters specified in control file

        Functions performed:
            qcck.do_rangecheck*
            qcck.do_CSATcheck
            qcck.do_7500check
            qcck.do_diurnalcheck*
            qcck.do_excludedates*
            qcck.do_excludehours*
            qcts.albedo
        """
    # make a copy of the L1 data
    ds2 = copy.deepcopy(ds1)
    # set some attributes for this level
    qcutils.UpdateGlobalAttributes(cf, ds2, "L2")
    ds2.globalattributes['Functions'] = ''
    # put the control file name into the global attributes
    ds2.globalattributes['controlfile_name'] = cf['controlfile_name']
    # apply the quality control checks (range, diurnal, exclude dates and exclude hours
    qcck.do_qcchecks(cf, ds2)
    # do the CSAT diagnostic check
    qcck.do_SONICcheck(cf, ds2)
    # do the IRGA diagnostic check
    qcck.do_IRGAcheck(cf, ds2)
    # constrain albedo estimates to full sun angles
    #qcts.albedo(cf,ds2)
    #log.info(' Finished the albedo constraints')    # apply linear corrections to the data
    #log.info(' Applying linear corrections ...')
    qcck.do_linear(cf, ds2)
    # check missing data and QC flags are consistent
    qcutils.CheckQCFlags(ds2)
    # write series statistics to file
    qcio.get_seriesstats(cf, ds2)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds2)

    return ds2
Exemplo n.º 4
0
def l2qc(cf,ds1):
    """
        Perform initial QA/QC on flux data
        Generates L2 from L1 data
        * check parameters specified in control file
        
        Functions performed:
            qcck.do_rangecheck*
            qcck.do_CSATcheck
            qcck.do_7500check
            qcck.do_diurnalcheck*
            qcck.do_excludedates*
            qcck.do_excludehours*
            qcts.albedo
        """
    # make a copy of the L1 data
    ds2 = copy.deepcopy(ds1)
    # set some attributes for this level    
    qcutils.UpdateGlobalAttributes(cf,ds2,"L2")
    ds2.globalattributes['Functions'] = ''
    # put the control file name into the global attributes
    ds2.globalattributes['controlfile_name'] = cf['controlfile_name']
    # apply the quality control checks (range, diurnal, exclude dates and exclude hours
    qcck.do_qcchecks(cf,ds2)
    # do the CSAT diagnostic check
    qcck.do_CSATcheck(cf,ds2)
    # do the IRGA diagnostic check
    qcck.do_IRGAcheck(cf,ds2)
    # constrain albedo estimates to full sun angles
    #qcts.albedo(cf,ds2)
    #log.info(' Finished the albedo constraints')    # apply linear corrections to the data
    #log.info(' Applying linear corrections ...')
    qcck.do_linear(cf,ds2)
    # write series statistics to file
    qcio.get_seriesstats(cf,ds2)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds2)
    
    return ds2
Exemplo n.º 5
0
def l3qc(cf,ds2):
    """
        Corrections
        Generates L3 from L2 data
        
        Functions performed:
            qcts.AddMetVars (optional)
            qcts.CorrectSWC (optional*)
            qcck.do_linear (all sites)
            qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x
            qcts.TaFromTv (optional)
            qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x
            qcts.CoordRotation2D (all sites)
            qcts.MassmanApprox (optional*)y
            qcts.Massman (optional*)y
            qcts.CalculateFluxes (used if Massman not optioned)x
            qcts.CalculateFluxesRM (used if Massman optioned)y
            qcts.FhvtoFh (all sites)
            qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.CalculateNetRadiation (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fsd (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fn (optional*)
            qcts.InterpolateOverMissing (optional)
            AverageSeriesByElements (optional)
            qcts.CorrectFgForStorage (all sites)
            qcts.Average3SeriesByElements (optional)
            qcts.CalculateAvailableEnergy (optional)
            qcck.do_qcchecks (all sites)
            qcck.gaps (optional)
            
            *:  requires ancillary measurements for paratmerisation
            +:  each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov
            x:  required together in option set
            y:  required together in option set
        """
    # make a copy of the L2 data
    ds3 = copy.deepcopy(ds2)
    # set some attributes for this level    
    qcutils.UpdateGlobalAttributes(cf,ds3,"L3")
    # initialise the global attribute to document the functions used
    ds3.globalattributes['Functions'] = ''
    # put the control file name into the global attributes
    ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
    # check to see if we have any imports
    qcgf.ImportSeries(cf,ds3)
    # correct measured soil water content using empirical relationship to collected samples
    qcts.CorrectSWC(cf,ds3)
    # apply linear corrections to the data
    qcck.do_linear(cf,ds3)
    # merge whatever humidities are available
    qcts.MergeHumidities(cf,ds3,convert_units=True)
    # get the air temperature from the CSAT virtual temperature
    qcts.TaFromTv(cf,ds3)
    # merge the HMP and corrected CSAT data
    qcts.MergeSeries(cf,ds3,'Ta',[0,10],convert_units=True)
    qcutils.CheckUnits(ds3,"Ta","C",convert_units=True)
    # calculate humidities (absolute, specific and relative) from whatever is available
    qcts.CalculateHumidities(ds3)
    # merge the 7500 CO2 concentration
    qcts.MergeSeries(cf,ds3,'Cc',[0,10],convert_units=True)
    # PRI - disable CO2 units conversion from whatever to mg/m3
    #     - this step is, as far as I can see, redundant, see qcts.Fc_WPL()
    #qcutils.CheckUnits(ds3,"Cc","mg/m3",convert_units=True)
    # add relevant meteorological values to L3 data
    qcts.CalculateMeteorologicalVariables(ds3)
    # check to see if the user wants to use the fluxes in the L2 file
    if not qcutils.cfoptionskeylogical(cf,Key="UseL2Fluxes",default=False):
        # check the covariancve units and change if necessary
        qcts.CheckCovarianceUnits(ds3)
        # do the 2D coordinate rotation
        qcts.CoordRotation2D(cf,ds3)
        # do the Massman frequency attenuation correction
        qcts.MassmanStandard(cf,ds3)
        # calculate the fluxes
        qcts.CalculateFluxes(cf,ds3)
        # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
        qcts.FhvtoFh(cf,ds3)
        # correct the H2O & CO2 flux due to effects of flux on density measurements
        qcts.Fe_WPL(cf,ds3)
        qcts.Fc_WPL(cf,ds3)
    # convert CO2 units if required
    qcutils.ConvertCO2Units(cf,ds3,Cc='Cc')
    # calculate Fc storage term - single height only at present
    qcts.CalculateFcStorage(cf,ds3)
    # convert Fc and Fc_storage units if required
    qcutils.ConvertFcUnits(cf,ds3,Fc='Fc',Fc_storage='Fc_storage')
    # correct Fc for storage term - only recommended if storage calculated from profile available
    qcts.CorrectFcForStorage(cf,ds3)
    # merge the incoming shortwave radiation
    qcts.MergeSeries(cf,ds3,'Fsd',[0,10])
    # calculate the net radiation from the Kipp and Zonen CNR1
    qcts.CalculateNetRadiation(cf,ds3,Fn_out='Fn_KZ',Fsd_in='Fsd',Fsu_in='Fsu',Fld_in='Fld',Flu_in='Flu')
    qcts.MergeSeries(cf,ds3,'Fn',[0,10])
    # combine wind speed from the Wind Sentry and  the CSAT
    qcts.MergeSeries(cf,ds3,'Ws',[0,10])
    # combine wind direction from the Wind Sentry and  the CSAT
    qcts.MergeSeries(cf,ds3,'Wd',[0,10])
    # correct soil heat flux for storage
    #    ... either average the raw ground heat flux, soil temperature and moisture
    #        and then do the correction (OzFlux "standard")
    qcts.AverageSeriesByElements(cf,ds3,'Ts')
    qcts.AverageSeriesByElements(cf,ds3,'Sws')
    if qcutils.cfoptionskeylogical(cf,Key='CorrectIndividualFg'):
        #    ... or correct the individual ground heat flux measurements (James' method)
            qcts.CorrectIndividualFgForStorage(cf,ds3)
            qcts.AverageSeriesByElements(cf,ds3,'Fg')
    else:
        qcts.AverageSeriesByElements(cf,ds3,'Fg')
        qcts.CorrectFgForStorage(cf,ds3,Fg_out='Fg',Fg_in='Fg',Ts_in='Ts',Sws_in='Sws')
    # calculate the available energy
    qcts.CalculateAvailableEnergy(ds3,Fa_out='Fa',Fn_in='Fn',Fg_in='Fg')
    # create new series using MergeSeries or AverageSeries
    qcck.CreateNewSeries(cf,ds3)
    # create a series of daily averaged soil moisture interpolated back to the time step
    #qcts.DailyAverageSws_Interpolated(cf,ds3,Sws_out='Sws_daily',Sws_in='Sws')
    # re-apply the quality control checks (range, diurnal and rules)
    qcck.do_qcchecks(cf,ds3)
    # coordinate gaps in the three main fluxes
    qcck.CoordinateFluxGaps(cf,ds3)
    # coordinate gaps in Ah_7500_Av with Fc
    qcck.CoordinateAh7500AndFcGaps(cf,ds3)
    # get the statistics for the QC flags and write these to an Excel spreadsheet
    qcio.get_seriesstats(cf,ds3)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds3)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds3)

    return ds3
Exemplo n.º 6
0
def l3qc(cf, ds2):
    """
        Corrections
        Generates L3 from L2 data
        
        Functions performed:
            qcts.AddMetVars (optional)
            qcts.CorrectSWC (optional*)
            qcck.do_linear (all sites)
            qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x
            qcts.TaFromTv (optional)
            qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x
            qcts.CoordRotation2D (all sites)
            qcts.MassmanApprox (optional*)y
            qcts.Massman (optional*)y
            qcts.CalculateFluxes (used if Massman not optioned)x
            qcts.CalculateFluxesRM (used if Massman optioned)y
            qcts.FhvtoFh (all sites)
            qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.CalculateNetRadiation (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fsd (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fn (optional*)
            qcts.InterpolateOverMissing (optional)
            AverageSeriesByElements (optional)
            qcts.CorrectFgForStorage (all sites)
            qcts.Average3SeriesByElements (optional)
            qcts.CalculateAvailableEnergy (optional)
            qcck.do_qcchecks (all sites)
            qcck.gaps (optional)
            
            *:  requires ancillary measurements for paratmerisation
            +:  each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov
            x:  required together in option set
            y:  required together in option set
        """
    # make a copy of the L2 data
    ds3 = copy.deepcopy(ds2)
    # set some attributes for this level
    qcutils.UpdateGlobalAttributes(cf, ds3, "L3")
    # initialise the global attribute to document the functions used
    ds3.globalattributes['Functions'] = ''
    # put the control file name into the global attributes
    ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
    # check to see if we have any imports
    qcgf.ImportSeries(cf, ds3)
    # correct measured soil water content using empirical relationship to collected samples
    qcts.CorrectSWC(cf, ds3)
    # apply linear corrections to the data
    qcck.do_linear(cf, ds3)
    # merge whatever humidities are available
    qcts.MergeHumidities(cf, ds3, convert_units=True)
    # get the air temperature from the CSAT virtual temperature
    qcts.TaFromTv(cf, ds3)
    # merge the HMP and corrected CSAT data
    qcts.MergeSeries(cf, ds3, 'Ta', [0, 10], convert_units=True)
    qcutils.CheckUnits(ds3, "Ta", "C", convert_units=True)
    # calculate humidities (absolute, specific and relative) from whatever is available
    qcts.CalculateHumidities(ds3)
    # merge the 7500 CO2 concentration
    qcts.MergeSeries(cf, ds3, 'Cc', [0, 10], convert_units=True)
    qcutils.CheckUnits(ds3, "Cc", "mg/m3", convert_units=True)
    # add relevant meteorological values to L3 data
    qcts.CalculateMeteorologicalVariables(ds3)
    # check to see if the user wants to use the fluxes in the L2 file
    if not qcutils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False):
        # check the covariancve units and change if necessary
        qcts.CheckCovarianceUnits(ds3)
        # do the 2D coordinate rotation
        qcts.CoordRotation2D(cf, ds3)
        # do the Massman frequency attenuation correction
        qcts.MassmanStandard(cf, ds3)
        # calculate the fluxes
        qcts.CalculateFluxes(cf, ds3)
        # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
        qcts.FhvtoFh(cf, ds3)
        # correct the H2O & CO2 flux due to effects of flux on density measurements
        qcts.Fe_WPL(cf, ds3)
        qcts.Fc_WPL(cf, ds3)
    # convert CO2 units if required
    qcutils.ConvertCO2Units(cf, ds3, Cc='Cc')
    # calculate Fc storage term - single height only at present
    qcts.CalculateFcStorage(cf, ds3)
    # convert Fc and Fc_storage units if required
    qcutils.ConvertFcUnits(cf, ds3, Fc='Fc', Fc_storage='Fc_storage')
    # correct Fc for storage term - only recommended if storage calculated from profile available
    qcts.CorrectFcForStorage(cf, ds3)
    # merge the incoming shortwave radiation
    qcts.MergeSeries(cf, ds3, 'Fsd', [0, 10])
    # calculate the net radiation from the Kipp and Zonen CNR1
    qcts.CalculateNetRadiation(cf,
                               ds3,
                               Fn_out='Fn_KZ',
                               Fsd_in='Fsd',
                               Fsu_in='Fsu',
                               Fld_in='Fld',
                               Flu_in='Flu')
    qcts.MergeSeries(cf, ds3, 'Fn', [0, 10])
    # combine wind speed from the Wind Sentry and  the CSAT
    qcts.MergeSeries(cf, ds3, 'Ws', [0, 10])
    # combine wind direction from the Wind Sentry and  the CSAT
    qcts.MergeSeries(cf, ds3, 'Wd', [0, 10])
    # correct soil heat flux for storage
    #    ... either average the raw ground heat flux, soil temperature and moisture
    #        and then do the correction (OzFlux "standard")
    qcts.AverageSeriesByElements(cf, ds3, 'Ts')
    qcts.AverageSeriesByElements(cf, ds3, 'Sws')
    if qcutils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'):
        #    ... or correct the individual ground heat flux measurements (James' method)
        qcts.CorrectIndividualFgForStorage(cf, ds3)
        qcts.AverageSeriesByElements(cf, ds3, 'Fg')
    else:
        qcts.AverageSeriesByElements(cf, ds3, 'Fg')
        qcts.CorrectFgForStorage(cf,
                                 ds3,
                                 Fg_out='Fg',
                                 Fg_in='Fg',
                                 Ts_in='Ts',
                                 Sws_in='Sws')
    # calculate the available energy
    qcts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg')
    # create new series using MergeSeries or AverageSeries
    qcck.CreateNewSeries(cf, ds3)
    # create a series of daily averaged soil moisture interpolated back to the time step
    #qcts.DailyAverageSws_Interpolated(cf,ds3,Sws_out='Sws_daily',Sws_in='Sws')
    # re-apply the quality control checks (range, diurnal and rules)
    qcck.do_qcchecks(cf, ds3)
    # coordinate gaps in the three main fluxes
    qcck.CoordinateFluxGaps(cf, ds3)
    # coordinate gaps in Ah_7500_Av with Fc
    qcck.CoordinateAh7500AndFcGaps(cf, ds3)
    # get the statistics for the QC flags and write these to an Excel spreadsheet
    qcio.get_seriesstats(cf, ds3)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds3)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds3)

    return ds3
Exemplo n.º 7
0
def l3qc(cf, ds2):
    """
    """
    # make a copy of the L2 data
    ds3 = copy.deepcopy(ds2)
    # set some attributes for this level
    qcutils.UpdateGlobalAttributes(cf, ds3, "L3")
    # put the control file name into the global attributes
    ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
    # check to see if we have any imports
    qcgf.ImportSeries(cf, ds3)
    # apply linear corrections to the data
    qcck.do_linear(cf, ds3)
    # ************************
    # *** Merge humidities ***
    # ************************
    # merge whatever humidities are available
    qcts.MergeHumidities(cf, ds3, convert_units=True)
    # **************************
    # *** Merge temperatures ***
    # **************************
    # get the air temperature from the CSAT virtual temperature
    qcts.TaFromTv(cf, ds3)
    # merge the HMP and corrected CSAT data
    qcts.MergeSeries(cf, ds3, "Ta", convert_units=True)
    qcutils.CheckUnits(ds3, "Ta", "C", convert_units=True)
    # ***************************
    # *** Calcuate humidities ***
    # ***************************
    # calculate humidities (absolute, specific and relative) from whatever is available
    qcts.CalculateHumidities(ds3)
    # ********************************
    # *** Merge CO2 concentrations ***
    # ********************************
    # merge the 7500 CO2 concentration
    # PRI 09/08/2017 possibly the ugliest thing I have done yet
    # This needs to be abstracted to a general alias checking routine at the
    # start of the L3 processing so that possible aliases are mapped to a single
    # set of variable names.
    if "CO2" in cf["Variables"]:
        CO2 = "CO2"
    elif "Cc" in cf["Variables"]:
        CO2 = "Cc"
    else:
        msg = "Label for CO2 ('CO2','Cc') not found in control file"
        logger.error(msg)
        return
    qcts.MergeSeries(cf, ds3, CO2, convert_units=True)
    # ******************************************
    # *** Calculate meteorological variables ***
    # ******************************************
    # Update meteorological variables
    qcts.CalculateMeteorologicalVariables(ds3)
    # *************************************************
    # *** Calculate fluxes from covariances section ***
    # *************************************************
    # check to see if the user wants to use the fluxes in the L2 file
    if not qcutils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False):
        # check the covariance units and change if necessary
        qcts.CheckCovarianceUnits(ds3)
        # do the 2D coordinate rotation
        qcts.CoordRotation2D(cf, ds3)
        # do the Massman frequency attenuation correction
        qcts.MassmanStandard(cf, ds3)
        # calculate the fluxes
        qcts.CalculateFluxes(cf, ds3)
        # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
        qcts.FhvtoFh(cf, ds3)
        # correct the H2O & CO2 flux due to effects of flux on density measurements
        qcts.Fe_WPL(cf, ds3)
        qcts.Fc_WPL(cf, ds3)
    # **************************************
    # *** Calculate Monin-Obukhov length ***
    # **************************************
    qcts.CalculateMoninObukhovLength(ds3)
    # **************************
    # *** CO2 and Fc section ***
    # **************************
    # convert CO2 units if required
    qcutils.ConvertCO2Units(cf, ds3, CO2=CO2)
    # calculate Fc storage term - single height only at present
    qcts.CalculateFcStorageSinglePoint(cf, ds3, Fc_out='Fc_single', CO2_in=CO2)
    # convert Fc and Fc_storage units if required
    qcutils.ConvertFcUnits(cf, ds3)
    # merge Fc and Fc_storage series if required
    merge_list = [
        label for label in cf["Variables"].keys() if label[0:2] == "Fc"
        and "MergeSeries" in cf["Variables"][label].keys()
    ]
    for label in merge_list:
        qcts.MergeSeries(cf, ds3, label, save_originals=True)
    # correct Fc for storage term - only recommended if storage calculated from profile available
    qcts.CorrectFcForStorage(cf, ds3)
    # *************************
    # *** Radiation section ***
    # *************************
    # merge the incoming shortwave radiation
    qcts.MergeSeries(cf, ds3, 'Fsd')
    # calculate the net radiation from the Kipp and Zonen CNR1
    qcts.CalculateNetRadiation(cf,
                               ds3,
                               Fn_out='Fn_KZ',
                               Fsd_in='Fsd',
                               Fsu_in='Fsu',
                               Fld_in='Fld',
                               Flu_in='Flu')
    qcts.MergeSeries(cf, ds3, 'Fn')
    # ****************************************
    # *** Wind speed and direction section ***
    # ****************************************
    # combine wind speed from the Wind Sentry and the SONIC
    qcts.MergeSeries(cf, ds3, 'Ws')
    # combine wind direction from the Wind Sentry and the SONIC
    qcts.MergeSeries(cf, ds3, 'Wd')
    # ********************
    # *** Soil section ***
    # ********************
    # correct soil heat flux for storage
    #    ... either average the raw ground heat flux, soil temperature and moisture
    #        and then do the correction (OzFlux "standard")
    qcts.AverageSeriesByElements(cf, ds3, 'Ts')
    qcts.AverageSeriesByElements(cf, ds3, 'Sws')
    if qcutils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'):
        #    ... or correct the individual ground heat flux measurements (James' method)
        qcts.CorrectIndividualFgForStorage(cf, ds3)
        qcts.AverageSeriesByElements(cf, ds3, 'Fg')
    else:
        qcts.AverageSeriesByElements(cf, ds3, 'Fg')
        qcts.CorrectFgForStorage(cf,
                                 ds3,
                                 Fg_out='Fg',
                                 Fg_in='Fg',
                                 Ts_in='Ts',
                                 Sws_in='Sws')
    # calculate the available energy
    qcts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg')
    # create new series using MergeSeries or AverageSeries
    qcck.CreateNewSeries(cf, ds3)
    # re-apply the quality control checks (range, diurnal and rules)
    qcck.do_qcchecks(cf, ds3)
    # coordinate gaps in the three main fluxes
    qcck.CoordinateFluxGaps(cf, ds3)
    # coordinate gaps in Ah_7500_Av with Fc
    qcck.CoordinateAh7500AndFcGaps(cf, ds3)
    # check missing data and QC flags are consistent
    qcutils.CheckQCFlags(ds3)
    # get the statistics for the QC flags and write these to an Excel spreadsheet
    qcio.get_seriesstats(cf, ds3)
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds3)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds3)

    return ds3
Exemplo n.º 8
0
def l3qc(cf, ds2):
    """
        Corrections
        Generates L3 from L2 data
        
        Functions performed:
            qcts.AddMetVars (optional)
            qcts.CorrectSWC (optional*)
            qcck.do_linear (all sites)
            qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x
            qcts.TaFromTv (optional)
            qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x
            qcts.CoordRotation2D (all sites)
            qcts.MassmanApprox (optional*)y
            qcts.Massman (optional*)y
            qcts.CalculateFluxes (used if Massman not optioned)x
            qcts.CalculateFluxesRM (used if Massman optioned)y
            qcts.FhvtoFh (all sites)
            qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.CalculateNetRadiation (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fsd (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fn (optional*)
            qcts.InterpolateOverMissing (optional)
            AverageSeriesByElements (optional)
            qcts.CorrectFgForStorage (all sites)
            qcts.Average3SeriesByElements (optional)
            qcts.CalculateAvailableEnergy (optional)
            qcck.do_qcchecks (all sites)
            qcck.gaps (optional)
            
            *:  requires ancillary measurements for paratmerisation
            +:  each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov
            x:  required together in option set
            y:  required together in option set
        """
    # make a copy of the L2 data
    ds3 = copy.deepcopy(ds2)
    ds3.globalattributes['nc_level'] = 'L3'
    ds3.globalattributes['EPDversion'] = sys.version
    ds3.globalattributes['QC_version_history'] = cfg.__doc__
    # put the control file name into the global attributes
    ds3.globalattributes['controlfile_name'] = cf['controlfile_name']

    # calculate NDVI
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='NDVI') and cf['Functions']['NDVI'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', calculateNDVI'
        except:
            ds3.globalattributes['L3Functions'] = 'calculateNDVI'

        log.info(' Calculating NDVI from component reflectances ...')
        qcts.CalculateNDVI(cf, ds3)

    # bypass soil temperature correction for Sws (when Ts bad)
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='BypassSwsTcorr'
                          ) and cf['Functions']['BypassSwsTcorr'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', BypassSwsTcorr'
        except:
            ds3.globalattributes['L3Functions'] = 'BypassSwsTcorr'

        log.info(' Re-computing Sws without temperature correction ...')
        qcts.BypassTcorr(cf, ds3)

    # correct measured soil water content using empirical relationship to collected samples
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='CorrectSWC') and cf['Functions']['CorrectSWC'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CorrectSWC'
        except:
            ds3.globalattributes['L3Functions'] = 'CorrectSWC'

        log.info(' Correcting soil moisture data ...')
        qcts.CorrectSWC(cf, ds3)

    # apply linear corrections to the data
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', do_linear'
        except:
            ds3.globalattributes['L3Functions'] = 'do_linear'

        log.info(' Applying linear corrections ...')
        qcck.do_linear(cf, ds3)

    # determine HMP Ah if not output by datalogger
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='CalculateAh'
                          ) and cf['Functions']['CalculateAh'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CalculateAh'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateAh'

        log.info(' Adding HMP Ah to database')
        qcts.CalculateAhHMP(cf, ds3)

    # merge the HMP and corrected 7500 data
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='MergeSeriesAhTa'
                          ) and cf['Functions']['MergeSeriesAhTa'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', MergeSeriesAhTaCc'
        except:
            ds3.globalattributes['L3Functions'] = 'MergeSeriesAhTaCc'

        qcts.MergeSeries(cf, ds3, 'Ah', [0, 10])
        qcts.MergeSeries(cf, ds3, 'Cc', [0, 10])

        # get the air temperature from the CSAT virtual temperature
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', TaFromTv'
        except:
            ds3.globalattributes['L3Functions'] = 'TaFromTv'

        qcts.TaFromTv(cf, ds3)

        # merge the HMP and corrected CSAT data
        qcts.MergeSeries(cf, ds3, 'Ta', [0, 10])

    # add relevant meteorological values to L3 data
    if (qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections')
            and cf['Functions']['Corrections']
            == 'True') or (qcutils.cfkeycheck(
                cf, Base='Functions', ThisOne='CalculateMetVars')
                           and cf['Functions']['CalculateMetVars'] == 'True'):
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CalculateMetVars'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateMetVars'

        log.info(' Adding standard met variables to database')
        qcts.CalculateMeteorologicalVariables(ds3)

    # do the 2D coordinate rotation
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CoordRotation2D'
        except:
            ds3.globalattributes['L3Functions'] = 'CoordRotation2D'

        qcts.CoordRotation2D(cf, ds3)

    # do the Massman frequency attenuation correction
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', Massman'
        except:
            ds3.globalattributes['L3Functions'] = 'Massman'

        qcts.MassmanStandard(cf, ds3)

    # calculate the fluxes
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CalculateFluxes'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateFluxes'

        qcts.CalculateFluxes(cf, ds3)

    # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', FhvtoFh'
        except:
            ds3.globalattributes['L3Functions'] = 'FhvtoFh'

        qcts.FhvtoFh(cf, ds3)

    # correct the H2O & CO2 flux due to effects of flux on density measurements
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        if qcutils.cfkeycheck(
                cf, Base='Functions',
                ThisOne='WPLcov') and cf['Functions']['WPLcov'] == 'True':
            try:
                ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                    'L3Functions'] + ', WPLcov'
            except:
                ds3.globalattributes['L3Functions'] = 'WPLcov'

            qcts.do_WPL(cf, ds3, cov='True')
        else:
            try:
                ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                    'L3Functions'] + ', WPL'
            except:
                ds3.globalattributes['L3Functions'] = 'WPL'

            qcts.do_WPL(cf, ds3)

    # calculate the net radiation from the Kipp and Zonen CNR1
    if qcutils.cfkeycheck(
            cf, Base='Functions', ThisOne='CalculateNetRadiation'
    ) and cf['Functions']['CalculateNetRadiation'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CalculateNetRadiation'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateNetRadiation'

        qcts.MergeSeries(cf, ds3, 'Fsd', [0, 10])
        qcts.CalculateNetRadiation(ds3, 'Fn_KZ', 'Fsd', 'Fsu', 'Fld', 'Flu')
        qcts.MergeSeries(cf, ds3, 'Fn', [0, 10])

    # combine wind speed from the CSAT and the Wind Sentry
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='MergeSeriesWS'
                          ) and cf['Functions']['MergeSeriesWS'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', MergeSeriesWS'
        except:
            ds3.globalattributes['L3Functions'] = 'MergeSeriesWS'

        qcts.MergeSeries(cf, ds3, 'Ws', [0, 10])

    # average the soil temperature data
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        if 'SoilAverage' not in ds3.globalattributes['L3Functions']:
            try:
                ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                    'L3Functions'] + ', SoilAverage'
            except:
                ds3.globalattributes['L3Functions'] = 'SoilAverage'

        # interpolate over any ramaining gaps up to 3 hours in length
        qcts.AverageSeriesByElementsI(cf, ds3, 'Ts')
        qcts.AverageSeriesByElementsI(cf, ds3, 'Sws')

    # correct the measured soil heat flux for storage in the soil layer above the sensor
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CorrectFgForStorage'
        except:
            ds3.globalattributes['L3Functions'] = 'CorrectFgForStorage'

        if qcutils.cfkeycheck(
                cf, Base='Functions', ThisOne='IndividualFgCorrection'
        ) and cf['Functions']['IndividualFgCorrection'] == 'True':
            qcts.CorrectIndividualFgForStorage(cf, ds3)
            qcts.AverageSeriesByElementsI(cf, ds3, 'Fg')
        else:
            qcts.AverageSeriesByElementsI(cf, ds3, 'Fg')
            qcts.CorrectGroupFgForStorage(cf, ds3)

    # calculate the available energy
    if qcutils.cfkeycheck(
            cf, Base='Functions', ThisOne='CalculateAvailableEnergy'
    ) and cf['Functions']['CalculateAvailableEnergy'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CalculateAvailableEnergy'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateAvailableEnergy'

        qcts.CalculateAvailableEnergy(ds3)

    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='DiagnosticMode'):
        if cf['Functions']['DiagnosticMode'] == 'False':
            qcutils.prepOzFluxVars(cf, ds3)
    else:
        qcutils.prepOzFluxVars(cf, ds3)

    # calculate specific humidity and saturated specific humidity profile
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='qTprofile') and cf['Functions']['qTprofile'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', qTprofile'
        except:
            ds3.globalattributes['L3Functions'] = 'qTprofile'

        qcts.CalculateSpecificHumidityProfile(cf, ds3)

    # calculate Penman-Monteith inversion
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='PenmanMonteith'
                          ) and cf['Functions']['PenmanMonteith'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', PenmanMonteith'
        except:
            ds3.globalattributes['L3Functions'] = 'PenmanMonteith'

        qcts.do_PenmanMonteith(cf, ds3)

    # calculate bulk Richardson numbers
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='bulkRichardson'
                          ) and cf['Functions']['bulkRichardson'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', bulkRichardson'
        except:
            ds3.globalattributes['L3Functions'] = 'bulkRichardson'

        qcts.do_bulkRichardson(cf, ds3)

    # re-apply the quality control checks (range, diurnal and rules)
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        ds3.globalattributes['L3Functions'] = ds3.globalattributes[
            'L3Functions'] + ', do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours)'
        qcck.do_qcchecks(cf, ds3)

    # quality control checks (range, diurnal and rules) without flux post-processing
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='QCChecks') and cf['Functions']['QCChecks'] == 'True':
        qcck.do_qcchecks(cf, ds3)

    # apply the ustar filter
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='ustarFilter'
                          ) and cf['Functions']['ustarFilter'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', ustarFilter'
        except:
            ds3.globalattributes['L3Functions'] = 'ustarFilter'

        qcts.FilterFcByUstar(cf, ds3)

    # coordinate gaps in the three main fluxes
    if qcutils.cfkeycheck(
            cf, Base='Functions', ThisOne='CoordinateFluxGaps'
    ) and cf['Functions']['CoordinateFluxGaps'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CoordinateFluxGaps'
        except:
            ds3.globalattributes['L3Functions'] = 'CoordinateFluxGaps'

        qcck.CoordinateFluxGaps(cf, ds3)

    # coordinate gaps in Ah_7500_Av with Fc
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CoordinateAh7500AndFcGaps'
        except:
            ds3.globalattributes['L3Functions'] = 'CoordinateAh7500AndFcGaps'

        qcck.CoordinateAh7500AndFcGaps(cf, ds3)

    # calcluate ET at observation interval
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='CalculateET'
                          ) and cf['Functions']['CalculateET'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', CalculateET'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateET'

        log.info(' Calculating ET')
        qcts.CalculateET(cf, ds3, 'L3')

    # run MOST (Buckingham Pi) 2d footprint model (Kljun et al. 2004)
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='footprint') and cf['Functions']['footprint'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', footprint'
        except:
            ds3.globalattributes['L3Functions'] = 'footprint'

        qcts.do_footprint_2d(cf, ds3)

    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='Corrections'
                          ) and cf['Functions']['Corrections'] == 'True':
        qcio.get_seriesstats(cf, ds3)

    # convert Fc [mgCO2 m-2 s-1] to Fc_co2 [mgCO2 m-2 s-1], Fc_c [mgC m-2 s-1], NEE [umol m-2 s-1] and NEP = - NEE
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='convertFc') and cf['Functions']['convertFc'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', convertFc'
        except:
            ds3.globalattributes['L3Functions'] = 'convertFc'

        qcts.ConvertFc(cf, ds3)

    # convert Fc [mgCO2 m-2 s-1] to Fc [umol m-2 s-1]
    if qcutils.cfkeycheck(
            cf, Base='Functions',
            ThisOne='JasonFc') and cf['Functions']['JasonFc'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', convertFc (umol only)'
        except:
            ds3.globalattributes['L3Functions'] = 'convertFc (umol only)'

        qcts.ConvertFcJason(cf, ds3)

    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds3)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds3)

    # compute water-use efficiency from flux-gradient similarity (appendix A, Scanlon & Sahu 2008)
    if qcutils.cfkeycheck(cf, Base='Functions',
                          ThisOne='wue') and cf['Functions']['wue'] == 'True':
        try:
            ds3.globalattributes[
                'L3Functions'] = ds3.globalattributes['L3Functions'] + ', wue'
        except:
            ds3.globalattributes['L3Functions'] = 'wue'

        log.info(
            ' Calculating water-use efficiency from flux-gradient similarity')
        qcts.CalculateWUEfromSimilarity(cf, ds3)

    # compute climatology for L3 data
    if qcutils.cfkeycheck(cf, Base='Functions', ThisOne='climatology'
                          ) and cf['Functions']['climatology'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes[
                'L3Functions'] + ', climatology'
        except:
            ds3.globalattributes['L3Functions'] = 'climatology'

        qcts.do_climatology(cf, ds3)

    if qcutils.cfkeycheck(cf, Base='Functions',
                          ThisOne='Sums') and cf['Functions']['Sums'] == 'L3':
        try:
            ds3.globalattributes[
                'L3Functions'] = ds3.globalattributes['L5Functions'] + ', Sums'
        except:
            ds3.globalattributes['L3Functions'] = 'Sums'

        qcts.do_sums(cf, ds3)

    try:
        ds3.globalattributes['Functions'] = ds3.globalattributes[
            'Functions'] + ', ' + ds3.globalattributes['L3Functions']
    except:
        ds3.globalattributes['Functions'] = ds3.globalattributes['L3Functions']

    return ds3
Exemplo n.º 9
0
def l3qc(cf,ds2):
    """
        Corrections
        Generates L3 from L2 data
        
        Functions performed:
            qcts.AddMetVars (optional)
            qcts.CorrectSWC (optional*)
            qcck.do_linear (all sites)
            qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x
            qcts.TaFromTv (optional)
            qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x
            qcts.CoordRotation2D (all sites)
            qcts.MassmanApprox (optional*)y
            qcts.Massman (optional*)y
            qcts.CalculateFluxes (used if Massman not optioned)x
            qcts.CalculateFluxesRM (used if Massman optioned)y
            qcts.FhvtoFh (all sites)
            qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x
            qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y
            qcts.CalculateNetRadiation (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fsd (optional)
            qcutils.GetMergeList + qcts.MergeSeries Fn (optional*)
            qcts.InterpolateOverMissing (optional)
            AverageSeriesByElements (optional)
            qcts.CorrectFgForStorage (all sites)
            qcts.Average3SeriesByElements (optional)
            qcts.CalculateAvailableEnergy (optional)
            qcck.do_qcchecks (all sites)
            qcck.gaps (optional)
            
            *:  requires ancillary measurements for paratmerisation
            +:  each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov
            x:  required together in option set
            y:  required together in option set
        """
    # make a copy of the L2 data
    ds3 = copy.deepcopy(ds2)
    ds3.globalattributes['nc_level'] = 'L3'
    ds3.globalattributes['EPDversion'] = sys.version
    ds3.globalattributes['QC_version_history'] = cfg.__doc__
    # put the control file name into the global attributes
    ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
    
    # calculate NDVI
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='NDVI') and cf['Functions']['NDVI'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', calculateNDVI'
        except:
            ds3.globalattributes['L3Functions'] = 'calculateNDVI'
        
        log.info(' Calculating NDVI from component reflectances ...')
        qcts.CalculateNDVI(cf,ds3)
    
    # bypass soil temperature correction for Sws (when Ts bad)
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='BypassSwsTcorr') and cf['Functions']['BypassSwsTcorr'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', BypassSwsTcorr'
        except:
            ds3.globalattributes['L3Functions'] = 'BypassSwsTcorr'
        
        log.info(' Re-computing Sws without temperature correction ...')
        qcts.BypassTcorr(cf,ds3)
    
    # correct measured soil water content using empirical relationship to collected samples
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CorrectSWC') and cf['Functions']['CorrectSWC'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CorrectSWC'
        except:
            ds3.globalattributes['L3Functions'] = 'CorrectSWC'
        
        log.info(' Correcting soil moisture data ...')
        qcts.CorrectSWC(cf,ds3)
    
    # apply linear corrections to the data
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', do_linear'
        except:
            ds3.globalattributes['L3Functions'] = 'do_linear'
        
        log.info(' Applying linear corrections ...')
        qcck.do_linear(cf,ds3)
    
    # determine HMP Ah if not output by datalogger
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CalculateAh') and cf['Functions']['CalculateAh'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CalculateAh'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateAh'
        
        log.info(' Adding HMP Ah to database')
        qcts.CalculateAhHMP(cf,ds3)
    
    # merge the HMP and corrected 7500 data
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='MergeSeriesAhTa') and cf['Functions']['MergeSeriesAhTa'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', MergeSeriesAhTaCc'
        except:
            ds3.globalattributes['L3Functions'] = 'MergeSeriesAhTaCc'
        
        qcts.MergeSeries(cf,ds3,'Ah',[0,10])
        qcts.MergeSeries(cf,ds3,'Cc',[0,10])
        
    # get the air temperature from the CSAT virtual temperature
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', TaFromTv'
        except:
            ds3.globalattributes['L3Functions'] = 'TaFromTv'
        
        qcts.TaFromTv(cf,ds3)
        
    # merge the HMP and corrected CSAT data
        qcts.MergeSeries(cf,ds3,'Ta',[0,10])
    
    # add relevant meteorological values to L3 data
    if (qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True') or (qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CalculateMetVars') and cf['Functions']['CalculateMetVars'] == 'True'):
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CalculateMetVars'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateMetVars'
            
        log.info(' Adding standard met variables to database')
        qcts.CalculateMeteorologicalVariables(ds3)
    
    # do the 2D coordinate rotation
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CoordRotation2D'
        except:
            ds3.globalattributes['L3Functions'] = 'CoordRotation2D'
        
        qcts.CoordRotation2D(cf,ds3)
    
    # do the Massman frequency attenuation correction
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', Massman'
        except:
            ds3.globalattributes['L3Functions'] = 'Massman'
        
        qcts.MassmanStandard(cf,ds3)
    
    # calculate the fluxes
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CalculateFluxes'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateFluxes'
        
        qcts.CalculateFluxes(cf,ds3)
    
    # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', FhvtoFh'
        except:
            ds3.globalattributes['L3Functions'] = 'FhvtoFh'
        
        qcts.FhvtoFh(cf,ds3)
    
    # correct the H2O & CO2 flux due to effects of flux on density measurements
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='WPLcov') and cf['Functions']['WPLcov'] == 'True':
            try:
                ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', WPLcov'
            except:
                ds3.globalattributes['L3Functions'] = 'WPLcov'
            
            qcts.do_WPL(cf,ds3,cov='True')
        else:
            try:
                ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', WPL'
            except:
                ds3.globalattributes['L3Functions'] = 'WPL'
            
            qcts.do_WPL(cf,ds3)
    
    # calculate the net radiation from the Kipp and Zonen CNR1
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CalculateNetRadiation') and cf['Functions']['CalculateNetRadiation'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CalculateNetRadiation'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateNetRadiation'
        
        qcts.MergeSeries(cf,ds3,'Fsd',[0,10])
        qcts.CalculateNetRadiation(ds3,'Fn_KZ','Fsd','Fsu','Fld','Flu')
        qcts.MergeSeries(cf,ds3,'Fn',[0,10])
    
    # combine wind speed from the CSAT and the Wind Sentry
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='MergeSeriesWS') and cf['Functions']['MergeSeriesWS'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', MergeSeriesWS'
        except:
            ds3.globalattributes['L3Functions'] = 'MergeSeriesWS'
        
        qcts.MergeSeries(cf,ds3,'Ws',[0,10])
    
    # average the soil temperature data
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        if 'SoilAverage' not in ds3.globalattributes['L3Functions']:
            try:
                ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', SoilAverage'
            except:
                ds3.globalattributes['L3Functions'] = 'SoilAverage'
            
        # interpolate over any ramaining gaps up to 3 hours in length
        qcts.AverageSeriesByElementsI(cf,ds3,'Ts')
        qcts.AverageSeriesByElementsI(cf,ds3,'Sws')
        
    # correct the measured soil heat flux for storage in the soil layer above the sensor
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CorrectFgForStorage'
        except:
            ds3.globalattributes['L3Functions'] = 'CorrectFgForStorage'
        
        if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='IndividualFgCorrection') and cf['Functions']['IndividualFgCorrection'] == 'True':
            qcts.CorrectIndividualFgForStorage(cf,ds3)
            qcts.AverageSeriesByElementsI(cf,ds3,'Fg')
        else:
            qcts.AverageSeriesByElementsI(cf,ds3,'Fg')
            qcts.CorrectGroupFgForStorage(cf,ds3)
    
    # calculate the available energy
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CalculateAvailableEnergy') and cf['Functions']['CalculateAvailableEnergy'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CalculateAvailableEnergy'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateAvailableEnergy'
        
        qcts.CalculateAvailableEnergy(ds3)
    
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='DiagnosticMode'):
        if cf['Functions']['DiagnosticMode'] == 'False':
            qcutils.prepOzFluxVars(cf,ds3)
    else:
        qcutils.prepOzFluxVars(cf,ds3)
    
    # calculate specific humidity and saturated specific humidity profile
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='qTprofile') and cf['Functions']['qTprofile'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', qTprofile'
        except:
            ds3.globalattributes['L3Functions'] = 'qTprofile'
        
        qcts.CalculateSpecificHumidityProfile(cf,ds3)
    
    # calculate Penman-Monteith inversion
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='PenmanMonteith') and cf['Functions']['PenmanMonteith'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', PenmanMonteith'
        except:
            ds3.globalattributes['L3Functions'] = 'PenmanMonteith'
        
        qcts.do_PenmanMonteith(cf,ds3)
    
    # calculate bulk Richardson numbers
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='bulkRichardson') and cf['Functions']['bulkRichardson'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', bulkRichardson'
        except:
            ds3.globalattributes['L3Functions'] = 'bulkRichardson'
        
        qcts.do_bulkRichardson(cf,ds3)
    
    # re-apply the quality control checks (range, diurnal and rules)
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', do_qccheck(RangeCheck, diurnalcheck, excludedates, excludehours)'
        qcck.do_qcchecks(cf,ds3)
    
    # apply the ustar filter
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='ustarFilter') and cf['Functions']['ustarFilter'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', ustarFilter'
        except:
            ds3.globalattributes['L3Functions'] = 'ustarFilter'
        
        qcts.FilterFcByUstar(cf,ds3)
    
    # coordinate gaps in the three main fluxes
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CoordinateFluxGaps') and cf['Functions']['CoordinateFluxGaps'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CoordinateFluxGaps'
        except:
            ds3.globalattributes['L3Functions'] = 'CoordinateFluxGaps'
        
        qcck.CoordinateFluxGaps(cf,ds3)
    
    # coordinate gaps in Ah_7500_Av with Fc
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CoordinateAh7500AndFcGaps'
        except:
            ds3.globalattributes['L3Functions'] = 'CoordinateAh7500AndFcGaps'
        
        qcck.CoordinateAh7500AndFcGaps(cf,ds3)
    
    # calcluate ET at observation interval
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='CalculateET') and cf['Functions']['CalculateET'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', CalculateET'
        except:
            ds3.globalattributes['L3Functions'] = 'CalculateET'
        
        log.info(' Calculating ET')
        qcts.CalculateET(cf,ds3,'L3')
    
    # run MOST (Buckingham Pi) 2d footprint model (Kljun et al. 2004)
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='footprint') and cf['Functions']['footprint'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', footprint'
        except:
            ds3.globalattributes['L3Functions'] = 'footprint'
        
        qcts.do_footprint_2d(cf,ds3)
    
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Corrections') and cf['Functions']['Corrections'] == 'True':
        qcio.get_seriesstats(cf,ds3)
    
    # convert Fc [mgCO2 m-2 s-1] to Fc_co2 [mgCO2 m-2 s-1], Fc_c [mgC m-2 s-1], NEE [umol m-2 s-1] and NEP = - NEE
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='convertFc') and cf['Functions']['convertFc'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', convertFc'
        except:
            ds3.globalattributes['L3Functions'] = 'convertFc'
        
        qcts.ConvertFc(cf,ds3)
    
    # convert Fc [mgCO2 m-2 s-1] to Fc [umol m-2 s-1]
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='JasonFc') and cf['Functions']['JasonFc'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', convertFc (umol only)'
        except:
            ds3.globalattributes['L3Functions'] = 'convertFc (umol only)'
        
        qcts.ConvertFcJason(cf,ds3)
    
    # write the percentage of good data as a variable attribute
    qcutils.get_coverage_individual(ds3)
    # write the percentage of good data for groups
    qcutils.get_coverage_groups(ds3)
    
    # compute climatology for L3 data
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='climatology') and cf['Functions']['climatology'] == 'True':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L3Functions']+', climatology'
        except:
            ds3.globalattributes['L3Functions'] = 'climatology'
        
        qcts.do_climatology(cf,ds3)
    
    if qcutils.cfkeycheck(cf,Base='Functions',ThisOne='Sums') and cf['Functions']['Sums'] == 'L3':
        try:
            ds3.globalattributes['L3Functions'] = ds3.globalattributes['L5Functions']+', Sums'
        except:
            ds3.globalattributes['L3Functions'] = 'Sums'
        
        qcts.do_sums(cf,ds3)
    
    try:
        ds3.globalattributes['Functions'] = ds3.globalattributes['Functions'] + ', ' + ds3.globalattributes['L3Functions']
    except:
        ds3.globalattributes['Functions'] = ds3.globalattributes['L3Functions']
    
    return ds3