def CoordinateAh7500AndFcGaps(cf,ds,Fcvar='Fc'): '''Cleans up Ah_7500_Av based upon Fc gaps to for QA check on Ah_7500_Av v Ah_HMP.''' if not qcutils.cfoptionskeylogical(cf,Key='CoordinateAh7500&FcGaps'): return log.info(' Doing the Ah_7500 check') if qcutils.cfkeycheck(cf,Base='FunctionArgs',ThisOne='AhcheckFc'): Fclist = ast.literal_eval(cf['FunctionArgs']['AhcheckFc']) Fcvar = Fclist[0] # index1 Index of bad Ah_7500_Av observations index1 = numpy.where((ds.series['Ah_7500_Av']['Flag']!=0) & (ds.series['Ah_7500_Av']['Flag']!=10)) # index2 Index of bad Fc observations index2 = numpy.where((ds.series[Fcvar]['Flag']!=0) & (ds.series[Fcvar]['Flag']!=10)) ds.series['Ah_7500_Av']['Data'][index2] = numpy.float64(c.missing_value) ds.series['Ah_7500_Av']['Flag'][index2] = ds.series[Fcvar]['Flag'][index2] ds.series['Ah_7500_Av']['Flag'][index1] = ds.series['Ah_7500_Av']['Flag'][index1] if 'CoordinateAh7500AndFcGaps' not in ds.globalattributes['Functions']: ds.globalattributes['Functions'] = ds.globalattributes['Functions']+',CoordinateAh7500AndFcGaps'
def CoordinateFluxGaps(cf,ds,Fc_in='Fc',Fe_in='Fe',Fh_in='Fh'): if not qcutils.cfoptionskeylogical(cf,Key='CoordinateFluxGaps'): return if qcutils.cfkeycheck(cf,Base='FunctionArgs',ThisOne='gapsvars'): vars = ast.literal_eval(cf['FunctionArgs']['gapsvars']) Fc_in = vars[0] Fe_in = vars[1] Fh_in = vars[2] Fc,f,a = qcutils.GetSeriesasMA(ds,Fc_in) Fe,f,a = qcutils.GetSeriesasMA(ds,Fe_in) Fh,f,a = qcutils.GetSeriesasMA(ds,Fh_in) # April 2015 PRI - changed numpy.ma.where to numpy.where index = numpy.where((numpy.ma.getmaskarray(Fc)==True)| (numpy.ma.getmaskarray(Fe)==True)| (numpy.ma.getmaskarray(Fh)==True))[0] #index = numpy.ma.where((numpy.ma.getmaskarray(Fc)==True)| #(numpy.ma.getmaskarray(Fe)==True)| #(numpy.ma.getmaskarray(Fh)==True))[0] # the following for ... in loop is not necessary for i in range(len(index)): j = index[i] if Fc.mask[j]==False: Fc.mask[j]=True Fc[j] = numpy.float64(c.missing_value) ds.series[Fc_in]['Flag'][j] = numpy.int32(19) if Fe.mask[j]==False: Fe.mask[j]=True Fe[j] = numpy.float64(c.missing_value) ds.series[Fe_in]['Flag'][j] = numpy.int32(19) if Fh.mask[j]==False: Fh.mask[j]=True Fh[j] = numpy.float64(c.missing_value) ds.series[Fh_in]['Flag'][j] = numpy.int32(19) ds.series[Fc_in]['Data']=numpy.ma.filled(Fc,float(c.missing_value)) ds.series[Fe_in]['Data']=numpy.ma.filled(Fe,float(c.missing_value)) ds.series[Fh_in]['Data']=numpy.ma.filled(Fh,float(c.missing_value)) log.info(' Finished gap co-ordination')
def l3qc(cf,ds2): """ Corrections Generates L3 from L2 data Functions performed: qcts.AddMetVars (optional) qcts.CorrectSWC (optional*) qcck.do_linear (all sites) qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x qcts.TaFromTv (optional) qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x qcts.CoordRotation2D (all sites) qcts.MassmanApprox (optional*)y qcts.Massman (optional*)y qcts.CalculateFluxes (used if Massman not optioned)x qcts.CalculateFluxesRM (used if Massman optioned)y qcts.FhvtoFh (all sites) qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y qcts.CalculateNetRadiation (optional) qcutils.GetMergeList + qcts.MergeSeries Fsd (optional) qcutils.GetMergeList + qcts.MergeSeries Fn (optional*) qcts.InterpolateOverMissing (optional) AverageSeriesByElements (optional) qcts.CorrectFgForStorage (all sites) qcts.Average3SeriesByElements (optional) qcts.CalculateAvailableEnergy (optional) qcck.do_qcchecks (all sites) qcck.gaps (optional) *: requires ancillary measurements for paratmerisation +: each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov x: required together in option set y: required together in option set """ # make a copy of the L2 data ds3 = copy.deepcopy(ds2) # set some attributes for this level qcutils.UpdateGlobalAttributes(cf,ds3,"L3") # initialise the global attribute to document the functions used ds3.globalattributes['Functions'] = '' # put the control file name into the global attributes ds3.globalattributes['controlfile_name'] = cf['controlfile_name'] # check to see if we have any imports qcgf.ImportSeries(cf,ds3) # correct measured soil water content using empirical relationship to collected samples qcts.CorrectSWC(cf,ds3) # apply linear corrections to the data qcck.do_linear(cf,ds3) # merge whatever humidities are available qcts.MergeHumidities(cf,ds3,convert_units=True) # get the air temperature from the CSAT virtual temperature qcts.TaFromTv(cf,ds3) # merge the HMP and corrected CSAT data qcts.MergeSeries(cf,ds3,'Ta',[0,10],convert_units=True) qcutils.CheckUnits(ds3,"Ta","C",convert_units=True) # calculate humidities (absolute, specific and relative) from whatever is available qcts.CalculateHumidities(ds3) # merge the 7500 CO2 concentration qcts.MergeSeries(cf,ds3,'Cc',[0,10],convert_units=True) # PRI - disable CO2 units conversion from whatever to mg/m3 # - this step is, as far as I can see, redundant, see qcts.Fc_WPL() #qcutils.CheckUnits(ds3,"Cc","mg/m3",convert_units=True) # add relevant meteorological values to L3 data qcts.CalculateMeteorologicalVariables(ds3) # check to see if the user wants to use the fluxes in the L2 file if not qcutils.cfoptionskeylogical(cf,Key="UseL2Fluxes",default=False): # check the covariancve units and change if necessary qcts.CheckCovarianceUnits(ds3) # do the 2D coordinate rotation qcts.CoordRotation2D(cf,ds3) # do the Massman frequency attenuation correction qcts.MassmanStandard(cf,ds3) # calculate the fluxes qcts.CalculateFluxes(cf,ds3) # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual) qcts.FhvtoFh(cf,ds3) # correct the H2O & CO2 flux due to effects of flux on density measurements qcts.Fe_WPL(cf,ds3) qcts.Fc_WPL(cf,ds3) # convert CO2 units if required qcutils.ConvertCO2Units(cf,ds3,Cc='Cc') # calculate Fc storage term - single height only at present qcts.CalculateFcStorage(cf,ds3) # convert Fc and Fc_storage units if required qcutils.ConvertFcUnits(cf,ds3,Fc='Fc',Fc_storage='Fc_storage') # correct Fc for storage term - only recommended if storage calculated from profile available qcts.CorrectFcForStorage(cf,ds3) # merge the incoming shortwave radiation qcts.MergeSeries(cf,ds3,'Fsd',[0,10]) # calculate the net radiation from the Kipp and Zonen CNR1 qcts.CalculateNetRadiation(cf,ds3,Fn_out='Fn_KZ',Fsd_in='Fsd',Fsu_in='Fsu',Fld_in='Fld',Flu_in='Flu') qcts.MergeSeries(cf,ds3,'Fn',[0,10]) # combine wind speed from the Wind Sentry and the CSAT qcts.MergeSeries(cf,ds3,'Ws',[0,10]) # combine wind direction from the Wind Sentry and the CSAT qcts.MergeSeries(cf,ds3,'Wd',[0,10]) # correct soil heat flux for storage # ... either average the raw ground heat flux, soil temperature and moisture # and then do the correction (OzFlux "standard") qcts.AverageSeriesByElements(cf,ds3,'Ts') qcts.AverageSeriesByElements(cf,ds3,'Sws') if qcutils.cfoptionskeylogical(cf,Key='CorrectIndividualFg'): # ... or correct the individual ground heat flux measurements (James' method) qcts.CorrectIndividualFgForStorage(cf,ds3) qcts.AverageSeriesByElements(cf,ds3,'Fg') else: qcts.AverageSeriesByElements(cf,ds3,'Fg') qcts.CorrectFgForStorage(cf,ds3,Fg_out='Fg',Fg_in='Fg',Ts_in='Ts',Sws_in='Sws') # calculate the available energy qcts.CalculateAvailableEnergy(ds3,Fa_out='Fa',Fn_in='Fn',Fg_in='Fg') # create new series using MergeSeries or AverageSeries qcck.CreateNewSeries(cf,ds3) # create a series of daily averaged soil moisture interpolated back to the time step #qcts.DailyAverageSws_Interpolated(cf,ds3,Sws_out='Sws_daily',Sws_in='Sws') # re-apply the quality control checks (range, diurnal and rules) qcck.do_qcchecks(cf,ds3) # coordinate gaps in the three main fluxes qcck.CoordinateFluxGaps(cf,ds3) # coordinate gaps in Ah_7500_Av with Fc qcck.CoordinateAh7500AndFcGaps(cf,ds3) # get the statistics for the QC flags and write these to an Excel spreadsheet qcio.get_seriesstats(cf,ds3) # write the percentage of good data as a variable attribute qcutils.get_coverage_individual(ds3) # write the percentage of good data for groups qcutils.get_coverage_groups(ds3) return ds3
def l3qc(cf, ds2): """ Corrections Generates L3 from L2 data Functions performed: qcts.AddMetVars (optional) qcts.CorrectSWC (optional*) qcck.do_linear (all sites) qcutils.GetMergeList + qcts.MergeSeries Ah_EC (optional)x qcts.TaFromTv (optional) qcutils.GetMergeList + qcts.MergeSeries Ta_EC (optional)x qcts.CoordRotation2D (all sites) qcts.MassmanApprox (optional*)y qcts.Massman (optional*)y qcts.CalculateFluxes (used if Massman not optioned)x qcts.CalculateFluxesRM (used if Massman optioned)y qcts.FhvtoFh (all sites) qcts.Fe_WPL (WPL computed on fluxes, as with Campbell algorithm)+x qcts.Fc_WPL (WPL computed on fluxes, as with Campbell algorithm)+x qcts.Fe_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y qcts.Fc_WPLcov (WPL computed on kinematic fluxes (ie, covariances), as with WPL80)+y qcts.CalculateNetRadiation (optional) qcutils.GetMergeList + qcts.MergeSeries Fsd (optional) qcutils.GetMergeList + qcts.MergeSeries Fn (optional*) qcts.InterpolateOverMissing (optional) AverageSeriesByElements (optional) qcts.CorrectFgForStorage (all sites) qcts.Average3SeriesByElements (optional) qcts.CalculateAvailableEnergy (optional) qcck.do_qcchecks (all sites) qcck.gaps (optional) *: requires ancillary measurements for paratmerisation +: each site requires one pair, either Fe_WPL & Fc_WPL (default) or Fe_WPLCov & FcWPLCov x: required together in option set y: required together in option set """ # make a copy of the L2 data ds3 = copy.deepcopy(ds2) # set some attributes for this level qcutils.UpdateGlobalAttributes(cf, ds3, "L3") # initialise the global attribute to document the functions used ds3.globalattributes['Functions'] = '' # put the control file name into the global attributes ds3.globalattributes['controlfile_name'] = cf['controlfile_name'] # check to see if we have any imports qcgf.ImportSeries(cf, ds3) # correct measured soil water content using empirical relationship to collected samples qcts.CorrectSWC(cf, ds3) # apply linear corrections to the data qcck.do_linear(cf, ds3) # merge whatever humidities are available qcts.MergeHumidities(cf, ds3, convert_units=True) # get the air temperature from the CSAT virtual temperature qcts.TaFromTv(cf, ds3) # merge the HMP and corrected CSAT data qcts.MergeSeries(cf, ds3, 'Ta', [0, 10], convert_units=True) qcutils.CheckUnits(ds3, "Ta", "C", convert_units=True) # calculate humidities (absolute, specific and relative) from whatever is available qcts.CalculateHumidities(ds3) # merge the 7500 CO2 concentration qcts.MergeSeries(cf, ds3, 'Cc', [0, 10], convert_units=True) qcutils.CheckUnits(ds3, "Cc", "mg/m3", convert_units=True) # add relevant meteorological values to L3 data qcts.CalculateMeteorologicalVariables(ds3) # check to see if the user wants to use the fluxes in the L2 file if not qcutils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False): # check the covariancve units and change if necessary qcts.CheckCovarianceUnits(ds3) # do the 2D coordinate rotation qcts.CoordRotation2D(cf, ds3) # do the Massman frequency attenuation correction qcts.MassmanStandard(cf, ds3) # calculate the fluxes qcts.CalculateFluxes(cf, ds3) # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual) qcts.FhvtoFh(cf, ds3) # correct the H2O & CO2 flux due to effects of flux on density measurements qcts.Fe_WPL(cf, ds3) qcts.Fc_WPL(cf, ds3) # convert CO2 units if required qcutils.ConvertCO2Units(cf, ds3, Cc='Cc') # calculate Fc storage term - single height only at present qcts.CalculateFcStorage(cf, ds3) # convert Fc and Fc_storage units if required qcutils.ConvertFcUnits(cf, ds3, Fc='Fc', Fc_storage='Fc_storage') # correct Fc for storage term - only recommended if storage calculated from profile available qcts.CorrectFcForStorage(cf, ds3) # merge the incoming shortwave radiation qcts.MergeSeries(cf, ds3, 'Fsd', [0, 10]) # calculate the net radiation from the Kipp and Zonen CNR1 qcts.CalculateNetRadiation(cf, ds3, Fn_out='Fn_KZ', Fsd_in='Fsd', Fsu_in='Fsu', Fld_in='Fld', Flu_in='Flu') qcts.MergeSeries(cf, ds3, 'Fn', [0, 10]) # combine wind speed from the Wind Sentry and the CSAT qcts.MergeSeries(cf, ds3, 'Ws', [0, 10]) # combine wind direction from the Wind Sentry and the CSAT qcts.MergeSeries(cf, ds3, 'Wd', [0, 10]) # correct soil heat flux for storage # ... either average the raw ground heat flux, soil temperature and moisture # and then do the correction (OzFlux "standard") qcts.AverageSeriesByElements(cf, ds3, 'Ts') qcts.AverageSeriesByElements(cf, ds3, 'Sws') if qcutils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'): # ... or correct the individual ground heat flux measurements (James' method) qcts.CorrectIndividualFgForStorage(cf, ds3) qcts.AverageSeriesByElements(cf, ds3, 'Fg') else: qcts.AverageSeriesByElements(cf, ds3, 'Fg') qcts.CorrectFgForStorage(cf, ds3, Fg_out='Fg', Fg_in='Fg', Ts_in='Ts', Sws_in='Sws') # calculate the available energy qcts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg') # create new series using MergeSeries or AverageSeries qcck.CreateNewSeries(cf, ds3) # create a series of daily averaged soil moisture interpolated back to the time step #qcts.DailyAverageSws_Interpolated(cf,ds3,Sws_out='Sws_daily',Sws_in='Sws') # re-apply the quality control checks (range, diurnal and rules) qcck.do_qcchecks(cf, ds3) # coordinate gaps in the three main fluxes qcck.CoordinateFluxGaps(cf, ds3) # coordinate gaps in Ah_7500_Av with Fc qcck.CoordinateAh7500AndFcGaps(cf, ds3) # get the statistics for the QC flags and write these to an Excel spreadsheet qcio.get_seriesstats(cf, ds3) # write the percentage of good data as a variable attribute qcutils.get_coverage_individual(ds3) # write the percentage of good data for groups qcutils.get_coverage_groups(ds3) return ds3
def l3qc(cf, ds2): """ """ # make a copy of the L2 data ds3 = copy.deepcopy(ds2) # set some attributes for this level qcutils.UpdateGlobalAttributes(cf, ds3, "L3") # put the control file name into the global attributes ds3.globalattributes['controlfile_name'] = cf['controlfile_name'] # check to see if we have any imports qcgf.ImportSeries(cf, ds3) # apply linear corrections to the data qcck.do_linear(cf, ds3) # ************************ # *** Merge humidities *** # ************************ # merge whatever humidities are available qcts.MergeHumidities(cf, ds3, convert_units=True) # ************************** # *** Merge temperatures *** # ************************** # get the air temperature from the CSAT virtual temperature qcts.TaFromTv(cf, ds3) # merge the HMP and corrected CSAT data qcts.MergeSeries(cf, ds3, "Ta", convert_units=True) qcutils.CheckUnits(ds3, "Ta", "C", convert_units=True) # *************************** # *** Calcuate humidities *** # *************************** # calculate humidities (absolute, specific and relative) from whatever is available qcts.CalculateHumidities(ds3) # ******************************** # *** Merge CO2 concentrations *** # ******************************** # merge the 7500 CO2 concentration # PRI 09/08/2017 possibly the ugliest thing I have done yet # This needs to be abstracted to a general alias checking routine at the # start of the L3 processing so that possible aliases are mapped to a single # set of variable names. if "CO2" in cf["Variables"]: CO2 = "CO2" elif "Cc" in cf["Variables"]: CO2 = "Cc" else: msg = "Label for CO2 ('CO2','Cc') not found in control file" logger.error(msg) return qcts.MergeSeries(cf, ds3, CO2, convert_units=True) # ****************************************** # *** Calculate meteorological variables *** # ****************************************** # Update meteorological variables qcts.CalculateMeteorologicalVariables(ds3) # ************************************************* # *** Calculate fluxes from covariances section *** # ************************************************* # check to see if the user wants to use the fluxes in the L2 file if not qcutils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False): # check the covariance units and change if necessary qcts.CheckCovarianceUnits(ds3) # do the 2D coordinate rotation qcts.CoordRotation2D(cf, ds3) # do the Massman frequency attenuation correction qcts.MassmanStandard(cf, ds3) # calculate the fluxes qcts.CalculateFluxes(cf, ds3) # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual) qcts.FhvtoFh(cf, ds3) # correct the H2O & CO2 flux due to effects of flux on density measurements qcts.Fe_WPL(cf, ds3) qcts.Fc_WPL(cf, ds3) # ************************************** # *** Calculate Monin-Obukhov length *** # ************************************** qcts.CalculateMoninObukhovLength(ds3) # ************************** # *** CO2 and Fc section *** # ************************** # convert CO2 units if required qcutils.ConvertCO2Units(cf, ds3, CO2=CO2) # calculate Fc storage term - single height only at present qcts.CalculateFcStorageSinglePoint(cf, ds3, Fc_out='Fc_single', CO2_in=CO2) # convert Fc and Fc_storage units if required qcutils.ConvertFcUnits(cf, ds3) # merge Fc and Fc_storage series if required merge_list = [ label for label in cf["Variables"].keys() if label[0:2] == "Fc" and "MergeSeries" in cf["Variables"][label].keys() ] for label in merge_list: qcts.MergeSeries(cf, ds3, label, save_originals=True) # correct Fc for storage term - only recommended if storage calculated from profile available qcts.CorrectFcForStorage(cf, ds3) # ************************* # *** Radiation section *** # ************************* # merge the incoming shortwave radiation qcts.MergeSeries(cf, ds3, 'Fsd') # calculate the net radiation from the Kipp and Zonen CNR1 qcts.CalculateNetRadiation(cf, ds3, Fn_out='Fn_KZ', Fsd_in='Fsd', Fsu_in='Fsu', Fld_in='Fld', Flu_in='Flu') qcts.MergeSeries(cf, ds3, 'Fn') # **************************************** # *** Wind speed and direction section *** # **************************************** # combine wind speed from the Wind Sentry and the SONIC qcts.MergeSeries(cf, ds3, 'Ws') # combine wind direction from the Wind Sentry and the SONIC qcts.MergeSeries(cf, ds3, 'Wd') # ******************** # *** Soil section *** # ******************** # correct soil heat flux for storage # ... either average the raw ground heat flux, soil temperature and moisture # and then do the correction (OzFlux "standard") qcts.AverageSeriesByElements(cf, ds3, 'Ts') qcts.AverageSeriesByElements(cf, ds3, 'Sws') if qcutils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'): # ... or correct the individual ground heat flux measurements (James' method) qcts.CorrectIndividualFgForStorage(cf, ds3) qcts.AverageSeriesByElements(cf, ds3, 'Fg') else: qcts.AverageSeriesByElements(cf, ds3, 'Fg') qcts.CorrectFgForStorage(cf, ds3, Fg_out='Fg', Fg_in='Fg', Ts_in='Ts', Sws_in='Sws') # calculate the available energy qcts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg') # create new series using MergeSeries or AverageSeries qcck.CreateNewSeries(cf, ds3) # re-apply the quality control checks (range, diurnal and rules) qcck.do_qcchecks(cf, ds3) # coordinate gaps in the three main fluxes qcck.CoordinateFluxGaps(cf, ds3) # coordinate gaps in Ah_7500_Av with Fc qcck.CoordinateAh7500AndFcGaps(cf, ds3) # check missing data and QC flags are consistent qcutils.CheckQCFlags(ds3) # get the statistics for the QC flags and write these to an Excel spreadsheet qcio.get_seriesstats(cf, ds3) # write the percentage of good data as a variable attribute qcutils.get_coverage_individual(ds3) # write the percentage of good data for groups qcutils.get_coverage_groups(ds3) return ds3