def l2qc(cf, ds1): """ Perform initial QA/QC on flux data Generates L2 from L1 data * check parameters specified in control file Functions performed: pfp_ck.do_rangecheck* pfp_ck.do_CSATcheck pfp_ck.do_7500check pfp_ck.do_diurnalcheck* pfp_ck.do_excludedates* pfp_ck.do_excludehours* pfp_ts.albedo """ # make a copy of the L1 data ds2 = copy.deepcopy(ds1) # set some attributes for this level pfp_utils.UpdateGlobalAttributes(cf, ds2, "L2") # apply linear corrections to the data pfp_ck.do_linear(cf, ds2) # apply the quality control checks (range, diurnal, exclude dates and exclude hours pfp_ck.do_qcchecks(cf, ds2) # do the CSAT diagnostic check pfp_ck.do_SONICcheck(cf, ds2) # do the IRGA diagnostic check pfp_ck.do_IRGAcheck(cf, ds2) # check missing data and QC flags are consistent pfp_utils.CheckQCFlags(ds2) # write series statistics to file pfp_io.get_seriesstats(cf, ds2) # write the percentage of good data as a variable attribute pfp_utils.get_coverage_individual(ds2) return ds2
def l1qc(cf): """ Purpose: Reads input files, either an Excel workbook or a collection of CSV files, and returns the data as a data structure. Usage: Side effects: Returns a data structure containing the data specified in the L1 control file. Author: PRI Date: February 2020 """ # get a new data structure ds = pfp_io.DataStructure() # parse the L1 control file l1_info = pfp_compliance.ParseL1ControlFile(cf) # return if parsing throws an error if not pfp_compliance.check_status_ok(ds, l1_info): return ds # check the Excel workbook xl_data = pfp_compliance.CheckExcelWorkbook(l1_info) # copy data from the worksheets to individual data structures ds_dict = pfp_io.ExcelToDataStructures(xl_data, l1_info) # merge the individual data structures to a single data structure ds = pfp_ts.MergeDataStructures(ds_dict, l1_info) # write the processing level to a global attribute ds.globalattributes["nc_level"] = "L1" # calculate variances from standard deviations and vice versa pfp_ts.CalculateStandardDeviations(ds) # create new variables using user defined functions pfp_ts.DoFunctions(ds, l1_info["read_excel"]) # check missing data and QC flags are consistent pfp_utils.CheckQCFlags(ds) return ds
def l1qc(cf): # get the data series from the Excel file in_filename = pfp_io.get_infilenamefromcf(cf) if not pfp_utils.file_exists(in_filename, mode="quiet"): msg = " Input file " + in_filename + " not found ..." logger.error(msg) ds1 = pfp_io.DataStructure() ds1.returncodes = {"value": 1, "message": msg} return ds1 file_name, file_extension = os.path.splitext(in_filename) if "csv" in file_extension.lower(): ds1 = pfp_io.csv_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Excel datetime from the Python datetime objects #pfp_utils.get_xldatefromdatetime(ds1) else: ds1 = pfp_io.xl_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Python datetime objects from the Excel datetime #pfp_utils.get_datetime_from_xldate(ds1) # get the netCDF attributes from the control file #pfp_ts.do_attributes(cf,ds1) pfp_utils.get_datetime(cf, ds1) # round the Python datetime to the nearest second pfp_utils.round_datetime(ds1, mode="nearest_second") #check for gaps in the Python datetime series and fix if present fixtimestepmethod = pfp_utils.get_keyvaluefromcf(cf, ["options"], "FixTimeStepMethod", default="round") if pfp_utils.CheckTimeStep(ds1): pfp_utils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod) # recalculate the Excel datetime #pfp_utils.get_xldatefromdatetime(ds1) # get the Year, Month, Day etc from the Python datetime #pfp_utils.get_ymdhmsfromdatetime(ds1) # write the processing level to a global attribute ds1.globalattributes['nc_level'] = str("L1") # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds1.globalattributes.keys(): ds1.globalattributes['start_date'] = str( ds1.series['DateTime']['Data'][0]) if 'end_date' not in ds1.globalattributes.keys(): ds1.globalattributes['end_date'] = str( ds1.series['DateTime']['Data'][-1]) # calculate variances from standard deviations and vice versa pfp_ts.CalculateStandardDeviations(cf, ds1) # create new variables using user defined functions pfp_ts.DoFunctions(cf, ds1) # create a series of synthetic downwelling shortwave radiation #pfp_ts.get_synthetic_fsd(ds1) # check missing data and QC flags are consistent pfp_utils.CheckQCFlags(ds1) return ds1
def l3qc(cf, ds2): """ """ # make a copy of the L2 data ds3 = copy.deepcopy(ds2) # set some attributes for this level pfp_utils.UpdateGlobalAttributes(cf, ds3, "L3") # put the control file name into the global attributes ds3.globalattributes['controlfile_name'] = cf['controlfile_name'] # check to see if we have any imports pfp_gf.ImportSeries(cf, ds3) # apply linear corrections to the data pfp_ck.do_linear(cf, ds3) # ************************ # *** Merge humidities *** # ************************ # merge whatever humidities are available pfp_ts.MergeHumidities(cf, ds3, convert_units=True) # ************************** # *** Merge temperatures *** # ************************** # get the air temperature from the CSAT virtual temperature pfp_ts.TaFromTv(cf, ds3) # merge the HMP and corrected CSAT data pfp_ts.MergeSeries(cf, ds3, "Ta", convert_units=True) pfp_utils.CheckUnits(ds3, "Ta", "C", convert_units=True) # *************************** # *** Calcuate humidities *** # *************************** # calculate humidities (absolute, specific and relative) from whatever is available pfp_ts.CalculateHumidities(ds3) # ******************************** # *** Merge CO2 concentrations *** # ******************************** # merge the 7500 CO2 concentration # PRI 09/08/2017 possibly the ugliest thing I have done yet # This needs to be abstracted to a general alias checking routine at the # start of the L3 processing so that possible aliases are mapped to a single # set of variable names. if "CO2" in cf["Variables"]: CO2 = "CO2" elif "Cc" in cf["Variables"]: CO2 = "Cc" else: msg = "Label for CO2 ('CO2','Cc') not found in control file" logger.error(msg) return pfp_ts.MergeSeries(cf, ds3, CO2, convert_units=True) # ****************************************** # *** Calculate meteorological variables *** # ****************************************** # Update meteorological variables pfp_ts.CalculateMeteorologicalVariables(ds3) # ************************************************* # *** Calculate fluxes from covariances section *** # ************************************************* # check to see if the user wants to use the fluxes in the L2 file if not pfp_utils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False): # check the covariance units and change if necessary pfp_ts.CheckCovarianceUnits(ds3) # do the 2D coordinate rotation pfp_ts.CoordRotation2D(cf, ds3) # do the Massman frequency attenuation correction pfp_ts.MassmanStandard(cf, ds3) # calculate the fluxes pfp_ts.CalculateFluxes(cf, ds3) # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual) pfp_ts.FhvtoFh(cf, ds3) # correct the H2O & CO2 flux due to effects of flux on density measurements pfp_ts.Fe_WPL(cf, ds3) pfp_ts.Fc_WPL(cf, ds3) # ************************************** # *** Calculate Monin-Obukhov length *** # ************************************** pfp_ts.CalculateMoninObukhovLength(ds3) # ************************** # *** CO2 and Fc section *** # ************************** # convert CO2 units if required pfp_utils.ConvertCO2Units(cf, ds3, CO2=CO2) # calculate Fc storage term - single height only at present pfp_ts.CalculateFcStorageSinglePoint(cf, ds3, Fc_out='Fc_single', CO2_in=CO2) # convert Fc and Fc_storage units if required pfp_utils.ConvertFcUnits(cf, ds3) # merge Fc and Fc_storage series if required merge_list = [ label for label in cf["Variables"].keys() if label[0:2] == "Fc" and "MergeSeries" in cf["Variables"][label].keys() ] for label in merge_list: pfp_ts.MergeSeries(cf, ds3, label, save_originals=True) # correct Fc for storage term - only recommended if storage calculated from profile available pfp_ts.CorrectFcForStorage(cf, ds3) # ************************* # *** Radiation section *** # ************************* # merge the incoming shortwave radiation pfp_ts.MergeSeries(cf, ds3, 'Fsd') # calculate the net radiation from the Kipp and Zonen CNR1 pfp_ts.CalculateNetRadiation(cf, ds3, Fn_out='Fn_KZ', Fsd_in='Fsd', Fsu_in='Fsu', Fld_in='Fld', Flu_in='Flu') pfp_ts.MergeSeries(cf, ds3, 'Fn') # **************************************** # *** Wind speed and direction section *** # **************************************** # combine wind speed from the Wind Sentry and the SONIC pfp_ts.MergeSeries(cf, ds3, 'Ws') # combine wind direction from the Wind Sentry and the SONIC pfp_ts.MergeSeries(cf, ds3, 'Wd') # ******************** # *** Soil section *** # ******************** # correct soil heat flux for storage # ... either average the raw ground heat flux, soil temperature and moisture # and then do the correction (OzFlux "standard") pfp_ts.AverageSeriesByElements(cf, ds3, 'Ts') pfp_ts.AverageSeriesByElements(cf, ds3, 'Sws') if pfp_utils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'): # ... or correct the individual ground heat flux measurements (James' method) pfp_ts.CorrectIndividualFgForStorage(cf, ds3) pfp_ts.AverageSeriesByElements(cf, ds3, 'Fg') else: pfp_ts.AverageSeriesByElements(cf, ds3, 'Fg') pfp_ts.CorrectFgForStorage(cf, ds3, Fg_out='Fg', Fg_in='Fg', Ts_in='Ts', Sws_in='Sws') # calculate the available energy pfp_ts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg') # create new series using MergeSeries or AverageSeries pfp_ck.CreateNewSeries(cf, ds3) # re-apply the quality control checks (range, diurnal and rules) pfp_ck.do_qcchecks(cf, ds3) # coordinate gaps in the three main fluxes pfp_ck.CoordinateFluxGaps(cf, ds3) # coordinate gaps in Ah_7500_Av with Fc pfp_ck.CoordinateAh7500AndFcGaps(cf, ds3) # check missing data and QC flags are consistent pfp_utils.CheckQCFlags(ds3) # get the statistics for the QC flags and write these to an Excel spreadsheet pfp_io.get_seriesstats(cf, ds3) # write the percentage of good data as a variable attribute pfp_utils.get_coverage_individual(ds3) # write the percentage of good data for groups pfp_utils.get_coverage_groups(ds3) return ds3
def l3qc(cf, ds2): """ """ # make a copy of the L2 data ds3 = copy.deepcopy(ds2) # set some attributes for this level pfp_utils.UpdateGlobalAttributes(cf, ds3, "L3") # check to see if we have any imports pfp_gf.ImportSeries(cf,ds3) # apply linear corrections to the data pfp_ck.do_linear(cf,ds3) # parse the control file for information on how the user wants to do the gap filling l3_info = pfp_compliance.ParseL3ControlFile(cf, ds3) if l3_info["status"]["value"] != 0: logger.error(l3_info["status"]["message"]) return ds3 # ************************ # *** Merge humidities *** # ************************ # merge whatever humidities are available pfp_ts.MergeHumidities(cf, ds3, convert_units=True) # ************************** # *** Merge temperatures *** # ************************** # get the air temperature from the CSAT virtual temperature pfp_ts.TaFromTv(cf, ds3) # merge the HMP and corrected CSAT data pfp_ts.CombineSeries(cf, ds3, "Ta", convert_units=True) pfp_utils.CheckUnits(ds3, "Ta", "degC", convert_units=True) # *************************** # *** Calcuate humidities *** # *************************** # calculate humidities (absolute, specific and relative) from whatever is available pfp_ts.CalculateHumidities(ds3) # ******************************** # *** Merge CO2 concentrations *** # ******************************** # merge the CO2 concentration pfp_ts.CombineSeries(cf, ds3, l3_info["CO2"]["label"], convert_units=True) # ****************************************** # *** Calculate meteorological variables *** # ****************************************** # Update meteorological variables pfp_ts.CalculateMeteorologicalVariables(ds3, l3_info) # ************************************************* # *** Calculate fluxes from covariances section *** # ************************************************* # check to see if the user wants to use the fluxes in the L2 file if not pfp_utils.get_optionskeyaslogical(cf, "UseL2Fluxes", default=False): # check the covariance units and change if necessary pfp_ts.CheckCovarianceUnits(ds3) # do the 2D coordinate rotation pfp_ts.CoordRotation2D(cf, ds3) # do the Massman frequency attenuation correction pfp_ts.MassmanStandard(cf, ds3) # calculate the fluxes pfp_ts.CalculateFluxes(cf, ds3) # approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual) pfp_ts.FhvtoFh(cf, ds3) # correct the H2O & CO2 flux due to effects of flux on density measurements if pfp_ts.Fe_WPL(cf, ds3): return ds3 if pfp_ts.Fco2_WPL(cf, ds3): return ds3 # ************************** # *** CO2 and Fc section *** # ************************** # convert CO2 units if required pfp_utils.ConvertCO2Units(cf, ds3) # calculate Fco2 storage term - single height only at present pfp_ts.CalculateFco2StorageSinglePoint(cf, ds3, l3_info["CO2"]["label"]) # convert Fco2 units if required pfp_utils.ConvertFco2Units(cf, ds3) # merge Fco2 and Fco2_storage series if required pfp_ts.CombineSeries(cf, ds3, l3_info["Fco2"]["combine_list"], save_originals=True) # correct Fco2 for storage term - only recommended if storage calculated from profile available pfp_ts.CorrectFco2ForStorage(cf, ds3) # ************************* # *** Radiation section *** # ************************* # merge the incoming shortwave radiation pfp_ts.CombineSeries(cf, ds3, "Fsd") # calculate the net radiation from the Kipp and Zonen CNR1 pfp_ts.CalculateNetRadiation(cf, ds3) pfp_ts.CombineSeries(cf, ds3, "Fn") # **************************************** # *** Wind speed and direction section *** # **************************************** # combine wind speed from the Wind Sentry and the SONIC pfp_ts.CombineSeries(cf,ds3, "Ws") # combine wind direction from the Wind Sentry and the SONIC pfp_ts.CombineSeries(cf,ds3, "Wd") # ******************** # *** Soil section *** # ******************** # correct soil heat flux for storage # ... either average the raw ground heat flux, soil temperature and moisture # and then do the correction (OzFlux "standard") pfp_ts.CombineSeries(cf, ds3, "Ts") pfp_ts.CombineSeries(cf, ds3, "Sws") if pfp_utils.get_optionskeyaslogical(cf, "CorrectIndividualFg"): # ... or correct the individual ground heat flux measurements (James' method) pfp_ts.CorrectIndividualFgForStorage(cf, ds3) pfp_ts.CombineSeries(cf, ds3, "Fg") else: pfp_ts.CombineSeries(cf, ds3, "Fg") pfp_ts.CorrectFgForStorage(cf, ds3) # calculate the available energy pfp_ts.CalculateAvailableEnergy(ds3) # create new series using MergeSeries or AverageSeries pfp_ck.CreateNewSeries(cf, ds3) # Calculate Monin-Obukhov length pfp_ts.CalculateMoninObukhovLength(ds3) # re-apply the quality control checks (range, diurnal and rules) pfp_ck.do_qcchecks(cf, ds3) # check missing data and QC flags are consistent pfp_utils.CheckQCFlags(ds3) # get the statistics for the QC flags and write these to an Excel spreadsheet pfp_io.get_seriesstats(cf, ds3) # write the percentage of good data as a variable attribute pfp_utils.get_coverage_individual(ds3) # write the percentage of good data for groups pfp_utils.get_coverage_groups(ds3) # remove intermediate series from the data structure pfp_ts.RemoveIntermediateSeries(ds3, l3_info) return ds3