def l2qc(cf, ds1):
"""
Perform initial QA/QC on flux data
Generates L2 from L1 data
* check parameters specified in control file
Functions performed:
pfp_ck.do_rangecheck*
pfp_ck.do_CSATcheck
pfp_ck.do_7500check
pfp_ck.do_diurnalcheck*
pfp_ck.do_excludedates*
pfp_ck.do_excludehours*
pfp_ts.albedo
"""
# make a copy of the L1 data
ds2 = copy.deepcopy(ds1)
# set some attributes for this level
pfp_utils.UpdateGlobalAttributes(cf, ds2, "L2")
# apply linear corrections to the data
pfp_ck.do_linear(cf, ds2)
# apply the quality control checks (range, diurnal, exclude dates and exclude hours
pfp_ck.do_qcchecks(cf, ds2)
# do the CSAT diagnostic check
pfp_ck.do_SONICcheck(cf, ds2)
# do the IRGA diagnostic check
pfp_ck.do_IRGAcheck(cf, ds2)
# check missing data and QC flags are consistent
pfp_utils.CheckQCFlags(ds2)
# write series statistics to file
pfp_io.get_seriesstats(cf, ds2)
# write the percentage of good data as a variable attribute
pfp_utils.get_coverage_individual(ds2)
return ds2
def l1qc(cf):
"""
Purpose:
Reads input files, either an Excel workbook or a collection of CSV files,
and returns the data as a data structure.
Usage:
Side effects:
Returns a data structure containing the data specified in the L1
control file.
Author: PRI
Date: February 2020
"""
# get a new data structure
ds = pfp_io.DataStructure()
# parse the L1 control file
l1_info = pfp_compliance.ParseL1ControlFile(cf)
# return if parsing throws an error
if not pfp_compliance.check_status_ok(ds, l1_info): return ds
# check the Excel workbook
xl_data = pfp_compliance.CheckExcelWorkbook(l1_info)
# copy data from the worksheets to individual data structures
ds_dict = pfp_io.ExcelToDataStructures(xl_data, l1_info)
# merge the individual data structures to a single data structure
ds = pfp_ts.MergeDataStructures(ds_dict, l1_info)
# write the processing level to a global attribute
ds.globalattributes["nc_level"] = "L1"
# calculate variances from standard deviations and vice versa
pfp_ts.CalculateStandardDeviations(ds)
# create new variables using user defined functions
pfp_ts.DoFunctions(ds, l1_info["read_excel"])
# check missing data and QC flags are consistent
pfp_utils.CheckQCFlags(ds)
return ds
def l1qc(cf):
# get the data series from the Excel file
in_filename = pfp_io.get_infilenamefromcf(cf)
if not pfp_utils.file_exists(in_filename, mode="quiet"):
msg = " Input file " + in_filename + " not found ..."
logger.error(msg)
ds1 = pfp_io.DataStructure()
ds1.returncodes = {"value": 1, "message": msg}
return ds1
file_name, file_extension = os.path.splitext(in_filename)
if "csv" in file_extension.lower():
ds1 = pfp_io.csv_read_series(cf)
if ds1.returncodes["value"] != 0:
return ds1
# get a series of Excel datetime from the Python datetime objects
#pfp_utils.get_xldatefromdatetime(ds1)
else:
ds1 = pfp_io.xl_read_series(cf)
if ds1.returncodes["value"] != 0:
return ds1
# get a series of Python datetime objects from the Excel datetime
#pfp_utils.get_datetime_from_xldate(ds1)
# get the netCDF attributes from the control file
#pfp_ts.do_attributes(cf,ds1)
pfp_utils.get_datetime(cf, ds1)
# round the Python datetime to the nearest second
pfp_utils.round_datetime(ds1, mode="nearest_second")
#check for gaps in the Python datetime series and fix if present
fixtimestepmethod = pfp_utils.get_keyvaluefromcf(cf, ["options"],
"FixTimeStepMethod",
default="round")
if pfp_utils.CheckTimeStep(ds1):
pfp_utils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod)
# recalculate the Excel datetime
#pfp_utils.get_xldatefromdatetime(ds1)
# get the Year, Month, Day etc from the Python datetime
#pfp_utils.get_ymdhmsfromdatetime(ds1)
# write the processing level to a global attribute
ds1.globalattributes['nc_level'] = str("L1")
# get the start and end date from the datetime series unless they were
# given in the control file
if 'start_date' not in ds1.globalattributes.keys():
ds1.globalattributes['start_date'] = str(
ds1.series['DateTime']['Data'][0])
if 'end_date' not in ds1.globalattributes.keys():
ds1.globalattributes['end_date'] = str(
ds1.series['DateTime']['Data'][-1])
# calculate variances from standard deviations and vice versa
pfp_ts.CalculateStandardDeviations(cf, ds1)
# create new variables using user defined functions
pfp_ts.DoFunctions(cf, ds1)
# create a series of synthetic downwelling shortwave radiation
#pfp_ts.get_synthetic_fsd(ds1)
# check missing data and QC flags are consistent
pfp_utils.CheckQCFlags(ds1)
return ds1
def l3qc(cf, ds2):
"""
"""
# make a copy of the L2 data
ds3 = copy.deepcopy(ds2)
# set some attributes for this level
pfp_utils.UpdateGlobalAttributes(cf, ds3, "L3")
# put the control file name into the global attributes
ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
# check to see if we have any imports
pfp_gf.ImportSeries(cf, ds3)
# apply linear corrections to the data
pfp_ck.do_linear(cf, ds3)
# ************************
# *** Merge humidities ***
# ************************
# merge whatever humidities are available
pfp_ts.MergeHumidities(cf, ds3, convert_units=True)
# **************************
# *** Merge temperatures ***
# **************************
# get the air temperature from the CSAT virtual temperature
pfp_ts.TaFromTv(cf, ds3)
# merge the HMP and corrected CSAT data
pfp_ts.MergeSeries(cf, ds3, "Ta", convert_units=True)
pfp_utils.CheckUnits(ds3, "Ta", "C", convert_units=True)
# ***************************
# *** Calcuate humidities ***
# ***************************
# calculate humidities (absolute, specific and relative) from whatever is available
pfp_ts.CalculateHumidities(ds3)
# ********************************
# *** Merge CO2 concentrations ***
# ********************************
# merge the 7500 CO2 concentration
# PRI 09/08/2017 possibly the ugliest thing I have done yet
# This needs to be abstracted to a general alias checking routine at the
# start of the L3 processing so that possible aliases are mapped to a single
# set of variable names.
if "CO2" in cf["Variables"]:
CO2 = "CO2"
elif "Cc" in cf["Variables"]:
CO2 = "Cc"
else:
msg = "Label for CO2 ('CO2','Cc') not found in control file"
logger.error(msg)
return
pfp_ts.MergeSeries(cf, ds3, CO2, convert_units=True)
# ******************************************
# *** Calculate meteorological variables ***
# ******************************************
# Update meteorological variables
pfp_ts.CalculateMeteorologicalVariables(ds3)
# *************************************************
# *** Calculate fluxes from covariances section ***
# *************************************************
# check to see if the user wants to use the fluxes in the L2 file
if not pfp_utils.cfoptionskeylogical(cf, Key="UseL2Fluxes", default=False):
# check the covariance units and change if necessary
pfp_ts.CheckCovarianceUnits(ds3)
# do the 2D coordinate rotation
pfp_ts.CoordRotation2D(cf, ds3)
# do the Massman frequency attenuation correction
pfp_ts.MassmanStandard(cf, ds3)
# calculate the fluxes
pfp_ts.CalculateFluxes(cf, ds3)
# approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
pfp_ts.FhvtoFh(cf, ds3)
# correct the H2O & CO2 flux due to effects of flux on density measurements
pfp_ts.Fe_WPL(cf, ds3)
pfp_ts.Fc_WPL(cf, ds3)
# **************************************
# *** Calculate Monin-Obukhov length ***
# **************************************
pfp_ts.CalculateMoninObukhovLength(ds3)
# **************************
# *** CO2 and Fc section ***
# **************************
# convert CO2 units if required
pfp_utils.ConvertCO2Units(cf, ds3, CO2=CO2)
# calculate Fc storage term - single height only at present
pfp_ts.CalculateFcStorageSinglePoint(cf,
ds3,
Fc_out='Fc_single',
CO2_in=CO2)
# convert Fc and Fc_storage units if required
pfp_utils.ConvertFcUnits(cf, ds3)
# merge Fc and Fc_storage series if required
merge_list = [
label for label in cf["Variables"].keys() if label[0:2] == "Fc"
and "MergeSeries" in cf["Variables"][label].keys()
]
for label in merge_list:
pfp_ts.MergeSeries(cf, ds3, label, save_originals=True)
# correct Fc for storage term - only recommended if storage calculated from profile available
pfp_ts.CorrectFcForStorage(cf, ds3)
# *************************
# *** Radiation section ***
# *************************
# merge the incoming shortwave radiation
pfp_ts.MergeSeries(cf, ds3, 'Fsd')
# calculate the net radiation from the Kipp and Zonen CNR1
pfp_ts.CalculateNetRadiation(cf,
ds3,
Fn_out='Fn_KZ',
Fsd_in='Fsd',
Fsu_in='Fsu',
Fld_in='Fld',
Flu_in='Flu')
pfp_ts.MergeSeries(cf, ds3, 'Fn')
# ****************************************
# *** Wind speed and direction section ***
# ****************************************
# combine wind speed from the Wind Sentry and the SONIC
pfp_ts.MergeSeries(cf, ds3, 'Ws')
# combine wind direction from the Wind Sentry and the SONIC
pfp_ts.MergeSeries(cf, ds3, 'Wd')
# ********************
# *** Soil section ***
# ********************
# correct soil heat flux for storage
# ... either average the raw ground heat flux, soil temperature and moisture
# and then do the correction (OzFlux "standard")
pfp_ts.AverageSeriesByElements(cf, ds3, 'Ts')
pfp_ts.AverageSeriesByElements(cf, ds3, 'Sws')
if pfp_utils.cfoptionskeylogical(cf, Key='CorrectIndividualFg'):
# ... or correct the individual ground heat flux measurements (James' method)
pfp_ts.CorrectIndividualFgForStorage(cf, ds3)
pfp_ts.AverageSeriesByElements(cf, ds3, 'Fg')
else:
pfp_ts.AverageSeriesByElements(cf, ds3, 'Fg')
pfp_ts.CorrectFgForStorage(cf,
ds3,
Fg_out='Fg',
Fg_in='Fg',
Ts_in='Ts',
Sws_in='Sws')
# calculate the available energy
pfp_ts.CalculateAvailableEnergy(ds3, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg')
# create new series using MergeSeries or AverageSeries
pfp_ck.CreateNewSeries(cf, ds3)
# re-apply the quality control checks (range, diurnal and rules)
pfp_ck.do_qcchecks(cf, ds3)
# coordinate gaps in the three main fluxes
pfp_ck.CoordinateFluxGaps(cf, ds3)
# coordinate gaps in Ah_7500_Av with Fc
pfp_ck.CoordinateAh7500AndFcGaps(cf, ds3)
# check missing data and QC flags are consistent
pfp_utils.CheckQCFlags(ds3)
# get the statistics for the QC flags and write these to an Excel spreadsheet
pfp_io.get_seriesstats(cf, ds3)
# write the percentage of good data as a variable attribute
pfp_utils.get_coverage_individual(ds3)
# write the percentage of good data for groups
pfp_utils.get_coverage_groups(ds3)
return ds3
def l3qc(cf, ds2):
"""
"""
# make a copy of the L2 data
ds3 = copy.deepcopy(ds2)
# set some attributes for this level
pfp_utils.UpdateGlobalAttributes(cf, ds3, "L3")
# check to see if we have any imports
pfp_gf.ImportSeries(cf,ds3)
# apply linear corrections to the data
pfp_ck.do_linear(cf,ds3)
# parse the control file for information on how the user wants to do the gap filling
l3_info = pfp_compliance.ParseL3ControlFile(cf, ds3)
if l3_info["status"]["value"] != 0:
logger.error(l3_info["status"]["message"])
return ds3
# ************************
# *** Merge humidities ***
# ************************
# merge whatever humidities are available
pfp_ts.MergeHumidities(cf, ds3, convert_units=True)
# **************************
# *** Merge temperatures ***
# **************************
# get the air temperature from the CSAT virtual temperature
pfp_ts.TaFromTv(cf, ds3)
# merge the HMP and corrected CSAT data
pfp_ts.CombineSeries(cf, ds3, "Ta", convert_units=True)
pfp_utils.CheckUnits(ds3, "Ta", "degC", convert_units=True)
# ***************************
# *** Calcuate humidities ***
# ***************************
# calculate humidities (absolute, specific and relative) from whatever is available
pfp_ts.CalculateHumidities(ds3)
# ********************************
# *** Merge CO2 concentrations ***
# ********************************
# merge the CO2 concentration
pfp_ts.CombineSeries(cf, ds3, l3_info["CO2"]["label"], convert_units=True)
# ******************************************
# *** Calculate meteorological variables ***
# ******************************************
# Update meteorological variables
pfp_ts.CalculateMeteorologicalVariables(ds3, l3_info)
# *************************************************
# *** Calculate fluxes from covariances section ***
# *************************************************
# check to see if the user wants to use the fluxes in the L2 file
if not pfp_utils.get_optionskeyaslogical(cf, "UseL2Fluxes", default=False):
# check the covariance units and change if necessary
pfp_ts.CheckCovarianceUnits(ds3)
# do the 2D coordinate rotation
pfp_ts.CoordRotation2D(cf, ds3)
# do the Massman frequency attenuation correction
pfp_ts.MassmanStandard(cf, ds3)
# calculate the fluxes
pfp_ts.CalculateFluxes(cf, ds3)
# approximate wT from virtual wT using wA (ref: Campbell OPECSystem manual)
pfp_ts.FhvtoFh(cf, ds3)
# correct the H2O & CO2 flux due to effects of flux on density measurements
if pfp_ts.Fe_WPL(cf, ds3):
return ds3
if pfp_ts.Fco2_WPL(cf, ds3):
return ds3
# **************************
# *** CO2 and Fc section ***
# **************************
# convert CO2 units if required
pfp_utils.ConvertCO2Units(cf, ds3)
# calculate Fco2 storage term - single height only at present
pfp_ts.CalculateFco2StorageSinglePoint(cf, ds3, l3_info["CO2"]["label"])
# convert Fco2 units if required
pfp_utils.ConvertFco2Units(cf, ds3)
# merge Fco2 and Fco2_storage series if required
pfp_ts.CombineSeries(cf, ds3, l3_info["Fco2"]["combine_list"], save_originals=True)
# correct Fco2 for storage term - only recommended if storage calculated from profile available
pfp_ts.CorrectFco2ForStorage(cf, ds3)
# *************************
# *** Radiation section ***
# *************************
# merge the incoming shortwave radiation
pfp_ts.CombineSeries(cf, ds3, "Fsd")
# calculate the net radiation from the Kipp and Zonen CNR1
pfp_ts.CalculateNetRadiation(cf, ds3)
pfp_ts.CombineSeries(cf, ds3, "Fn")
# ****************************************
# *** Wind speed and direction section ***
# ****************************************
# combine wind speed from the Wind Sentry and the SONIC
pfp_ts.CombineSeries(cf,ds3, "Ws")
# combine wind direction from the Wind Sentry and the SONIC
pfp_ts.CombineSeries(cf,ds3, "Wd")
# ********************
# *** Soil section ***
# ********************
# correct soil heat flux for storage
# ... either average the raw ground heat flux, soil temperature and moisture
# and then do the correction (OzFlux "standard")
pfp_ts.CombineSeries(cf, ds3, "Ts")
pfp_ts.CombineSeries(cf, ds3, "Sws")
if pfp_utils.get_optionskeyaslogical(cf, "CorrectIndividualFg"):
# ... or correct the individual ground heat flux measurements (James' method)
pfp_ts.CorrectIndividualFgForStorage(cf, ds3)
pfp_ts.CombineSeries(cf, ds3, "Fg")
else:
pfp_ts.CombineSeries(cf, ds3, "Fg")
pfp_ts.CorrectFgForStorage(cf, ds3)
# calculate the available energy
pfp_ts.CalculateAvailableEnergy(ds3)
# create new series using MergeSeries or AverageSeries
pfp_ck.CreateNewSeries(cf, ds3)
# Calculate Monin-Obukhov length
pfp_ts.CalculateMoninObukhovLength(ds3)
# re-apply the quality control checks (range, diurnal and rules)
pfp_ck.do_qcchecks(cf, ds3)
# check missing data and QC flags are consistent
pfp_utils.CheckQCFlags(ds3)
# get the statistics for the QC flags and write these to an Excel spreadsheet
pfp_io.get_seriesstats(cf, ds3)
# write the percentage of good data as a variable attribute
pfp_utils.get_coverage_individual(ds3)
# write the percentage of good data for groups
pfp_utils.get_coverage_groups(ds3)
# remove intermediate series from the data structure
pfp_ts.RemoveIntermediateSeries(ds3, l3_info)
return ds3