def do_l6qc(self): """ Call qcls.l6qc function to partition NEE into GPP and ER. """ logging.info(" Starting L6 processing ...") cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return infilename = qcio.get_infilenamefromcf(cf) if len(infilename)==0: self.do_progress(text='An error occurred, check the console ...'); return if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return ds5 = qcio.nc_read_series(infilename) if len(ds5.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del ds5; return ds5.globalattributes['controlfile_name'] = cf['controlfile_name'] self.update_startenddate(str(ds5.series['DateTime']['Data'][0]), str(ds5.series['DateTime']['Data'][-1])) sitename = ds5.globalattributes['site_name'] self.do_progress(text='Doing L6 partitioning: '+sitename+' ...') if "Options" not in cf: cf["Options"]={} cf["Options"]["call_mode"] = "interactive" ds6 = qcls.l6qc(cf,ds5) self.do_progress(text='Finished L6: '+sitename) logging.info(' Finished L6: '+sitename) self.do_progress(text='Saving L6 partitioned data ...') # put up the progress message outfilename = qcio.get_outfilenamefromcf(cf) if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return ncFile = qcio.nc_open_write(outfilename) outputlist = qcio.get_outputlistfromcf(cf,'nc') qcio.nc_write_series(ncFile,ds6,outputlist=outputlist) # save the L6 data self.do_progress(text='Finished saving L6 partitioned data') # tell the user we are done logging.info(' Finished saving L6 partitioned data') logging.info("")
def do_plotL3L4(self): """ Plot L3 (QA/QC and Corrected) and L4 (Gap Filled) data in blue and red, respectively Control File for do_l4qc function used. If L4 Control File not loaded, requires control file selection. """ if 'ds3' not in dir(self) or 'ds4' not in dir(self): self.cf = qcio.load_controlfile(path='controlfiles') if len(self.cf)==0: self.do_progress(text='Waiting for input ...') return l3filename = qcio.get_infilenamefromcf(self.cf) if not qcutils.file_exists(l3filename): self.do_progress(text='An error occurred, check the console ...'); return self.ds3 = qcio.nc_read_series(l3filename) if len(self.ds3.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds3; return l4filename = qcio.get_outfilenamefromcf(self.cf) self.ds4 = qcio.nc_read_series(l4filename) if len(self.ds4.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds4; return self.update_startenddate(str(self.ds3.series['DateTime']['Data'][0]), str(self.ds3.series['DateTime']['Data'][-1])) self.do_progress(text='Plotting L3 and L4 QC ...') cfname = self.ds4.globalattributes['controlfile_name'] self.cf = qcio.get_controlfilecontents(cfname) for nFig in self.cf['Plots'].keys(): si = qcutils.GetDateIndex(self.ds3.series['DateTime']['Data'],self.plotstartEntry.get(), ts=self.ds3.globalattributes['time_step'],default=0,match='exact') ei = qcutils.GetDateIndex(self.ds3.series['DateTime']['Data'],self.plotendEntry.get(), ts=self.ds3.globalattributes['time_step'],default=-1,match='exact') qcplot.plottimeseries(self.cf,nFig,self.ds3,self.ds4,si,ei) self.do_progress(text='Finished plotting L4') logging.info(' Finished plotting L4, check the GUI')
def do_l4qc(self): """ Call qcls.l4qc_gapfill function Performs L4 gap filling on L3 met data or Ingests L4 gap filled fluxes performed in external SOLO-ANN and c omputes daily sums Outputs L4 netCDF file to ncData folder Outputs L4 netCDF file to OzFlux folder ControlFiles: L4_year.txt or L4b.txt ControlFile contents (see ControlFile/Templates/L4.txt and ControlFile/Templates/L4b.txt for examples): [General]: Python control parameters (SOLO) Site characteristics parameters (Gap filling) [Files]: L3 input file name and path (Gap filling) L4 input file name and path (SOLO) L4 output file name and ncData folder path (both) L4 OzFlux output file name and OzFlux folder path [Variables]: Variable subset list for OzFlux output file (where available) """ logging.info(" Starting L4 processing ...") cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return infilename = qcio.get_infilenamefromcf(cf) if len(infilename)==0: self.do_progress(text='An error occurred, check the console ...'); return if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return ds3 = qcio.nc_read_series(infilename) if len(ds3.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del ds3; return ds3.globalattributes['controlfile_name'] = cf['controlfile_name'] self.update_startenddate(str(ds3.series['DateTime']['Data'][0]), str(ds3.series['DateTime']['Data'][-1])) sitename = ds3.globalattributes['site_name'] self.do_progress(text='Doing L4 gap filling drivers: '+sitename+' ...') if "Options" not in cf: cf["Options"]={} cf["Options"]["call_mode"] = "interactive" ds4 = qcls.l4qc(cf,ds3) if ds4.returncodes["alternate"]=="quit" or ds4.returncodes["solo"]=="quit": self.do_progress(text='Quitting L4: '+sitename) logging.info(' Quitting L4: '+sitename) else: self.do_progress(text='Finished L4: '+sitename) logging.info(' Finished L4: '+sitename) self.do_progress(text='Saving L4 gap filled data ...') # put up the progress message outfilename = qcio.get_outfilenamefromcf(cf) if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return ncFile = qcio.nc_open_write(outfilename) outputlist = qcio.get_outputlistfromcf(cf,'nc') qcio.nc_write_series(ncFile,ds4,outputlist=outputlist) # save the L4 data self.do_progress(text='Finished saving L4 gap filled data') # tell the user we are done logging.info(' Finished saving L4 gap filled data') logging.info("")
def l1qc(cf): # get the data series from the Excel file in_filename = qcio.get_infilenamefromcf(cf) if not qcutils.file_exists(in_filename, mode="quiet"): msg = " Input file " + in_filename + " not found ..." logger.error(msg) ds1 = qcio.DataStructure() ds1.returncodes = {"value": 1, "message": msg} return ds1 file_name, file_extension = os.path.splitext(in_filename) if "csv" in file_extension.lower(): ds1 = qcio.csv_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Excel datetime from the Python datetime objects qcutils.get_xldatefromdatetime(ds1) else: ds1 = qcio.xl_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Python datetime objects from the Excel datetime qcutils.get_datetimefromxldate(ds1) # get the netCDF attributes from the control file qcts.do_attributes(cf, ds1) # round the Python datetime to the nearest second qcutils.round_datetime(ds1, mode="nearest_second") #check for gaps in the Python datetime series and fix if present fixtimestepmethod = qcutils.get_keyvaluefromcf(cf, ["options"], "FixTimeStepMethod", default="round") if qcutils.CheckTimeStep(ds1): qcutils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod) # recalculate the Excel datetime qcutils.get_xldatefromdatetime(ds1) # get the Year, Month, Day etc from the Python datetime qcutils.get_ymdhmsfromdatetime(ds1) # write the processing level to a global attribute ds1.globalattributes['nc_level'] = str("L1") # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds1.globalattributes.keys(): ds1.globalattributes['start_date'] = str( ds1.series['DateTime']['Data'][0]) if 'end_date' not in ds1.globalattributes.keys(): ds1.globalattributes['end_date'] = str( ds1.series['DateTime']['Data'][-1]) # calculate variances from standard deviations and vice versa qcts.CalculateStandardDeviations(cf, ds1) # create new variables using user defined functions qcts.DoFunctions(cf, ds1) # create a series of synthetic downwelling shortwave radiation qcts.get_synthetic_fsd(ds1) # check missing data and QC flags are consistent qcutils.CheckQCFlags(ds1) return ds1
def do_l2qc(self): """ Call qcls.l2qc function Performs L2 QA/QC processing on raw data Outputs L2 netCDF file to ncData folder ControlFiles: L2_year.txt or L2.txt ControlFile contents (see ControlFile/Templates/L2.txt for example): [General]: Enter list of functions to be performed [Files]: L1 input file name and path L2 output file name and path [Variables]: Variable names and parameters for: Range check to set upper and lower rejection limits Diurnal check to reject observations by time of day that are outside specified standard deviation limits Timestamps for excluded dates Timestamps for excluded hours [Plots]: Variable lists for plot generation """ logging.info(" Starting L2 processing ...") self.do_progress(text='Load L2 Control File ...') self.cf = qcio.load_controlfile(path='controlfiles') if len(self.cf)==0: logging.info( " L2: no control file chosen") self.do_progress(text='Waiting for input ...') return infilename = qcio.get_infilenamefromcf(self.cf) if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return self.do_progress(text='Doing L2 QC ...') self.ds1 = qcio.nc_read_series(infilename) if len(self.ds1.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds1; return self.update_startenddate(str(self.ds1.series['DateTime']['Data'][0]), str(self.ds1.series['DateTime']['Data'][-1])) self.ds2 = qcls.l2qc(self.cf,self.ds1) logging.info(' Finished L2 QC process') self.do_progress(text='Finished L2 QC process') self.do_progress(text='Saving L2 QC ...') # put up the progress message outfilename = qcio.get_outfilenamefromcf(self.cf) if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return ncFile = qcio.nc_open_write(outfilename) qcio.nc_write_series(ncFile,self.ds2) # save the L2 data self.do_progress(text='Finished saving L2 QC data') # tdo_progressell the user we are done logging.info(' Finished saving L2 QC data') logging.info("")
def l1qc(cf): # get the data series from the Excel file in_filename = qcio.get_infilenamefromcf(cf) if not qcutils.file_exists(in_filename,mode="quiet"): msg = " Input file "+in_filename+" not found ..." log.error(msg) ds1 = qcio.DataStructure() ds1.returncodes = {"value":1,"message":msg} return ds1 file_name,file_extension = os.path.splitext(in_filename) if "csv" in file_extension.lower(): ds1 = qcio.csv_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Excel datetime from the Python datetime objects qcutils.get_xldatefromdatetime(ds1) else: ds1 = qcio.xl_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Python datetime objects from the Excel datetime qcutils.get_datetimefromxldate(ds1) # get the netCDF attributes from the control file qcts.do_attributes(cf,ds1) # round the Python datetime to the nearest second qcutils.round_datetime(ds1,mode="nearest_second") #check for gaps in the Python datetime series and fix if present fixtimestepmethod = qcutils.get_keyvaluefromcf(cf,["options"],"FixTimeStepMethod",default="round") if qcutils.CheckTimeStep(ds1): qcutils.FixTimeStep(ds1,fixtimestepmethod=fixtimestepmethod) # recalculate the Excel datetime qcutils.get_xldatefromdatetime(ds1) # get the Year, Month, Day etc from the Python datetime qcutils.get_ymdhmsfromdatetime(ds1) # write the processing level to a global attribute ds1.globalattributes['nc_level'] = str("L1") # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds1.globalattributes.keys(): ds1.globalattributes['start_date'] = str(ds1.series['DateTime']['Data'][0]) if 'end_date' not in ds1.globalattributes.keys(): ds1.globalattributes['end_date'] = str(ds1.series['DateTime']['Data'][-1]) # calculate variances from standard deviations and vice versa qcts.CalculateStandardDeviations(cf,ds1) # create new variables using user defined functions qcts.DoFunctions(cf,ds1) # create a series of synthetic downwelling shortwave radiation qcts.get_synthetic_fsd(ds1) return ds1
def do_plotL1L2(self): """ Plot L1 (raw) and L2 (QA/QC) data in blue and red, respectively Control File for do_l2qc function used. If L2 Control File not loaded, requires control file selection. """ if 'ds1' not in dir(self) or 'ds2' not in dir(self): self.cf = qcio.load_controlfile(path='controlfiles') if len(self.cf)==0: self.do_progress(text='Waiting for input ...'); return l1filename = qcio.get_infilenamefromcf(self.cf) if not qcutils.file_exists(l1filename): self.do_progress(text='An error occurred, check the console ...'); return self.ds1 = qcio.nc_read_series(l1filename) if len(self.ds1.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds1; return l2filename = qcio.get_outfilenamefromcf(self.cf) self.ds2 = qcio.nc_read_series(l2filename) if len(self.ds2.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds2; return self.update_startenddate(str(self.ds1.series['DateTime']['Data'][0]), str(self.ds1.series['DateTime']['Data'][-1])) self.do_progress(text='Plotting L1 & L2 QC ...') cfname = self.ds2.globalattributes['controlfile_name'] self.cf = qcio.get_controlfilecontents(cfname) for nFig in self.cf['Plots'].keys(): si = qcutils.GetDateIndex(self.ds1.series['DateTime']['Data'],self.plotstartEntry.get(), ts=self.ds1.globalattributes['time_step'],default=0,match='exact') ei = qcutils.GetDateIndex(self.ds1.series['DateTime']['Data'],self.plotendEntry.get(), ts=self.ds1.globalattributes['time_step'],default=-1,match='exact') plt_cf = self.cf['Plots'][str(nFig)] if 'Type' in plt_cf.keys(): if str(plt_cf['Type']).lower() =='xy': self.do_progress(text='Plotting L1 and L2 XY ...') qcplot.plotxy(self.cf,nFig,plt_cf,self.ds1,self.ds2,si,ei) else: self.do_progress(text='Plotting L1 and L2 QC ...') qcplot.plottimeseries(self.cf,nFig,self.ds1,self.ds2,si,ei) else: self.do_progress(text='Plotting L1 and L2 QC ...') qcplot.plottimeseries(self.cf,nFig,self.ds1,self.ds2,si,ei) self.do_progress(text='Finished plotting L1 and L2') logging.info(' Finished plotting L1 and L2, check the GUI')
def l1qc_read_files(cf): # get the data series from the Excel file in_filename = qcio.get_infilenamefromcf(cf) if not qcutils.file_exists(in_filename, mode="quiet"): msg = " Input file " + in_filename + " not found ..." log.error(msg) ds1 = qcio.DataStructure() ds1.returncodes = {"value": 1, "message": msg} return ds1 file_name, file_extension = os.path.splitext(in_filename) if "csv" in file_extension.lower(): ds1 = qcio.csv_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Excel datetime from the Python datetime objects qcutils.get_xldatefromdatetime(ds1) else: ds1 = qcio.xl_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Python datetime objects from the Excel datetime qcutils.get_datetimefromxldate(ds1) return ds1
def climatology(cf): nc_filename = qcio.get_infilenamefromcf(cf) if not qcutils.file_exists(nc_filename): return xl_filename = nc_filename.replace(".nc","_Climatology.xls") xlFile = xlwt.Workbook() ds = qcio.nc_read_series(nc_filename) # calculate Fa if it is not in the data structure if "Fa" not in ds.series.keys(): if "Fn" in ds.series.keys() and "Fg" in ds.series.keys(): qcts.CalculateAvailableEnergy(ds,Fa_out='Fa',Fn_in='Fn',Fg_in='Fg') else: log.warning(" Climatology: Fn or Fg not in data struicture") # get the time step ts = int(ds.globalattributes['time_step']) # get the site name SiteName = ds.globalattributes['site_name'] # get the datetime series dt = ds.series['DateTime']['Data'] Hdh = ds.series['Hdh']['Data'] Month = ds.series['Month']['Data'] # get the initial start and end dates StartDate = str(dt[0]) EndDate = str(dt[-1]) # find the start index of the first whole day (time=00:30) si = qcutils.GetDateIndex(dt,StartDate,ts=ts,default=0,match='startnextday') # find the end index of the last whole day (time=00:00) ei = qcutils.GetDateIndex(dt,EndDate,ts=ts,default=-1,match='endpreviousday') # get local views of the datetime series ldt = dt[si:ei+1] Hdh = Hdh[si:ei+1] Month = Month[si:ei+1] # get the number of time steps in a day and the number of days in the data ntsInDay = int(24.0*60.0/float(ts)) nDays = int(len(ldt))/ntsInDay for ThisOne in cf['Variables'].keys(): if "AltVarName" in cf['Variables'][ThisOne].keys(): ThisOne = cf['Variables'][ThisOne]["AltVarName"] if ThisOne in ds.series.keys(): log.info(" Doing climatology for "+ThisOne) data,f,a = qcutils.GetSeriesasMA(ds,ThisOne,si=si,ei=ei) if numpy.ma.count(data)==0: log.warning(" No data for "+ThisOne+", skipping ...") continue fmt_str = get_formatstring(cf,ThisOne,fmt_def='') xlSheet = xlFile.add_sheet(ThisOne) Av_all = do_diurnalstats(Month,Hdh,data,xlSheet,format_string=fmt_str,ts=ts) # now do it for each day # we want to preserve any data that has been truncated by the use of the "startnextday" # and "endpreviousday" match options used above. Here we revisit the start and end indices # and adjust these backwards and forwards respectively if data has been truncated. nDays_daily = nDays ei_daily = ei si_daily = si sdate = ldt[0] edate = ldt[-1] # is there data after the current end date? if dt[-1]>ldt[-1]: # if so, push the end index back by 1 day so it is included ei_daily = ei + ntsInDay nDays_daily = nDays_daily + 1 edate = ldt[-1]+datetime.timedelta(days=1) # is there data before the current start date? if dt[0]<ldt[0]: # if so, push the start index back by 1 day so it is included si_daily = si - ntsInDay nDays_daily = nDays_daily + 1 sdate = ldt[0]-datetime.timedelta(days=1) # get the data and use the "pad" option to add missing data if required to # complete the extra days data,f,a = qcutils.GetSeriesasMA(ds,ThisOne,si=si_daily,ei=ei_daily,mode="pad") data_daily = data.reshape(nDays_daily,ntsInDay) xlSheet = xlFile.add_sheet(ThisOne+'(day)') write_data_1columnpertimestep(xlSheet, data_daily, ts, startdate=sdate, format_string=fmt_str) data_daily_i = do_2dinterpolation(data_daily) xlSheet = xlFile.add_sheet(ThisOne+'i(day)') write_data_1columnpertimestep(xlSheet, data_daily_i, ts, startdate=sdate, format_string=fmt_str) elif ThisOne=="EF": log.info(" Doing evaporative fraction") EF = numpy.ma.zeros([48,12]) + float(c.missing_value) Hdh,f,a = qcutils.GetSeriesasMA(ds,'Hdh',si=si,ei=ei) Fa,f,a = qcutils.GetSeriesasMA(ds,'Fa',si=si,ei=ei) Fe,f,a = qcutils.GetSeriesasMA(ds,'Fe',si=si,ei=ei) for m in range(1,13): mi = numpy.where(Month==m)[0] Fa_Num,Hr,Fa_Av,Sd,Mx,Mn = get_diurnalstats(Hdh[mi],Fa[mi],ts) Fe_Num,Hr,Fe_Av,Sd,Mx,Mn = get_diurnalstats(Hdh[mi],Fe[mi],ts) index = numpy.ma.where((Fa_Num>4)&(Fe_Num>4)) EF[:,m-1][index] = Fe_Av[index]/Fa_Av[index] # reject EF values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf,'EF') EF = numpy.ma.filled(numpy.ma.masked_where((EF>upr)|(EF<lwr),EF),float(c.missing_value)) # write the EF to the Excel file xlSheet = xlFile.add_sheet('EF') write_data_1columnpermonth(xlSheet, EF, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values EFi = do_2dinterpolation(EF) xlSheet = xlFile.add_sheet('EFi') write_data_1columnpermonth(xlSheet, EFi, ts, format_string='0.00') # now do EF for each day Fa,f,a = qcutils.GetSeriesasMA(ds,'Fa',si=si,ei=ei) Fe,f,a = qcutils.GetSeriesasMA(ds,'Fe',si=si,ei=ei) EF = Fe/Fa EF = numpy.ma.filled(numpy.ma.masked_where((EF>upr)|(EF<lwr),EF),float(c.missing_value)) EF_daily = EF.reshape(nDays,ntsInDay) xlSheet = xlFile.add_sheet('EF(day)') write_data_1columnpertimestep(xlSheet, EF_daily, ts, startdate=ldt[0], format_string='0.00') EFi = do_2dinterpolation(EF_daily) xlSheet = xlFile.add_sheet('EFi(day)') write_data_1columnpertimestep(xlSheet, EFi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne=="BR": log.info(" Doing Bowen ratio") BR = numpy.ma.zeros([48,12]) + float(c.missing_value) Fe,f,a = qcutils.GetSeriesasMA(ds,'Fe',si=si,ei=ei) Fh,f,a = qcutils.GetSeriesasMA(ds,'Fh',si=si,ei=ei) for m in range(1,13): mi = numpy.where(Month==m)[0] Fh_Num,Hr,Fh_Av,Sd,Mx,Mn = get_diurnalstats(Hdh[mi],Fh[mi],ts) Fe_Num,Hr,Fe_Av,Sd,Mx,Mn = get_diurnalstats(Hdh[mi],Fe[mi],ts) index = numpy.ma.where((Fh_Num>4)&(Fe_Num>4)) BR[:,m-1][index] = Fh_Av[index]/Fe_Av[index] # reject BR values greater than upper limit or less than lower limit upr,lwr = get_rangecheck_limit(cf,'BR') BR = numpy.ma.filled(numpy.ma.masked_where((BR>upr)|(BR<lwr),BR),float(c.missing_value)) # write the BR to the Excel file xlSheet = xlFile.add_sheet('BR') write_data_1columnpermonth(xlSheet, BR, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values BRi = do_2dinterpolation(BR) xlSheet = xlFile.add_sheet('BRi') write_data_1columnpermonth(xlSheet, BRi, ts, format_string='0.00') # now do BR for each day ... Fe,f,a = qcutils.GetSeriesasMA(ds,'Fe',si=si,ei=ei) Fh,f,a = qcutils.GetSeriesasMA(ds,'Fh',si=si,ei=ei) BR = Fh/Fe BR = numpy.ma.filled(numpy.ma.masked_where((BR>upr)|(BR<lwr),BR),float(c.missing_value)) BR_daily = BR.reshape(nDays,ntsInDay) xlSheet = xlFile.add_sheet('BR(day)') write_data_1columnpertimestep(xlSheet, BR_daily, ts, startdate=ldt[0], format_string='0.00') BRi = do_2dinterpolation(BR_daily) xlSheet = xlFile.add_sheet('BRi(day)') write_data_1columnpertimestep(xlSheet, BRi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne=="WUE": log.info(" Doing ecosystem WUE") WUE = numpy.ma.zeros([48,12]) + float(c.missing_value) Fe,f,a = qcutils.GetSeriesasMA(ds,'Fe',si=si,ei=ei) Fc,f,a = qcutils.GetSeriesasMA(ds,'Fc',si=si,ei=ei) for m in range(1,13): mi = numpy.where(Month==m)[0] Fc_Num,Hr,Fc_Av,Sd,Mx,Mn = get_diurnalstats(Hdh[mi],Fc[mi],ts) Fe_Num,Hr,Fe_Av,Sd,Mx,Mn = get_diurnalstats(Hdh[mi],Fe[mi],ts) index = numpy.ma.where((Fc_Num>4)&(Fe_Num>4)) WUE[:,m-1][index] = Fc_Av[index]/Fe_Av[index] # reject WUE values greater than upper limit or less than lower limit upr,lwr = get_rangecheck_limit(cf,'WUE') WUE = numpy.ma.filled(numpy.ma.masked_where((WUE>upr)|(WUE<lwr),WUE),float(c.missing_value)) # write the WUE to the Excel file xlSheet = xlFile.add_sheet('WUE') write_data_1columnpermonth(xlSheet, WUE, ts, format_string='0.00000') # do the 2D interpolation to fill missing EF values WUEi = do_2dinterpolation(WUE) xlSheet = xlFile.add_sheet('WUEi') write_data_1columnpermonth(xlSheet, WUEi, ts, format_string='0.00000') # now do WUE for each day ... Fe,f,a = qcutils.GetSeriesasMA(ds,'Fe',si=si,ei=ei) Fc,f,a = qcutils.GetSeriesasMA(ds,'Fc',si=si,ei=ei) WUE = Fc/Fe WUE = numpy.ma.filled(numpy.ma.masked_where((WUE>upr)|(WUE<lwr),WUE),float(c.missing_value)) WUE_daily = WUE.reshape(nDays,ntsInDay) xlSheet = xlFile.add_sheet('WUE(day)') write_data_1columnpertimestep(xlSheet, WUE_daily, ts, startdate=ldt[0], format_string='0.00000') WUEi = do_2dinterpolation(WUE_daily) xlSheet = xlFile.add_sheet('WUEi(day)') write_data_1columnpertimestep(xlSheet, WUEi, ts, startdate=ldt[0], format_string='0.00000') else: log.warning(" qcclim.climatology: requested variable "+ThisOne+" not in data structure") continue log.info(" Saving Excel file "+xl_filename) xlFile.save(xl_filename)
for level in level_list: if level.lower()=="l1": # L1 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] logging.info('Starting L1 processing with '+cfname) cf = qcio.get_controlfilecontents(cfname) qcio.xl2nc(cf,'L1') logging.info('Finished L1 processing with '+cfname) elif level.lower()=="l2": # L2 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] logging.info('Starting L2 processing with '+cfname) cf = qcio.get_controlfilecontents(cfname) infilename = qcio.get_infilenamefromcf(cf) ds1 = qcio.nc_read_series(infilename) ds2 = qcls.l2qc(cf,ds1) outfilename = qcio.get_outfilenamefromcf(cf) ncFile = qcio.nc_open_write(outfilename) qcio.nc_write_series(ncFile,ds2) logging.info('Finished L2 processing with '+cfname) elif level.lower()=="l3": # L3 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] logging.info('Starting L3 processing with '+cfname) cf = qcio.get_controlfilecontents(cfname) infilename = qcio.get_infilenamefromcf(cf) ds2 = qcio.nc_read_series(infilename) ds3 = qcls.l3qc(cf,ds2)
def climatology(cf): nc_filename = qcio.get_infilenamefromcf(cf) if not qcutils.file_exists(nc_filename): return xl_filename = nc_filename.replace(".nc", "_Climatology.xls") xlFile = xlwt.Workbook() ds = qcio.nc_read_series(nc_filename) # calculate Fa if it is not in the data structure if "Fa" not in ds.series.keys(): if "Fn" in ds.series.keys() and "Fg" in ds.series.keys(): qcts.CalculateAvailableEnergy(ds, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg') else: log.warning(" Climatology: Fn or Fg not in data struicture") # get the time step ts = int(ds.globalattributes['time_step']) # get the site name SiteName = ds.globalattributes['site_name'] # get the datetime series dt = ds.series['DateTime']['Data'] Hdh = ds.series['Hdh']['Data'] Month = ds.series['Month']['Data'] # get the initial start and end dates StartDate = str(dt[0]) EndDate = str(dt[-1]) # find the start index of the first whole day (time=00:30) si = qcutils.GetDateIndex(dt, StartDate, ts=ts, default=0, match='startnextday') # find the end index of the last whole day (time=00:00) ei = qcutils.GetDateIndex(dt, EndDate, ts=ts, default=-1, match='endpreviousday') # get local views of the datetime series ldt = dt[si:ei + 1] Hdh = Hdh[si:ei + 1] Month = Month[si:ei + 1] # get the number of time steps in a day and the number of days in the data ntsInDay = int(24.0 * 60.0 / float(ts)) nDays = int(len(ldt)) / ntsInDay for ThisOne in cf['Variables'].keys(): if "AltVarName" in cf['Variables'][ThisOne].keys(): ThisOne = cf['Variables'][ThisOne]["AltVarName"] if ThisOne in ds.series.keys(): log.info(" Doing climatology for " + ThisOne) data, f, a = qcutils.GetSeriesasMA(ds, ThisOne, si=si, ei=ei) if numpy.ma.count(data) == 0: log.warning(" No data for " + ThisOne + ", skipping ...") continue fmt_str = get_formatstring(cf, ThisOne, fmt_def='') xlSheet = xlFile.add_sheet(ThisOne) Av_all = do_diurnalstats(Month, Hdh, data, xlSheet, format_string=fmt_str, ts=ts) # now do it for each day # we want to preserve any data that has been truncated by the use of the "startnextday" # and "endpreviousday" match options used above. Here we revisit the start and end indices # and adjust these backwards and forwards respectively if data has been truncated. nDays_daily = nDays ei_daily = ei si_daily = si sdate = ldt[0] edate = ldt[-1] # is there data after the current end date? if dt[-1] > ldt[-1]: # if so, push the end index back by 1 day so it is included ei_daily = ei + ntsInDay nDays_daily = nDays_daily + 1 edate = ldt[-1] + datetime.timedelta(days=1) # is there data before the current start date? if dt[0] < ldt[0]: # if so, push the start index back by 1 day so it is included si_daily = si - ntsInDay nDays_daily = nDays_daily + 1 sdate = ldt[0] - datetime.timedelta(days=1) # get the data and use the "pad" option to add missing data if required to # complete the extra days data, f, a = qcutils.GetSeriesasMA(ds, ThisOne, si=si_daily, ei=ei_daily, mode="pad") data_daily = data.reshape(nDays_daily, ntsInDay) xlSheet = xlFile.add_sheet(ThisOne + '(day)') write_data_1columnpertimestep(xlSheet, data_daily, ts, startdate=sdate, format_string=fmt_str) data_daily_i = do_2dinterpolation(data_daily) xlSheet = xlFile.add_sheet(ThisOne + 'i(day)') write_data_1columnpertimestep(xlSheet, data_daily_i, ts, startdate=sdate, format_string=fmt_str) elif ThisOne == "EF": log.info(" Doing evaporative fraction") EF = numpy.ma.zeros([48, 12]) + float(c.missing_value) Hdh, f, a = qcutils.GetSeriesasMA(ds, 'Hdh', si=si, ei=ei) Fa, f, a = qcutils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei) Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fa_Num, Hr, Fa_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fa[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fa_Num > 4) & (Fe_Num > 4)) EF[:, m - 1][index] = Fe_Av[index] / Fa_Av[index] # reject EF values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'EF') EF = numpy.ma.filled( numpy.ma.masked_where((EF > upr) | (EF < lwr), EF), float(c.missing_value)) # write the EF to the Excel file xlSheet = xlFile.add_sheet('EF') write_data_1columnpermonth(xlSheet, EF, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values EFi = do_2dinterpolation(EF) xlSheet = xlFile.add_sheet('EFi') write_data_1columnpermonth(xlSheet, EFi, ts, format_string='0.00') # now do EF for each day Fa, f, a = qcutils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei) Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) EF = Fe / Fa EF = numpy.ma.filled( numpy.ma.masked_where((EF > upr) | (EF < lwr), EF), float(c.missing_value)) EF_daily = EF.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('EF(day)') write_data_1columnpertimestep(xlSheet, EF_daily, ts, startdate=ldt[0], format_string='0.00') EFi = do_2dinterpolation(EF_daily) xlSheet = xlFile.add_sheet('EFi(day)') write_data_1columnpertimestep(xlSheet, EFi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne == "BR": log.info(" Doing Bowen ratio") BR = numpy.ma.zeros([48, 12]) + float(c.missing_value) Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fh, f, a = qcutils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fh_Num, Hr, Fh_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fh[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fh_Num > 4) & (Fe_Num > 4)) BR[:, m - 1][index] = Fh_Av[index] / Fe_Av[index] # reject BR values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'BR') BR = numpy.ma.filled( numpy.ma.masked_where((BR > upr) | (BR < lwr), BR), float(c.missing_value)) # write the BR to the Excel file xlSheet = xlFile.add_sheet('BR') write_data_1columnpermonth(xlSheet, BR, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values BRi = do_2dinterpolation(BR) xlSheet = xlFile.add_sheet('BRi') write_data_1columnpermonth(xlSheet, BRi, ts, format_string='0.00') # now do BR for each day ... Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fh, f, a = qcutils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei) BR = Fh / Fe BR = numpy.ma.filled( numpy.ma.masked_where((BR > upr) | (BR < lwr), BR), float(c.missing_value)) BR_daily = BR.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('BR(day)') write_data_1columnpertimestep(xlSheet, BR_daily, ts, startdate=ldt[0], format_string='0.00') BRi = do_2dinterpolation(BR_daily) xlSheet = xlFile.add_sheet('BRi(day)') write_data_1columnpertimestep(xlSheet, BRi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne == "WUE": log.info(" Doing ecosystem WUE") WUE = numpy.ma.zeros([48, 12]) + float(c.missing_value) Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fc, f, a = qcutils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fc_Num, Hr, Fc_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fc[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fc_Num > 4) & (Fe_Num > 4)) WUE[:, m - 1][index] = Fc_Av[index] / Fe_Av[index] # reject WUE values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'WUE') WUE = numpy.ma.filled( numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE), float(c.missing_value)) # write the WUE to the Excel file xlSheet = xlFile.add_sheet('WUE') write_data_1columnpermonth(xlSheet, WUE, ts, format_string='0.00000') # do the 2D interpolation to fill missing EF values WUEi = do_2dinterpolation(WUE) xlSheet = xlFile.add_sheet('WUEi') write_data_1columnpermonth(xlSheet, WUEi, ts, format_string='0.00000') # now do WUE for each day ... Fe, f, a = qcutils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fc, f, a = qcutils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei) WUE = Fc / Fe WUE = numpy.ma.filled( numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE), float(c.missing_value)) WUE_daily = WUE.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('WUE(day)') write_data_1columnpertimestep(xlSheet, WUE_daily, ts, startdate=ldt[0], format_string='0.00000') WUEi = do_2dinterpolation(WUE_daily) xlSheet = xlFile.add_sheet('WUEi(day)') write_data_1columnpertimestep(xlSheet, WUEi, ts, startdate=ldt[0], format_string='0.00000') else: log.warning(" qcclim.climatology: requested variable " + ThisOne + " not in data structure") continue log.info(" Saving Excel file " + xl_filename) xlFile.save(xl_filename)
cfname = cf_batch["Levels"][level][i] logging.info('Starting L1 processing with '+cfname) cf = qcio.get_controlfilecontents(cfname) ds1 = qcls.l1qc(cf) outfilename = qcio.get_outfilenamefromcf(cf) ncFile = qcio.nc_open_write(outfilename) qcio.nc_write_series(ncFile,ds1) logging.info('Finished L1 processing with '+cfname) logging.info('') elif level.lower()=="l2": # L2 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] logging.info('Starting L2 processing with '+cfname) cf = qcio.get_controlfilecontents(cfname) infilename = qcio.get_infilenamefromcf(cf) ds1 = qcio.nc_read_series(infilename) ds2 = qcls.l2qc(cf,ds1) outfilename = qcio.get_outfilenamefromcf(cf) ncFile = qcio.nc_open_write(outfilename) qcio.nc_write_series(ncFile,ds2) logging.info('Finished L2 processing with '+cfname) logging.info('') elif level.lower()=="l3": # L3 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] logging.info('Starting L3 processing with '+cfname) cf = qcio.get_controlfilecontents(cfname) infilename = qcio.get_infilenamefromcf(cf) ds2 = qcio.nc_read_series(infilename)
def do_l3qc(self): """ Call qcls.l3qc_sitename function Performs L3 Corrections and QA/QC processing on L2 data Outputs L3 netCDF file to ncData folder Outputs L3 netCDF file to OzFlux folder Available corrections: * corrections requiring ancillary measurements or samples marked with an asterisk Linear correction fixed slope linearly shifting slope Conversion of virtual temperature to actual temperature 2D Coordinate rotation Massman correction for frequency attenuation* Webb, Pearman and Leuning correction for flux effects on density measurements Conversion of virtual heat flux to actual heat flux Correction of soil moisture content to empirical calibration curve* Addition of soil heat storage to ground ground heat flux* ControlFiles: L3_year.txt or L3a.txt ControlFile contents (see ControlFile/Templates/L3.txt for example): [General]: Python control parameters [Files]: L2 input file name and path L3 output file name and ncData folder path L3 OzFlux output file name and OzFlux folder path [Massman] (where available): Constants used in frequency attenuation correction zmd: instrument height (z) less zero-plane displacement height (d), m z0: aerodynamic roughness length, m angle: angle from CSAT mounting point between CSAT and IRGA mid-path, degrees CSATarm: distance from CSAT mounting point to CSAT mid-path, m IRGAarm: distance from CSAT mounting point to IRGA mid-path, m [Soil]: Constants used in correcting Fg for storage and in empirical corrections of soil water content FgDepth: Heat flux plate depth, m BulkDensity: Soil bulk density, kg/m3 OrganicContent: Soil organic content, fraction SwsDefault Constants for empirical corrections using log(sensor) and exp(sensor) functions (SWC_a0, SWC_a1, SWC_b0, SWC_b1, SWC_t, TDR_a0, TDR_a1, TDR_b0, TDR_b1, TDR_t) Variable and attributes lists (empSWCin, empSWCout, empTDRin, empTDRout, linTDRin, SWCattr, TDRattr) [Output]: Variable subset list for OzFlux output file [Variables]: Variable names and parameters for: Range check to set upper and lower rejection limits Diurnal check to reject observations by time of day that are outside specified standard deviation limits Timestamps, slope, and offset for Linear correction [Plots]: Variable lists for plot generation """ logging.info(" Starting L3 processing ...") self.cf = qcio.load_controlfile(path='controlfiles') if len(self.cf)==0: logging.info( " L3: no control file chosen") self.do_progress(text='Waiting for input ...') return infilename = qcio.get_infilenamefromcf(self.cf) if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return self.ds2 = qcio.nc_read_series(infilename) if len(self.ds2.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds2; return self.update_startenddate(str(self.ds2.series['DateTime']['Data'][0]), str(self.ds2.series['DateTime']['Data'][-1])) self.do_progress(text='Doing L3 QC & Corrections ...') self.ds3 = qcls.l3qc(self.cf,self.ds2) self.do_progress(text='Finished L3') txtstr = ' Finished L3: Standard processing for site: ' txtstr = txtstr+self.ds3.globalattributes['site_name'].replace(' ','') logging.info(txtstr) self.do_progress(text='Saving L3 QC & Corrected NetCDF data ...') # put up the progress message outfilename = qcio.get_outfilenamefromcf(self.cf) if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return ncFile = qcio.nc_open_write(outfilename) outputlist = qcio.get_outputlistfromcf(self.cf,'nc') qcio.nc_write_series(ncFile,self.ds3,outputlist=outputlist) # save the L3 data self.do_progress(text='Finished saving L3 QC & Corrected NetCDF data') # tell the user we are done logging.info(' Finished saving L3 QC & Corrected NetCDF data') logging.info("")
import qcplot import qcutils import statsmodels.api as sm import sys import time nfig = 0 plotwidth = 10.9 plotheight = 7.5 # load the control file cf = qcio.load_controlfile(path='../controlfiles') if len(cf) == 0: sys.exit() min_n = int(cf["General"]["minimum_number"]) min_r = float(cf["General"]["minimum_correlation"]) # get the input file name fname = qcio.get_infilenamefromcf(cf) if not os.path.exists(fname): print " compare_ah: Input netCDF file " + fname + " doesn't exist" sys.exit() # read the input file and return the data structure ds = qcio.nc_read_series(fname) if len(ds.series.keys()) == 0: print time.strftime('%X') + ' netCDF file ' + fname + ' not found' sys.exit() # get the site name SiteName = ds.globalattributes['site_name'] # get the time step ts = int(ds.globalattributes['time_step']) # get the datetime series DateTime = ds.series['DateTime']['Data'] # get the initial start and end dates