def do_plotfluxnet(self,mode="standard"): """ Plot FluxNet style time series of data.""" self.do_progress(text='Doing FluxNet plots ...') if mode=="standard": stdname = "controlfiles/standard/fluxnet.txt" if os.path.exists(stdname): cf = qcio.get_controlfilecontents(stdname) filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file') if len(filename)==0 or not os.path.exists(filename): self.do_progress(text='Waiting for input ...'); return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/" cf["Files"]["in_filename"] = ntpath.split(filename)[1] else: self.do_progress(text='Loading control file ...') cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return else: self.do_progress(text='Loading control file ...') cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return self.do_progress(text='Plotting FluxNet style plots ...') qcplot.plot_fluxnet(cf) self.do_progress(text='Finished FluxNet plotting') logging.info(' Finished FluxNet plotting')
def do_plotfingerprint(self,mode="standard"): """ Plot fingerprint""" logging.info(' Starting fingerprint plot') self.do_progress(text='Doing fingerprint plot ...') if mode=="standard": stdname = "controlfiles/standard/fingerprint.txt" if os.path.exists(stdname): cf = qcio.get_controlfilecontents(stdname) filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file') if len(filename)==0 or not os.path.exists(filename): self.do_progress(text='Waiting for input ...'); return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/" cf["Files"]["in_filename"] = ntpath.split(filename)[1] else: self.do_progress(text='Loading control file ...') cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return else: self.do_progress(text='Loading control file ...') cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return if "Options" not in cf: cf["Options"]={} cf["Options"]["call_mode"] = "interactive" self.do_progress(text='Plotting fingerprint ...') qcplot.plot_fingerprint(cf) self.do_progress(text='Finished plotting fingerprint') logging.info(' Finished plotting fingerprint') logging.info("")
def do_climatology(self,mode="standard"): """ Calls qcclim.climatology """ logging.info(' Starting climatology') self.do_progress(text='Doing climatology ...') if mode=="standard": stdname = "controlfiles/standard/climatology.txt" if os.path.exists(stdname): cf = qcio.get_controlfilecontents(stdname) self.do_progress(text='Opening input file ...') filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file') if len(filename)==0: logging.info( " Climatology: no input file chosen") self.do_progress(text='Waiting for input ...') return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/" cf["Files"]["in_filename"] = ntpath.split(filename)[1] else: self.do_progress(text='Loading control file ...') cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return else: self.do_progress(text='Loading control file ...') cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return self.do_progress(text='Doing the climatology') qcclim.climatology(cf) self.do_progress(text='Finished climatology') logging.info(' Finished climatology') logging.info("")
def compare_eddypro(): epname = qcio.get_filename_dialog(title='Choose an EddyPro full output file') ofname = qcio.get_filename_dialog(title='Choose an L3 output file') ds_ep = qcio.read_eddypro_full(epname) ds_of = qcio.nc_read_series(ofname) dt_ep = ds_ep.series['DateTime']['Data'] dt_of = ds_of.series['DateTime']['Data'] start_datetime = max([dt_ep[0],dt_of[0]]) end_datetime = min([dt_ep[-1],dt_of[-1]]) si_of = qcutils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact') ei_of = qcutils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact') si_ep = qcutils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact') ei_ep = qcutils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact') us_of = qcutils.GetVariableAsDictionary(ds_of,'ustar',si=si_of,ei=ei_of) us_ep = qcutils.GetVariableAsDictionary(ds_ep,'ustar',si=si_ep,ei=ei_ep) Fh_of = qcutils.GetVariableAsDictionary(ds_of,'Fh',si=si_of,ei=ei_of) Fh_ep = qcutils.GetVariableAsDictionary(ds_ep,'Fh',si=si_ep,ei=ei_ep) Fe_of = qcutils.GetVariableAsDictionary(ds_of,'Fe',si=si_of,ei=ei_of) Fe_ep = qcutils.GetVariableAsDictionary(ds_ep,'Fe',si=si_ep,ei=ei_ep) Fc_of = qcutils.GetVariableAsDictionary(ds_of,'Fc',si=si_of,ei=ei_of) Fc_ep = qcutils.GetVariableAsDictionary(ds_ep,'Fc',si=si_ep,ei=ei_ep) # copy the range check values from the OFQC attributes to the EP attributes for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]): for item in ["rangecheck_upper", "rangecheck_lower"]: if item in of["Attr"]: ep["Attr"][item] = of["Attr"][item] # apply QC to the EddyPro data qcck.ApplyRangeCheckToVariable(us_ep) qcck.ApplyRangeCheckToVariable(Fc_ep) qcck.ApplyRangeCheckToVariable(Fe_ep) qcck.ApplyRangeCheckToVariable(Fh_ep) # plot the comparison plt.ion() fig = plt.figure(1,figsize=(8,8)) qcplot.xyplot(us_ep["Data"],us_of["Data"],sub=[2,2,1],regr=2,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)') qcplot.xyplot(Fh_ep["Data"],Fh_of["Data"],sub=[2,2,2],regr=2,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)') qcplot.xyplot(Fe_ep["Data"],Fe_of["Data"],sub=[2,2,3],regr=2,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)') qcplot.xyplot(Fc_ep["Data"],Fc_of["Data"],sub=[2,2,4],regr=2,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)') plt.tight_layout() plt.draw() plt.ioff()
def do_nc2xls(self): """ Calls qcio.nc_2xls. """ logging.info(" Starting conversion to Excel file") self.do_progress(text="Choosing netCDF file ...") ncfilename = qcio.get_filename_dialog(path="../Sites",title="Choose a netCDF file") if len(ncfilename)==0: self.do_progress(text="Waiting for input ..."); return self.do_progress(text="Converting netCDF file to Excel file") qcio.nc_2xls(ncfilename,outputlist=None) self.do_progress(text="Finished converting netCDF file") logging.info(" Finished converting netCDF file") logging.info("")
def do_nc2reddyproc(self): """ Calls qcio.reddyproc_write_csv.""" logging.info(' Starting conversion to REddyProc CSV file') self.do_progress(text="Choosing netCDF file ...") ncfilename = qcio.get_filename_dialog(path="../Sites",title="Choose a netCDF file") if len(ncfilename)==0 or not os.path.exists(ncfilename): self.do_progress(text="Waiting for input ..."); return self.do_progress(text='Converting nc to REddyProc CSV ...') qcio.reddyproc_write_csv(ncfilename) logging.info(' Finished conversion') self.do_progress(text='Finished conversion') logging.info("")
def compare_eddypro(): epname = qcio.get_filename_dialog(title='Choose an EddyPro full output file') ofname = qcio.get_filename_dialog(title='Choose an L3 output file') ds_ep = qcio.read_eddypro_full(epname) ds_of = qcio.nc_read_series(ofname) dt_ep = ds_ep.series['DateTime']['Data'] dt_of = ds_of.series['DateTime']['Data'] si = dt_of.index(dt_ep[0]) ei = dt_of.index(dt_ep[-1]) us_of,f,a = qcutils.GetSeriesasMA(ds_of,'ustar',si=si,ei=ei) us_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'ustar') Fh_of,f,a = qcutils.GetSeriesasMA(ds_of,'Fh',si=si,ei=ei) Fh_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'Fh') Fe_of,f,a = qcutils.GetSeriesasMA(ds_of,'Fe',si=si,ei=ei) Fe_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'Fe') Fc_of,f,a = qcutils.GetSeriesasMA(ds_of,'Fc',si=si,ei=ei) Fc_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'Fc') us_of.mask = numpy.ma.mask_or(us_of.mask,us_ep.mask) us_ep.mask = numpy.ma.mask_or(us_of.mask,us_ep.mask) Fh_of.mask = numpy.ma.mask_or(Fh_of.mask,Fh_ep.mask) Fh_ep.mask = numpy.ma.mask_or(Fh_of.mask,Fh_ep.mask) Fe_of.mask = numpy.ma.mask_or(Fe_of.mask,Fe_ep.mask) Fe_ep.mask = numpy.ma.mask_or(Fe_of.mask,Fe_ep.mask) Fc_of.mask = numpy.ma.mask_or(Fc_of.mask,Fc_ep.mask) Fc_ep.mask = numpy.ma.mask_or(Fc_of.mask,Fc_ep.mask) plt.ion() fig = plt.figure(1,figsize=(8,8)) qcplot.xyplot(us_ep,us_of,sub=[2,2,1],regr=1,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)') qcplot.xyplot(Fh_ep,Fh_of,sub=[2,2,2],regr=1,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)') qcplot.xyplot(Fe_ep,Fe_of,sub=[2,2,3],regr=1,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)') qcplot.xyplot(Fc_ep,Fc_of,sub=[2,2,4],regr=1,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)') plt.tight_layout() plt.draw() plt.ioff()
def do_plotquickcheck(self): """ Plot quickcheck""" self.do_progress(text='Loading control file ...') stdname = "controlfiles/standard/quickcheck.txt" if os.path.exists(stdname): cf = qcio.get_controlfilecontents(stdname) filename = qcio.get_filename_dialog(path='../Sites',title='Choose an input file') if len(filename)==0: self.do_progress(text='Waiting for input ...'); return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/" cf["Files"]["in_filename"] = ntpath.split(filename)[1] else: cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return self.do_progress(text='Plotting quickcheck ...') qcplot.plot_quickcheck(cf) self.do_progress(text='Finished plotting quickcheck') logging.info(' Finished plotting quickcheck')
def do_cpd(self): """ Calls qccpd.cpd_main Compares the results OzFluxQC (L3) with those from EddyPro (full output). """ logging.info(' Starting estimation u* threshold using CPD') self.do_progress(text='Estimating u* threshold using CPD ...') stdname = "controlfiles/standard/cpd.txt" if os.path.exists(stdname): cf = qcio.get_controlfilecontents(stdname) filename = qcio.get_filename_dialog(path='../Sites',title='Choose an input nc file') if len(filename)==0: self.do_progress(text='Waiting for input ...'); return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/" cf["Files"]["in_filename"] = ntpath.split(filename)[1] else: cf = qcio.load_controlfile(path='controlfiles') if len(cf)==0: self.do_progress(text='Waiting for input ...'); return if "Options" not in cf: cf["Options"]={} cf["Options"]["call_mode"] = "interactive" qccpd.cpd_main(cf) self.do_progress(text='Finished estimating u* threshold') logging.info(' Finished estimating u* threshold') logging.info("")
# coding: utf-8 import qcio import qcutils import statsmodels.api as sm duname=qcio.get_filename_dialog() drname=qcio.get_filename_dialog() ds_du=qcio.nc_read_series(duname) ds_dr=qcio.nc_read_series(drname) dt_du=ds.series["DateTime"]["Data"] dt_du=ds_du.series["DateTime"]["Data"] dt_dr=ds_dr.series["DateTime"]["Data"] print dt_du[0],dt_dr[0] print dt_du[-1],dt_dr[-1] ts=ds_du.globalattributes["time_step"] si=qcutils.GetDateIndex(dt_du,str(dt_du[0]),ts=ts,match="startnextday") ei=qcutils.GetDateIndex(dt_du,str(dt_dr[-1]),ts=ts,match="endpreviousday") print si,ei Fsd_du,f=qcutils.GetSeriesasMA(ds_du,'Fsd',si=si,ei=ei) Fsd_dr,f=qcutils.GetSeriesasMA(ds_dr,'Fsd',si=si,ei=ei) plot(Fsd_dr,Fsd_du) ei=qcutils.GetDateIndex(ds_du,"2009-01-01 00:00",ts=ts,match="endpreviousday") ei=qcutils.GetDateIndex(dt_du,"2009-01-01 00:00",ts=ts,match="endpreviousday") fig=figure(1) Fsd_du_2008,f=qcutils.GetSeriesasMA(ds_du,'Fsd',si=si,ei=ei) Fsd_dr_2008,f=qcutils.GetSeriesasMA(ds_dr,'Fsd',si=si,ei=ei) plot(Fsd_du_2008,Fsd_dr_2008,'b.') Fsd_du_2008.mask=(Fsd_du_2008.mask==True)|(Fsd_dr_2008.mask==True) Fsd_dr_2008.mask=(Fsd_du_2008.mask==True)|(Fsd_dr_2008.mask==True) Fsd_du_2008=numpy.ma.compressed(Fsd_du_2008) Fsd_dr_2008=numpy.ma.compressed(Fsd_dr_2008) x=Fsd_du_2008
def compare_eddypro(): epname = qcio.get_filename_dialog( title='Choose an EddyPro full output file') ofname = qcio.get_filename_dialog(title='Choose an L3 output file') ds_ep = qcio.read_eddypro_full(epname) ds_of = qcio.nc_read_series(ofname) dt_ep = ds_ep.series['DateTime']['Data'] dt_of = ds_of.series['DateTime']['Data'] start_datetime = max([dt_ep[0], dt_of[0]]) end_datetime = min([dt_ep[-1], dt_of[-1]]) si_of = qcutils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact') ei_of = qcutils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact') si_ep = qcutils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact') ei_ep = qcutils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact') us_of = qcutils.GetVariableAsDictionary(ds_of, 'ustar', si=si_of, ei=ei_of) us_ep = qcutils.GetVariableAsDictionary(ds_ep, 'ustar', si=si_ep, ei=ei_ep) Fh_of = qcutils.GetVariableAsDictionary(ds_of, 'Fh', si=si_of, ei=ei_of) Fh_ep = qcutils.GetVariableAsDictionary(ds_ep, 'Fh', si=si_ep, ei=ei_ep) Fe_of = qcutils.GetVariableAsDictionary(ds_of, 'Fe', si=si_of, ei=ei_of) Fe_ep = qcutils.GetVariableAsDictionary(ds_ep, 'Fe', si=si_ep, ei=ei_ep) Fc_of = qcutils.GetVariableAsDictionary(ds_of, 'Fc', si=si_of, ei=ei_of) Fc_ep = qcutils.GetVariableAsDictionary(ds_ep, 'Fc', si=si_ep, ei=ei_ep) # copy the range check values from the OFQC attributes to the EP attributes for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]): for item in ["rangecheck_upper", "rangecheck_lower"]: if item in of["Attr"]: ep["Attr"][item] = of["Attr"][item] # apply QC to the EddyPro data qcck.ApplyRangeCheckToVariable(us_ep) qcck.ApplyRangeCheckToVariable(Fc_ep) qcck.ApplyRangeCheckToVariable(Fe_ep) qcck.ApplyRangeCheckToVariable(Fh_ep) # plot the comparison plt.ion() fig = plt.figure(1, figsize=(8, 8)) qcplot.xyplot(us_ep["Data"], us_of["Data"], sub=[2, 2, 1], regr=2, xlabel='u*_EP (m/s)', ylabel='u*_OF (m/s)') qcplot.xyplot(Fh_ep["Data"], Fh_of["Data"], sub=[2, 2, 2], regr=2, xlabel='Fh_EP (W/m2)', ylabel='Fh_OF (W/m2)') qcplot.xyplot(Fe_ep["Data"], Fe_of["Data"], sub=[2, 2, 3], regr=2, xlabel='Fe_EP (W/m2)', ylabel='Fe_OF (W/m2)') qcplot.xyplot(Fc_ep["Data"], Fc_of["Data"], sub=[2, 2, 4], regr=2, xlabel='Fc_EP (umol/m2/s)', ylabel='Fc_OF (umol/m2/s)') plt.tight_layout() plt.draw() plt.ioff()
import numpy import os import sys # check the scripts directory is present if not os.path.exists("../scripts/"): print("erai2nc: the scripts directory is missing") sys.exit() # since the scripts directory is there, try importing the modules sys.path.append('../scripts') import qcio import qcutils aws_name = qcio.get_filename_dialog(path="/mnt/OzFlux/Sites") ds_aws_30minute = qcio.nc_read_series(aws_name) has_gaps = qcutils.CheckTimeStep(ds_aws_30minute) if has_gaps: print("Problems found with time step") qcutils.FixTimeStep(ds_aws_30minute) qcutils.get_ymdhmsfromdatetime(ds_aws_30minute) dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"] ddt = [ dt_aws_30minute[i + 1] - dt_aws_30minute[i] for i in range(0, len(dt_aws_30minute) - 1) ] print("Minimum time step is", min(ddt), " Maximum time step is", max(ddt)) dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"] start_date = dt_aws_30minute[0] end_date = dt_aws_30minute[-1]
import csv import matplotlib.pyplot as plt import numpy import os import qcio import qcutils import qcts from scipy.signal import butter,filtfilt import subprocess import sys sys.path.append("/usr/local/lib/python2.7/dist-packages") from ffnet import ffnet, mlgraph fname = qcio.get_filename_dialog() ds = qcio.nc_read_series(fname) nRecs =int(ds.globalattributes['nc_nrecs']) Fc,f=qcutils.GetSeriesasMA(ds,'Fc') Month,f=qcutils.GetSeriesasMA(ds,'Month') us,f=qcutils.GetSeriesasMA(ds,'ustar') Fsd,f=qcutils.GetSeriesasMA(ds,'Fsd') dt = ds.series['DateTime']['Data'] qcts.InterpolateOverMissing(ds,'Sws') Sws,f=qcutils.GetSeriesasMA(ds,'Sws') qcts.InterpolateOverMissing(ds,'Ts') Ts,f=qcutils.GetSeriesasMA(ds,'Ts') b,a = butter(3,0.1) Sws_notMA,WasMA = qcutils.MAtoSeries(Sws)
import numpy import os import sys # check the scripts directory is present if not os.path.exists("../scripts/"): print "erai2nc: the scripts directory is missing" sys.exit() # since the scripts directory is there, try importing the modules sys.path.append('../scripts') import qcio import qcutils aws_name=qcio.get_filename_dialog(path="/mnt/OzFlux/Sites") ds_aws_30minute = qcio.nc_read_series(aws_name) has_gaps = qcutils.CheckTimeStep(ds_aws_30minute) if has_gaps: print "Problems found with time step" qcutils.FixTimeStep(ds_aws_30minute) qcutils.get_ymdhmsfromdatetime(ds_aws_30minute) dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"] ddt=[dt_aws_30minute[i+1]-dt_aws_30minute[i] for i in range(0,len(dt_aws_30minute)-1)] print "Minimum time step is",min(ddt)," Maximum time step is",max(ddt) dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"] start_date = dt_aws_30minute[0] end_date = dt_aws_30minute[-1] si_wholehour = qcutils.GetDateIndex(dt_aws_30minute,str(start_date),ts=30,match="startnexthour") ei_wholehour = qcutils.GetDateIndex(dt_aws_30minute,str(end_date),ts=30,match="endprevioushour") start_date = dt_aws_30minute[si_wholehour] end_date = dt_aws_30minute[ei_wholehour]