def do_plotfluxnet(self,mode="standard"):
""" Plot FluxNet style time series of data."""
self.do_progress(text='Doing FluxNet plots ...')
if mode=="standard":
stdname = "controlfiles/standard/fluxnet.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file')
if len(filename)==0 or not os.path.exists(filename):
self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Plotting FluxNet style plots ...')
qcplot.plot_fluxnet(cf)
self.do_progress(text='Finished FluxNet plotting')
logging.info(' Finished FluxNet plotting')
def do_plotfingerprint(self,mode="standard"):
""" Plot fingerprint"""
logging.info(' Starting fingerprint plot')
self.do_progress(text='Doing fingerprint plot ...')
if mode=="standard":
stdname = "controlfiles/standard/fingerprint.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file')
if len(filename)==0 or not os.path.exists(filename):
self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
self.do_progress(text='Plotting fingerprint ...')
qcplot.plot_fingerprint(cf)
self.do_progress(text='Finished plotting fingerprint')
logging.info(' Finished plotting fingerprint')
logging.info("")
def do_climatology(self,mode="standard"):
"""
Calls qcclim.climatology
"""
logging.info(' Starting climatology')
self.do_progress(text='Doing climatology ...')
if mode=="standard":
stdname = "controlfiles/standard/climatology.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
self.do_progress(text='Opening input file ...')
filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file')
if len(filename)==0:
logging.info( " Climatology: no input file chosen")
self.do_progress(text='Waiting for input ...')
return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Doing the climatology')
qcclim.climatology(cf)
self.do_progress(text='Finished climatology')
logging.info(' Finished climatology')
logging.info("")
def compare_eddypro():
epname = qcio.get_filename_dialog(title='Choose an EddyPro full output file')
ofname = qcio.get_filename_dialog(title='Choose an L3 output file')
ds_ep = qcio.read_eddypro_full(epname)
ds_of = qcio.nc_read_series(ofname)
dt_ep = ds_ep.series['DateTime']['Data']
dt_of = ds_of.series['DateTime']['Data']
start_datetime = max([dt_ep[0],dt_of[0]])
end_datetime = min([dt_ep[-1],dt_of[-1]])
si_of = qcutils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact')
ei_of = qcutils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact')
si_ep = qcutils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact')
ei_ep = qcutils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact')
us_of = qcutils.GetVariableAsDictionary(ds_of,'ustar',si=si_of,ei=ei_of)
us_ep = qcutils.GetVariableAsDictionary(ds_ep,'ustar',si=si_ep,ei=ei_ep)
Fh_of = qcutils.GetVariableAsDictionary(ds_of,'Fh',si=si_of,ei=ei_of)
Fh_ep = qcutils.GetVariableAsDictionary(ds_ep,'Fh',si=si_ep,ei=ei_ep)
Fe_of = qcutils.GetVariableAsDictionary(ds_of,'Fe',si=si_of,ei=ei_of)
Fe_ep = qcutils.GetVariableAsDictionary(ds_ep,'Fe',si=si_ep,ei=ei_ep)
Fc_of = qcutils.GetVariableAsDictionary(ds_of,'Fc',si=si_of,ei=ei_of)
Fc_ep = qcutils.GetVariableAsDictionary(ds_ep,'Fc',si=si_ep,ei=ei_ep)
# copy the range check values from the OFQC attributes to the EP attributes
for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]):
for item in ["rangecheck_upper", "rangecheck_lower"]:
if item in of["Attr"]:
ep["Attr"][item] = of["Attr"][item]
# apply QC to the EddyPro data
qcck.ApplyRangeCheckToVariable(us_ep)
qcck.ApplyRangeCheckToVariable(Fc_ep)
qcck.ApplyRangeCheckToVariable(Fe_ep)
qcck.ApplyRangeCheckToVariable(Fh_ep)
# plot the comparison
plt.ion()
fig = plt.figure(1,figsize=(8,8))
qcplot.xyplot(us_ep["Data"],us_of["Data"],sub=[2,2,1],regr=2,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)')
qcplot.xyplot(Fh_ep["Data"],Fh_of["Data"],sub=[2,2,2],regr=2,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)')
qcplot.xyplot(Fe_ep["Data"],Fe_of["Data"],sub=[2,2,3],regr=2,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)')
qcplot.xyplot(Fc_ep["Data"],Fc_of["Data"],sub=[2,2,4],regr=2,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)')
plt.tight_layout()
plt.draw()
plt.ioff()
def do_nc2xls(self):
""" Calls qcio.nc_2xls. """
logging.info(" Starting conversion to Excel file")
self.do_progress(text="Choosing netCDF file ...")
ncfilename = qcio.get_filename_dialog(path="../Sites",title="Choose a netCDF file")
if len(ncfilename)==0: self.do_progress(text="Waiting for input ..."); return
self.do_progress(text="Converting netCDF file to Excel file")
qcio.nc_2xls(ncfilename,outputlist=None)
self.do_progress(text="Finished converting netCDF file")
logging.info(" Finished converting netCDF file")
logging.info("")
def do_nc2reddyproc(self):
""" Calls qcio.reddyproc_write_csv."""
logging.info(' Starting conversion to REddyProc CSV file')
self.do_progress(text="Choosing netCDF file ...")
ncfilename = qcio.get_filename_dialog(path="../Sites",title="Choose a netCDF file")
if len(ncfilename)==0 or not os.path.exists(ncfilename):
self.do_progress(text="Waiting for input ..."); return
self.do_progress(text='Converting nc to REddyProc CSV ...')
qcio.reddyproc_write_csv(ncfilename)
logging.info(' Finished conversion')
self.do_progress(text='Finished conversion')
logging.info("")
def compare_eddypro():
epname = qcio.get_filename_dialog(title='Choose an EddyPro full output file')
ofname = qcio.get_filename_dialog(title='Choose an L3 output file')
ds_ep = qcio.read_eddypro_full(epname)
ds_of = qcio.nc_read_series(ofname)
dt_ep = ds_ep.series['DateTime']['Data']
dt_of = ds_of.series['DateTime']['Data']
si = dt_of.index(dt_ep[0])
ei = dt_of.index(dt_ep[-1])
us_of,f,a = qcutils.GetSeriesasMA(ds_of,'ustar',si=si,ei=ei)
us_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'ustar')
Fh_of,f,a = qcutils.GetSeriesasMA(ds_of,'Fh',si=si,ei=ei)
Fh_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'Fh')
Fe_of,f,a = qcutils.GetSeriesasMA(ds_of,'Fe',si=si,ei=ei)
Fe_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'Fe')
Fc_of,f,a = qcutils.GetSeriesasMA(ds_of,'Fc',si=si,ei=ei)
Fc_ep,f,a = qcutils.GetSeriesasMA(ds_ep,'Fc')
us_of.mask = numpy.ma.mask_or(us_of.mask,us_ep.mask)
us_ep.mask = numpy.ma.mask_or(us_of.mask,us_ep.mask)
Fh_of.mask = numpy.ma.mask_or(Fh_of.mask,Fh_ep.mask)
Fh_ep.mask = numpy.ma.mask_or(Fh_of.mask,Fh_ep.mask)
Fe_of.mask = numpy.ma.mask_or(Fe_of.mask,Fe_ep.mask)
Fe_ep.mask = numpy.ma.mask_or(Fe_of.mask,Fe_ep.mask)
Fc_of.mask = numpy.ma.mask_or(Fc_of.mask,Fc_ep.mask)
Fc_ep.mask = numpy.ma.mask_or(Fc_of.mask,Fc_ep.mask)
plt.ion()
fig = plt.figure(1,figsize=(8,8))
qcplot.xyplot(us_ep,us_of,sub=[2,2,1],regr=1,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)')
qcplot.xyplot(Fh_ep,Fh_of,sub=[2,2,2],regr=1,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)')
qcplot.xyplot(Fe_ep,Fe_of,sub=[2,2,3],regr=1,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)')
qcplot.xyplot(Fc_ep,Fc_of,sub=[2,2,4],regr=1,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)')
plt.tight_layout()
plt.draw()
plt.ioff()
def do_plotquickcheck(self):
""" Plot quickcheck"""
self.do_progress(text='Loading control file ...')
stdname = "controlfiles/standard/quickcheck.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose an input file')
if len(filename)==0: self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Plotting quickcheck ...')
qcplot.plot_quickcheck(cf)
self.do_progress(text='Finished plotting quickcheck')
logging.info(' Finished plotting quickcheck')
def do_cpd(self):
"""
Calls qccpd.cpd_main
Compares the results OzFluxQC (L3) with those from EddyPro (full output).
"""
logging.info(' Starting estimation u* threshold using CPD')
self.do_progress(text='Estimating u* threshold using CPD ...')
stdname = "controlfiles/standard/cpd.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose an input nc file')
if len(filename)==0: self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
qccpd.cpd_main(cf)
self.do_progress(text='Finished estimating u* threshold')
logging.info(' Finished estimating u* threshold')
logging.info("")
# coding: utf-8 import qcio import qcutils import statsmodels.api as sm duname=qcio.get_filename_dialog() drname=qcio.get_filename_dialog() ds_du=qcio.nc_read_series(duname) ds_dr=qcio.nc_read_series(drname) dt_du=ds.series["DateTime"]["Data"] dt_du=ds_du.series["DateTime"]["Data"] dt_dr=ds_dr.series["DateTime"]["Data"] print dt_du[0],dt_dr[0] print dt_du[-1],dt_dr[-1] ts=ds_du.globalattributes["time_step"] si=qcutils.GetDateIndex(dt_du,str(dt_du[0]),ts=ts,match="startnextday") ei=qcutils.GetDateIndex(dt_du,str(dt_dr[-1]),ts=ts,match="endpreviousday") print si,ei Fsd_du,f=qcutils.GetSeriesasMA(ds_du,'Fsd',si=si,ei=ei) Fsd_dr,f=qcutils.GetSeriesasMA(ds_dr,'Fsd',si=si,ei=ei) plot(Fsd_dr,Fsd_du) ei=qcutils.GetDateIndex(ds_du,"2009-01-01 00:00",ts=ts,match="endpreviousday") ei=qcutils.GetDateIndex(dt_du,"2009-01-01 00:00",ts=ts,match="endpreviousday") fig=figure(1) Fsd_du_2008,f=qcutils.GetSeriesasMA(ds_du,'Fsd',si=si,ei=ei) Fsd_dr_2008,f=qcutils.GetSeriesasMA(ds_dr,'Fsd',si=si,ei=ei) plot(Fsd_du_2008,Fsd_dr_2008,'b.') Fsd_du_2008.mask=(Fsd_du_2008.mask==True)|(Fsd_dr_2008.mask==True) Fsd_dr_2008.mask=(Fsd_du_2008.mask==True)|(Fsd_dr_2008.mask==True) Fsd_du_2008=numpy.ma.compressed(Fsd_du_2008) Fsd_dr_2008=numpy.ma.compressed(Fsd_dr_2008) x=Fsd_du_2008
def compare_eddypro():
epname = qcio.get_filename_dialog(
title='Choose an EddyPro full output file')
ofname = qcio.get_filename_dialog(title='Choose an L3 output file')
ds_ep = qcio.read_eddypro_full(epname)
ds_of = qcio.nc_read_series(ofname)
dt_ep = ds_ep.series['DateTime']['Data']
dt_of = ds_of.series['DateTime']['Data']
start_datetime = max([dt_ep[0], dt_of[0]])
end_datetime = min([dt_ep[-1], dt_of[-1]])
si_of = qcutils.GetDateIndex(dt_of,
str(start_datetime),
ts=30,
default=0,
match='exact')
ei_of = qcutils.GetDateIndex(dt_of,
str(end_datetime),
ts=30,
default=len(dt_of),
match='exact')
si_ep = qcutils.GetDateIndex(dt_ep,
str(start_datetime),
ts=30,
default=0,
match='exact')
ei_ep = qcutils.GetDateIndex(dt_ep,
str(end_datetime),
ts=30,
default=len(dt_ep),
match='exact')
us_of = qcutils.GetVariableAsDictionary(ds_of, 'ustar', si=si_of, ei=ei_of)
us_ep = qcutils.GetVariableAsDictionary(ds_ep, 'ustar', si=si_ep, ei=ei_ep)
Fh_of = qcutils.GetVariableAsDictionary(ds_of, 'Fh', si=si_of, ei=ei_of)
Fh_ep = qcutils.GetVariableAsDictionary(ds_ep, 'Fh', si=si_ep, ei=ei_ep)
Fe_of = qcutils.GetVariableAsDictionary(ds_of, 'Fe', si=si_of, ei=ei_of)
Fe_ep = qcutils.GetVariableAsDictionary(ds_ep, 'Fe', si=si_ep, ei=ei_ep)
Fc_of = qcutils.GetVariableAsDictionary(ds_of, 'Fc', si=si_of, ei=ei_of)
Fc_ep = qcutils.GetVariableAsDictionary(ds_ep, 'Fc', si=si_ep, ei=ei_ep)
# copy the range check values from the OFQC attributes to the EP attributes
for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of],
[us_ep, Fh_ep, Fe_ep, Fc_ep]):
for item in ["rangecheck_upper", "rangecheck_lower"]:
if item in of["Attr"]:
ep["Attr"][item] = of["Attr"][item]
# apply QC to the EddyPro data
qcck.ApplyRangeCheckToVariable(us_ep)
qcck.ApplyRangeCheckToVariable(Fc_ep)
qcck.ApplyRangeCheckToVariable(Fe_ep)
qcck.ApplyRangeCheckToVariable(Fh_ep)
# plot the comparison
plt.ion()
fig = plt.figure(1, figsize=(8, 8))
qcplot.xyplot(us_ep["Data"],
us_of["Data"],
sub=[2, 2, 1],
regr=2,
xlabel='u*_EP (m/s)',
ylabel='u*_OF (m/s)')
qcplot.xyplot(Fh_ep["Data"],
Fh_of["Data"],
sub=[2, 2, 2],
regr=2,
xlabel='Fh_EP (W/m2)',
ylabel='Fh_OF (W/m2)')
qcplot.xyplot(Fe_ep["Data"],
Fe_of["Data"],
sub=[2, 2, 3],
regr=2,
xlabel='Fe_EP (W/m2)',
ylabel='Fe_OF (W/m2)')
qcplot.xyplot(Fc_ep["Data"],
Fc_of["Data"],
sub=[2, 2, 4],
regr=2,
xlabel='Fc_EP (umol/m2/s)',
ylabel='Fc_OF (umol/m2/s)')
plt.tight_layout()
plt.draw()
plt.ioff()
import numpy
import os
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print("erai2nc: the scripts directory is missing")
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import qcio
import qcutils
aws_name = qcio.get_filename_dialog(path="/mnt/OzFlux/Sites")
ds_aws_30minute = qcio.nc_read_series(aws_name)
has_gaps = qcutils.CheckTimeStep(ds_aws_30minute)
if has_gaps:
print("Problems found with time step")
qcutils.FixTimeStep(ds_aws_30minute)
qcutils.get_ymdhmsfromdatetime(ds_aws_30minute)
dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"]
ddt = [
dt_aws_30minute[i + 1] - dt_aws_30minute[i]
for i in range(0,
len(dt_aws_30minute) - 1)
]
print("Minimum time step is", min(ddt), " Maximum time step is", max(ddt))
dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"]
start_date = dt_aws_30minute[0]
end_date = dt_aws_30minute[-1]
import csv
import matplotlib.pyplot as plt
import numpy
import os
import qcio
import qcutils
import qcts
from scipy.signal import butter,filtfilt
import subprocess
import sys
sys.path.append("/usr/local/lib/python2.7/dist-packages")
from ffnet import ffnet, mlgraph
fname = qcio.get_filename_dialog()
ds = qcio.nc_read_series(fname)
nRecs =int(ds.globalattributes['nc_nrecs'])
Fc,f=qcutils.GetSeriesasMA(ds,'Fc')
Month,f=qcutils.GetSeriesasMA(ds,'Month')
us,f=qcutils.GetSeriesasMA(ds,'ustar')
Fsd,f=qcutils.GetSeriesasMA(ds,'Fsd')
dt = ds.series['DateTime']['Data']
qcts.InterpolateOverMissing(ds,'Sws')
Sws,f=qcutils.GetSeriesasMA(ds,'Sws')
qcts.InterpolateOverMissing(ds,'Ts')
Ts,f=qcutils.GetSeriesasMA(ds,'Ts')
b,a = butter(3,0.1)
Sws_notMA,WasMA = qcutils.MAtoSeries(Sws)
import numpy
import os
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print "erai2nc: the scripts directory is missing"
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import qcio
import qcutils
aws_name=qcio.get_filename_dialog(path="/mnt/OzFlux/Sites")
ds_aws_30minute = qcio.nc_read_series(aws_name)
has_gaps = qcutils.CheckTimeStep(ds_aws_30minute)
if has_gaps:
print "Problems found with time step"
qcutils.FixTimeStep(ds_aws_30minute)
qcutils.get_ymdhmsfromdatetime(ds_aws_30minute)
dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"]
ddt=[dt_aws_30minute[i+1]-dt_aws_30minute[i] for i in range(0,len(dt_aws_30minute)-1)]
print "Minimum time step is",min(ddt)," Maximum time step is",max(ddt)
dt_aws_30minute = ds_aws_30minute.series["DateTime"]["Data"]
start_date = dt_aws_30minute[0]
end_date = dt_aws_30minute[-1]
si_wholehour = qcutils.GetDateIndex(dt_aws_30minute,str(start_date),ts=30,match="startnexthour")
ei_wholehour = qcutils.GetDateIndex(dt_aws_30minute,str(end_date),ts=30,match="endprevioushour")
start_date = dt_aws_30minute[si_wholehour]
end_date = dt_aws_30minute[ei_wholehour]
