def do_plotfluxnet(self,mode="standard"):
""" Plot FluxNet style time series of data."""
self.do_progress(text='Doing FluxNet plots ...')
if mode=="standard":
stdname = "controlfiles/standard/fluxnet.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file')
if len(filename)==0 or not os.path.exists(filename):
self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Plotting FluxNet style plots ...')
qcplot.plot_fluxnet(cf)
self.do_progress(text='Finished FluxNet plotting')
logging.info(' Finished FluxNet plotting')
def do_climatology(self,mode="standard"):
"""
Calls qcclim.climatology
"""
logging.info(' Starting climatology')
self.do_progress(text='Doing climatology ...')
if mode=="standard":
stdname = "controlfiles/standard/climatology.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
self.do_progress(text='Opening input file ...')
filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file')
if len(filename)==0:
logging.info( " Climatology: no input file chosen")
self.do_progress(text='Waiting for input ...')
return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Doing the climatology')
qcclim.climatology(cf)
self.do_progress(text='Finished climatology')
logging.info(' Finished climatology')
logging.info("")
def do_plotfingerprint(self,mode="standard"):
""" Plot fingerprint"""
logging.info(' Starting fingerprint plot')
self.do_progress(text='Doing fingerprint plot ...')
if mode=="standard":
stdname = "controlfiles/standard/fingerprint.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose a netCDF file')
if len(filename)==0 or not os.path.exists(filename):
self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
else:
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
self.do_progress(text='Plotting fingerprint ...')
qcplot.plot_fingerprint(cf)
self.do_progress(text='Finished plotting fingerprint')
logging.info(' Finished plotting fingerprint')
logging.info("")
def do_l6qc(self):
"""
Call qcls.l6qc function to partition NEE into GPP and ER.
"""
logging.info(" Starting L6 processing ...")
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
infilename = qcio.get_infilenamefromcf(cf)
if len(infilename)==0: self.do_progress(text='An error occurred, check the console ...'); return
if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return
ds5 = qcio.nc_read_series(infilename)
if len(ds5.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del ds5; return
ds5.globalattributes['controlfile_name'] = cf['controlfile_name']
self.update_startenddate(str(ds5.series['DateTime']['Data'][0]),
str(ds5.series['DateTime']['Data'][-1]))
sitename = ds5.globalattributes['site_name']
self.do_progress(text='Doing L6 partitioning: '+sitename+' ...')
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
ds6 = qcls.l6qc(cf,ds5)
self.do_progress(text='Finished L6: '+sitename)
logging.info(' Finished L6: '+sitename)
self.do_progress(text='Saving L6 partitioned data ...') # put up the progress message
outfilename = qcio.get_outfilenamefromcf(cf)
if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return
ncFile = qcio.nc_open_write(outfilename)
outputlist = qcio.get_outputlistfromcf(cf,'nc')
qcio.nc_write_series(ncFile,ds6,outputlist=outputlist) # save the L6 data
self.do_progress(text='Finished saving L6 partitioned data') # tell the user we are done
logging.info(' Finished saving L6 partitioned data')
logging.info("")
def do_l4qc(self):
"""
Call qcls.l4qc_gapfill function
Performs L4 gap filling on L3 met data
or
Ingests L4 gap filled fluxes performed in external SOLO-ANN and c
omputes daily sums
Outputs L4 netCDF file to ncData folder
Outputs L4 netCDF file to OzFlux folder
ControlFiles:
L4_year.txt
or
L4b.txt
ControlFile contents (see ControlFile/Templates/L4.txt and
ControlFile/Templates/L4b.txt for examples):
[General]:
Python control parameters (SOLO)
Site characteristics parameters (Gap filling)
[Files]:
L3 input file name and path (Gap filling)
L4 input file name and path (SOLO)
L4 output file name and ncData folder path (both)
L4 OzFlux output file name and OzFlux folder path
[Variables]:
Variable subset list for OzFlux output file (where
available)
"""
logging.info(" Starting L4 processing ...")
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
infilename = qcio.get_infilenamefromcf(cf)
if len(infilename)==0: self.do_progress(text='An error occurred, check the console ...'); return
if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return
ds3 = qcio.nc_read_series(infilename)
if len(ds3.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del ds3; return
ds3.globalattributes['controlfile_name'] = cf['controlfile_name']
self.update_startenddate(str(ds3.series['DateTime']['Data'][0]),
str(ds3.series['DateTime']['Data'][-1]))
sitename = ds3.globalattributes['site_name']
self.do_progress(text='Doing L4 gap filling drivers: '+sitename+' ...')
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
ds4 = qcls.l4qc(cf,ds3)
if ds4.returncodes["alternate"]=="quit" or ds4.returncodes["solo"]=="quit":
self.do_progress(text='Quitting L4: '+sitename)
logging.info(' Quitting L4: '+sitename)
else:
self.do_progress(text='Finished L4: '+sitename)
logging.info(' Finished L4: '+sitename)
self.do_progress(text='Saving L4 gap filled data ...') # put up the progress message
outfilename = qcio.get_outfilenamefromcf(cf)
if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return
ncFile = qcio.nc_open_write(outfilename)
outputlist = qcio.get_outputlistfromcf(cf,'nc')
qcio.nc_write_series(ncFile,ds4,outputlist=outputlist) # save the L4 data
self.do_progress(text='Finished saving L4 gap filled data') # tell the user we are done
logging.info(' Finished saving L4 gap filled data')
logging.info("")
def do_plotL3L4(self):
"""
Plot L3 (QA/QC and Corrected) and L4 (Gap Filled) data in blue and
red, respectively
Control File for do_l4qc function used.
If L4 Control File not loaded, requires control file selection.
"""
if 'ds3' not in dir(self) or 'ds4' not in dir(self):
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0:
self.do_progress(text='Waiting for input ...')
return
l3filename = qcio.get_infilenamefromcf(self.cf)
if not qcutils.file_exists(l3filename): self.do_progress(text='An error occurred, check the console ...'); return
self.ds3 = qcio.nc_read_series(l3filename)
if len(self.ds3.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds3; return
l4filename = qcio.get_outfilenamefromcf(self.cf)
self.ds4 = qcio.nc_read_series(l4filename)
if len(self.ds4.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds4; return
self.update_startenddate(str(self.ds3.series['DateTime']['Data'][0]),
str(self.ds3.series['DateTime']['Data'][-1]))
self.do_progress(text='Plotting L3 and L4 QC ...')
cfname = self.ds4.globalattributes['controlfile_name']
self.cf = qcio.get_controlfilecontents(cfname)
for nFig in self.cf['Plots'].keys():
si = qcutils.GetDateIndex(self.ds3.series['DateTime']['Data'],self.plotstartEntry.get(),
ts=self.ds3.globalattributes['time_step'],default=0,match='exact')
ei = qcutils.GetDateIndex(self.ds3.series['DateTime']['Data'],self.plotendEntry.get(),
ts=self.ds3.globalattributes['time_step'],default=-1,match='exact')
qcplot.plottimeseries(self.cf,nFig,self.ds3,self.ds4,si,ei)
self.do_progress(text='Finished plotting L4')
logging.info(' Finished plotting L4, check the GUI')
def do_nc2smap(self):
""" Calls qcio.smap_write_csv. """
logging.info(' Starting conversion to SMAP CSV file')
self.do_progress(text='Load control file ...')
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Converting nc to SMAP CSV ...')
qcio.smap_write_csv(self.cf)
logging.info(' Finished conversion')
self.do_progress(text='Finished conversion')
logging.info("")
def do_l2qc(self):
"""
Call qcls.l2qc function
Performs L2 QA/QC processing on raw data
Outputs L2 netCDF file to ncData folder
ControlFiles:
L2_year.txt
or
L2.txt
ControlFile contents (see ControlFile/Templates/L2.txt for example):
[General]:
Enter list of functions to be performed
[Files]:
L1 input file name and path
L2 output file name and path
[Variables]:
Variable names and parameters for:
Range check to set upper and lower rejection limits
Diurnal check to reject observations by time of day that
are outside specified standard deviation limits
Timestamps for excluded dates
Timestamps for excluded hours
[Plots]:
Variable lists for plot generation
"""
logging.info(" Starting L2 processing ...")
self.do_progress(text='Load L2 Control File ...')
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0:
logging.info( " L2: no control file chosen")
self.do_progress(text='Waiting for input ...')
return
infilename = qcio.get_infilenamefromcf(self.cf)
if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return
self.do_progress(text='Doing L2 QC ...')
self.ds1 = qcio.nc_read_series(infilename)
if len(self.ds1.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds1; return
self.update_startenddate(str(self.ds1.series['DateTime']['Data'][0]),
str(self.ds1.series['DateTime']['Data'][-1]))
self.ds2 = qcls.l2qc(self.cf,self.ds1)
logging.info(' Finished L2 QC process')
self.do_progress(text='Finished L2 QC process')
self.do_progress(text='Saving L2 QC ...') # put up the progress message
outfilename = qcio.get_outfilenamefromcf(self.cf)
if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return
ncFile = qcio.nc_open_write(outfilename)
qcio.nc_write_series(ncFile,self.ds2) # save the L2 data
self.do_progress(text='Finished saving L2 QC data') # tdo_progressell the user we are done
logging.info(' Finished saving L2 QC data')
logging.info("")
def do_ncconcat(self):
"""
Calls qcio.nc_concatenate
"""
logging.info(' Starting concatenation of netCDF files')
self.do_progress(text='Loading control file ...')
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Concatenating files')
qcio.nc_concatenate(cf)
self.do_progress(text='Finished concatenating files')
logging.info(' Finished concatenating files')
logging.info("")
def do_xl2nc(self):
"""
Calls qcio.xl2nc
"""
logging.info(" Starting L1 processing ...")
self.do_progress(text='Loading control file ...')
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Reading Excel file & writing to netCDF')
rcode = qcio.xl2nc(self.cf,"L1")
if rcode==1:
self.do_progress(text='Finished writing to netCDF ...')
logging.info(' Finished writing to netCDF ...')
else:
self.do_progress(text='An error occurred, check the console ...')
def do_plotquickcheck(self):
""" Plot quickcheck"""
self.do_progress(text='Loading control file ...')
stdname = "controlfiles/standard/quickcheck.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose an input file')
if len(filename)==0: self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
self.do_progress(text='Plotting quickcheck ...')
qcplot.plot_quickcheck(cf)
self.do_progress(text='Finished plotting quickcheck')
logging.info(' Finished plotting quickcheck')
def do_plotL6_summary(self):
"""
Plot L6 summary.
"""
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0:
self.do_progress(text='Waiting for input ...')
return
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
l6filename = qcio.get_outfilenamefromcf(cf)
if not qcutils.file_exists(l6filename): self.do_progress(text='An error occurred, check the console ...'); return
ds6 = qcio.nc_read_series(l6filename)
if len(ds6.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del ds6; return
self.update_startenddate(str(ds6.series['DateTime']['Data'][0]),
str(ds6.series['DateTime']['Data'][-1]))
self.do_progress(text='Plotting L6 summary ...')
qcgf.ImportSeries(cf,ds6)
qcrp.L6_summary(cf,ds6)
self.do_progress(text='Finished plotting L6 summary')
logging.info(' Finished plotting L6 summary, check the GUI')
def do_plotL1L2(self):
"""
Plot L1 (raw) and L2 (QA/QC) data in blue and red, respectively
Control File for do_l2qc function used.
If L2 Control File not loaded, requires control file selection.
"""
if 'ds1' not in dir(self) or 'ds2' not in dir(self):
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0: self.do_progress(text='Waiting for input ...'); return
l1filename = qcio.get_infilenamefromcf(self.cf)
if not qcutils.file_exists(l1filename): self.do_progress(text='An error occurred, check the console ...'); return
self.ds1 = qcio.nc_read_series(l1filename)
if len(self.ds1.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds1; return
l2filename = qcio.get_outfilenamefromcf(self.cf)
self.ds2 = qcio.nc_read_series(l2filename)
if len(self.ds2.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds2; return
self.update_startenddate(str(self.ds1.series['DateTime']['Data'][0]),
str(self.ds1.series['DateTime']['Data'][-1]))
self.do_progress(text='Plotting L1 & L2 QC ...')
cfname = self.ds2.globalattributes['controlfile_name']
self.cf = qcio.get_controlfilecontents(cfname)
for nFig in self.cf['Plots'].keys():
si = qcutils.GetDateIndex(self.ds1.series['DateTime']['Data'],self.plotstartEntry.get(),
ts=self.ds1.globalattributes['time_step'],default=0,match='exact')
ei = qcutils.GetDateIndex(self.ds1.series['DateTime']['Data'],self.plotendEntry.get(),
ts=self.ds1.globalattributes['time_step'],default=-1,match='exact')
plt_cf = self.cf['Plots'][str(nFig)]
if 'Type' in plt_cf.keys():
if str(plt_cf['Type']).lower() =='xy':
self.do_progress(text='Plotting L1 and L2 XY ...')
qcplot.plotxy(self.cf,nFig,plt_cf,self.ds1,self.ds2,si,ei)
else:
self.do_progress(text='Plotting L1 and L2 QC ...')
qcplot.plottimeseries(self.cf,nFig,self.ds1,self.ds2,si,ei)
else:
self.do_progress(text='Plotting L1 and L2 QC ...')
qcplot.plottimeseries(self.cf,nFig,self.ds1,self.ds2,si,ei)
self.do_progress(text='Finished plotting L1 and L2')
logging.info(' Finished plotting L1 and L2, check the GUI')
def do_plotL3L3(self):
"""
Plot L3 (QA/QC and Corrected) data
Control File for do_l3qc function used.
If L3 Control File not loaded, requires control file selection.
"""
if 'ds3' not in dir(self):
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0: self.do_progress(text='Waiting for input ...'); return
l3filename = qcio.get_outfilenamefromcf(self.cf)
self.ds3 = qcio.nc_read_series(l3filename)
if len(self.ds3.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds3; return
self.update_startenddate(str(self.ds3.series['DateTime']['Data'][0]),
str(self.ds3.series['DateTime']['Data'][-1]))
self.do_progress(text='Plotting L3 QC ...')
cfname = self.ds3.globalattributes['controlfile_name']
self.cf = qcio.get_controlfilecontents(cfname)
for nFig in self.cf['Plots'].keys():
si = qcutils.GetDateIndex(self.ds3.series['DateTime']['Data'],self.plotstartEntry.get(),
ts=self.ds3.globalattributes['time_step'],default=0,match='exact')
ei = qcutils.GetDateIndex(self.ds3.series['DateTime']['Data'],self.plotendEntry.get(),
ts=self.ds3.globalattributes['time_step'],default=-1,match='exact')
plt_cf = self.cf['Plots'][str(nFig)]
if 'Type' in plt_cf.keys():
if str(plt_cf['Type']).lower() =='xy':
self.do_progress(text='Plotting L3 XY ...')
qcplot.plotxy(self.cf,nFig,plt_cf,self.ds3,self.ds3,si,ei)
else:
self.do_progress(text='Plotting L3 QC ...')
SeriesList = ast.literal_eval(plt_cf['Variables'])
qcplot.plottimeseries(self.cf,nFig,self.ds3,self.ds3,si,ei)
else:
self.do_progress(text='Plotting L3 QC ...')
qcplot.plottimeseries(self.cf,nFig,self.ds3,self.ds3,si,ei)
self.do_progress(text='Finished plotting L3')
logging.info(' Finished plotting L3, check the GUI')
def do_cpd(self):
"""
Calls qccpd.cpd_main
Compares the results OzFluxQC (L3) with those from EddyPro (full output).
"""
logging.info(' Starting estimation u* threshold using CPD')
self.do_progress(text='Estimating u* threshold using CPD ...')
stdname = "controlfiles/standard/cpd.txt"
if os.path.exists(stdname):
cf = qcio.get_controlfilecontents(stdname)
filename = qcio.get_filename_dialog(path='../Sites',title='Choose an input nc file')
if len(filename)==0: self.do_progress(text='Waiting for input ...'); return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = ntpath.split(filename)[0]+"/"
cf["Files"]["in_filename"] = ntpath.split(filename)[1]
else:
cf = qcio.load_controlfile(path='controlfiles')
if len(cf)==0: self.do_progress(text='Waiting for input ...'); return
if "Options" not in cf: cf["Options"]={}
cf["Options"]["call_mode"] = "interactive"
qccpd.cpd_main(cf)
self.do_progress(text='Finished estimating u* threshold')
logging.info(' Finished estimating u* threshold')
logging.info("")
import scipy
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print "erai2nc: the scripts directory is missing"
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import meteorologicalfunctions as mf
import qcio
import qcutils
do_plots = True
# load the control file contents
cf = qcio.load_controlfile(path="../controlfiles/MODIS/")
# get the DAP file name
evi_url = cf["Files"]["evi_path"]
evi_file = cf["Files"]["evi_file"]
evi_filename = os.path.join(evi_url, evi_file)
# open the EVI file on the AusCover THREDDS server
nc_file = netCDF4.Dataset(evi_filename, "r")
# get the latitude and longitude resolution
lat_resolution = getattr(nc_file, "geospatial_lat_resolution")
lon_resolution = getattr(nc_file, "geospatial_lon_resolution")
# read latitude, longitude and time from the MODIS file
lat = nc_file.variables["latitude"][:]
lon = nc_file.variables["longitude"][:]
modis_time = nc_file.variables["time"][:]
# get a Python datetime series from the netCDF time (UTC)
modis_time_units = getattr(nc_file.variables["time"], "units")
import qccpd
import qcio
import qcls
import qcutils
#log = qcutils.startlog('batch','logfiles/batch.log')
logging.basicConfig(filename='logfiles/batchprocess.log',level=logging.DEBUG)
console = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
console.setFormatter(formatter)
console.setLevel(logging.INFO)
logging.getLogger('').addHandler(console)
# get the batch processing control file
if len(sys.argv)==1:
cf_batch = qcio.load_controlfile(path='controlfiles')
if len(cf_batch)==0: sys.exit()
else:
cfname = sys.argv[1]
if os.path.exists(cfname):
cf_batch = qcio.get_controlfilecontents(cfname)
else:
logging.error("Control file "+cfname+" does not exist")
sys.exit()
level_list = ['L1','L2','L3','concatenate','climatology','cpd','L4','L5','L6']
if "Options" in cf_batch:
if "levels" in cf_batch["Options"]: level_list = ast.literal_eval(cf_batch["Options"]["levels"])
for level in level_list:
if level.lower()=="l1":
# L1 processing
xl_sheet.write(0,col+1,year)
# write the data, one row per site, one column per year
for row, site in enumerate(site_list):
xl_sheet.write(row+1,0,site)
for col, year in enumerate(year_list):
if stat in data[site][year].keys():
xl_sheet.write(row+1,col+1,data[site][year][stat])
else:
xl_sheet.write(row+1,col+1,"")
# save the workbook
xl_book.save(xl_file_path)
return
# read the control file file
cf = qcio.load_controlfile(path='../controlfiles')
xl_file_path = cf["Files"]["xl_file_path"]
xl_sheet_name = cf["Files"]["xl_sheet_name"]
isd_base_path = cf["Files"]["isd_base_path"]
out_base_path = cf["Files"]["out_base_path"]
max_interp_length = cf["Files"]["max_interp_length"]
# read the site master spreadsheet
site_info = read_site_master(xl_file_path, xl_sheet_name)
# get a list of sites
site_list = site_info.keys()
# creat a dictionary to hold the ISD site time steps
isd_time_steps = OrderedDict()
for site in site_list:
# construct the output file path
fluxnet_id = site_info[site]["FluxNet ID"]
import pytz
from scipy.interpolate import InterpolatedUnivariateSpline
#from scipy.interpolate import interp1d
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print "erai2nc: the scripts directory is missing"
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import meteorologicalfunctions as mf
import pysolar
import qcio
import qcutils
cf = qcio.load_controlfile(path='../controlfiles/ERAI/',
title='Choose a control file')
cf_list = [cf["L1"][n] for n in cf["L1"].keys()]
#cf_list = sorted(glob.glob("../controlfiles/ERAI/L1/USA/*"))
for cf_name in cf_list:
print "Processing control file " + cf_name
cf = qcio.get_controlfilecontents(cf_name)
erai_name = os.path.join(cf["Files"]["erai_path"],
cf["Files"]["erai_file"])
erai_timestep = 180
erai_file = netCDF4.Dataset(erai_name)
latitude = erai_file.variables["latitude"][:]
longitude = erai_file.variables["longitude"][:]
lat_resolution = abs(latitude[-1] - latitude[0]) / (len(latitude) - 1)
lon_resolution = abs(longitude[-1] - longitude[0]) / (len(longitude) - 1)
# get the time and convert to Python datetime object
indices.append(idx)
else:
for item in a:
idx = b.index(item,idx+1)
indices.append(idx)
return indices
logging.basicConfig(filename='../logfiles/access2nc.log',level=logging.DEBUG)
console = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
console.setFormatter(formatter)
console.setLevel(logging.INFO)
logging.getLogger('').addHandler(console)
# get the control file
cf = qcio.load_controlfile(path='../controlfiles')
if len(cf)==0: sys.exit()
var_list = cf["Variables"].keys()
site_list = cf["Sites"].keys()
for site in site_list:
logging.info("Starting site: "+site)
# get the input file mask
infilename = cf["Sites"][site]["in_filepath"]+cf["Sites"][site]["in_filename"]
# get a sorted list of files that match the mask in the control file
file_list = sorted(glob.glob(infilename))
# number of files to process
nFiles = len(file_list)
logging.info("Processing "+str(nFiles)+" files")
# output file name
outfilename = cf["Sites"][site]["out_filepath"]+cf["Sites"][site]["out_filename"]
# interpolate to 30 minutes or not
import glob
import ntpath
import os
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print("erai2nc: the scripts directory is missing")
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import qcio
cf = qcio.load_controlfile(path="../controlfiles")
base_pattern = "HM01X_Data*.txt"
in_file_pattern = cf["Files"]["In"]["file_path"] + base_pattern
in_file_list = sorted(glob.glob(in_file_pattern))
for in_filename in in_file_list:
p = ntpath.basename(in_filename).split("_")
out_filename = cf["Files"]["Out"]["file_path"] + p[0] + "_" + p[
1] + "_" + p[2] + ".csv"
# open the output file
out_file = open(out_filename, "a")
# open the first input file and append
print("Adding contents of " + in_filename + " to " + out_filename)
for line in open(in_filename):
out_file.write(line)
# now loop over the folders containing the files to be appended
add_path_list = [
cf["Files"]["Add"][i] for i in list(cf["Files"]["Add"].keys())
]
import glob
import ntpath
import os
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print "erai2nc: the scripts directory is missing"
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import qcio
cf = qcio.load_controlfile(path="../controlfiles")
base_pattern = "HM01X_Data*.txt"
in_file_pattern = cf["Files"]["In"]["file_path"]+base_pattern
in_file_list = sorted(glob.glob(in_file_pattern))
for in_filename in in_file_list:
p = ntpath.basename(in_filename).split("_")
out_filename = cf["Files"]["Out"]["file_path"]+p[0]+"_"+p[1]+"_"+p[2]+".csv"
# open the output file
out_file = open(out_filename,"a")
# open the first input file and append
print "Adding contents of "+in_filename+" to "+out_filename
for line in open(in_filename):
out_file.write(line)
# now loop over the folders containing the files to be appended
add_path_list = [cf["Files"]["Add"][i] for i in cf["Files"]["Add"].keys()]
bom_id = in_filename.split("_")[2]
for add_path in add_path_list:
# build the file name
import scipy
import sys
# check the scripts directory is present
if not os.path.exists("../scripts/"):
print "erai2nc: the scripts directory is missing"
sys.exit()
# since the scripts directory is there, try importing the modules
sys.path.append('../scripts')
import meteorologicalfunctions as mf
import qcio
import qcutils
do_plots = True
# load the control file contents
cf = qcio.load_controlfile(path="../controlfiles/MODIS/")
# get the DAP file name
evi_url = cf["Files"]["evi_path"]
evi_file = cf["Files"]["evi_file"]
evi_filename = os.path.join(evi_url,evi_file)
# open the EVI file on the AusCover THREDDS server
nc_file = netCDF4.Dataset(evi_filename,"r")
# get the latitude and longitude resolution
lat_resolution = getattr(nc_file,"geospatial_lat_resolution")
lon_resolution = getattr(nc_file,"geospatial_lon_resolution")
# read latitude, longitude and time from the MODIS file
lat = nc_file.variables["latitude"][:]
lon = nc_file.variables["longitude"][:]
modis_time = nc_file.variables["time"][:]
# get a Python datetime series from the netCDF time (UTC)
modis_time_units = getattr(nc_file.variables["time"],"units")
t = time.localtime()
rundatetime = datetime.datetime(t[0],t[1],t[2],t[3],t[4],t[5]).strftime("%Y%m%d%H%M")
log_filename = 'logfiles/batchprocess_'+rundatetime+'.log'
logging.basicConfig(filename=log_filename,
format='%(asctime)s %(levelname)s %(message)s',
datefmt = '%H:%M:%S',
level=logging.DEBUG)
console = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', '%H:%M:%S')
console.setFormatter(formatter)
console.setLevel(logging.INFO)
logging.getLogger('').addHandler(console)
# get the batch processing control file
if len(sys.argv)==1:
cf_batch = qcio.load_controlfile(path='controlfiles')
if len(cf_batch)==0: sys.exit()
else:
cfname = sys.argv[1]
if os.path.exists(cfname):
cf_batch = qcio.get_controlfilecontents(cfname)
else:
logging.error("Control file "+cfname+" does not exist")
sys.exit()
level_list = ['L1','L2','L3','concatenate','climatology','cpd','L4','L5','L6']
if "Options" in cf_batch:
if "levels" in cf_batch["Options"]: level_list = ast.literal_eval(cf_batch["Options"]["levels"])
for level in level_list:
if level.lower()=="l1":
# L1 processing
def do_l3qc(self):
"""
Call qcls.l3qc_sitename function
Performs L3 Corrections and QA/QC processing on L2 data
Outputs L3 netCDF file to ncData folder
Outputs L3 netCDF file to OzFlux folder
Available corrections:
* corrections requiring ancillary measurements or samples
marked with an asterisk
Linear correction
fixed slope
linearly shifting slope
Conversion of virtual temperature to actual temperature
2D Coordinate rotation
Massman correction for frequency attenuation*
Webb, Pearman and Leuning correction for flux effects on density
measurements
Conversion of virtual heat flux to actual heat flux
Correction of soil moisture content to empirical calibration
curve*
Addition of soil heat storage to ground ground heat flux*
ControlFiles:
L3_year.txt
or
L3a.txt
ControlFile contents (see ControlFile/Templates/L3.txt for example):
[General]:
Python control parameters
[Files]:
L2 input file name and path
L3 output file name and ncData folder path
L3 OzFlux output file name and OzFlux folder path
[Massman] (where available):
Constants used in frequency attenuation correction
zmd: instrument height (z) less zero-plane displacement
height (d), m
z0: aerodynamic roughness length, m
angle: angle from CSAT mounting point between CSAT and
IRGA mid-path, degrees
CSATarm: distance from CSAT mounting point to CSAT
mid-path, m
IRGAarm: distance from CSAT mounting point to IRGA
mid-path, m
[Soil]:
Constants used in correcting Fg for storage and in empirical
corrections of soil water content
FgDepth: Heat flux plate depth, m
BulkDensity: Soil bulk density, kg/m3
OrganicContent: Soil organic content, fraction
SwsDefault
Constants for empirical corrections using log(sensor)
and exp(sensor) functions (SWC_a0, SWC_a1, SWC_b0,
SWC_b1, SWC_t, TDR_a0, TDR_a1, TDR_b0, TDR_b1,
TDR_t)
Variable and attributes lists (empSWCin, empSWCout,
empTDRin, empTDRout, linTDRin, SWCattr, TDRattr)
[Output]:
Variable subset list for OzFlux output file
[Variables]:
Variable names and parameters for:
Range check to set upper and lower rejection limits
Diurnal check to reject observations by time of day that
are outside specified standard deviation limits
Timestamps, slope, and offset for Linear correction
[Plots]:
Variable lists for plot generation
"""
logging.info(" Starting L3 processing ...")
self.cf = qcio.load_controlfile(path='controlfiles')
if len(self.cf)==0:
logging.info( " L3: no control file chosen")
self.do_progress(text='Waiting for input ...')
return
infilename = qcio.get_infilenamefromcf(self.cf)
if not qcutils.file_exists(infilename): self.do_progress(text='An error occurred, check the console ...'); return
self.ds2 = qcio.nc_read_series(infilename)
if len(self.ds2.series.keys())==0: self.do_progress(text='An error occurred, check the console ...'); del self.ds2; return
self.update_startenddate(str(self.ds2.series['DateTime']['Data'][0]),
str(self.ds2.series['DateTime']['Data'][-1]))
self.do_progress(text='Doing L3 QC & Corrections ...')
self.ds3 = qcls.l3qc(self.cf,self.ds2)
self.do_progress(text='Finished L3')
txtstr = ' Finished L3: Standard processing for site: '
txtstr = txtstr+self.ds3.globalattributes['site_name'].replace(' ','')
logging.info(txtstr)
self.do_progress(text='Saving L3 QC & Corrected NetCDF data ...') # put up the progress message
outfilename = qcio.get_outfilenamefromcf(self.cf)
if len(outfilename)==0: self.do_progress(text='An error occurred, check the console ...'); return
ncFile = qcio.nc_open_write(outfilename)
outputlist = qcio.get_outputlistfromcf(self.cf,'nc')
qcio.nc_write_series(ncFile,self.ds3,outputlist=outputlist) # save the L3 data
self.do_progress(text='Finished saving L3 QC & Corrected NetCDF data') # tell the user we are done
logging.info(' Finished saving L3 QC & Corrected NetCDF data')
logging.info("")
