def do_file_convert_nc2xls():
"""
Purpose:
Convert a PFP-style netCDF file to an Excel workbook.
Usage:
Side effects:
Creates an Excel workbook in the same directory as the netCDF file.
Author: PRI
Date: Back in the day
Mods:
August 2018: rewrite for use with new GUI
"""
logger.info(" Starting conversion to Excel file")
try:
ncfilename = pfp_io.get_filename_dialog(file_path="../Sites",
title="Choose a netCDF file",
ext="*.nc")
if len(ncfilename) == 0:
logger.info(" No file selected, cancelling ...")
return
logger.info(" Converting netCDF file to Excel file")
pfp_io.nc_2xls(ncfilename, outputlist=None)
logger.info(" Finished converting netCDF file")
logger.info("")
except Exception:
msg = " Error converting to Excel file, see below for details ..."
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_plot_fcvsustar():
"""
Purpose:
Plot Fc versus u*.
Usage:
pfp_top_level.do_plot_fcvsustar()
Side effects:
Annual and seasonal plots of Fc versus u* to the screen and creates .PNG
hardcopies of the plots.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
logger.info("Starting Fc versus u* plots")
try:
file_path = pfp_io.get_filename_dialog(file_path="../Sites",
title="Choose a netCDF file")
if len(file_path) == 0 or not os.path.isfile(file_path):
return
# read the netCDF file
ds = pfp_io.nc_read_series(file_path)
if ds.returncodes["value"] != 0: return
logger.info("Plotting Fc versus u* ...")
pfp_plot.plot_fcvsustar(ds)
logger.info(" Finished plotting Fc versus u*")
logger.info("")
except Exception:
error_message = " An error occured while plotting Fc versus u*, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def compare_eddypro():
epname = pfp_io.get_filename_dialog(title='Choose an EddyPro full output file')
ofname = pfp_io.get_filename_dialog(title='Choose an L3 output file')
ds_ep = pfp_io.read_eddypro_full(epname)
ds_of = pfp_io.nc_read_series(ofname)
dt_ep = ds_ep.series['DateTime']['Data']
dt_of = ds_of.series['DateTime']['Data']
start_datetime = max([dt_ep[0],dt_of[0]])
end_datetime = min([dt_ep[-1],dt_of[-1]])
si_of = pfp_utils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact')
ei_of = pfp_utils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact')
si_ep = pfp_utils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact')
ei_ep = pfp_utils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact')
us_of = pfp_utils.GetVariable(ds_of,'ustar',start=si_of,end=ei_of)
us_ep = pfp_utils.GetVariable(ds_ep,'ustar',start=si_ep,end=ei_ep)
Fh_of = pfp_utils.GetVariable(ds_of,'Fh',start=si_of,end=ei_of)
Fh_ep = pfp_utils.GetVariable(ds_ep,'Fh',start=si_ep,end=ei_ep)
Fe_of = pfp_utils.GetVariable(ds_of,'Fe',start=si_of,end=ei_of)
Fe_ep = pfp_utils.GetVariable(ds_ep,'Fe',start=si_ep,end=ei_ep)
Fc_of = pfp_utils.GetVariable(ds_of,'Fc',start=si_of,end=ei_of)
Fc_ep = pfp_utils.GetVariable(ds_ep,'Fc',start=si_ep,end=ei_ep)
# copy the range check values from the OFQC attributes to the EP attributes
for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]):
for item in ["rangecheck_upper", "rangecheck_lower"]:
if item in of["Attr"]:
ep["Attr"][item] = of["Attr"][item]
# apply QC to the EddyPro data
pfp_ck.ApplyRangeCheckToVariable(us_ep)
pfp_ck.ApplyRangeCheckToVariable(Fc_ep)
pfp_ck.ApplyRangeCheckToVariable(Fe_ep)
pfp_ck.ApplyRangeCheckToVariable(Fh_ep)
# plot the comparison
plt.ion()
fig = plt.figure(1,figsize=(8,8))
pfp_plot.xyplot(us_ep["Data"],us_of["Data"],sub=[2,2,1],regr=2,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)')
pfp_plot.xyplot(Fh_ep["Data"],Fh_of["Data"],sub=[2,2,2],regr=2,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)')
pfp_plot.xyplot(Fe_ep["Data"],Fe_of["Data"],sub=[2,2,3],regr=2,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)')
pfp_plot.xyplot(Fc_ep["Data"],Fc_of["Data"],sub=[2,2,4],regr=2,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)')
plt.tight_layout()
plt.draw()
plt.ioff()
def do_utilities_ustar_mpt(cfg=None, mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Moving Point Threshold (MPT) method.
This code calls the original FluxNet MPT C code. The executable for this is
in PyFluxPro/mpt/bin.
Side effects:
Calls pfp_mpt.mpt_main
"""
try:
logger.info(" Starting u* threshold detection (MPT)")
if mode == "standard":
stdname = "controlfiles/standard/mpt.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path='../Sites', title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" MPT: no input file chosen")
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_MPT.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
logger.info(" Doing u* threshold detection (MPT)")
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
pfp_mpt.mpt_main(cfg)
logger.info(" Finished u* threshold detection (MPT)")
logger.info("")
except Exception:
error_message = " An error occured while doing MPT u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_nc2ecostress(cfg=None):
"""
Purpose:
Convert a PFP-style netCDF file to an ECOSTRESS CSV file.
Usage:
Side effects:
Creates a CSV file in the same directory as the netCDF file.
Author: PRI
Date: Back in the day
Mods:
September 2018: rewrite for use with new GUI
"""
logger.info(" Starting conversion to ECOSTRESS file")
try:
if not cfg:
# check to see if there is an nc2ecostress.txt control file in controlfiles/standard
# if there is
# open controlfiles/standard/nc2csv_ecostress.txt
# ask for netCDF file name
# add [Files] section to control file
stdname = "controlfiles/standard/nc2csv_ecostress.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
cfg["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
result = pfp_io.write_csv_ecostress(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occurred, check the log messages")
logger.error("")
except Exception:
error_message = " Error converting to ECOSTRESS format, see below for details ... "
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_cpd_barr(cfg=None, mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Change Point Detection method described in
Barr et al. 2013, AFM 171-172, pp31-45.
This code is a line-by-line translation of the original Barr MATLAB scripts
into Python.
"""
try:
logger.info(" Starting CPD u* threshold detection (Barr)")
if mode == "standard":
stdname = "controlfiles/standard/cpd2.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD (Barr): no input file chosen")
return
if "Files" not in cfg:
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD_Barr.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path='controlfiles')
if len(cfg) == 0:
return
logger.info("Doing CPD u* threshold detection (Barr)")
pfp_cpd2.cpd2_main(cfg)
logger.info(" Finished CPD u* threshold detection (Barr)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold (Barr), see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_mpt(mode="standard"):
"""
Calls pfp_mpt.mpt_main
Calculate the u* threshold using the Moving Point Threshold (MPT) method.
"""
try:
logger.info(" Starting u* threshold detection (MPT)")
if mode == "standard":
stdname = "controlfiles/standard/mpt.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path='../Sites', title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" MPT: no input file chosen")
return
if "Files" not in dir(cf):
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_MPT.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
logger.info(" Doing u* threshold detection (MPT)")
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "interactive"
pfp_mpt.mpt_main(cf)
logger.info(" Finished u* threshold detection (MPT)")
logger.info("")
except Exception:
error_message = " An error occured while doing MPT u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_plot_timeseries():
"""
Purpose:
Plot time series of data, usually L3 and above.
Usage:
pfp_top_level.do_plot_timeseries()
Side effects:
Plots timeseries to the screen and creates .PNG hardcopies of
the plots.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
try:
logger.info("Starting timeseries plot")
stdname = "controlfiles/standard/fluxnet.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(file_path="../Sites",
title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = os.path.split(filename)[0] + "/"
cf["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
logger.info("Loaded control file ...")
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
logger.info("Plotting time series ...")
pfp_plot.plot_fluxnet(cf)
logger.info(" Finished plotting time series")
logger.info("")
except Exception:
error_message = " An error occured while plotting time series, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_cpd1(mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Change Point Detection method described in
Barr et al. 2013, AFM 171-172, pp31-45.
This code is the original implementation by Ian McHugh.
"""
try:
logger.info(" Starting CPD u* threshold detection (McHugh)")
if mode == "standard":
stdname = "controlfiles/standard/cpd1.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD (McHugh): no input file chosen")
return
if "Files" not in cf:
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD_McHugh.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path='controlfiles')
if len(cf) == 0:
return
logger.info("Doing CPD u* threshold detection (McHugh)")
pfp_cpd1.cpd1_main(cf)
logger.info(" Finished CPD u* threshold detection (McHugh)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold (McHugh), see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_ncupdate(cfg=None):
"""
Purpose:
Convert from original netCDF files to V1 (October 2018).
Usage:
Author: PRI
Date: October 2018
"""
logger.info(" Starting conversion of netCDF")
if not cfg:
# check to see if there is an nc2ecostress.txt control file in controlfiles/standard
# if there is
# open controlfiles/standard/nc2csv_ecostress.txt
# ask for netCDF file name
# add [Files] section to control file
stdname = os.path.join("controlfiles", "standard",
"map_old_to_new.txt")
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(file_path="../OzFlux/Sites",
title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
cfg["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
result = pfp_compliance.nc_update(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occured, check the log messages")
logger.error("")
return
def do_utilities_climatology(cfg=None, mode="standard"):
try:
logger.info(" Starting climatology")
if mode == "standard":
stdname = "controlfiles/standard/climatology.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title='Choose a netCDF file')
if not os.path.exists(filename):
logger.info(" Climatology: no input file chosen")
return
if "Files" not in cfg:
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_Climatology.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
logger.info("Doing the climatology")
pfp_clim.climatology(cfg)
logger.info(" Finished climatology")
logger.info("")
except Exception:
error_message = " An error occured while doing climatology, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_cpd(mode="standard"):
try:
logger.info(" Starting u* threshold detection (CPD)")
if mode == "standard":
stdname = "controlfiles/standard/cpd.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD: no input file chosen")
return
if "Files" not in cf:
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path='controlfiles')
if len(cf) == 0:
return
logger.info("Doing u* threshold detection (CPD)")
pfp_cpd.cpd_main(cf)
logger.info(" Finished u* threshold detection (CPD)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_nc2biomet(cfg, mode="standard"):
"""
Purpose:
Convert a PFP-style netCDF file to an EddyPro biomet CSV file.
Usage:
Side effects:
Creates a CSV file in the same directory as the netCDF file.
Author: PRI
Date: Back in the day
Mods:
March 2020: rewrite for use with new GUI
April 2020: routine can be invoked from File/Convert menu or
by loading control file and using Run/Current.
The latter method allows the user to modify the
control file before running it.
"""
logger.info(" Starting conversion to EddyPro biomet file")
try:
# check to see if the user chose a standard or a custom run
if cfg is None and mode == "standard":
# standard run so we use the control file in PyFluxPro/controlfiles/standard
stdname = "controlfiles/standard/nc2csv_biomet.txt"
# check to see if the standard control file exists
if os.path.exists(stdname):
# standard control file exists so read it
cfg = pfp_io.get_controlfilecontents(stdname)
# then ask the user to choose a netCDF file
filename = pfp_io.get_filename_dialog(
file_path=".", title='Choose a netCDF file')
# check that the netCDF file exists
if not os.path.exists(filename):
# return if no file chosen
logger.info(" Write biomet CSV file: no input file chosen")
return
# add a [Files] section to the control file ...
if "Files" not in cfg:
cfg["Files"] = {}
# ... and put the file path, input file name and output file name in [Files]
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_biomet.csv")
else:
# issue an error mesage and return if the standard control file does not exist
msg = " Write biomet CSV file: standard control file 'nc2csv_biomet.txt' does not exist"
logger.error(msg)
return
elif cfg is not None and mode == "custom":
# custom run so we proceed with the user's control file
pass
else:
# tell the user we got the wrong input options and return
msg = " Write biomet CSV file: wrong input options"
logger.error(msg)
return
# add the [Options] section and populate it
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cfg["Options"]["show_plots"] = "Yes"
# do the business
result = pfp_io.write_csv_ep_biomet(cfg)
# check everything went well
if result == 1:
# looks good
logger.info(" Finished converting netCDF file")
logger.info("")
else:
# or not
logger.error("")
logger.error(" An error occurred, check the log messages")
logger.error("")
except Exception:
# tell the user if something goes wrong and put the exception in the log window
error_message = " Error converting to BIOMET format, see below for details ... "
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
