def do_L5_batch(cf_level):
#logger = pfp_log.change_logger_filename("pfp_log", "L5")
for i in list(cf_level.keys()):
if not os.path.isfile(cf_level[i]):
msg = " Control file " + cf_level[i] + " not found"
logger.error(msg)
continue
cf_file_name = os.path.split(cf_level[i])
msg = "Starting L5 processing with " + cf_file_name[1]
logger.info(msg)
try:
cf_l5 = pfp_io.get_controlfilecontents(cf_level[i])
if "Options" not in cf_l5:
cf_l5["Options"] = {}
cf_l5["Options"]["call_mode"] = "batch"
cf_l5["Options"]["show_plots"] = "No"
infilename = pfp_io.get_infilenamefromcf(cf_l5)
ds4 = pfp_io.nc_read_series(infilename)
if ds4.returncodes["value"] != 0: return
ds5 = pfp_levels.l5qc(None, cf_l5, ds4)
outfilename = pfp_io.get_outfilenamefromcf(cf_l5)
nc_file = pfp_io.nc_open_write(outfilename)
if nc_file is None: return
pfp_io.nc_write_series(nc_file, ds5)
msg = "Finished L5 processing with " + cf_file_name[1]
logger.info(msg)
# now plot the fingerprints for the L5 files
cf_fp = pfp_io.get_controlfilecontents("controlfiles/standard/fingerprint.txt")
if "Files" not in dir(cf_fp):
cf_fp["Files"] = {}
file_name = pfp_io.get_outfilenamefromcf(cf_l5)
file_path = ntpath.split(file_name)[0] + "/"
cf_fp["Files"]["file_path"] = file_path
cf_fp["Files"]["in_filename"] = ntpath.split(file_name)[1]
if "plot_path" in cf_l5["Files"]:
cf_fp["Files"]["plot_path"] = cf_l5["Files"]["plot_path"]
else:
cf_fp["Files"]["plot_path"] = file_path[:file_path.index("Data")] + "Plots/"
if "Options" not in cf_fp:
cf_fp["Options"] = {}
cf_fp["Options"]["call_mode"] = "batch"
cf_fp["Options"]["show_plots"] = "No"
msg = "Doing fingerprint plots using " + cf_fp["Files"]["in_filename"]
logger.info(msg)
pfp_plot.plot_fingerprint(cf_fp)
msg = "Finished fingerprint plots"
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during L5 with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_L4_batch(cf_level):
for i in cf_level.keys():
if not os.path.isfile(cf_level[i]):
msg = " Control file " + cf_level[i] + " not found"
logger.error(msg)
continue
cf_file_name = os.path.split(cf_level[i])
msg = "Starting L4 processing with " + cf_file_name[1]
logger.info(msg)
try:
cf_l4 = pfp_io.get_controlfilecontents(cf_level[i])
if "Options" not in cf_l4:
cf_l4["Options"] = {}
cf_l4["Options"]["call_mode"] = "batch"
cf_l4["Options"]["show_plots"] = "No"
infilename = pfp_io.get_infilenamefromcf(cf_l4)
ds3 = pfp_io.nc_read_series(infilename)
ds4 = pfp_levels.l4qc(None, cf_l4, ds3)
outfilename = pfp_io.get_outfilenamefromcf(cf_l4)
outputlist = pfp_io.get_outputlistfromcf(cf_l4, "nc")
ncFile = pfp_io.nc_open_write(outfilename)
pfp_io.nc_write_series(ncFile, ds4, outputlist=outputlist)
msg = "Finished L4 processing with " + cf_file_name[1]
logger.info(msg)
# now plot the fingerprints for the L4 files
cf_fp = pfp_io.get_controlfilecontents("controlfiles/standard/fingerprint.txt")
if "Files" not in dir(cf_fp):
cf_fp["Files"] = {}
file_name = pfp_io.get_outfilenamefromcf(cf_l4)
file_path = ntpath.split(file_name)[0] + "/"
cf_fp["Files"]["file_path"] = file_path
cf_fp["Files"]["in_filename"] = ntpath.split(file_name)[1]
if "plot_path" in cf_l4["Files"]:
cf_fp["Files"]["plot_path"] = cf_l4["Files"]["plot_path"]
else:
cf_fp["Files"]["plot_path"] = file_path[:file_path.index("Data")] + "Plots/"
if "Options" not in cf_fp:
cf_fp["Options"] = {}
cf_fp["Options"]["call_mode"] = "batch"
cf_fp["Options"]["show_plots"] = "No"
msg = "Doing fingerprint plots using " + cf_fp["Files"]["in_filename"]
logger.info(msg)
pfp_plot.plot_fingerprint(cf_fp)
logger.info("Finished fingerprint plots")
logger.info("")
except Exception:
msg = "Error occurred during L4 with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_concatenate_batch(cf_level):
logger = pfp_log.change_logger_filename("pfp_log", "concatenate")
for i in cf_level.keys():
if not os.path.isfile(cf_level[i]):
msg = " Control file " + cf_level[i] + " not found"
logger.error(msg)
continue
cf_file_name = os.path.split(cf_level[i])
msg = "Starting concatenation with " + cf_file_name[1]
logger.info(msg)
try:
cf_cc = pfp_io.get_controlfilecontents(cf_level[i])
info = pfp_compliance.ParseConcatenateControlFile(cf_cc)
if not info["NetCDFConcatenate"]["OK"]:
msg = " Error occurred parsing the control file " + cf_file_name[1]
logger.error(msg)
continue
pfp_io.NetCDFConcatenate(info)
msg = "Finished concatenation with " + cf_file_name[1]
logger.info(msg)
# now plot the fingerprints for the concatenated files
opt = pfp_utils.get_keyvaluefromcf(cf_cc, ["Options"], "DoFingerprints", default="yes")
if opt.lower() == "no":
continue
cf_fp = pfp_io.get_controlfilecontents("controlfiles/standard/fingerprint.txt")
if "Files" not in dir(cf_fp):
cf_fp["Files"] = {}
file_name = cf_cc["Files"]["Out"]["ncFileName"]
file_path = ntpath.split(file_name)[0] + "/"
cf_fp["Files"]["file_path"] = file_path
cf_fp["Files"]["in_filename"] = ntpath.split(file_name)[1]
if "plot_path" in cf_cc["Files"]:
cf_fp["Files"]["plot_path"] = cf_cc["Files"]["plot_path"]
else:
cf_fp["Files"]["plot_path"] = file_path[:file_path.index("Data")] + "Plots/"
if "Options" not in cf_fp:
cf_fp["Options"] = {}
cf_fp["Options"]["call_mode"] = "batch"
cf_fp["Options"]["show_plots"] = "No"
msg = "Doing fingerprint plots using " + cf_fp["Files"]["in_filename"]
logger.info(msg)
pfp_plot.plot_fingerprint(cf_fp)
msg = "Finished fingerprint plots"
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during concatenation with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_L6_batch(cf_level):
logger = pfp_log.change_logger_filename("pfp_log", "L6")
for i in cf_level.keys():
if not os.path.isfile(cf_level[i]):
msg = " Control file " + cf_level[i] + " not found"
logger.error(msg)
continue
cf_file_name = os.path.split(cf_level[i])
msg = "Starting L6 processing with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "batch"
cf["Options"]["show_plots"] = "No"
infilename = pfp_io.get_infilenamefromcf(cf)
ds5 = pfp_io.nc_read_series(infilename)
ds6 = pfp_levels.l6qc(None, cf, ds5)
outfilename = pfp_io.get_outfilenamefromcf(cf)
ncFile = pfp_io.nc_open_write(outfilename)
pfp_io.nc_write_series(ncFile, ds6)
msg = "Finished L6 processing with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during L6 with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_L3_batch(cf_level):
#logger = pfp_log.change_logger_filename("pfp_log", "L3")
for i in list(cf_level.keys()):
cf_file_name = os.path.split(cf_level[i])
msg = "Starting L3 processing with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
infilename = pfp_io.get_infilenamefromcf(cf)
ds2 = pfp_io.nc_read_series(infilename)
if ds2.returncodes["value"] != 0: return
ds3 = pfp_levels.l3qc(cf, ds2)
outfilename = pfp_io.get_outfilenamefromcf(cf)
nc_file = pfp_io.nc_open_write(outfilename)
if nc_file is None: return
pfp_io.nc_write_series(nc_file, ds3)
msg = "Finished L3 processing with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during L3 processing " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_utilities_ustar_mpt(cfg=None, mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Moving Point Threshold (MPT) method.
This code calls the original FluxNet MPT C code. The executable for this is
in PyFluxPro/mpt/bin.
Side effects:
Calls pfp_mpt.mpt_main
"""
try:
logger.info(" Starting u* threshold detection (MPT)")
if mode == "standard":
stdname = "controlfiles/standard/mpt.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path='../Sites', title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" MPT: no input file chosen")
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_MPT.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
logger.info(" Doing u* threshold detection (MPT)")
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
pfp_mpt.mpt_main(cfg)
logger.info(" Finished u* threshold detection (MPT)")
logger.info("")
except Exception:
error_message = " An error occured while doing MPT u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_reddyproc_batch(cf_level):
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting REddyProc output with " + cf_file_name[1]
logger.info(msg)
cf = pfp_io.get_controlfilecontents(cf_level[i])
pfp_io.reddyproc_write_csv(cf)
msg = "Finished REddyProc output with " + cf_file_name[1]
logger.info(msg)
logger.info("")
return
def do_file_convert_nc2ecostress(cfg=None):
"""
Purpose:
Convert a PFP-style netCDF file to an ECOSTRESS CSV file.
Usage:
Side effects:
Creates a CSV file in the same directory as the netCDF file.
Author: PRI
Date: Back in the day
Mods:
September 2018: rewrite for use with new GUI
"""
logger.info(" Starting conversion to ECOSTRESS file")
try:
if not cfg:
# check to see if there is an nc2ecostress.txt control file in controlfiles/standard
# if there is
# open controlfiles/standard/nc2csv_ecostress.txt
# ask for netCDF file name
# add [Files] section to control file
stdname = "controlfiles/standard/nc2csv_ecostress.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
cfg["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
result = pfp_io.write_csv_ecostress(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occurred, check the log messages")
logger.error("")
except Exception:
error_message = " Error converting to ECOSTRESS format, see below for details ... "
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_fluxnet_batch(cf_level):
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting FluxNet output with " + cf_file_name[1]
logger.info(msg)
cf = pfp_io.get_controlfilecontents(cf_level[i])
pfp_io.fn_write_csv(cf)
msg = "Finished FluxNet output with " + cf_file_name[1]
logger.info(msg)
logger.info("")
return
def do_utilities_ustar_cpd_barr(cfg=None, mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Change Point Detection method described in
Barr et al. 2013, AFM 171-172, pp31-45.
This code is a line-by-line translation of the original Barr MATLAB scripts
into Python.
"""
try:
logger.info(" Starting CPD u* threshold detection (Barr)")
if mode == "standard":
stdname = "controlfiles/standard/cpd2.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD (Barr): no input file chosen")
return
if "Files" not in cfg:
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD_Barr.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path='controlfiles')
if len(cfg) == 0:
return
logger.info("Doing CPD u* threshold detection (Barr)")
pfp_cpd2.cpd2_main(cfg)
logger.info(" Finished CPD u* threshold detection (Barr)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold (Barr), see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_mpt(mode="standard"):
"""
Calls pfp_mpt.mpt_main
Calculate the u* threshold using the Moving Point Threshold (MPT) method.
"""
try:
logger.info(" Starting u* threshold detection (MPT)")
if mode == "standard":
stdname = "controlfiles/standard/mpt.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path='../Sites', title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" MPT: no input file chosen")
return
if "Files" not in dir(cf):
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_MPT.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
logger.info(" Doing u* threshold detection (MPT)")
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "interactive"
pfp_mpt.mpt_main(cf)
logger.info(" Finished u* threshold detection (MPT)")
logger.info("")
except Exception:
error_message = " An error occured while doing MPT u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_plot_timeseries():
"""
Purpose:
Plot time series of data, usually L3 and above.
Usage:
pfp_top_level.do_plot_timeseries()
Side effects:
Plots timeseries to the screen and creates .PNG hardcopies of
the plots.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
try:
logger.info("Starting timeseries plot")
stdname = "controlfiles/standard/fluxnet.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(file_path="../Sites",
title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = os.path.split(filename)[0] + "/"
cf["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
logger.info("Loaded control file ...")
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
logger.info("Plotting time series ...")
pfp_plot.plot_fluxnet(cf)
logger.info(" Finished plotting time series")
logger.info("")
except Exception:
error_message = " An error occured while plotting time series, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_ncupdate(cfg=None):
"""
Purpose:
Convert from original netCDF files to V1 (October 2018).
Usage:
Author: PRI
Date: October 2018
"""
logger.info(" Starting conversion of netCDF")
if not cfg:
# check to see if there is an nc2ecostress.txt control file in controlfiles/standard
# if there is
# open controlfiles/standard/nc2csv_ecostress.txt
# ask for netCDF file name
# add [Files] section to control file
stdname = os.path.join("controlfiles", "standard",
"map_old_to_new.txt")
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(file_path="../OzFlux/Sites",
title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
cfg["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
result = pfp_compliance.nc_update(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occured, check the log messages")
logger.error("")
return
def do_utilities_ustar_cpd1(mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Change Point Detection method described in
Barr et al. 2013, AFM 171-172, pp31-45.
This code is the original implementation by Ian McHugh.
"""
try:
logger.info(" Starting CPD u* threshold detection (McHugh)")
if mode == "standard":
stdname = "controlfiles/standard/cpd1.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD (McHugh): no input file chosen")
return
if "Files" not in cf:
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD_McHugh.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path='controlfiles')
if len(cf) == 0:
return
logger.info("Doing CPD u* threshold detection (McHugh)")
pfp_cpd1.cpd1_main(cf)
logger.info(" Finished CPD u* threshold detection (McHugh)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold (McHugh), see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_ecostress_batch(cf_level):
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting ECOSTRESS output with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
pfp_io.write_csv_ecostress(cf)
msg = "Finished ECOSTRESS output with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during ECOSTRESS output with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_utilities_climatology(cfg=None, mode="standard"):
try:
logger.info(" Starting climatology")
if mode == "standard":
stdname = "controlfiles/standard/climatology.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title='Choose a netCDF file')
if not os.path.exists(filename):
logger.info(" Climatology: no input file chosen")
return
if "Files" not in cfg:
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_Climatology.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
logger.info("Doing the climatology")
pfp_clim.climatology(cfg)
logger.info(" Finished climatology")
logger.info("")
except Exception:
error_message = " An error occured while doing climatology, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_ncupdate():
"""
Purpose:
Convert from OFQC netCDF files to PFP V1 (October 2018).
Usage:
Author: PRI
Date: October 2018
"""
logger.info(" Starting conversion of netCDF")
try:
# get a list of netCDF files to update
file_names = QtWidgets.QFileDialog.getOpenFileNames(
caption="Choose netCDF files", filter="*.nc")[0]
if len(file_names) == 0: return
# get the control file
stdname = os.path.join("controlfiles", "standard", "nc_cleanup.txt")
cfg = pfp_io.get_controlfilecontents(stdname)
if len(cfg) == 0: return
# loop over the selected files
for file_name in file_names:
# make the [Files] section
cfg["Files"] = {
"file_path": os.path.join(os.path.split(file_name)[0], ""),
"in_filename": os.path.split(file_name)[1]
}
# make the [Options] section
cfg["Options"] = {"call_mode": "interactive", "show_plots": "Yes"}
result = pfp_compliance.nc_update(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occured, check the log messages")
logger.error("")
except Exception:
msg = " Error running netCDF update, see below for details ..."
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_L1_batch(cf_level):
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
logger.info("Starting L1 processing with %s", cf_file_name[1])
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
ds1 = pfp_levels.l1qc(cf)
outfilename = pfp_io.get_outfilenamefromcf(cf)
ncFile = pfp_io.nc_open_write(outfilename)
pfp_io.nc_write_series(ncFile, ds1)
msg = "Finished L1 processing with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during L1 processing " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_utilities_ustar_cpd(mode="standard"):
try:
logger.info(" Starting u* threshold detection (CPD)")
if mode == "standard":
stdname = "controlfiles/standard/cpd.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD: no input file chosen")
return
if "Files" not in cf:
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path='controlfiles')
if len(cf) == 0:
return
logger.info("Doing u* threshold detection (CPD)")
pfp_cpd.cpd_main(cf)
logger.info(" Finished u* threshold detection (CPD)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_climatology_batch(cf_level):
for i in cf_level.keys():
if not os.path.isfile(cf_level[i]):
msg = " Control file " + cf_level[i] + " not found"
logger.error(msg)
continue
cf_file_name = os.path.split(cf_level[i])
msg = "Starting climatology with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
pfp_clim.climatology(cf)
msg = "Finished climatology with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during climatology with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_mpt_batch(cf_level):
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting MPT with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "batch"
cf["Options"]["show_plots"] = "No"
pfp_mpt.mpt_main(cf)
msg = "Finished MPT with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during MPT with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_cpd1_batch(cf_level):
logger = pfp_log.change_logger_filename("pfp_log", "cpd1")
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting CPD (McHugh) with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "batch"
cf["Options"]["show_plots"] = "No"
pfp_cpd1.cpd1_main(cf)
msg = "Finished CPD (McHugh) with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during CPD (McHugh) with " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_L2_batch(cf_level):
logger = pfp_log.change_logger_filename("pfp_log", "L2")
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting L2 processing with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
infilename = pfp_io.get_infilenamefromcf(cf)
ds1 = pfp_io.nc_read_series(infilename)
ds2 = pfp_levels.l2qc(cf, ds1)
outfilename = pfp_io.get_outfilenamefromcf(cf)
ncFile = pfp_io.nc_open_write(outfilename)
pfp_io.nc_write_series(ncFile, ds2)
msg = "Finished L2 processing with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during L2 processing " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
def do_L3_batch(cf_level):
for i in cf_level.keys():
cf_file_name = os.path.split(cf_level[i])
msg = "Starting L3 processing with " + cf_file_name[1]
logger.info(msg)
try:
cf = pfp_io.get_controlfilecontents(cf_level[i])
infilename = pfp_io.get_infilenamefromcf(cf)
ds2 = pfp_io.nc_read_series(infilename)
ds3 = pfp_levels.l3qc(cf, ds2)
outfilename = pfp_io.get_outfilenamefromcf(cf)
outputlist = pfp_io.get_outputlistfromcf(cf, "nc")
ncFile = pfp_io.nc_open_write(outfilename)
pfp_io.nc_write_series(ncFile, ds3, outputlist=outputlist)
msg = "Finished L3 processing with " + cf_file_name[1]
logger.info(msg)
logger.info("")
except Exception:
msg = "Error occurred during L3 processing " + cf_file_name[1]
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
continue
return
t = time.localtime()
rundatetime = datetime.datetime(t[0], t[1], t[2], t[3], t[4],
t[5]).strftime("%Y%m%d%H%M")
log_file_name = 'batchprocess_' + rundatetime + '.log'
logger = pfp_log.init_logger(logger_name="pfp_log",
log_file_name=log_file_name)
# get the batch processing control file
if len(sys.argv) == 1:
cf_batch = pfp_io.load_controlfile(path='controlfiles')
if len(cf_batch) == 0: sys.exit()
else:
cfname = sys.argv[1]
if os.path.exists(cfname):
cf_batch = pfp_io.get_controlfilecontents(cfname)
else:
logger.error("Control file " + cfname + " does not exist")
sys.exit()
level_list = [
'L1', 'L2', 'L3', 'concatenate', 'climatology', 'cpd', 'L4', 'L5', 'L6'
]
if "Options" in cf_batch:
if "levels" in cf_batch["Options"]:
level_list = ast.literal_eval(cf_batch["Options"]["levels"])
for level in level_list:
if level.lower() not in [
"l1", "l2", "l3", "ecostress", "fluxnet", "reddyproc",
"concatenate", "climatology", "cpd", "l4", "l5", "l6"
]:
def do_file_convert_nc2biomet(cfg, mode="standard"):
"""
Purpose:
Convert a PFP-style netCDF file to an EddyPro biomet CSV file.
Usage:
Side effects:
Creates a CSV file in the same directory as the netCDF file.
Author: PRI
Date: Back in the day
Mods:
March 2020: rewrite for use with new GUI
April 2020: routine can be invoked from File/Convert menu or
by loading control file and using Run/Current.
The latter method allows the user to modify the
control file before running it.
"""
logger.info(" Starting conversion to EddyPro biomet file")
try:
# check to see if the user chose a standard or a custom run
if cfg is None and mode == "standard":
# standard run so we use the control file in PyFluxPro/controlfiles/standard
stdname = "controlfiles/standard/nc2csv_biomet.txt"
# check to see if the standard control file exists
if os.path.exists(stdname):
# standard control file exists so read it
cfg = pfp_io.get_controlfilecontents(stdname)
# then ask the user to choose a netCDF file
filename = pfp_io.get_filename_dialog(
file_path=".", title='Choose a netCDF file')
# check that the netCDF file exists
if not os.path.exists(filename):
# return if no file chosen
logger.info(" Write biomet CSV file: no input file chosen")
return
# add a [Files] section to the control file ...
if "Files" not in cfg:
cfg["Files"] = {}
# ... and put the file path, input file name and output file name in [Files]
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_biomet.csv")
else:
# issue an error mesage and return if the standard control file does not exist
msg = " Write biomet CSV file: standard control file 'nc2csv_biomet.txt' does not exist"
logger.error(msg)
return
elif cfg is not None and mode == "custom":
# custom run so we proceed with the user's control file
pass
else:
# tell the user we got the wrong input options and return
msg = " Write biomet CSV file: wrong input options"
logger.error(msg)
return
# add the [Options] section and populate it
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cfg["Options"]["show_plots"] = "Yes"
# do the business
result = pfp_io.write_csv_ep_biomet(cfg)
# check everything went well
if result == 1:
# looks good
logger.info(" Finished converting netCDF file")
logger.info("")
else:
# or not
logger.error("")
logger.error(" An error occurred, check the log messages")
logger.error("")
except Exception:
# tell the user if something goes wrong and put the exception in the log window
error_message = " Error converting to BIOMET format, see below for details ... "
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
if len(cfg_file_path) == 0:
sys.exit()
else:
# control file name on the command line
if not os.path.exists(sys.argv[1]):
# control file doesn't exist
logger.error("Control file %s does not exist", sys.argv[1])
sys.exit()
else:
cfg_file_path = sys.argv[1]
# read the control file file
"""
The controlfiles organises the input and output data path, the sheet
from the site_master (e.g. all sites or individual sites) and the sa_limit.
"""
cf = pfp_io.get_controlfilecontents(cfg_file_path, mode="verbose")
xl_file_path = cf["Files"]["xl_file_path"]
xl_sheet_name = cf["Files"]["xl_sheet_name"]
era5_path = cf["Files"]["era5_path"]
out_base_path = cf["Files"]["out_base_path"]
out_process = cf["Files"]["out_process"]
concat_control_path = cf["Files"]["concat_control"]
site_sa_limit = cf["Options"]["site_sa_limit"]
# get the site information from the site master spreadsheet
site_info = read_site_master(xl_file_path, xl_sheet_name)
# get a list of sites
site_list = list(site_info.keys())
# and a list of the ERA5 files to be processed
era5_list = sorted(glob.glob(era5_path))
if out_process == 'append':
era5_files = era5_list[-1:]