def do_utilities_ustar_mpt(cfg=None, mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Moving Point Threshold (MPT) method.
This code calls the original FluxNet MPT C code. The executable for this is
in PyFluxPro/mpt/bin.
Side effects:
Calls pfp_mpt.mpt_main
"""
try:
logger.info(" Starting u* threshold detection (MPT)")
if mode == "standard":
stdname = "controlfiles/standard/mpt.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path='../Sites', title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" MPT: no input file chosen")
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_MPT.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
logger.info(" Doing u* threshold detection (MPT)")
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
pfp_mpt.mpt_main(cfg)
logger.info(" Finished u* threshold detection (MPT)")
logger.info("")
except Exception:
error_message = " An error occured while doing MPT u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_cpd_barr(cfg=None, mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Change Point Detection method described in
Barr et al. 2013, AFM 171-172, pp31-45.
This code is a line-by-line translation of the original Barr MATLAB scripts
into Python.
"""
try:
logger.info(" Starting CPD u* threshold detection (Barr)")
if mode == "standard":
stdname = "controlfiles/standard/cpd2.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD (Barr): no input file chosen")
return
if "Files" not in cfg:
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD_Barr.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path='controlfiles')
if len(cfg) == 0:
return
logger.info("Doing CPD u* threshold detection (Barr)")
pfp_cpd2.cpd2_main(cfg)
logger.info(" Finished CPD u* threshold detection (Barr)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold (Barr), see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_concatenate(cfg=None):
"""
Purpose:
Top level routine for concatenating multiple, single-year files into
a single, multiple-year file.
NOTE: The input files must be listed in the control file in chronological
order.
Usage:
pfp_top_level.do_file_concatenate()
Side effects:
Creates a single netCDF file containing the contents of the input files.
Author: PRI
Date: Back in the day
Mods:
June 2018: rewrite for use with new GUI.
"""
logger.info(" Starting concatenation of netCDF files")
if not cfg:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
logger.info("Quitting concatenation (no control file)")
return
pfp_io.nc_concatenate(cfg)
logger.info(" Finished concatenating files")
logger.info("")
return
def do_utilities_ustar_mpt(mode="standard"):
"""
Calls pfp_mpt.mpt_main
Calculate the u* threshold using the Moving Point Threshold (MPT) method.
"""
try:
logger.info(" Starting u* threshold detection (MPT)")
if mode == "standard":
stdname = "controlfiles/standard/mpt.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path='../Sites', title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" MPT: no input file chosen")
return
if "Files" not in dir(cf):
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_MPT.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
logger.info(" Doing u* threshold detection (MPT)")
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "interactive"
pfp_mpt.mpt_main(cf)
logger.info(" Finished u* threshold detection (MPT)")
logger.info("")
except Exception:
error_message = " An error occured while doing MPT u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_ustar_cpd1(mode="standard"):
"""
Purpose:
Calculate the u* threshold using the Change Point Detection method described in
Barr et al. 2013, AFM 171-172, pp31-45.
This code is the original implementation by Ian McHugh.
"""
try:
logger.info(" Starting CPD u* threshold detection (McHugh)")
if mode == "standard":
stdname = "controlfiles/standard/cpd1.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD (McHugh): no input file chosen")
return
if "Files" not in cf:
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD_McHugh.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path='controlfiles')
if len(cf) == 0:
return
logger.info("Doing CPD u* threshold detection (McHugh)")
pfp_cpd1.cpd1_main(cf)
logger.info(" Finished CPD u* threshold detection (McHugh)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold (McHugh), see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_utilities_climatology(cfg=None, mode="standard"):
try:
logger.info(" Starting climatology")
if mode == "standard":
stdname = "controlfiles/standard/climatology.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title='Choose a netCDF file')
if not os.path.exists(filename):
logger.info(" Climatology: no input file chosen")
return
if "Files" not in cfg:
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cfg["Files"]["in_filename"] = in_filename
cfg["Files"]["out_filename"] = in_filename.replace(
".nc", "_Climatology.xls")
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
elif mode == "custom":
# we get here via Run/Current and should have a control file
if cfg == None:
return
else:
logger.info("Loading control file ...")
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
logger.info("Doing the climatology")
pfp_clim.climatology(cfg)
logger.info(" Finished climatology")
logger.info("")
except Exception:
error_message = " An error occured while doing climatology, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_nc2ecostress(cfg=None):
"""
Purpose:
Convert a PFP-style netCDF file to an ECOSTRESS CSV file.
Usage:
Side effects:
Creates a CSV file in the same directory as the netCDF file.
Author: PRI
Date: Back in the day
Mods:
September 2018: rewrite for use with new GUI
"""
logger.info(" Starting conversion to ECOSTRESS file")
try:
if not cfg:
# check to see if there is an nc2ecostress.txt control file in controlfiles/standard
# if there is
# open controlfiles/standard/nc2csv_ecostress.txt
# ask for netCDF file name
# add [Files] section to control file
stdname = "controlfiles/standard/nc2csv_ecostress.txt"
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
cfg["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
result = pfp_io.write_csv_ecostress(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occurred, check the log messages")
logger.error("")
except Exception:
error_message = " Error converting to ECOSTRESS format, see below for details ... "
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_run_l3(cfg=None):
"""
Purpose:
Top level routine for running the L23 post-processing.
Usage:
pfp_top_level.do_l3()
Side effects:
Creates an L3 netCDF file.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
try:
logger.info("Starting L3 processing")
if not cfg:
cfg = pfp_io.load_controlfile()
if len(cfg) == 0:
logger.info("Quiting L3 processing (no control file)")
return
in_filepath = pfp_io.get_infilenamefromcf(cfg)
if not pfp_utils.file_exists(in_filepath):
in_filename = os.path.split(in_filepath)
logger.error("File " + in_filename[1] + " not found")
return
ds2 = pfp_io.nc_read_series(in_filepath)
ds3 = pfp_levels.l3qc(cfg, ds2)
if ds3.returncodes["value"] != 0:
logger.error("An error occurred during L3 processing")
logger.error("")
return
out_filepath = pfp_io.get_outfilenamefromcf(cfg)
nc_file = pfp_io.nc_open_write(out_filepath)
pfp_io.nc_write_series(nc_file, ds3)
logger.info("Finished L3 processing")
if "Plots" in list(cfg.keys()):
logger.info("Plotting L3 data")
for nFig in cfg['Plots'].keys():
plt_cf = cfg['Plots'][str(nFig)]
if 'Type' in plt_cf.keys():
if str(plt_cf['Type']).lower() == 'xy':
pfp_plot.plotxy(cfg, nFig, plt_cf, ds2, ds3)
else:
pfp_plot.plottimeseries(cfg, nFig, ds2, ds3)
else:
pfp_plot.plottimeseries(cfg, nFig, ds2, ds3)
logger.info("Finished plotting L3 data")
except Exception:
msg = " Error running L3, see below for details ..."
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
logger.info("")
return
def do_utilities_ustar_cpd(mode="standard"):
try:
logger.info(" Starting u* threshold detection (CPD)")
if mode == "standard":
stdname = "controlfiles/standard/cpd.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(
file_path="../Sites", title="Choose a netCDF file")
if not os.path.exists(filename):
logger.info(" CPD: no input file chosen")
return
if "Files" not in cf:
cf["Files"] = {}
cf["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
in_filename = os.path.split(filename)[1]
cf["Files"]["in_filename"] = in_filename
cf["Files"]["out_filename"] = in_filename.replace(
".nc", "_CPD.xls")
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
else:
logger.info("Loading control file ...")
cf = pfp_io.load_controlfile(path='controlfiles')
if len(cf) == 0:
return
logger.info("Doing u* threshold detection (CPD)")
pfp_cpd.cpd_main(cf)
logger.info(" Finished u* threshold detection (CPD)")
logger.info("")
except Exception:
error_message = " An error occured while doing CPD u* threshold, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_run_l6(main_gui, cfg=None):
"""
Purpose:
Top level routine for running the L6 gap filling.
Usage:
pfp_top_level.do_run_l6()
Side effects:
Creates an L6 netCDF file with NEE partitioned into GPP and ER.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
try:
logger.info("Starting L6 processing")
if not cfg:
cfg = pfp_io.load_controlfile(path='controlfiles')
if len(cfg) == 0:
logger.info("Quiting L6 processing (no control file)")
return
in_filepath = pfp_io.get_infilenamefromcf(cfg)
if not pfp_utils.file_exists(in_filepath):
in_filename = os.path.split(in_filepath)
logger.error("File " + in_filename[1] + " not found")
return
ds5 = pfp_io.nc_read_series(in_filepath)
#ds5.globalattributes['controlfile_name'] = cfg['controlfile_name']
sitename = ds5.globalattributes['site_name']
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cfg["Options"]["show_plots"] = "Yes"
ds6 = pfp_levels.l6qc(main_gui, cfg, ds5)
if ds6.returncodes["value"] != 0:
logger.info("Quitting L6: " + sitename)
else:
logger.info("Finished L6: " + sitename)
out_filepath = pfp_io.get_outfilenamefromcf(cfg)
nc_file = pfp_io.nc_open_write(out_filepath)
pfp_io.nc_write_series(nc_file, ds6)
logger.info("Finished saving L6 gap filled data")
logger.info("")
except Exception:
msg = " Error running L6, see below for details ..."
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_plot_timeseries():
"""
Purpose:
Plot time series of data, usually L3 and above.
Usage:
pfp_top_level.do_plot_timeseries()
Side effects:
Plots timeseries to the screen and creates .PNG hardcopies of
the plots.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
try:
logger.info("Starting timeseries plot")
stdname = "controlfiles/standard/fluxnet.txt"
if os.path.exists(stdname):
cf = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(file_path="../Sites",
title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cf): cf["Files"] = {}
cf["Files"]["file_path"] = os.path.split(filename)[0] + "/"
cf["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cf = pfp_io.load_controlfile(path="controlfiles")
if len(cf) == 0:
return
logger.info("Loaded control file ...")
if "Options" not in cf:
cf["Options"] = {}
cf["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
logger.info("Plotting time series ...")
pfp_plot.plot_fluxnet(cf)
logger.info(" Finished plotting time series")
logger.info("")
except Exception:
error_message = " An error occured while plotting time series, see below for details ..."
logger.error(error_message)
error_message = traceback.format_exc()
logger.error(error_message)
return
def do_file_convert_ncupdate(cfg=None):
"""
Purpose:
Convert from original netCDF files to V1 (October 2018).
Usage:
Author: PRI
Date: October 2018
"""
logger.info(" Starting conversion of netCDF")
if not cfg:
# check to see if there is an nc2ecostress.txt control file in controlfiles/standard
# if there is
# open controlfiles/standard/nc2csv_ecostress.txt
# ask for netCDF file name
# add [Files] section to control file
stdname = os.path.join("controlfiles", "standard",
"map_old_to_new.txt")
if os.path.exists(stdname):
cfg = pfp_io.get_controlfilecontents(stdname)
filename = pfp_io.get_filename_dialog(file_path="../OzFlux/Sites",
title="Choose a netCDF file")
if len(filename) == 0:
return
if "Files" not in dir(cfg):
cfg["Files"] = {}
cfg["Files"]["file_path"] = os.path.join(
os.path.split(filename)[0], "")
cfg["Files"]["in_filename"] = os.path.split(filename)[1]
else:
cfg = pfp_io.load_controlfile(path="controlfiles")
if len(cfg) == 0:
return
if "Options" not in cfg:
cfg["Options"] = {}
cfg["Options"]["call_mode"] = "interactive"
cf["Options"]["show_plots"] = "Yes"
result = pfp_compliance.nc_update(cfg)
if result == 0:
logger.info(" Finished converting netCDF file")
logger.info("")
else:
logger.error("")
logger.error(" An error occured, check the log messages")
logger.error("")
return
def do_run_l1(cfg=None):
"""
Purpose:
Top level routine for running the L1 data import.
Usage:
pfp_top_level.do_l1()
Side effects:
Creates an L1 netCDF file.
Author: PRI
Date: Back in the day
Mods:
December 2017: rewrite for use with new GUI
"""
try:
logger.info("Starting L1 processing")
if not cfg:
cfg = pfp_io.load_controlfile()
if len(cfg) == 0:
logger.info("Quiting L1 processing (no control file)")
return
ds1 = pfp_levels.l1qc(cfg)
if ds1.returncodes["value"] == 0:
outfilename = pfp_io.get_outfilenamefromcf(cfg)
ncFile = pfp_io.nc_open_write(outfilename)
pfp_io.nc_write_series(ncFile, ds1)
logger.info("Finished L1 processing")
else:
msg = "An error occurred during L1 processing"
logger.error(msg)
logger.info("")
except Exception:
msg = " Error running L1, see below for details ..."
logger.error(msg)
error_message = traceback.format_exc()
logger.error(error_message)
return
deletechars=deletechars)
# get the datetime from the character date string
dt = numpy.array([dateutil.parser.parse(s) for s in tmp["Date"]])
names.remove("Date")
# create the data dictionary
evi = {"DateTime": dt}
# load the data into the dictionary
for item in names:
evi[item] = numpy.array(tmp[item], dtype=numpy.float64)
# return the data dictionary
return evi
# load the control file
app = Qt.QApplication([])
cf = pfp_io.load_controlfile(path="../controlfiles/MODIS/")
deletechars = set("""~!@#$^&=+~\|]}[{';: ?.>,<""")
ts = 30
evi_time_units = "seconds since 1970-01-01 00:00:00.0"
do_plots = True
sites = cf["Sites"].keys()
site = sites[0]
site_timezone = cf["Sites"][site]["site_timezone"]
site_latitude = float(cf["Sites"][site]["site_latitude"])
site_longitude = float(cf["Sites"][site]["site_longitude"])
site_timestep = int(cf["Sites"][site]["site_timestep"])
evi_min = float(cf["Sites"][site]["evi_min"])
evi_max = float(cf["Sites"][site]["evi_max"])
sgnp = int(cf["Sites"][site]["sg_num_points"])
import pfp_io
import pfp_log
import pfp_ls
import pfp_plot
import pfp_utils
t = time.localtime()
rundatetime = datetime.datetime(t[0], t[1], t[2], t[3], t[4],
t[5]).strftime("%Y%m%d%H%M")
log_file_name = 'batchprocess_' + rundatetime + '.log'
logger = pfp_log.init_logger(logger_name="pfp_log",
log_file_name=log_file_name)
# get the batch processing control file
if len(sys.argv) == 1:
cf_batch = pfp_io.load_controlfile(path='controlfiles')
if len(cf_batch) == 0: sys.exit()
else:
cfname = sys.argv[1]
if os.path.exists(cfname):
cf_batch = pfp_io.get_controlfilecontents(cfname)
else:
logger.error("Control file " + cfname + " does not exist")
sys.exit()
level_list = [
'L1', 'L2', 'L3', 'concatenate', 'climatology', 'cpd', 'L4', 'L5', 'L6'
]
if "Options" in cf_batch:
if "levels" in cf_batch["Options"]:
level_list = ast.literal_eval(cf_batch["Options"]["levels"])
