def do_file_convert_nc2xls(): """ Purpose: Convert a PFP-style netCDF file to an Excel workbook. Usage: Side effects: Creates an Excel workbook in the same directory as the netCDF file. Author: PRI Date: Back in the day Mods: August 2018: rewrite for use with new GUI """ logger.info(" Starting conversion to Excel file") try: ncfilename = pfp_io.get_filename_dialog(file_path="../Sites", title="Choose a netCDF file", ext="*.nc") if len(ncfilename) == 0: logger.info(" No file selected, cancelling ...") return logger.info(" Converting netCDF file to Excel file") pfp_io.nc_2xls(ncfilename, outputlist=None) logger.info(" Finished converting netCDF file") logger.info("") except Exception: msg = " Error converting to Excel file, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def do_plot_fcvsustar(): """ Purpose: Plot Fc versus u*. Usage: pfp_top_level.do_plot_fcvsustar() Side effects: Annual and seasonal plots of Fc versus u* to the screen and creates .PNG hardcopies of the plots. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ logger.info("Starting Fc versus u* plots") try: file_path = pfp_io.get_filename_dialog(file_path="../Sites", title="Choose a netCDF file") if len(file_path) == 0 or not os.path.isfile(file_path): return # read the netCDF file ds = pfp_io.nc_read_series(file_path) if ds.returncodes["value"] != 0: return logger.info("Plotting Fc versus u* ...") pfp_plot.plot_fcvsustar(ds) logger.info(" Finished plotting Fc versus u*") logger.info("") except Exception: error_message = " An error occured while plotting Fc versus u*, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def compare_eddypro(): epname = pfp_io.get_filename_dialog(title='Choose an EddyPro full output file') ofname = pfp_io.get_filename_dialog(title='Choose an L3 output file') ds_ep = pfp_io.read_eddypro_full(epname) ds_of = pfp_io.nc_read_series(ofname) dt_ep = ds_ep.series['DateTime']['Data'] dt_of = ds_of.series['DateTime']['Data'] start_datetime = max([dt_ep[0],dt_of[0]]) end_datetime = min([dt_ep[-1],dt_of[-1]]) si_of = pfp_utils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact') ei_of = pfp_utils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact') si_ep = pfp_utils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact') ei_ep = pfp_utils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact') us_of = pfp_utils.GetVariable(ds_of,'ustar',start=si_of,end=ei_of) us_ep = pfp_utils.GetVariable(ds_ep,'ustar',start=si_ep,end=ei_ep) Fh_of = pfp_utils.GetVariable(ds_of,'Fh',start=si_of,end=ei_of) Fh_ep = pfp_utils.GetVariable(ds_ep,'Fh',start=si_ep,end=ei_ep) Fe_of = pfp_utils.GetVariable(ds_of,'Fe',start=si_of,end=ei_of) Fe_ep = pfp_utils.GetVariable(ds_ep,'Fe',start=si_ep,end=ei_ep) Fc_of = pfp_utils.GetVariable(ds_of,'Fc',start=si_of,end=ei_of) Fc_ep = pfp_utils.GetVariable(ds_ep,'Fc',start=si_ep,end=ei_ep) # copy the range check values from the OFQC attributes to the EP attributes for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]): for item in ["rangecheck_upper", "rangecheck_lower"]: if item in of["Attr"]: ep["Attr"][item] = of["Attr"][item] # apply QC to the EddyPro data pfp_ck.ApplyRangeCheckToVariable(us_ep) pfp_ck.ApplyRangeCheckToVariable(Fc_ep) pfp_ck.ApplyRangeCheckToVariable(Fe_ep) pfp_ck.ApplyRangeCheckToVariable(Fh_ep) # plot the comparison plt.ion() fig = plt.figure(1,figsize=(8,8)) pfp_plot.xyplot(us_ep["Data"],us_of["Data"],sub=[2,2,1],regr=2,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)') pfp_plot.xyplot(Fh_ep["Data"],Fh_of["Data"],sub=[2,2,2],regr=2,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)') pfp_plot.xyplot(Fe_ep["Data"],Fe_of["Data"],sub=[2,2,3],regr=2,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)') pfp_plot.xyplot(Fc_ep["Data"],Fc_of["Data"],sub=[2,2,4],regr=2,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)') plt.tight_layout() plt.draw() plt.ioff()
def do_utilities_ustar_mpt(cfg=None, mode="standard"): """ Purpose: Calculate the u* threshold using the Moving Point Threshold (MPT) method. This code calls the original FluxNet MPT C code. The executable for this is in PyFluxPro/mpt/bin. Side effects: Calls pfp_mpt.mpt_main """ try: logger.info(" Starting u* threshold detection (MPT)") if mode == "standard": stdname = "controlfiles/standard/mpt.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path='../Sites', title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" MPT: no input file chosen") return if "Files" not in dir(cfg): cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_MPT.xls") else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return elif mode == "custom": # we get here via Run/Current and should have a control file if cfg == None: return else: logger.info("Loading control file ...") cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return logger.info(" Doing u* threshold detection (MPT)") if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" pfp_mpt.mpt_main(cfg) logger.info(" Finished u* threshold detection (MPT)") logger.info("") except Exception: error_message = " An error occured while doing MPT u* threshold, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_file_convert_nc2ecostress(cfg=None): """ Purpose: Convert a PFP-style netCDF file to an ECOSTRESS CSV file. Usage: Side effects: Creates a CSV file in the same directory as the netCDF file. Author: PRI Date: Back in the day Mods: September 2018: rewrite for use with new GUI """ logger.info(" Starting conversion to ECOSTRESS file") try: if not cfg: # check to see if there is an nc2ecostress.txt control file in controlfiles/standard # if there is # open controlfiles/standard/nc2csv_ecostress.txt # ask for netCDF file name # add [Files] section to control file stdname = "controlfiles/standard/nc2csv_ecostress.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if len(filename) == 0: return if "Files" not in dir(cfg): cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") cfg["Files"]["in_filename"] = os.path.split(filename)[1] else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cf["Options"]["show_plots"] = "Yes" result = pfp_io.write_csv_ecostress(cfg) if result == 0: logger.info(" Finished converting netCDF file") logger.info("") else: logger.error("") logger.error(" An error occurred, check the log messages") logger.error("") except Exception: error_message = " Error converting to ECOSTRESS format, see below for details ... " logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_ustar_cpd_barr(cfg=None, mode="standard"): """ Purpose: Calculate the u* threshold using the Change Point Detection method described in Barr et al. 2013, AFM 171-172, pp31-45. This code is a line-by-line translation of the original Barr MATLAB scripts into Python. """ try: logger.info(" Starting CPD u* threshold detection (Barr)") if mode == "standard": stdname = "controlfiles/standard/cpd2.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" CPD (Barr): no input file chosen") return if "Files" not in cfg: cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_CPD_Barr.xls") else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return elif mode == "custom": # we get here via Run/Current and should have a control file if cfg == None: return else: logger.info("Loading control file ...") cfg = pfp_io.load_controlfile(path='controlfiles') if len(cfg) == 0: return logger.info("Doing CPD u* threshold detection (Barr)") pfp_cpd2.cpd2_main(cfg) logger.info(" Finished CPD u* threshold detection (Barr)") logger.info("") except Exception: error_message = " An error occured while doing CPD u* threshold (Barr), see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_ustar_mpt(mode="standard"): """ Calls pfp_mpt.mpt_main Calculate the u* threshold using the Moving Point Threshold (MPT) method. """ try: logger.info(" Starting u* threshold detection (MPT)") if mode == "standard": stdname = "controlfiles/standard/mpt.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path='../Sites', title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" MPT: no input file chosen") return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cf["Files"]["in_filename"] = in_filename cf["Files"]["out_filename"] = in_filename.replace( ".nc", "_MPT.xls") else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return else: logger.info("Loading control file ...") cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return logger.info(" Doing u* threshold detection (MPT)") if "Options" not in cf: cf["Options"] = {} cf["Options"]["call_mode"] = "interactive" pfp_mpt.mpt_main(cf) logger.info(" Finished u* threshold detection (MPT)") logger.info("") except Exception: error_message = " An error occured while doing MPT u* threshold, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_plot_timeseries(): """ Purpose: Plot time series of data, usually L3 and above. Usage: pfp_top_level.do_plot_timeseries() Side effects: Plots timeseries to the screen and creates .PNG hardcopies of the plots. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting timeseries plot") stdname = "controlfiles/standard/fluxnet.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog(file_path="../Sites", title="Choose a netCDF file") if len(filename) == 0: return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = os.path.split(filename)[0] + "/" cf["Files"]["in_filename"] = os.path.split(filename)[1] else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return logger.info("Loaded control file ...") if "Options" not in cf: cf["Options"] = {} cf["Options"]["call_mode"] = "interactive" cf["Options"]["show_plots"] = "Yes" logger.info("Plotting time series ...") pfp_plot.plot_fluxnet(cf) logger.info(" Finished plotting time series") logger.info("") except Exception: error_message = " An error occured while plotting time series, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_ustar_cpd1(mode="standard"): """ Purpose: Calculate the u* threshold using the Change Point Detection method described in Barr et al. 2013, AFM 171-172, pp31-45. This code is the original implementation by Ian McHugh. """ try: logger.info(" Starting CPD u* threshold detection (McHugh)") if mode == "standard": stdname = "controlfiles/standard/cpd1.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" CPD (McHugh): no input file chosen") return if "Files" not in cf: cf["Files"] = {} cf["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cf["Files"]["in_filename"] = in_filename cf["Files"]["out_filename"] = in_filename.replace( ".nc", "_CPD_McHugh.xls") else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return else: logger.info("Loading control file ...") cf = pfp_io.load_controlfile(path='controlfiles') if len(cf) == 0: return logger.info("Doing CPD u* threshold detection (McHugh)") pfp_cpd1.cpd1_main(cf) logger.info(" Finished CPD u* threshold detection (McHugh)") logger.info("") except Exception: error_message = " An error occured while doing CPD u* threshold (McHugh), see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_file_convert_ncupdate(cfg=None): """ Purpose: Convert from original netCDF files to V1 (October 2018). Usage: Author: PRI Date: October 2018 """ logger.info(" Starting conversion of netCDF") if not cfg: # check to see if there is an nc2ecostress.txt control file in controlfiles/standard # if there is # open controlfiles/standard/nc2csv_ecostress.txt # ask for netCDF file name # add [Files] section to control file stdname = os.path.join("controlfiles", "standard", "map_old_to_new.txt") if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog(file_path="../OzFlux/Sites", title="Choose a netCDF file") if len(filename) == 0: return if "Files" not in dir(cfg): cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") cfg["Files"]["in_filename"] = os.path.split(filename)[1] else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cf["Options"]["show_plots"] = "Yes" result = pfp_compliance.nc_update(cfg) if result == 0: logger.info(" Finished converting netCDF file") logger.info("") else: logger.error("") logger.error(" An error occured, check the log messages") logger.error("") return
def do_utilities_climatology(cfg=None, mode="standard"): try: logger.info(" Starting climatology") if mode == "standard": stdname = "controlfiles/standard/climatology.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title='Choose a netCDF file') if not os.path.exists(filename): logger.info(" Climatology: no input file chosen") return if "Files" not in cfg: cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_Climatology.xls") else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return elif mode == "custom": # we get here via Run/Current and should have a control file if cfg == None: return else: logger.info("Loading control file ...") cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return logger.info("Doing the climatology") pfp_clim.climatology(cfg) logger.info(" Finished climatology") logger.info("") except Exception: error_message = " An error occured while doing climatology, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_ustar_cpd(mode="standard"): try: logger.info(" Starting u* threshold detection (CPD)") if mode == "standard": stdname = "controlfiles/standard/cpd.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" CPD: no input file chosen") return if "Files" not in cf: cf["Files"] = {} cf["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cf["Files"]["in_filename"] = in_filename cf["Files"]["out_filename"] = in_filename.replace( ".nc", "_CPD.xls") else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return else: logger.info("Loading control file ...") cf = pfp_io.load_controlfile(path='controlfiles') if len(cf) == 0: return logger.info("Doing u* threshold detection (CPD)") pfp_cpd.cpd_main(cf) logger.info(" Finished u* threshold detection (CPD)") logger.info("") except Exception: error_message = " An error occured while doing CPD u* threshold, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_file_convert_nc2biomet(cfg, mode="standard"): """ Purpose: Convert a PFP-style netCDF file to an EddyPro biomet CSV file. Usage: Side effects: Creates a CSV file in the same directory as the netCDF file. Author: PRI Date: Back in the day Mods: March 2020: rewrite for use with new GUI April 2020: routine can be invoked from File/Convert menu or by loading control file and using Run/Current. The latter method allows the user to modify the control file before running it. """ logger.info(" Starting conversion to EddyPro biomet file") try: # check to see if the user chose a standard or a custom run if cfg is None and mode == "standard": # standard run so we use the control file in PyFluxPro/controlfiles/standard stdname = "controlfiles/standard/nc2csv_biomet.txt" # check to see if the standard control file exists if os.path.exists(stdname): # standard control file exists so read it cfg = pfp_io.get_controlfilecontents(stdname) # then ask the user to choose a netCDF file filename = pfp_io.get_filename_dialog( file_path=".", title='Choose a netCDF file') # check that the netCDF file exists if not os.path.exists(filename): # return if no file chosen logger.info(" Write biomet CSV file: no input file chosen") return # add a [Files] section to the control file ... if "Files" not in cfg: cfg["Files"] = {} # ... and put the file path, input file name and output file name in [Files] cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_biomet.csv") else: # issue an error mesage and return if the standard control file does not exist msg = " Write biomet CSV file: standard control file 'nc2csv_biomet.txt' does not exist" logger.error(msg) return elif cfg is not None and mode == "custom": # custom run so we proceed with the user's control file pass else: # tell the user we got the wrong input options and return msg = " Write biomet CSV file: wrong input options" logger.error(msg) return # add the [Options] section and populate it if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cfg["Options"]["show_plots"] = "Yes" # do the business result = pfp_io.write_csv_ep_biomet(cfg) # check everything went well if result == 1: # looks good logger.info(" Finished converting netCDF file") logger.info("") else: # or not logger.error("") logger.error(" An error occurred, check the log messages") logger.error("") except Exception: # tell the user if something goes wrong and put the exception in the log window error_message = " Error converting to BIOMET format, see below for details ... " logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return