def do_utilities_ustar_mpt(cfg=None, mode="standard"): """ Purpose: Calculate the u* threshold using the Moving Point Threshold (MPT) method. This code calls the original FluxNet MPT C code. The executable for this is in PyFluxPro/mpt/bin. Side effects: Calls pfp_mpt.mpt_main """ try: logger.info(" Starting u* threshold detection (MPT)") if mode == "standard": stdname = "controlfiles/standard/mpt.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path='../Sites', title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" MPT: no input file chosen") return if "Files" not in dir(cfg): cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_MPT.xls") else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return elif mode == "custom": # we get here via Run/Current and should have a control file if cfg == None: return else: logger.info("Loading control file ...") cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return logger.info(" Doing u* threshold detection (MPT)") if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" pfp_mpt.mpt_main(cfg) logger.info(" Finished u* threshold detection (MPT)") logger.info("") except Exception: error_message = " An error occured while doing MPT u* threshold, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_ustar_cpd_barr(cfg=None, mode="standard"): """ Purpose: Calculate the u* threshold using the Change Point Detection method described in Barr et al. 2013, AFM 171-172, pp31-45. This code is a line-by-line translation of the original Barr MATLAB scripts into Python. """ try: logger.info(" Starting CPD u* threshold detection (Barr)") if mode == "standard": stdname = "controlfiles/standard/cpd2.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" CPD (Barr): no input file chosen") return if "Files" not in cfg: cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_CPD_Barr.xls") else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return elif mode == "custom": # we get here via Run/Current and should have a control file if cfg == None: return else: logger.info("Loading control file ...") cfg = pfp_io.load_controlfile(path='controlfiles') if len(cfg) == 0: return logger.info("Doing CPD u* threshold detection (Barr)") pfp_cpd2.cpd2_main(cfg) logger.info(" Finished CPD u* threshold detection (Barr)") logger.info("") except Exception: error_message = " An error occured while doing CPD u* threshold (Barr), see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_file_concatenate(cfg=None): """ Purpose: Top level routine for concatenating multiple, single-year files into a single, multiple-year file. NOTE: The input files must be listed in the control file in chronological order. Usage: pfp_top_level.do_file_concatenate() Side effects: Creates a single netCDF file containing the contents of the input files. Author: PRI Date: Back in the day Mods: June 2018: rewrite for use with new GUI. """ logger.info(" Starting concatenation of netCDF files") if not cfg: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: logger.info("Quitting concatenation (no control file)") return pfp_io.nc_concatenate(cfg) logger.info(" Finished concatenating files") logger.info("") return
def do_utilities_ustar_mpt(mode="standard"): """ Calls pfp_mpt.mpt_main Calculate the u* threshold using the Moving Point Threshold (MPT) method. """ try: logger.info(" Starting u* threshold detection (MPT)") if mode == "standard": stdname = "controlfiles/standard/mpt.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path='../Sites', title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" MPT: no input file chosen") return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cf["Files"]["in_filename"] = in_filename cf["Files"]["out_filename"] = in_filename.replace( ".nc", "_MPT.xls") else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return else: logger.info("Loading control file ...") cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return logger.info(" Doing u* threshold detection (MPT)") if "Options" not in cf: cf["Options"] = {} cf["Options"]["call_mode"] = "interactive" pfp_mpt.mpt_main(cf) logger.info(" Finished u* threshold detection (MPT)") logger.info("") except Exception: error_message = " An error occured while doing MPT u* threshold, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_ustar_cpd1(mode="standard"): """ Purpose: Calculate the u* threshold using the Change Point Detection method described in Barr et al. 2013, AFM 171-172, pp31-45. This code is the original implementation by Ian McHugh. """ try: logger.info(" Starting CPD u* threshold detection (McHugh)") if mode == "standard": stdname = "controlfiles/standard/cpd1.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" CPD (McHugh): no input file chosen") return if "Files" not in cf: cf["Files"] = {} cf["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cf["Files"]["in_filename"] = in_filename cf["Files"]["out_filename"] = in_filename.replace( ".nc", "_CPD_McHugh.xls") else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return else: logger.info("Loading control file ...") cf = pfp_io.load_controlfile(path='controlfiles') if len(cf) == 0: return logger.info("Doing CPD u* threshold detection (McHugh)") pfp_cpd1.cpd1_main(cf) logger.info(" Finished CPD u* threshold detection (McHugh)") logger.info("") except Exception: error_message = " An error occured while doing CPD u* threshold (McHugh), see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_utilities_climatology(cfg=None, mode="standard"): try: logger.info(" Starting climatology") if mode == "standard": stdname = "controlfiles/standard/climatology.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title='Choose a netCDF file') if not os.path.exists(filename): logger.info(" Climatology: no input file chosen") return if "Files" not in cfg: cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cfg["Files"]["in_filename"] = in_filename cfg["Files"]["out_filename"] = in_filename.replace( ".nc", "_Climatology.xls") else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return elif mode == "custom": # we get here via Run/Current and should have a control file if cfg == None: return else: logger.info("Loading control file ...") cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return logger.info("Doing the climatology") pfp_clim.climatology(cfg) logger.info(" Finished climatology") logger.info("") except Exception: error_message = " An error occured while doing climatology, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_file_convert_nc2ecostress(cfg=None): """ Purpose: Convert a PFP-style netCDF file to an ECOSTRESS CSV file. Usage: Side effects: Creates a CSV file in the same directory as the netCDF file. Author: PRI Date: Back in the day Mods: September 2018: rewrite for use with new GUI """ logger.info(" Starting conversion to ECOSTRESS file") try: if not cfg: # check to see if there is an nc2ecostress.txt control file in controlfiles/standard # if there is # open controlfiles/standard/nc2csv_ecostress.txt # ask for netCDF file name # add [Files] section to control file stdname = "controlfiles/standard/nc2csv_ecostress.txt" if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if len(filename) == 0: return if "Files" not in dir(cfg): cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") cfg["Files"]["in_filename"] = os.path.split(filename)[1] else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cf["Options"]["show_plots"] = "Yes" result = pfp_io.write_csv_ecostress(cfg) if result == 0: logger.info(" Finished converting netCDF file") logger.info("") else: logger.error("") logger.error(" An error occurred, check the log messages") logger.error("") except Exception: error_message = " Error converting to ECOSTRESS format, see below for details ... " logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_run_l3(cfg=None): """ Purpose: Top level routine for running the L23 post-processing. Usage: pfp_top_level.do_l3() Side effects: Creates an L3 netCDF file. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L3 processing") if not cfg: cfg = pfp_io.load_controlfile() if len(cfg) == 0: logger.info("Quiting L3 processing (no control file)") return in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds2 = pfp_io.nc_read_series(in_filepath) ds3 = pfp_levels.l3qc(cfg, ds2) if ds3.returncodes["value"] != 0: logger.error("An error occurred during L3 processing") logger.error("") return out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) pfp_io.nc_write_series(nc_file, ds3) logger.info("Finished L3 processing") if "Plots" in list(cfg.keys()): logger.info("Plotting L3 data") for nFig in cfg['Plots'].keys(): plt_cf = cfg['Plots'][str(nFig)] if 'Type' in plt_cf.keys(): if str(plt_cf['Type']).lower() == 'xy': pfp_plot.plotxy(cfg, nFig, plt_cf, ds2, ds3) else: pfp_plot.plottimeseries(cfg, nFig, ds2, ds3) else: pfp_plot.plottimeseries(cfg, nFig, ds2, ds3) logger.info("Finished plotting L3 data") except Exception: msg = " Error running L3, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) logger.info("") return
def do_utilities_ustar_cpd(mode="standard"): try: logger.info(" Starting u* threshold detection (CPD)") if mode == "standard": stdname = "controlfiles/standard/cpd.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog( file_path="../Sites", title="Choose a netCDF file") if not os.path.exists(filename): logger.info(" CPD: no input file chosen") return if "Files" not in cf: cf["Files"] = {} cf["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") in_filename = os.path.split(filename)[1] cf["Files"]["in_filename"] = in_filename cf["Files"]["out_filename"] = in_filename.replace( ".nc", "_CPD.xls") else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return else: logger.info("Loading control file ...") cf = pfp_io.load_controlfile(path='controlfiles') if len(cf) == 0: return logger.info("Doing u* threshold detection (CPD)") pfp_cpd.cpd_main(cf) logger.info(" Finished u* threshold detection (CPD)") logger.info("") except Exception: error_message = " An error occured while doing CPD u* threshold, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_run_l6(main_gui, cfg=None): """ Purpose: Top level routine for running the L6 gap filling. Usage: pfp_top_level.do_run_l6() Side effects: Creates an L6 netCDF file with NEE partitioned into GPP and ER. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L6 processing") if not cfg: cfg = pfp_io.load_controlfile(path='controlfiles') if len(cfg) == 0: logger.info("Quiting L6 processing (no control file)") return in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds5 = pfp_io.nc_read_series(in_filepath) #ds5.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds5.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cfg["Options"]["show_plots"] = "Yes" ds6 = pfp_levels.l6qc(main_gui, cfg, ds5) if ds6.returncodes["value"] != 0: logger.info("Quitting L6: " + sitename) else: logger.info("Finished L6: " + sitename) out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) pfp_io.nc_write_series(nc_file, ds6) logger.info("Finished saving L6 gap filled data") logger.info("") except Exception: msg = " Error running L6, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def do_plot_timeseries(): """ Purpose: Plot time series of data, usually L3 and above. Usage: pfp_top_level.do_plot_timeseries() Side effects: Plots timeseries to the screen and creates .PNG hardcopies of the plots. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting timeseries plot") stdname = "controlfiles/standard/fluxnet.txt" if os.path.exists(stdname): cf = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog(file_path="../Sites", title="Choose a netCDF file") if len(filename) == 0: return if "Files" not in dir(cf): cf["Files"] = {} cf["Files"]["file_path"] = os.path.split(filename)[0] + "/" cf["Files"]["in_filename"] = os.path.split(filename)[1] else: cf = pfp_io.load_controlfile(path="controlfiles") if len(cf) == 0: return logger.info("Loaded control file ...") if "Options" not in cf: cf["Options"] = {} cf["Options"]["call_mode"] = "interactive" cf["Options"]["show_plots"] = "Yes" logger.info("Plotting time series ...") pfp_plot.plot_fluxnet(cf) logger.info(" Finished plotting time series") logger.info("") except Exception: error_message = " An error occured while plotting time series, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_file_convert_ncupdate(cfg=None): """ Purpose: Convert from original netCDF files to V1 (October 2018). Usage: Author: PRI Date: October 2018 """ logger.info(" Starting conversion of netCDF") if not cfg: # check to see if there is an nc2ecostress.txt control file in controlfiles/standard # if there is # open controlfiles/standard/nc2csv_ecostress.txt # ask for netCDF file name # add [Files] section to control file stdname = os.path.join("controlfiles", "standard", "map_old_to_new.txt") if os.path.exists(stdname): cfg = pfp_io.get_controlfilecontents(stdname) filename = pfp_io.get_filename_dialog(file_path="../OzFlux/Sites", title="Choose a netCDF file") if len(filename) == 0: return if "Files" not in dir(cfg): cfg["Files"] = {} cfg["Files"]["file_path"] = os.path.join( os.path.split(filename)[0], "") cfg["Files"]["in_filename"] = os.path.split(filename)[1] else: cfg = pfp_io.load_controlfile(path="controlfiles") if len(cfg) == 0: return if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cf["Options"]["show_plots"] = "Yes" result = pfp_compliance.nc_update(cfg) if result == 0: logger.info(" Finished converting netCDF file") logger.info("") else: logger.error("") logger.error(" An error occured, check the log messages") logger.error("") return
def do_run_l1(cfg=None): """ Purpose: Top level routine for running the L1 data import. Usage: pfp_top_level.do_l1() Side effects: Creates an L1 netCDF file. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L1 processing") if not cfg: cfg = pfp_io.load_controlfile() if len(cfg) == 0: logger.info("Quiting L1 processing (no control file)") return ds1 = pfp_levels.l1qc(cfg) if ds1.returncodes["value"] == 0: outfilename = pfp_io.get_outfilenamefromcf(cfg) ncFile = pfp_io.nc_open_write(outfilename) pfp_io.nc_write_series(ncFile, ds1) logger.info("Finished L1 processing") else: msg = "An error occurred during L1 processing" logger.error(msg) logger.info("") except Exception: msg = " Error running L1, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
deletechars=deletechars) # get the datetime from the character date string dt = numpy.array([dateutil.parser.parse(s) for s in tmp["Date"]]) names.remove("Date") # create the data dictionary evi = {"DateTime": dt} # load the data into the dictionary for item in names: evi[item] = numpy.array(tmp[item], dtype=numpy.float64) # return the data dictionary return evi # load the control file app = Qt.QApplication([]) cf = pfp_io.load_controlfile(path="../controlfiles/MODIS/") deletechars = set("""~!@#$^&=+~\|]}[{';: ?.>,<""") ts = 30 evi_time_units = "seconds since 1970-01-01 00:00:00.0" do_plots = True sites = cf["Sites"].keys() site = sites[0] site_timezone = cf["Sites"][site]["site_timezone"] site_latitude = float(cf["Sites"][site]["site_latitude"]) site_longitude = float(cf["Sites"][site]["site_longitude"]) site_timestep = int(cf["Sites"][site]["site_timestep"]) evi_min = float(cf["Sites"][site]["evi_min"]) evi_max = float(cf["Sites"][site]["evi_max"]) sgnp = int(cf["Sites"][site]["sg_num_points"])
import pfp_io import pfp_log import pfp_ls import pfp_plot import pfp_utils t = time.localtime() rundatetime = datetime.datetime(t[0], t[1], t[2], t[3], t[4], t[5]).strftime("%Y%m%d%H%M") log_file_name = 'batchprocess_' + rundatetime + '.log' logger = pfp_log.init_logger(logger_name="pfp_log", log_file_name=log_file_name) # get the batch processing control file if len(sys.argv) == 1: cf_batch = pfp_io.load_controlfile(path='controlfiles') if len(cf_batch) == 0: sys.exit() else: cfname = sys.argv[1] if os.path.exists(cfname): cf_batch = pfp_io.get_controlfilecontents(cfname) else: logger.error("Control file " + cfname + " does not exist") sys.exit() level_list = [ 'L1', 'L2', 'L3', 'concatenate', 'climatology', 'cpd', 'L4', 'L5', 'L6' ] if "Options" in cf_batch: if "levels" in cf_batch["Options"]: level_list = ast.literal_eval(cf_batch["Options"]["levels"])