def do_L3_batch(cf_level): #logger = pfp_log.change_logger_filename("pfp_log", "L3") for i in list(cf_level.keys()): cf_file_name = os.path.split(cf_level[i]) msg = "Starting L3 processing with " + cf_file_name[1] logger.info(msg) try: cf = pfp_io.get_controlfilecontents(cf_level[i]) infilename = pfp_io.get_infilenamefromcf(cf) ds2 = pfp_io.nc_read_series(infilename) if ds2.returncodes["value"] != 0: return ds3 = pfp_levels.l3qc(cf, ds2) outfilename = pfp_io.get_outfilenamefromcf(cf) nc_file = pfp_io.nc_open_write(outfilename) if nc_file is None: return pfp_io.nc_write_series(nc_file, ds3) msg = "Finished L3 processing with " + cf_file_name[1] logger.info(msg) logger.info("") except Exception: msg = "Error occurred during L3 processing " + cf_file_name[1] logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) continue return
def do_plot_fcvsustar(): """ Purpose: Plot Fc versus u*. Usage: pfp_top_level.do_plot_fcvsustar() Side effects: Annual and seasonal plots of Fc versus u* to the screen and creates .PNG hardcopies of the plots. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ logger.info("Starting Fc versus u* plots") try: file_path = pfp_io.get_filename_dialog(file_path="../Sites", title="Choose a netCDF file") if len(file_path) == 0 or not os.path.isfile(file_path): return # read the netCDF file ds = pfp_io.nc_read_series(file_path) if ds.returncodes["value"] != 0: return logger.info("Plotting Fc versus u* ...") pfp_plot.plot_fcvsustar(ds) logger.info(" Finished plotting Fc versus u*") logger.info("") except Exception: error_message = " An error occured while plotting Fc versus u*, see below for details ..." logger.error(error_message) error_message = traceback.format_exc() logger.error(error_message) return
def do_audit_analysis(base_path): sites = sorted(os.listdir(base_path)) for item in sites: if not os.path.isdir(os.path.join(base_path, item)): sites.remove(item) site_info = OrderedDict() all_sites = {"start_date":datetime.datetime(3000,1,1,0,0), "end_date": datetime.datetime(2000,1,1,0,0)} n = 0 for site in sites: #portal_dir = os.path.join(base_path, site, "Data", "Processed") portal_dir = os.path.join(base_path, site, "Data", "All") file_mask = os.path.join(portal_dir, "*.nc") files = sorted(glob.glob(file_mask)) l3_name = os.path.join(portal_dir, site + "_L3.nc") if os.path.isfile(l3_name): print("Processing ", site) site_info[site] = {"file_name":l3_name} ds = pfp_io.nc_read_series(l3_name) site_info[site]["site_name"] = ds.globalattributes["site_name"] start_date = dateutil.parser.parse(ds.globalattributes["start_date"]) site_info[site]["start_date"] = start_date end_date = dateutil.parser.parse(ds.globalattributes["end_date"]) site_info[site]["end_date"] = end_date site_info[site]["X"] = numpy.array([start_date, end_date]) site_info[site]["Y"] = numpy.array([n+1, n+1]) n = n + 1 all_sites["start_date"] = min([all_sites["start_date"], site_info[site]["start_date"]]) all_sites["end_date"] = max([all_sites["end_date"], site_info[site]["end_date"]]) with open('audit_analysis.pickle', 'wb') as handle: pickle.dump([all_sites, site_info], handle, protocol=pickle.HIGHEST_PROTOCOL) return all_sites, site_info
def do_L6_batch(cf_level): logger = pfp_log.change_logger_filename("pfp_log", "L6") for i in cf_level.keys(): if not os.path.isfile(cf_level[i]): msg = " Control file " + cf_level[i] + " not found" logger.error(msg) continue cf_file_name = os.path.split(cf_level[i]) msg = "Starting L6 processing with " + cf_file_name[1] logger.info(msg) try: cf = pfp_io.get_controlfilecontents(cf_level[i]) if "Options" not in cf: cf["Options"] = {} cf["Options"]["call_mode"] = "batch" cf["Options"]["show_plots"] = "No" infilename = pfp_io.get_infilenamefromcf(cf) ds5 = pfp_io.nc_read_series(infilename) ds6 = pfp_levels.l6qc(None, cf, ds5) outfilename = pfp_io.get_outfilenamefromcf(cf) ncFile = pfp_io.nc_open_write(outfilename) pfp_io.nc_write_series(ncFile, ds6) msg = "Finished L6 processing with " + cf_file_name[1] logger.info(msg) logger.info("") except Exception: msg = "Error occurred during L6 with " + cf_file_name[1] logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) continue return
def ImportSeries(cf,ds): # check to see if there is an Imports section if "Imports" not in cf.keys(): return # number of records nRecs = int(ds.globalattributes["nc_nrecs"]) # get the start and end datetime ldt = ds.series["DateTime"]["Data"] start_date = ldt[0] end_date = ldt[-1] # loop over the series in the Imports section for label in cf["Imports"].keys(): import_filename = pfp_utils.get_keyvaluefromcf(cf,["Imports",label],"file_name",default="") if import_filename=="": msg = " ImportSeries: import filename not found in control file, skipping ..." logger.warning(msg) continue var_name = pfp_utils.get_keyvaluefromcf(cf,["Imports",label],"var_name",default="") if var_name=="": msg = " ImportSeries: variable name not found in control file, skipping ..." logger.warning(msg) continue ds_import = pfp_io.nc_read_series(import_filename) ts_import = ds_import.globalattributes["time_step"] ldt_import = ds_import.series["DateTime"]["Data"] si = pfp_utils.GetDateIndex(ldt_import,str(start_date),ts=ts_import,default=0,match="exact") ei = pfp_utils.GetDateIndex(ldt_import,str(end_date),ts=ts_import,default=len(ldt_import)-1,match="exact") data = numpy.ma.ones(nRecs)*float(c.missing_value) flag = numpy.ma.ones(nRecs) data_import,flag_import,attr_import = pfp_utils.GetSeriesasMA(ds_import,var_name,si=si,ei=ei) ldt_import = ldt_import[si:ei+1] index = pfp_utils.FindIndicesOfBInA(ldt_import,ldt) data[index] = data_import flag[index] = flag_import pfp_utils.CreateSeries(ds,label,data,flag,attr_import)
def mpt_main(cf): base_file_path = cf["Files"]["file_path"] nc_file_name = cf["Files"]["in_filename"] nc_file_path = os.path.join(base_file_path, nc_file_name) ds = pfp_io.nc_read_series(nc_file_path) out_file_paths = run_mpt_code(ds, nc_file_name) ustar_results = read_mpt_output(out_file_paths) mpt_file_path = nc_file_path.replace(".nc", "_MPT.xls") xl_write_mpt(mpt_file_path, ustar_results) return
def do_L5_batch(cf_level): #logger = pfp_log.change_logger_filename("pfp_log", "L5") for i in list(cf_level.keys()): if not os.path.isfile(cf_level[i]): msg = " Control file " + cf_level[i] + " not found" logger.error(msg) continue cf_file_name = os.path.split(cf_level[i]) msg = "Starting L5 processing with " + cf_file_name[1] logger.info(msg) try: cf_l5 = pfp_io.get_controlfilecontents(cf_level[i]) if "Options" not in cf_l5: cf_l5["Options"] = {} cf_l5["Options"]["call_mode"] = "batch" cf_l5["Options"]["show_plots"] = "No" infilename = pfp_io.get_infilenamefromcf(cf_l5) ds4 = pfp_io.nc_read_series(infilename) if ds4.returncodes["value"] != 0: return ds5 = pfp_levels.l5qc(None, cf_l5, ds4) outfilename = pfp_io.get_outfilenamefromcf(cf_l5) nc_file = pfp_io.nc_open_write(outfilename) if nc_file is None: return pfp_io.nc_write_series(nc_file, ds5) msg = "Finished L5 processing with " + cf_file_name[1] logger.info(msg) # now plot the fingerprints for the L5 files cf_fp = pfp_io.get_controlfilecontents("controlfiles/standard/fingerprint.txt") if "Files" not in dir(cf_fp): cf_fp["Files"] = {} file_name = pfp_io.get_outfilenamefromcf(cf_l5) file_path = ntpath.split(file_name)[0] + "/" cf_fp["Files"]["file_path"] = file_path cf_fp["Files"]["in_filename"] = ntpath.split(file_name)[1] if "plot_path" in cf_l5["Files"]: cf_fp["Files"]["plot_path"] = cf_l5["Files"]["plot_path"] else: cf_fp["Files"]["plot_path"] = file_path[:file_path.index("Data")] + "Plots/" if "Options" not in cf_fp: cf_fp["Options"] = {} cf_fp["Options"]["call_mode"] = "batch" cf_fp["Options"]["show_plots"] = "No" msg = "Doing fingerprint plots using " + cf_fp["Files"]["in_filename"] logger.info(msg) pfp_plot.plot_fingerprint(cf_fp) msg = "Finished fingerprint plots" logger.info(msg) logger.info("") except Exception: msg = "Error occurred during L5 with " + cf_file_name[1] logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) continue return
def do_run_l3(cfg=None): """ Purpose: Top level routine for running the L23 post-processing. Usage: pfp_top_level.do_l3() Side effects: Creates an L3 netCDF file. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L3 processing") if not cfg: cfg = pfp_io.load_controlfile() if len(cfg) == 0: logger.info("Quiting L3 processing (no control file)") return in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds2 = pfp_io.nc_read_series(in_filepath) ds3 = pfp_levels.l3qc(cfg, ds2) if ds3.returncodes["value"] != 0: logger.error("An error occurred during L3 processing") logger.error("") return out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) pfp_io.nc_write_series(nc_file, ds3) logger.info("Finished L3 processing") if "Plots" in list(cfg.keys()): logger.info("Plotting L3 data") for nFig in cfg['Plots'].keys(): plt_cf = cfg['Plots'][str(nFig)] if 'Type' in plt_cf.keys(): if str(plt_cf['Type']).lower() == 'xy': pfp_plot.plotxy(cfg, nFig, plt_cf, ds2, ds3) else: pfp_plot.plottimeseries(cfg, nFig, ds2, ds3) else: pfp_plot.plottimeseries(cfg, nFig, ds2, ds3) logger.info("Finished plotting L3 data") except Exception: msg = " Error running L3, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) logger.info("") return
def do_run_l5(main_gui, cfg): """ Purpose: Top level routine for running the L5 gap filling. Usage: pfp_top_level.do_run_l5() Side effects: Creates an L5 netCDF file with gap filled meteorology. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L5 processing") in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds4 = pfp_io.nc_read_series(in_filepath) if ds4.returncodes["value"] != 0: return #ds4.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds4.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" ds5 = pfp_levels.l5qc(main_gui, cfg, ds4) # check to see if all went well if ds5.returncodes["value"] != 0: # tell the user something went wrong logger.info("Quitting L5: " + sitename) # delete the output file if it exists out_filepath = pfp_io.get_outfilenamefromcf(cfg) if os.path.isfile(out_filepath): os.remove(out_filepath) else: # tell the user we are finished logger.info("Finished L5: " + sitename) # get the output file name from the control file out_filepath = pfp_io.get_outfilenamefromcf(cfg) # open it for writing nc_file = pfp_io.nc_open_write(out_filepath) if nc_file is None: return # write the output file pfp_io.nc_write_series(nc_file, ds5) logger.info("Finished saving L5 gap filled data") logger.info("") except Exception: msg = " Error running L5, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def do_L4_batch(cf_level): for i in cf_level.keys(): if not os.path.isfile(cf_level[i]): msg = " Control file " + cf_level[i] + " not found" logger.error(msg) continue cf_file_name = os.path.split(cf_level[i]) msg = "Starting L4 processing with " + cf_file_name[1] logger.info(msg) try: cf_l4 = pfp_io.get_controlfilecontents(cf_level[i]) if "Options" not in cf_l4: cf_l4["Options"] = {} cf_l4["Options"]["call_mode"] = "batch" cf_l4["Options"]["show_plots"] = "No" infilename = pfp_io.get_infilenamefromcf(cf_l4) ds3 = pfp_io.nc_read_series(infilename) ds4 = pfp_levels.l4qc(None, cf_l4, ds3) outfilename = pfp_io.get_outfilenamefromcf(cf_l4) outputlist = pfp_io.get_outputlistfromcf(cf_l4, "nc") ncFile = pfp_io.nc_open_write(outfilename) pfp_io.nc_write_series(ncFile, ds4, outputlist=outputlist) msg = "Finished L4 processing with " + cf_file_name[1] logger.info(msg) # now plot the fingerprints for the L4 files cf_fp = pfp_io.get_controlfilecontents("controlfiles/standard/fingerprint.txt") if "Files" not in dir(cf_fp): cf_fp["Files"] = {} file_name = pfp_io.get_outfilenamefromcf(cf_l4) file_path = ntpath.split(file_name)[0] + "/" cf_fp["Files"]["file_path"] = file_path cf_fp["Files"]["in_filename"] = ntpath.split(file_name)[1] if "plot_path" in cf_l4["Files"]: cf_fp["Files"]["plot_path"] = cf_l4["Files"]["plot_path"] else: cf_fp["Files"]["plot_path"] = file_path[:file_path.index("Data")] + "Plots/" if "Options" not in cf_fp: cf_fp["Options"] = {} cf_fp["Options"]["call_mode"] = "batch" cf_fp["Options"]["show_plots"] = "No" msg = "Doing fingerprint plots using " + cf_fp["Files"]["in_filename"] logger.info(msg) pfp_plot.plot_fingerprint(cf_fp) logger.info("Finished fingerprint plots") logger.info("") except Exception: msg = "Error occurred during L4 with " + cf_file_name[1] logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) continue return
def do_run_l6(main_gui, cfg=None): """ Purpose: Top level routine for running the L6 gap filling. Usage: pfp_top_level.do_run_l6() Side effects: Creates an L6 netCDF file with NEE partitioned into GPP and ER. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L6 processing") if not cfg: cfg = pfp_io.load_controlfile(path='controlfiles') if len(cfg) == 0: logger.info("Quiting L6 processing (no control file)") return in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds5 = pfp_io.nc_read_series(in_filepath) #ds5.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds5.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cfg["Options"]["show_plots"] = "Yes" ds6 = pfp_levels.l6qc(main_gui, cfg, ds5) if ds6.returncodes["value"] != 0: logger.info("Quitting L6: " + sitename) else: logger.info("Finished L6: " + sitename) out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) pfp_io.nc_write_series(nc_file, ds6) logger.info("Finished saving L6 gap filled data") logger.info("") except Exception: msg = " Error running L6, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def nc_update(cfg): """ Purpose: Update a PFP-style netCDF file by changing variable names and attributes. Usage: Author: PRI Date: October 2018 """ nc_file_path = pfp_io.get_infilenamefromcf(cfg) ds = pfp_io.nc_read_series(nc_file_path) change_variable_names(cfg, ds) copy_ws_wd(ds) remove_variables(cfg, ds) change_global_attributes(cfg, ds) nc_file = pfp_io.nc_open_write(nc_file_path) pfp_io.nc_write_series(nc_file, ds) return 0
def compare_eddypro(): epname = pfp_io.get_filename_dialog(title='Choose an EddyPro full output file') ofname = pfp_io.get_filename_dialog(title='Choose an L3 output file') ds_ep = pfp_io.read_eddypro_full(epname) ds_of = pfp_io.nc_read_series(ofname) dt_ep = ds_ep.series['DateTime']['Data'] dt_of = ds_of.series['DateTime']['Data'] start_datetime = max([dt_ep[0],dt_of[0]]) end_datetime = min([dt_ep[-1],dt_of[-1]]) si_of = pfp_utils.GetDateIndex(dt_of, str(start_datetime), ts=30, default=0, match='exact') ei_of = pfp_utils.GetDateIndex(dt_of, str(end_datetime), ts=30, default=len(dt_of), match='exact') si_ep = pfp_utils.GetDateIndex(dt_ep, str(start_datetime), ts=30, default=0, match='exact') ei_ep = pfp_utils.GetDateIndex(dt_ep, str(end_datetime), ts=30, default=len(dt_ep), match='exact') us_of = pfp_utils.GetVariable(ds_of,'ustar',start=si_of,end=ei_of) us_ep = pfp_utils.GetVariable(ds_ep,'ustar',start=si_ep,end=ei_ep) Fh_of = pfp_utils.GetVariable(ds_of,'Fh',start=si_of,end=ei_of) Fh_ep = pfp_utils.GetVariable(ds_ep,'Fh',start=si_ep,end=ei_ep) Fe_of = pfp_utils.GetVariable(ds_of,'Fe',start=si_of,end=ei_of) Fe_ep = pfp_utils.GetVariable(ds_ep,'Fe',start=si_ep,end=ei_ep) Fc_of = pfp_utils.GetVariable(ds_of,'Fc',start=si_of,end=ei_of) Fc_ep = pfp_utils.GetVariable(ds_ep,'Fc',start=si_ep,end=ei_ep) # copy the range check values from the OFQC attributes to the EP attributes for of, ep in zip([us_of, Fh_of, Fe_of, Fc_of], [us_ep, Fh_ep, Fe_ep, Fc_ep]): for item in ["rangecheck_upper", "rangecheck_lower"]: if item in of["Attr"]: ep["Attr"][item] = of["Attr"][item] # apply QC to the EddyPro data pfp_ck.ApplyRangeCheckToVariable(us_ep) pfp_ck.ApplyRangeCheckToVariable(Fc_ep) pfp_ck.ApplyRangeCheckToVariable(Fe_ep) pfp_ck.ApplyRangeCheckToVariable(Fh_ep) # plot the comparison plt.ion() fig = plt.figure(1,figsize=(8,8)) pfp_plot.xyplot(us_ep["Data"],us_of["Data"],sub=[2,2,1],regr=2,xlabel='u*_EP (m/s)',ylabel='u*_OF (m/s)') pfp_plot.xyplot(Fh_ep["Data"],Fh_of["Data"],sub=[2,2,2],regr=2,xlabel='Fh_EP (W/m2)',ylabel='Fh_OF (W/m2)') pfp_plot.xyplot(Fe_ep["Data"],Fe_of["Data"],sub=[2,2,3],regr=2,xlabel='Fe_EP (W/m2)',ylabel='Fe_OF (W/m2)') pfp_plot.xyplot(Fc_ep["Data"],Fc_of["Data"],sub=[2,2,4],regr=2,xlabel='Fc_EP (umol/m2/s)',ylabel='Fc_OF (umol/m2/s)') plt.tight_layout() plt.draw() plt.ioff()
def do_run_l4(main_gui, cfg): """ Purpose: Top level routine for running the L4 gap filling. Usage: pfp_top_level.do_run_l4() Side effects: Creates an L4 netCDF file with gap filled meteorology. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L4 processing") in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds3 = pfp_io.nc_read_series(in_filepath) if ds3.returncodes["value"] != 0: return #ds3.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds3.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" ds4 = pfp_levels.l4qc(main_gui, cfg, ds3) if ds4.returncodes["value"] != 0: logger.info("Quitting L4: " + sitename) else: logger.info("Finished L4: " + sitename) out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) if nc_file is None: return pfp_io.nc_write_series(nc_file, ds4) # save the L4 data logger.info("Finished saving L4 gap filled data") logger.info("") except Exception: msg = " Error running L4, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def do_L2_batch(cf_level): for i in cf_level.keys(): cf_file_name = os.path.split(cf_level[i]) msg = "Starting L2 processing with " + cf_file_name[1] logger.info(msg) try: cf = pfp_io.get_controlfilecontents(cf_level[i]) infilename = pfp_io.get_infilenamefromcf(cf) ds1 = pfp_io.nc_read_series(infilename) ds2 = pfp_levels.l2qc(cf, ds1) outfilename = pfp_io.get_outfilenamefromcf(cf) ncFile = pfp_io.nc_open_write(outfilename) pfp_io.nc_write_series(ncFile, ds2) msg = "Finished L2 processing with " + cf_file_name[1] logger.info(msg) logger.info("") except Exception: msg = "Error occurred during L2 processing " + cf_file_name[1] logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) continue return
def climatology(cf): nc_filename = pfp_io.get_infilenamefromcf(cf) if not pfp_utils.file_exists(nc_filename): return xl_filename = nc_filename.replace(".nc","_Climatology.xls") xlFile = xlwt.Workbook() ds = pfp_io.nc_read_series(nc_filename) # calculate Fa if it is not in the data structure got_Fa = True if "Fa" not in ds.series.keys(): if "Fn" in ds.series.keys() and "Fg" in ds.series.keys(): pfp_ts.CalculateAvailableEnergy(ds,Fa_out='Fa',Fn_in='Fn',Fg_in='Fg') else: got_Fa = False logger.warning(" Fn or Fg not in data struicture") # get the time step ts = int(ds.globalattributes['time_step']) # get the site name SiteName = ds.globalattributes['site_name'] # get the datetime series dt = ds.series['DateTime']['Data'] Hdh = numpy.array([(d.hour + d.minute/float(60)) for d in dt]) Month = numpy.array([d.month for d in dt]) # get the initial start and end dates StartDate = str(dt[0]) EndDate = str(dt[-1]) # find the start index of the first whole day (time=00:30) si = pfp_utils.GetDateIndex(dt,StartDate,ts=ts,default=0,match='startnextday') # find the end index of the last whole day (time=00:00) ei = pfp_utils.GetDateIndex(dt,EndDate,ts=ts,default=-1,match='endpreviousday') # get local views of the datetime series ldt = dt[si:ei+1] Hdh = Hdh[si:ei+1] Month = Month[si:ei+1] # get the number of time steps in a day and the number of days in the data ntsInDay = int(24.0*60.0/float(ts)) nDays = int(len(ldt))/ntsInDay for ThisOne in cf['Variables'].keys(): if "AltVarName" in cf['Variables'][ThisOne].keys(): ThisOne = cf['Variables'][ThisOne]["AltVarName"] if ThisOne in ds.series.keys(): logger.info(" Doing climatology for "+ThisOne) data,f,a = pfp_utils.GetSeriesasMA(ds,ThisOne,si=si,ei=ei) if numpy.ma.count(data)==0: logger.warning(" No data for "+ThisOne+", skipping ...") continue fmt_str = get_formatstring(cf,ThisOne,fmt_def='') xlSheet = xlFile.add_sheet(ThisOne) Av_all = do_diurnalstats(Month,Hdh,data,xlSheet,format_string=fmt_str,ts=ts) # now do it for each day # we want to preserve any data that has been truncated by the use of the "startnextday" # and "endpreviousday" match options used above. Here we revisit the start and end indices # and adjust these backwards and forwards respectively if data has been truncated. nDays_daily = nDays ei_daily = ei si_daily = si sdate = ldt[0] edate = ldt[-1] # is there data after the current end date? if dt[-1]>ldt[-1]: # if so, push the end index back by 1 day so it is included ei_daily = ei + ntsInDay nDays_daily = nDays_daily + 1 edate = ldt[-1]+datetime.timedelta(days=1) # is there data before the current start date? if dt[0]<ldt[0]: # if so, push the start index back by 1 day so it is included si_daily = si - ntsInDay nDays_daily = nDays_daily + 1 sdate = ldt[0]-datetime.timedelta(days=1) # get the data and use the "pad" option to add missing data if required to # complete the extra days data,f,a = pfp_utils.GetSeriesasMA(ds,ThisOne,si=si_daily,ei=ei_daily,mode="pad") data_daily = data.reshape(nDays_daily,ntsInDay) xlSheet = xlFile.add_sheet(ThisOne+'(day)') write_data_1columnpertimestep(xlSheet, data_daily, ts, startdate=sdate, format_string=fmt_str) data_daily_i = do_2dinterpolation(data_daily) xlSheet = xlFile.add_sheet(ThisOne+'i(day)') write_data_1columnpertimestep(xlSheet, data_daily_i, ts, startdate=sdate, format_string=fmt_str) else: logger.warning(" Requested variable "+ThisOne+" not in data structure") continue logger.info(" Saving Excel file "+os.path.split(xl_filename)[1]) xlFile.save(xl_filename)
for site in sites: sp = os.path.join(rp, site, "Data", "Portal") op = os.path.join(rp, site, "Data", "Processed") if not os.path.isdir(sp): print(sp + " , skipping site ...") continue files = sorted([f for f in os.listdir(sp) if ("L3" in f and ".nc" in f)]) if len(files) == 0: print("No files found in " + sp + " , skipping ...") continue for fn in files: ifp = os.path.join(sp, fn) print("Converting " + fn) cfg["Files"]["in_filename"] = ifp # read the input file ds1 = pfp_io.nc_read_series(ifp) # update the variable names change_variable_names(cfg, ds1) # make sure there are Ws and Wd series copy_ws_wd(ds1) # make sure we have all the variables we want ... ds2 = include_variables(cfg, ds1) # ... but not the ones we don't exclude_variables(cfg, ds2) # update the global attributes change_global_attributes(cfg, ds2) # update the variable attributes change_variable_attributes(cfg, ds2) # Fc single point storage consistent_Fc_storage(cfg, ds2, site) ofp = os.path.join(op, fn)
cfg_name = os.path.join("..", "controlfiles", "standard", "map_old_to_new.txt") if os.path.exists(cfg_name): cfg = ConfigObj(cfg_name) else: print " 'map_old_to_new' control file not found" rp = os.path.join(os.sep, "mnt", "OzFlux", "Sites") sites = sorted( [d for d in os.listdir(rp) if os.path.isdir(os.path.join(rp, d))]) for site in sites: sp = os.path.join(rp, site, "Data", "Portal") if not os.path.isdir(sp): print sp + " , skipping site ..." continue files = sorted([f for f in os.listdir(sp) if ("L3" in f and ".nc" in f)]) if len(files) == 0: print "No files found in " + sp + " , skipping ..." continue for file in files: fp = os.path.join(sp, file) print "Converting " + file ds = pfp_io.nc_read_series(fp) change_variable_names(cfg, ds) copy_ws_wd(ds) remove_variables(cfg, ds) change_global_attributes(cfg, ds) nf = pfp_io.nc_open_write(fp) pfp_io.nc_write_series(nf, ds)
def climatology(cf): nc_filename = pfp_io.get_infilenamefromcf(cf) if not pfp_utils.file_exists(nc_filename): return xl_filename = nc_filename.replace(".nc", "_Climatology.xls") xlFile = xlwt.Workbook() ds = pfp_io.nc_read_series(nc_filename) # calculate Fa if it is not in the data structure got_Fa = True if "Fa" not in ds.series.keys(): if "Fn" in ds.series.keys() and "Fg" in ds.series.keys(): pfp_ts.CalculateAvailableEnergy(ds, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg') else: got_Fa = False logger.warning(" Fn or Fg not in data struicture") # get the time step ts = int(ds.globalattributes['time_step']) # get the site name SiteName = ds.globalattributes['site_name'] # get the datetime series dt = ds.series['DateTime']['Data'] Hdh = numpy.array([(d.hour + d.minute / float(60)) for d in dt]) Month = numpy.array([d.month for d in dt]) # get the initial start and end dates StartDate = str(dt[0]) EndDate = str(dt[-1]) # find the start index of the first whole day (time=00:30) si = pfp_utils.GetDateIndex(dt, StartDate, ts=ts, default=0, match='startnextday') # find the end index of the last whole day (time=00:00) ei = pfp_utils.GetDateIndex(dt, EndDate, ts=ts, default=-1, match='endpreviousday') # get local views of the datetime series ldt = dt[si:ei + 1] Hdh = Hdh[si:ei + 1] Month = Month[si:ei + 1] # get the number of time steps in a day and the number of days in the data ntsInDay = int(24.0 * 60.0 / float(ts)) nDays = int(len(ldt)) / ntsInDay for ThisOne in cf['Variables'].keys(): if "AltVarName" in cf['Variables'][ThisOne].keys(): ThisOne = cf['Variables'][ThisOne]["AltVarName"] if ThisOne in ds.series.keys(): logger.info(" Doing climatology for " + ThisOne) data, f, a = pfp_utils.GetSeriesasMA(ds, ThisOne, si=si, ei=ei) if numpy.ma.count(data) == 0: logger.warning(" No data for " + ThisOne + ", skipping ...") continue fmt_str = get_formatstring(cf, ThisOne, fmt_def='') xlSheet = xlFile.add_sheet(ThisOne) Av_all = do_diurnalstats(Month, Hdh, data, xlSheet, format_string=fmt_str, ts=ts) # now do it for each day # we want to preserve any data that has been truncated by the use of the "startnextday" # and "endpreviousday" match options used above. Here we revisit the start and end indices # and adjust these backwards and forwards respectively if data has been truncated. nDays_daily = nDays ei_daily = ei si_daily = si sdate = ldt[0] edate = ldt[-1] # is there data after the current end date? if dt[-1] > ldt[-1]: # if so, push the end index back by 1 day so it is included ei_daily = ei + ntsInDay nDays_daily = nDays_daily + 1 edate = ldt[-1] + datetime.timedelta(days=1) # is there data before the current start date? if dt[0] < ldt[0]: # if so, push the start index back by 1 day so it is included si_daily = si - ntsInDay nDays_daily = nDays_daily + 1 sdate = ldt[0] - datetime.timedelta(days=1) # get the data and use the "pad" option to add missing data if required to # complete the extra days data, f, a = pfp_utils.GetSeriesasMA(ds, ThisOne, si=si_daily, ei=ei_daily, mode="pad") data_daily = data.reshape(nDays_daily, ntsInDay) xlSheet = xlFile.add_sheet(ThisOne + '(day)') write_data_1columnpertimestep(xlSheet, data_daily, ts, startdate=sdate, format_string=fmt_str) data_daily_i = do_2dinterpolation(data_daily) xlSheet = xlFile.add_sheet(ThisOne + 'i(day)') write_data_1columnpertimestep(xlSheet, data_daily_i, ts, startdate=sdate, format_string=fmt_str) elif ThisOne == "EF" and got_Fa: logger.info(" Doing evaporative fraction") EF = numpy.ma.zeros([48, 12]) + float(c.missing_value) Hdh, f, a = pfp_utils.GetSeriesasMA(ds, 'Hdh', si=si, ei=ei) Fa, f, a = pfp_utils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fa_Num, Hr, Fa_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fa[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fa_Num > 4) & (Fe_Num > 4)) EF[:, m - 1][index] = Fe_Av[index] / Fa_Av[index] # reject EF values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'EF') EF = numpy.ma.filled( numpy.ma.masked_where((EF > upr) | (EF < lwr), EF), float(c.missing_value)) # write the EF to the Excel file xlSheet = xlFile.add_sheet('EF') write_data_1columnpermonth(xlSheet, EF, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values EFi = do_2dinterpolation(EF) xlSheet = xlFile.add_sheet('EFi') write_data_1columnpermonth(xlSheet, EFi, ts, format_string='0.00') # now do EF for each day Fa, f, a = pfp_utils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) EF = Fe / Fa EF = numpy.ma.filled( numpy.ma.masked_where((EF > upr) | (EF < lwr), EF), float(c.missing_value)) EF_daily = EF.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('EF(day)') write_data_1columnpertimestep(xlSheet, EF_daily, ts, startdate=ldt[0], format_string='0.00') EFi = do_2dinterpolation(EF_daily) xlSheet = xlFile.add_sheet('EFi(day)') write_data_1columnpertimestep(xlSheet, EFi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne == "BR": logger.info(" Doing Bowen ratio") BR = numpy.ma.zeros([48, 12]) + float(c.missing_value) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fh, f, a = pfp_utils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fh_Num, Hr, Fh_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fh[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fh_Num > 4) & (Fe_Num > 4)) BR[:, m - 1][index] = Fh_Av[index] / Fe_Av[index] # reject BR values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'BR') BR = numpy.ma.filled( numpy.ma.masked_where((BR > upr) | (BR < lwr), BR), float(c.missing_value)) # write the BR to the Excel file xlSheet = xlFile.add_sheet('BR') write_data_1columnpermonth(xlSheet, BR, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values BRi = do_2dinterpolation(BR) xlSheet = xlFile.add_sheet('BRi') write_data_1columnpermonth(xlSheet, BRi, ts, format_string='0.00') # now do BR for each day ... Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fh, f, a = pfp_utils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei) BR = Fh / Fe BR = numpy.ma.filled( numpy.ma.masked_where((BR > upr) | (BR < lwr), BR), float(c.missing_value)) BR_daily = BR.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('BR(day)') write_data_1columnpertimestep(xlSheet, BR_daily, ts, startdate=ldt[0], format_string='0.00') BRi = do_2dinterpolation(BR_daily) xlSheet = xlFile.add_sheet('BRi(day)') write_data_1columnpertimestep(xlSheet, BRi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne == "WUE": logger.info(" Doing ecosystem WUE") WUE = numpy.ma.zeros([48, 12]) + float(c.missing_value) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fc, f, a = pfp_utils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fc_Num, Hr, Fc_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fc[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fc_Num > 4) & (Fe_Num > 4)) WUE[:, m - 1][index] = Fc_Av[index] / Fe_Av[index] # reject WUE values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'WUE') WUE = numpy.ma.filled( numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE), float(c.missing_value)) # write the WUE to the Excel file xlSheet = xlFile.add_sheet('WUE') write_data_1columnpermonth(xlSheet, WUE, ts, format_string='0.00000') # do the 2D interpolation to fill missing EF values WUEi = do_2dinterpolation(WUE) xlSheet = xlFile.add_sheet('WUEi') write_data_1columnpermonth(xlSheet, WUEi, ts, format_string='0.00000') # now do WUE for each day ... Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fc, f, a = pfp_utils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei) WUE = Fc / Fe WUE = numpy.ma.filled( numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE), float(c.missing_value)) WUE_daily = WUE.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('WUE(day)') write_data_1columnpertimestep(xlSheet, WUE_daily, ts, startdate=ldt[0], format_string='0.00000') WUEi = do_2dinterpolation(WUE_daily) xlSheet = xlFile.add_sheet('WUEi(day)') write_data_1columnpertimestep(xlSheet, WUEi, ts, startdate=ldt[0], format_string='0.00000') else: logger.warning(" Requested variable " + ThisOne + " not in data structure") continue logger.info(" Saving Excel file " + os.path.split(xl_filename)[1]) xlFile.save(xl_filename)
def gfalternate_createdict(cf, ds, series, ds_alt): """ Purpose: Creates a dictionary in ds to hold information about the alternate data used to gap fill the tower data. Usage: Side effects: Author: PRI Date: August 2014 """ # get the section of the control file containing the series section = pfp_utils.get_cfsection(cf, series=series, mode="quiet") # return without doing anything if the series isn't in a control file section if len(section)==0: msg = "GapFillFromAlternate: Series %s not found in control file, skipping ...", series logger.error(msg) return # create the alternate directory in the data structure if "alternate" not in dir(ds): ds.alternate = {} # name of alternate output series in ds output_list = cf[section][series]["GapFillFromAlternate"].keys() # loop over the outputs listed in the control file for output in output_list: # create the dictionary keys for this output ds.alternate[output] = {} ds.alternate[output]["label_tower"] = series # source name ds.alternate[output]["source"] = cf[section][series]["GapFillFromAlternate"][output]["source"] # site name ds.alternate[output]["site_name"] = ds.globalattributes["site_name"] # alternate data file name # first, look in the [Files] section for a generic file name file_list = cf["Files"].keys() lower_file_list = [item.lower() for item in file_list] if ds.alternate[output]["source"].lower() in lower_file_list: # found a generic file name i = lower_file_list.index(ds.alternate[output]["source"].lower()) ds.alternate[output]["file_name"] = cf["Files"][file_list[i]] else: # no generic file name found, look for a file name in the variable section ds.alternate[output]["file_name"] = cf[section][series]["GapFillFromAlternate"][output]["file_name"] # if the file has not already been read, do it now if ds.alternate[output]["file_name"] not in ds_alt: ds_alternate = pfp_io.nc_read_series(ds.alternate[output]["file_name"],fixtimestepmethod="round") gfalternate_matchstartendtimes(ds,ds_alternate) ds_alt[ds.alternate[output]["file_name"]] = ds_alternate # get the type of fit ds.alternate[output]["fit_type"] = "OLS" if "fit" in cf[section][series]["GapFillFromAlternate"][output]: if cf[section][series]["GapFillFromAlternate"][output]["fit"].lower() in ["ols","ols_thru0","mrev","replace","rma","odr"]: ds.alternate[output]["fit_type"] = cf[section][series]["GapFillFromAlternate"][output]["fit"] else: logger.info("gfAlternate: unrecognised fit option for series %s, used OLS", output) # correct for lag? if "lag" in cf[section][series]["GapFillFromAlternate"][output]: if cf[section][series]["GapFillFromAlternate"][output]["lag"].lower() in ["no","false"]: ds.alternate[output]["lag"] = "no" elif cf[section][series]["GapFillFromAlternate"][output]["lag"].lower() in ["yes","true"]: ds.alternate[output]["lag"] = "yes" else: logger.info("gfAlternate: unrecognised lag option for series %s", output) else: ds.alternate[output]["lag"] = "yes" # choose specific alternate variable? if "usevars" in cf[section][series]["GapFillFromAlternate"][output]: ds.alternate[output]["usevars"] = ast.literal_eval(cf[section][series]["GapFillFromAlternate"][output]["usevars"]) # alternate data variable name if different from name used in control file if "alternate_name" in cf[section][series]["GapFillFromAlternate"][output]: ds.alternate[output]["alternate_name"] = cf[section][series]["GapFillFromAlternate"][output]["alternate_name"] else: ds.alternate[output]["alternate_name"] = series # results of best fit for plotting later on ds.alternate[output]["results"] = {"startdate":[],"enddate":[],"No. points":[],"No. filled":[], "r":[],"Bias":[],"RMSE":[],"Frac Bias":[],"NMSE":[], "Avg (Tower)":[],"Avg (Alt)":[], "Var (Tower)":[],"Var (Alt)":[],"Var ratio":[]} # create an empty series in ds if the alternate output series doesn't exist yet if output not in ds.series.keys(): data,flag,attr = pfp_utils.MakeEmptySeries(ds,output) pfp_utils.CreateSeries(ds,output,data,flag,attr) pfp_utils.CreateSeries(ds,series+"_composite",data,flag,attr)
def CPD_run(cf): # Set input file and output path and create directories for plots and results path_out = cf['Files']['file_path'] file_in = os.path.join(cf['Files']['file_path'],cf['Files']['in_filename']) # if "out_filename" in cf['Files']: file_out = os.path.join(cf['Files']['file_path'],cf['Files']['out_filename']) else: file_out = os.path.join(cf['Files']['file_path'],cf['Files']['in_filename'].replace(".nc","_CPD.xls")) plot_path = "plots/" if "plot_path" in cf["Files"]: plot_path = os.path.join(cf["Files"]["plot_path"],"CPD/") if not os.path.isdir(plot_path): os.makedirs(plot_path) results_path = path_out if not os.path.isdir(results_path): os.makedirs(results_path) # get a dictionary of the variable names var_list = cf["Variables"].keys() names = {} for item in var_list: if "AltVarName" in cf["Variables"][item].keys(): names[item] = cf["Variables"][item]["AltVarName"] else: names[item] = item # read the netcdf file logger.info(' Reading netCDF file '+file_in) ds = pfp_io.nc_read_series(file_in) nrecs = int(ds.globalattributes["nc_nrecs"]) ts = int(ds.globalattributes["time_step"]) # get the datetime dt = ds.series["DateTime"]["Data"] # adjust the datetime so that the last time period in a year is correctly assigned. # e.g. last period for 2013 is 2014-01-01 00:00, here we make the year 2013 dt = dt - datetime.timedelta(minutes=ts) # now get the data d = {} f = {} for item in names.keys(): data,flag,attr = pfp_utils.GetSeries(ds,names[item]) d[item] = np.where(data==c.missing_value,np.nan,data) f[item] = flag # set all data to NaNs if any flag not 0 or 10 for item in f.keys(): for f_OK in [0,10]: idx = np.where(f[item]!=0)[0] if len(idx)!=0: for itemd in d.keys(): d[itemd][idx] = np.nan d["Year"] = np.array([ldt.year for ldt in dt]) df=pd.DataFrame(d,index=dt) # replace missing values with NaN df.replace(c.missing_value,np.nan) # Build dictionary of additional configs d={} d['radiation_threshold']=int(cf['Options']['Fsd_threshold']) d['num_bootstraps']=int(cf['Options']['Num_bootstraps']) d['flux_period']=int(ds.globalattributes["time_step"]) d['site_name']=ds.globalattributes["site_name"] d["call_mode"]=pfp_utils.get_keyvaluefromcf(cf,["Options"],"call_mode",default="interactive",mode="quiet") d["show_plots"] = pfp_utils.get_optionskeyaslogical(cf, "show_plots", default=True) d['plot_tclass'] = False if cf['Options']['Plot_TClass'] == 'True': d['plot_tclass'] = True if cf['Options']['Output_plots']=='True': d['plot_path']=plot_path if cf['Options']['Output_results']=='True': d['results_path']=results_path d["file_out"]=file_out return df,d
cf = pfp_io.get_controlfilecontents(cfname) ds1 = pfp_ls.l1qc(cf) outfilename = pfp_io.get_outfilenamefromcf(cf) ncFile = pfp_io.nc_open_write(outfilename) pfp_io.nc_write_series(ncFile, ds1) logger.info('Finished L1 processing with ' + cf_file_name[1]) logger.info('') elif level.lower() == "l2": # L2 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] cf_file_name = os.path.split(cfname) logger.info('Starting L2 processing with ' + cf_file_name[1]) cf = pfp_io.get_controlfilecontents(cfname) infilename = pfp_io.get_infilenamefromcf(cf) ds1 = pfp_io.nc_read_series(infilename) ds2 = pfp_ls.l2qc(cf, ds1) outfilename = pfp_io.get_outfilenamefromcf(cf) ncFile = pfp_io.nc_open_write(outfilename) pfp_io.nc_write_series(ncFile, ds2) logger.info('Finished L2 processing with ' + cf_file_name[1]) logger.info('') elif level.lower() == "l3": # L3 processing for i in cf_batch["Levels"][level].keys(): cfname = cf_batch["Levels"][level][i] cf_file_name = os.path.split(cfname) logger.info('Starting L3 processing with ' + cf_file_name[1]) cf = pfp_io.get_controlfilecontents(cfname) infilename = pfp_io.get_infilenamefromcf(cf) ds2 = pfp_io.nc_read_series(infilename)