def l1qc(cf): # get the data series from the Excel file in_filename = pfp_io.get_infilenamefromcf(cf) if not pfp_utils.file_exists(in_filename, mode="quiet"): msg = " Input file " + in_filename + " not found ..." logger.error(msg) ds1 = pfp_io.DataStructure() ds1.returncodes = {"value": 1, "message": msg} return ds1 file_name, file_extension = os.path.splitext(in_filename) if "csv" in file_extension.lower(): ds1 = pfp_io.csv_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Excel datetime from the Python datetime objects #pfp_utils.get_xldatefromdatetime(ds1) else: ds1 = pfp_io.xl_read_series(cf) if ds1.returncodes["value"] != 0: return ds1 # get a series of Python datetime objects from the Excel datetime #pfp_utils.get_datetime_from_xldate(ds1) # get the netCDF attributes from the control file #pfp_ts.do_attributes(cf,ds1) pfp_utils.get_datetime(cf, ds1) # round the Python datetime to the nearest second pfp_utils.round_datetime(ds1, mode="nearest_second") #check for gaps in the Python datetime series and fix if present fixtimestepmethod = pfp_utils.get_keyvaluefromcf(cf, ["options"], "FixTimeStepMethod", default="round") if pfp_utils.CheckTimeStep(ds1): pfp_utils.FixTimeStep(ds1, fixtimestepmethod=fixtimestepmethod) # recalculate the Excel datetime #pfp_utils.get_xldatefromdatetime(ds1) # get the Year, Month, Day etc from the Python datetime #pfp_utils.get_ymdhmsfromdatetime(ds1) # write the processing level to a global attribute ds1.globalattributes['nc_level'] = str("L1") # get the start and end date from the datetime series unless they were # given in the control file if 'start_date' not in ds1.globalattributes.keys(): ds1.globalattributes['start_date'] = str( ds1.series['DateTime']['Data'][0]) if 'end_date' not in ds1.globalattributes.keys(): ds1.globalattributes['end_date'] = str( ds1.series['DateTime']['Data'][-1]) # calculate variances from standard deviations and vice versa pfp_ts.CalculateStandardDeviations(cf, ds1) # create new variables using user defined functions pfp_ts.DoFunctions(cf, ds1) # create a series of synthetic downwelling shortwave radiation #pfp_ts.get_synthetic_fsd(ds1) # check missing data and QC flags are consistent pfp_utils.CheckQCFlags(ds1) return ds1
def do_run_l5(main_gui, cfg): """ Purpose: Top level routine for running the L5 gap filling. Usage: pfp_top_level.do_run_l5() Side effects: Creates an L5 netCDF file with gap filled meteorology. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L5 processing") in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds4 = pfp_io.nc_read_series(in_filepath) if ds4.returncodes["value"] != 0: return #ds4.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds4.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" ds5 = pfp_levels.l5qc(main_gui, cfg, ds4) # check to see if all went well if ds5.returncodes["value"] != 0: # tell the user something went wrong logger.info("Quitting L5: " + sitename) # delete the output file if it exists out_filepath = pfp_io.get_outfilenamefromcf(cfg) if os.path.isfile(out_filepath): os.remove(out_filepath) else: # tell the user we are finished logger.info("Finished L5: " + sitename) # get the output file name from the control file out_filepath = pfp_io.get_outfilenamefromcf(cfg) # open it for writing nc_file = pfp_io.nc_open_write(out_filepath) if nc_file is None: return # write the output file pfp_io.nc_write_series(nc_file, ds5) logger.info("Finished saving L5 gap filled data") logger.info("") except Exception: msg = " Error running L5, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def do_run_l3(cfg=None): """ Purpose: Top level routine for running the L23 post-processing. Usage: pfp_top_level.do_l3() Side effects: Creates an L3 netCDF file. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L3 processing") if not cfg: cfg = pfp_io.load_controlfile() if len(cfg) == 0: logger.info("Quiting L3 processing (no control file)") return in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds2 = pfp_io.nc_read_series(in_filepath) ds3 = pfp_levels.l3qc(cfg, ds2) if ds3.returncodes["value"] != 0: logger.error("An error occurred during L3 processing") logger.error("") return out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) pfp_io.nc_write_series(nc_file, ds3) logger.info("Finished L3 processing") if "Plots" in list(cfg.keys()): logger.info("Plotting L3 data") for nFig in cfg['Plots'].keys(): plt_cf = cfg['Plots'][str(nFig)] if 'Type' in plt_cf.keys(): if str(plt_cf['Type']).lower() == 'xy': pfp_plot.plotxy(cfg, nFig, plt_cf, ds2, ds3) else: pfp_plot.plottimeseries(cfg, nFig, ds2, ds3) else: pfp_plot.plottimeseries(cfg, nFig, ds2, ds3) logger.info("Finished plotting L3 data") except Exception: msg = " Error running L3, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) logger.info("") return
def do_run_l6(main_gui, cfg=None): """ Purpose: Top level routine for running the L6 gap filling. Usage: pfp_top_level.do_run_l6() Side effects: Creates an L6 netCDF file with NEE partitioned into GPP and ER. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L6 processing") if not cfg: cfg = pfp_io.load_controlfile(path='controlfiles') if len(cfg) == 0: logger.info("Quiting L6 processing (no control file)") return in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds5 = pfp_io.nc_read_series(in_filepath) #ds5.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds5.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" cfg["Options"]["show_plots"] = "Yes" ds6 = pfp_levels.l6qc(main_gui, cfg, ds5) if ds6.returncodes["value"] != 0: logger.info("Quitting L6: " + sitename) else: logger.info("Finished L6: " + sitename) out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) pfp_io.nc_write_series(nc_file, ds6) logger.info("Finished saving L6 gap filled data") logger.info("") except Exception: msg = " Error running L6, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def do_run_l4(main_gui, cfg): """ Purpose: Top level routine for running the L4 gap filling. Usage: pfp_top_level.do_run_l4() Side effects: Creates an L4 netCDF file with gap filled meteorology. Author: PRI Date: Back in the day Mods: December 2017: rewrite for use with new GUI """ try: logger.info("Starting L4 processing") in_filepath = pfp_io.get_infilenamefromcf(cfg) if not pfp_utils.file_exists(in_filepath): in_filename = os.path.split(in_filepath) logger.error("File " + in_filename[1] + " not found") return ds3 = pfp_io.nc_read_series(in_filepath) if ds3.returncodes["value"] != 0: return #ds3.globalattributes['controlfile_name'] = cfg['controlfile_name'] sitename = ds3.globalattributes['site_name'] if "Options" not in cfg: cfg["Options"] = {} cfg["Options"]["call_mode"] = "interactive" ds4 = pfp_levels.l4qc(main_gui, cfg, ds3) if ds4.returncodes["value"] != 0: logger.info("Quitting L4: " + sitename) else: logger.info("Finished L4: " + sitename) out_filepath = pfp_io.get_outfilenamefromcf(cfg) nc_file = pfp_io.nc_open_write(out_filepath) if nc_file is None: return pfp_io.nc_write_series(nc_file, ds4) # save the L4 data logger.info("Finished saving L4 gap filled data") logger.info("") except Exception: msg = " Error running L4, see below for details ..." logger.error(msg) error_message = traceback.format_exc() logger.error(error_message) return
def climatology(cf): nc_filename = pfp_io.get_infilenamefromcf(cf) if not pfp_utils.file_exists(nc_filename): return xl_filename = nc_filename.replace(".nc","_Climatology.xls") xlFile = xlwt.Workbook() ds = pfp_io.nc_read_series(nc_filename) # calculate Fa if it is not in the data structure got_Fa = True if "Fa" not in ds.series.keys(): if "Fn" in ds.series.keys() and "Fg" in ds.series.keys(): pfp_ts.CalculateAvailableEnergy(ds,Fa_out='Fa',Fn_in='Fn',Fg_in='Fg') else: got_Fa = False logger.warning(" Fn or Fg not in data struicture") # get the time step ts = int(ds.globalattributes['time_step']) # get the site name SiteName = ds.globalattributes['site_name'] # get the datetime series dt = ds.series['DateTime']['Data'] Hdh = numpy.array([(d.hour + d.minute/float(60)) for d in dt]) Month = numpy.array([d.month for d in dt]) # get the initial start and end dates StartDate = str(dt[0]) EndDate = str(dt[-1]) # find the start index of the first whole day (time=00:30) si = pfp_utils.GetDateIndex(dt,StartDate,ts=ts,default=0,match='startnextday') # find the end index of the last whole day (time=00:00) ei = pfp_utils.GetDateIndex(dt,EndDate,ts=ts,default=-1,match='endpreviousday') # get local views of the datetime series ldt = dt[si:ei+1] Hdh = Hdh[si:ei+1] Month = Month[si:ei+1] # get the number of time steps in a day and the number of days in the data ntsInDay = int(24.0*60.0/float(ts)) nDays = int(len(ldt))/ntsInDay for ThisOne in cf['Variables'].keys(): if "AltVarName" in cf['Variables'][ThisOne].keys(): ThisOne = cf['Variables'][ThisOne]["AltVarName"] if ThisOne in ds.series.keys(): logger.info(" Doing climatology for "+ThisOne) data,f,a = pfp_utils.GetSeriesasMA(ds,ThisOne,si=si,ei=ei) if numpy.ma.count(data)==0: logger.warning(" No data for "+ThisOne+", skipping ...") continue fmt_str = get_formatstring(cf,ThisOne,fmt_def='') xlSheet = xlFile.add_sheet(ThisOne) Av_all = do_diurnalstats(Month,Hdh,data,xlSheet,format_string=fmt_str,ts=ts) # now do it for each day # we want to preserve any data that has been truncated by the use of the "startnextday" # and "endpreviousday" match options used above. Here we revisit the start and end indices # and adjust these backwards and forwards respectively if data has been truncated. nDays_daily = nDays ei_daily = ei si_daily = si sdate = ldt[0] edate = ldt[-1] # is there data after the current end date? if dt[-1]>ldt[-1]: # if so, push the end index back by 1 day so it is included ei_daily = ei + ntsInDay nDays_daily = nDays_daily + 1 edate = ldt[-1]+datetime.timedelta(days=1) # is there data before the current start date? if dt[0]<ldt[0]: # if so, push the start index back by 1 day so it is included si_daily = si - ntsInDay nDays_daily = nDays_daily + 1 sdate = ldt[0]-datetime.timedelta(days=1) # get the data and use the "pad" option to add missing data if required to # complete the extra days data,f,a = pfp_utils.GetSeriesasMA(ds,ThisOne,si=si_daily,ei=ei_daily,mode="pad") data_daily = data.reshape(nDays_daily,ntsInDay) xlSheet = xlFile.add_sheet(ThisOne+'(day)') write_data_1columnpertimestep(xlSheet, data_daily, ts, startdate=sdate, format_string=fmt_str) data_daily_i = do_2dinterpolation(data_daily) xlSheet = xlFile.add_sheet(ThisOne+'i(day)') write_data_1columnpertimestep(xlSheet, data_daily_i, ts, startdate=sdate, format_string=fmt_str) else: logger.warning(" Requested variable "+ThisOne+" not in data structure") continue logger.info(" Saving Excel file "+os.path.split(xl_filename)[1]) xlFile.save(xl_filename)
def climatology(cf): nc_filename = pfp_io.get_infilenamefromcf(cf) if not pfp_utils.file_exists(nc_filename): return xl_filename = nc_filename.replace(".nc", "_Climatology.xls") xlFile = xlwt.Workbook() ds = pfp_io.nc_read_series(nc_filename) # calculate Fa if it is not in the data structure got_Fa = True if "Fa" not in ds.series.keys(): if "Fn" in ds.series.keys() and "Fg" in ds.series.keys(): pfp_ts.CalculateAvailableEnergy(ds, Fa_out='Fa', Fn_in='Fn', Fg_in='Fg') else: got_Fa = False logger.warning(" Fn or Fg not in data struicture") # get the time step ts = int(ds.globalattributes['time_step']) # get the site name SiteName = ds.globalattributes['site_name'] # get the datetime series dt = ds.series['DateTime']['Data'] Hdh = numpy.array([(d.hour + d.minute / float(60)) for d in dt]) Month = numpy.array([d.month for d in dt]) # get the initial start and end dates StartDate = str(dt[0]) EndDate = str(dt[-1]) # find the start index of the first whole day (time=00:30) si = pfp_utils.GetDateIndex(dt, StartDate, ts=ts, default=0, match='startnextday') # find the end index of the last whole day (time=00:00) ei = pfp_utils.GetDateIndex(dt, EndDate, ts=ts, default=-1, match='endpreviousday') # get local views of the datetime series ldt = dt[si:ei + 1] Hdh = Hdh[si:ei + 1] Month = Month[si:ei + 1] # get the number of time steps in a day and the number of days in the data ntsInDay = int(24.0 * 60.0 / float(ts)) nDays = int(len(ldt)) / ntsInDay for ThisOne in cf['Variables'].keys(): if "AltVarName" in cf['Variables'][ThisOne].keys(): ThisOne = cf['Variables'][ThisOne]["AltVarName"] if ThisOne in ds.series.keys(): logger.info(" Doing climatology for " + ThisOne) data, f, a = pfp_utils.GetSeriesasMA(ds, ThisOne, si=si, ei=ei) if numpy.ma.count(data) == 0: logger.warning(" No data for " + ThisOne + ", skipping ...") continue fmt_str = get_formatstring(cf, ThisOne, fmt_def='') xlSheet = xlFile.add_sheet(ThisOne) Av_all = do_diurnalstats(Month, Hdh, data, xlSheet, format_string=fmt_str, ts=ts) # now do it for each day # we want to preserve any data that has been truncated by the use of the "startnextday" # and "endpreviousday" match options used above. Here we revisit the start and end indices # and adjust these backwards and forwards respectively if data has been truncated. nDays_daily = nDays ei_daily = ei si_daily = si sdate = ldt[0] edate = ldt[-1] # is there data after the current end date? if dt[-1] > ldt[-1]: # if so, push the end index back by 1 day so it is included ei_daily = ei + ntsInDay nDays_daily = nDays_daily + 1 edate = ldt[-1] + datetime.timedelta(days=1) # is there data before the current start date? if dt[0] < ldt[0]: # if so, push the start index back by 1 day so it is included si_daily = si - ntsInDay nDays_daily = nDays_daily + 1 sdate = ldt[0] - datetime.timedelta(days=1) # get the data and use the "pad" option to add missing data if required to # complete the extra days data, f, a = pfp_utils.GetSeriesasMA(ds, ThisOne, si=si_daily, ei=ei_daily, mode="pad") data_daily = data.reshape(nDays_daily, ntsInDay) xlSheet = xlFile.add_sheet(ThisOne + '(day)') write_data_1columnpertimestep(xlSheet, data_daily, ts, startdate=sdate, format_string=fmt_str) data_daily_i = do_2dinterpolation(data_daily) xlSheet = xlFile.add_sheet(ThisOne + 'i(day)') write_data_1columnpertimestep(xlSheet, data_daily_i, ts, startdate=sdate, format_string=fmt_str) elif ThisOne == "EF" and got_Fa: logger.info(" Doing evaporative fraction") EF = numpy.ma.zeros([48, 12]) + float(c.missing_value) Hdh, f, a = pfp_utils.GetSeriesasMA(ds, 'Hdh', si=si, ei=ei) Fa, f, a = pfp_utils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fa_Num, Hr, Fa_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fa[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fa_Num > 4) & (Fe_Num > 4)) EF[:, m - 1][index] = Fe_Av[index] / Fa_Av[index] # reject EF values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'EF') EF = numpy.ma.filled( numpy.ma.masked_where((EF > upr) | (EF < lwr), EF), float(c.missing_value)) # write the EF to the Excel file xlSheet = xlFile.add_sheet('EF') write_data_1columnpermonth(xlSheet, EF, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values EFi = do_2dinterpolation(EF) xlSheet = xlFile.add_sheet('EFi') write_data_1columnpermonth(xlSheet, EFi, ts, format_string='0.00') # now do EF for each day Fa, f, a = pfp_utils.GetSeriesasMA(ds, 'Fa', si=si, ei=ei) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) EF = Fe / Fa EF = numpy.ma.filled( numpy.ma.masked_where((EF > upr) | (EF < lwr), EF), float(c.missing_value)) EF_daily = EF.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('EF(day)') write_data_1columnpertimestep(xlSheet, EF_daily, ts, startdate=ldt[0], format_string='0.00') EFi = do_2dinterpolation(EF_daily) xlSheet = xlFile.add_sheet('EFi(day)') write_data_1columnpertimestep(xlSheet, EFi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne == "BR": logger.info(" Doing Bowen ratio") BR = numpy.ma.zeros([48, 12]) + float(c.missing_value) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fh, f, a = pfp_utils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fh_Num, Hr, Fh_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fh[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fh_Num > 4) & (Fe_Num > 4)) BR[:, m - 1][index] = Fh_Av[index] / Fe_Av[index] # reject BR values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'BR') BR = numpy.ma.filled( numpy.ma.masked_where((BR > upr) | (BR < lwr), BR), float(c.missing_value)) # write the BR to the Excel file xlSheet = xlFile.add_sheet('BR') write_data_1columnpermonth(xlSheet, BR, ts, format_string='0.00') # do the 2D interpolation to fill missing EF values BRi = do_2dinterpolation(BR) xlSheet = xlFile.add_sheet('BRi') write_data_1columnpermonth(xlSheet, BRi, ts, format_string='0.00') # now do BR for each day ... Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fh, f, a = pfp_utils.GetSeriesasMA(ds, 'Fh', si=si, ei=ei) BR = Fh / Fe BR = numpy.ma.filled( numpy.ma.masked_where((BR > upr) | (BR < lwr), BR), float(c.missing_value)) BR_daily = BR.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('BR(day)') write_data_1columnpertimestep(xlSheet, BR_daily, ts, startdate=ldt[0], format_string='0.00') BRi = do_2dinterpolation(BR_daily) xlSheet = xlFile.add_sheet('BRi(day)') write_data_1columnpertimestep(xlSheet, BRi, ts, startdate=ldt[0], format_string='0.00') elif ThisOne == "WUE": logger.info(" Doing ecosystem WUE") WUE = numpy.ma.zeros([48, 12]) + float(c.missing_value) Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fc, f, a = pfp_utils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei) for m in range(1, 13): mi = numpy.where(Month == m)[0] Fc_Num, Hr, Fc_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fc[mi], ts) Fe_Num, Hr, Fe_Av, Sd, Mx, Mn = get_diurnalstats( Hdh[mi], Fe[mi], ts) index = numpy.ma.where((Fc_Num > 4) & (Fe_Num > 4)) WUE[:, m - 1][index] = Fc_Av[index] / Fe_Av[index] # reject WUE values greater than upper limit or less than lower limit upr, lwr = get_rangecheck_limit(cf, 'WUE') WUE = numpy.ma.filled( numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE), float(c.missing_value)) # write the WUE to the Excel file xlSheet = xlFile.add_sheet('WUE') write_data_1columnpermonth(xlSheet, WUE, ts, format_string='0.00000') # do the 2D interpolation to fill missing EF values WUEi = do_2dinterpolation(WUE) xlSheet = xlFile.add_sheet('WUEi') write_data_1columnpermonth(xlSheet, WUEi, ts, format_string='0.00000') # now do WUE for each day ... Fe, f, a = pfp_utils.GetSeriesasMA(ds, 'Fe', si=si, ei=ei) Fc, f, a = pfp_utils.GetSeriesasMA(ds, 'Fc', si=si, ei=ei) WUE = Fc / Fe WUE = numpy.ma.filled( numpy.ma.masked_where((WUE > upr) | (WUE < lwr), WUE), float(c.missing_value)) WUE_daily = WUE.reshape(nDays, ntsInDay) xlSheet = xlFile.add_sheet('WUE(day)') write_data_1columnpertimestep(xlSheet, WUE_daily, ts, startdate=ldt[0], format_string='0.00000') WUEi = do_2dinterpolation(WUE_daily) xlSheet = xlFile.add_sheet('WUEi(day)') write_data_1columnpertimestep(xlSheet, WUEi, ts, startdate=ldt[0], format_string='0.00000') else: logger.warning(" Requested variable " + ThisOne + " not in data structure") continue logger.info(" Saving Excel file " + os.path.split(xl_filename)[1]) xlFile.save(xl_filename)