def main(): # set up command line argument processing parser = argparse.ArgumentParser() # options parser.add_argument( "-v", "--verbose", help="increase output verbosity", action="store_true", default=False, ) parser.add_argument( "-n", "--dry-run", help="Process data but don't save results", action="store_true", default=False, ) parser.add_argument( "-p", "--aggregation-period", help="Number of Days to Aggregate (default=1)", nargs="?", type=int, choices=range(1, 5, 2), default=1, ) # positional arguments parser.add_argument("site", help="PhenoCam site name") parser.add_argument("roiname", help="ROI name, e.g. canopy_0001") # get args args = parser.parse_args() sitename = args.site roiname = args.roiname verbose = args.verbose dryrun = args.dry_run ndays = args.aggregation_period if verbose: print("site: {0}".format(sitename)) print("roiname: {0}".format(roiname)) print("verbose: {0}".format(verbose)) print("dryrun: {0}".format(dryrun)) print("period: {0}".format(ndays)) # read in config file for this ROI List if it exists config_file = "{0}_{1}.cfg".format(sitename, roiname) config_path = os.path.join(archive_dir, sitename, "ROI", config_file) if os.path.exists(config_path): # NOTE: should probably subclass safe config parser # and add gettime() method which checks for time validity cfgparser = configparser( defaults={ "nimage_threshold": str(default_nimage_threshold), "time_min": str(default_time_min), "time_max": str(default_time_max), "sunelev_min": str(default_sunelev_min), "brt_min": str(default_brt_min), "brt_max": str(default_brt_max), } ) cfgparser.read(config_path) if cfgparser.has_section("gcc90_calculation"): nimage_threshold = cfgparser.getint("gcc90_calculation", "nimage_threshold") time_max_str = cfgparser.get("gcc90_calculation", "time_max") [tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) time_min_str = cfgparser.get("gcc90_calculation", "time_min") [tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = cfgparser.getfloat("gcc90_calculation", "sunelev_min") brt_min = cfgparser.getint("gcc90_calculation", "brt_min") brt_max = cfgparser.getint("gcc90_calculation", "brt_max") else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max # print config values if verbose: print("") print("gcc config:") print("===========") print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname)) if os.path.exists(config_path): print("config file: {0}".format(config_file)) else: print("config file: None") print("nimage threshold: ", nimage_threshold) print("time of day min: ", time_min) print("time of day max: ", time_max) print("sun elev min: ", sunelev_min) print("aggregate days: ", ndays) print("minimum brightness: ", brt_min) print("maximum brightness: ", brt_max) # set up output filename outdir = os.path.join(archive_dir, sitename, "ROI") outfile = "{0}_{1}_{2}day.csv".format(sitename, roiname, ndays) outpath = os.path.join(outdir, outfile) if verbose: print("output file: ", outfile) # create gcc timeseries object as empty list gcc_ts = GCCTimeSeries( site=sitename, ROIListID=roiname, nday=ndays, nmin=nimage_threshold, tod_min=time_min, tod_max=time_max, sunelev_min=sunelev_min, brt_min=brt_min, brt_max=brt_max, ) # get roi timeseries for this site and roi roits = get_roi_timeseries(sitename, roiname) if verbose: print("") print("ROI timeseries info:") print("====================") print("site: ", roits.site) print("ROI list id: ", roits.roilistid) print("create date: ", roits.created_at) print("update date: ", roits.updated_at) print("nrows: ", len(roits.rows)) # make list of rows which match image selection criteria roits_rows = roits.select_rows( tod_min=time_min, tod_max=time_max, sunelev_min=sunelev_min, brt_min=brt_min, brt_max=brt_max, ) # check that some rows passed selection criteria nrows = len(roits_rows) if nrows == 0: print("No rows passed the selection criteria") return if verbose: print("Number of selected rows: {0}".format(nrows)) # make a list of dates for selected images img_date = [] for row in roits_rows: img_date.append(row["datetime"].date()) # list is ordered so find first and last dates dt_first = img_date[0] dt_last = img_date[nrows - 1] if verbose: print("date first: {}".format(dt_first)) print("date last: {}".format(dt_last)) # set up a generator which yields dates for the start # of the next nday period covering the date range of image gcc_dr = daterange2(dt_first, dt_last, ndays) # calculate offset for timeseries based on nday day_offset = ndays / 2 date_offset = timedelta(days=day_offset) # roits_ndx will be index into ROI timeseries roits_ndx = 0 # set up vars for accumulating stats img_cnt = 0 filenames = [] r_dn_vals = [] rcc_vals = [] g_dn_vals = [] gcc_vals = [] b_dn_vals = [] bcc_vals = [] solar_elev_vals = [] midday_delta_vals = [] # loop over nday time periods for gcc_ndx, start_date in enumerate(gcc_dr): end_date = start_date + timedelta(ndays) gcc_date = start_date + date_offset doy = gcc_date.timetuple().tm_yday midday_noon = datetime(gcc_date.year, gcc_date.month, gcc_date.day, 12, 0, 0) # get roits rows for this time period while ( roits_ndx < nrows and img_date[roits_ndx] >= start_date and img_date[roits_ndx] < end_date ): # skip this row if awbflag is 1 if roits_rows[roits_ndx]["awbflag"] == 1: if roits_ndx < nrows: roits_ndx += 1 continue else: break filenames.append(roits_rows[roits_ndx]["filename"]) r_dn = roits_rows[roits_ndx]["r_mean"] r_dn_vals.append(r_dn) g_dn = roits_rows[roits_ndx]["g_mean"] g_dn_vals.append(g_dn) b_dn = roits_rows[roits_ndx]["b_mean"] b_dn_vals.append(b_dn) dnsum = r_dn + g_dn + b_dn # NOTE: I'm recomputing gcc, rcc, bcc from DN values rather # than using value stored in roistats CSV if dnsum <= 0: rcc = np.nan bcc = np.nan gcc = np.nan else: img_cnt += 1 rcc = r_dn / dnsum bcc = b_dn / dnsum gcc = roits_rows[roits_ndx]["gcc"] solar_elev = roits_rows[roits_ndx]["solar_elev"] # note that rcc_vals can include NaN's rcc_vals.append(rcc) gcc_vals.append(gcc) bcc_vals.append(bcc) solar_elev_vals.append(solar_elev) midday_td = roits_rows[roits_ndx]["datetime"] - midday_noon midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds) midday_delta_vals.append(midday_td_secs) if roits_ndx < nrows: roits_ndx += 1 else: break # check to see if we got any (good) images if img_cnt == 0: # nodata for this time period image_count = 0 midday_filename = ND_STRING midday_r = ND_FLOAT midday_g = ND_FLOAT midday_b = ND_FLOAT midday_gcc = ND_FLOAT midday_rcc = ND_FLOAT r_mean = ND_FLOAT r_std = ND_FLOAT g_mean = ND_FLOAT g_std = ND_FLOAT b_mean = ND_FLOAT b_std = ND_FLOAT gcc_mean = ND_FLOAT gcc_std = ND_FLOAT gcc_50 = ND_FLOAT gcc_75 = ND_FLOAT gcc_90 = ND_FLOAT rcc_mean = ND_FLOAT rcc_std = ND_FLOAT rcc_50 = ND_FLOAT rcc_75 = ND_FLOAT rcc_90 = ND_FLOAT max_solar_elev = ND_FLOAT snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # got some good images but not enough - probably there # are cases where this will fail e.g. not images on the # midday of a 3-day aggregation period. elif img_cnt < nimage_threshold: # not enough images image_count = img_cnt # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_filename = filenames[mi_ndx] midday_r = r_dn_vals[mi_ndx] midday_g = g_dn_vals[mi_ndx] midday_b = b_dn_vals[mi_ndx] midday_gcc = gcc_vals[mi_ndx] midday_rcc = rcc_vals[mi_ndx] # no stats for this time interval r_mean = ND_FLOAT r_std = ND_FLOAT g_mean = ND_FLOAT g_std = ND_FLOAT b_mean = ND_FLOAT b_std = ND_FLOAT gcc_mean = ND_FLOAT gcc_std = ND_FLOAT gcc_50 = ND_FLOAT gcc_75 = ND_FLOAT gcc_90 = ND_FLOAT rcc_mean = ND_FLOAT rcc_std = ND_FLOAT rcc_50 = ND_FLOAT rcc_75 = ND_FLOAT rcc_90 = ND_FLOAT max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # stats for this period should be complete - only # snow flags are missing data else: # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_filename = filenames[mi_ndx] midday_r = r_dn_vals[mi_ndx] midday_g = g_dn_vals[mi_ndx] midday_b = b_dn_vals[mi_ndx] midday_gcc = gcc_vals[mi_ndx] midday_rcc = rcc_vals[mi_ndx] # get stats for this time interval image_count = img_cnt r_mean = np.nanmean(r_dn_vals) r_std = np.nanstd(r_dn_vals) g_mean = np.nanmean(g_dn_vals) g_std = np.nanstd(g_dn_vals) b_mean = np.nanmean(b_dn_vals) b_std = np.nanstd(b_dn_vals) gcc_mean = np.nanmean(gcc_vals) gcc_std = np.nanstd(gcc_vals) gcc_50 = quantile(gcc_vals, 0.5) gcc_75 = quantile(gcc_vals, 0.75) gcc_90 = quantile(gcc_vals, 0.9) rcc_mean = np.nanmean(rcc_vals) rcc_std = np.nanstd(rcc_vals) rcc_50 = quantile(rcc_vals, 0.5) rcc_75 = quantile(rcc_vals, 0.75) rcc_90 = quantile(rcc_vals, 0.9) max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # append to gcc timeseries gcc_ts_row = gcc_ts.insert_row( gcc_date, doy, image_count, midday_filename, midday_r, midday_g, midday_b, midday_gcc, midday_rcc, r_mean, r_std, g_mean, g_std, b_mean, b_std, gcc_mean, gcc_std, gcc_50, gcc_75, gcc_90, rcc_mean, rcc_std, rcc_50, rcc_75, rcc_90, max_solar_elev, snow_flag, outlierflag_gcc_mean, outlierflag_gcc_50, outlierflag_gcc_75, outlierflag_gcc_90, ) # print(result if verbose) if verbose: csvstr = gcc_ts.format_csvrow(gcc_ts_row) print(csvstr) # reset accumulated values img_cnt = 0 filenames = [] r_dn_vals = [] rcc_vals = [] g_dn_vals = [] gcc_vals = [] b_dn_vals = [] bcc_vals = [] solar_elev_vals = [] midday_delta_vals = [] if dryrun: nout = 0 else: nout = gcc_ts.writeCSV(outpath) print("Total: %d" % (nout,))
def main(): # set up command line argument processing parser = argparse.ArgumentParser( description="Generate a summary/aggregated NDVI file") # options parser.add_argument( "-v", "--verbose", help="increase output verbosity", action="store_true", default=False, ) parser.add_argument( "-n", "--dry-run", help="Process data but don't save results", action="store_true", default=False, ) parser.add_argument( "-p", "--aggregation-period", help="Number of Days to Aggregate (default=1)", nargs="?", type=int, choices=range(1, 5, 2), default=1, ) # positional arguments parser.add_argument("site", help="PhenoCam site name") parser.add_argument("roiname", help="ROI name, e.g. canopy_0001") # get args args = parser.parse_args() sitename = args.site roiname = args.roiname verbose = args.verbose dryrun = args.dry_run ndays = args.aggregation_period if verbose: print("site: {0}".format(sitename)) print("roiname: {0}".format(roiname)) print("verbose: {0}".format(verbose)) print("dryrun: {0}".format(dryrun)) print("period: {0}".format(ndays)) # read in config file for this ROI List if it exists config_file = "{0}_{1}.cfg".format(sitename, roiname) config_path = os.path.join(archive_dir, sitename, "ROI", config_file) if os.path.exists(config_path): # NOTE: should probably subclass safe config parser # and add gettime() method which checks for time validity cfgparser = configparser( defaults={ "nimage_threshold": str(default_nimage_threshold), "time_min": str(default_time_min), "time_max": str(default_time_max), "sunelev_min": str(default_sunelev_min), "brt_min": str(default_brt_min), "brt_max": str(default_brt_max), }) cfgparser.read(config_path) if cfgparser.has_section("gcc90_calculation"): nimage_threshold = cfgparser.getint("gcc90_calculation", "nimage_threshold") time_max_str = cfgparser.get("gcc90_calculation", "time_max") [tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) time_min_str = cfgparser.get("gcc90_calculation", "time_min") [tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = cfgparser.getfloat("gcc90_calculation", "sunelev_min") brt_min = cfgparser.getint("gcc90_calculation", "brt_min") brt_max = cfgparser.getint("gcc90_calculation", "brt_max") else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max # print config values if verbose: print("") print("gcc config:") print("===========") print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname)) if os.path.exists(config_path): print("config file: {0}".format(config_file)) else: print("config file: None") print("nimage threshold: ", nimage_threshold) print("time of day min: ", time_min) print("time of day max: ", time_max) print("sun elev min: ", sunelev_min) print("aggregate days: ", ndays) print("minimum brightness: ", brt_min) print("maximum brightness: ", brt_max) # set up output filename outdir = os.path.join(archive_dir, sitename, "ROI") outfile = "{0}_{1}_ndvi_{2}day.csv".format(sitename, roiname, ndays) outpath = os.path.join(outdir, outfile) if verbose: print("output file: ", outfile) # since this is "update" output file should already exist # if not just bail out if not os.path.exists(outpath): sys.stderr.write( "Existing NDVI summary file {0} not found.\n".format(outpath)) sys.exit(1) # read in existing CSV file ndvi_summary_ts = vi.NDVISummaryTimeSeries(site=sitename, ROIListID=roiname, ndays=ndays) ndvi_summary_ts.readCSV(outpath) nrows = len(ndvi_summary_ts.rows) print("Read {} rows".format(nrows)) sys.exit(0) # get NDVI timeseries for this site and roi ndvits = get_ndvi_timeseries(sitename, roiname) if verbose: print("") print("NDVI timeseries info:") print("=====================") print("site: ", ndvits.site) print("ROI list id: ", ndvits.roilistid) print("create date: ", ndvits.created_at) print("update date: ", ndvits.updated_at) print("nrows: ", len(ndvits.rows)) # make list of rows which match image selection criteria ndvits_rows = ndvits.select_rows( tod_min=time_min, tod_max=time_max, sunelev_min=sunelev_min, brt_min=brt_min, brt_max=brt_max, ) # check that some rows passed selection criteria nrows = len(ndvits_rows) if nrows == 0: print("No rows passed the selection criteria") return if verbose: print("Number of selected rows: {0}".format(nrows)) # make a list of dates for selected images img_date = [] for row in ndvits_rows: img_date.append(row["datetime"].date()) # list is ordered so find first and last dates dt_first = img_date[0] dt_last = img_date[nrows - 1] # set up a generator which yields dates for the start # of the next nday period covering the date range of image ndvi_dr = daterange2(dt_first, dt_last, ndays) # calculate offset for timeseries based on nday day_offset = ndays / 2 date_offset = timedelta(days=day_offset) # ndvits_ndx will be index into ROI timeseries ndvits_ndx = 0 # loop over nday time periods for ndvi_ndx, start_date in enumerate(ndvi_dr): # set up vars for accumulating stats for this period img_cnt = 0 rgb_filenames = [] ir_filenames = [] r_mean_vals = [] g_mean_vals = [] b_mean_vals = [] ir_mean_vals = [] gcc_vals = [] ndvi_vals = [] solar_elev_vals = [] midday_delta_vals = [] end_date = start_date + timedelta(ndays) ndvi_date = start_date + date_offset doy = ndvi_date.timetuple().tm_yday midday_noon = datetime(ndvi_date.year, ndvi_date.month, ndvi_date.day, 12, 0, 0) # get ndvits rows for this time period while (ndvits_ndx < nrows and img_date[ndvits_ndx] >= start_date and img_date[ndvits_ndx] < end_date): # # skip this row if awbflag is 1 # if ndvits_rows[ndvits_ndx]["awbflag"] == 1: # if ndvits_ndx < nrows: # ndvits_ndx += 1 # continue # else: # break # # filter on exposures # exposure_ir = ndvits_rows[ndvits_ndx]["exposure_ir"] # exposure_rgb = ndvits_rows[ndvits_ndx]["exposure_rgb"] # if exposure_ir/exposure_rgb > 0.8: # ndvits_ndx += 1 # continue # # filter on negative NDVI # if ndvits_rows[ndvits_ndx]["NDVI_c"] < 0: # ndvits_ndx += 1 # continue rgb_filenames.append(ndvits_rows[ndvits_ndx]["filename_rgb"]) ir_filenames.append(ndvits_rows[ndvits_ndx]["filename_ir"]) r_dn = ndvits_rows[ndvits_ndx]["r_mean"] r_mean_vals.append(r_dn) g_dn = ndvits_rows[ndvits_ndx]["g_mean"] g_mean_vals.append(g_dn) b_dn = ndvits_rows[ndvits_ndx]["b_mean"] b_mean_vals.append(b_dn) ir_dn = ndvits_rows[ndvits_ndx]["ir_mean"] ir_mean_vals.append(ir_dn) dnsum = r_dn + g_dn + b_dn # check that dnsum > 0 -- not sure why this is here! if dnsum <= 0: gcc = np.nan else: img_cnt += 1 gcc = ndvits_rows[ndvits_ndx]["gcc"] gcc_vals.append(gcc) ndvi = ndvits_rows[ndvits_ndx]["NDVI_c"] ndvi_vals.append(ndvi) solar_elev = ndvits_rows[ndvits_ndx]["solar_elev"] solar_elev_vals.append(solar_elev) midday_td = ndvits_rows[ndvits_ndx]["datetime"] - midday_noon midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds) midday_delta_vals.append(midday_td_secs) if ndvits_ndx < nrows: ndvits_ndx += 1 else: break # check to see if we got any (good) images if img_cnt == 0: # nodata for this time period image_count = 0 midday_rgb_filename = ND_STRING midday_ir_filename = ND_STRING midday_ndvi = ND_FLOAT gcc_90 = ND_FLOAT ndvi_mean = ND_FLOAT ndvi_std = ND_FLOAT ndvi_50 = ND_FLOAT ndvi_75 = ND_FLOAT ndvi_90 = ND_FLOAT max_solar_elev = ND_FLOAT snow_flag = ND_INT outlierflag_ndvi_mean = ND_INT outlierflag_ndvi_50 = ND_INT outlierflag_ndvi_75 = ND_INT outlierflag_ndvi_90 = ND_INT # got some good images but not enough - probably there # are cases where this will fail e.g. no images on the # midday of a 3-day aggregation period. elif img_cnt < nimage_threshold: # not enough images image_count = img_cnt # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_rgb_filename = rgb_filenames[mi_ndx] midday_ir_filename = ir_filenames[mi_ndx] midday_ndvi = ndvi_vals[mi_ndx] # no stats for this time interval gcc_90 = ND_FLOAT ndvi_mean = ND_FLOAT ndvi_std = ND_FLOAT ndvi_50 = ND_FLOAT ndvi_75 = ND_FLOAT ndvi_90 = ND_FLOAT max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_ndvi_mean = ND_INT outlierflag_ndvi_50 = ND_INT outlierflag_ndvi_75 = ND_INT outlierflag_ndvi_90 = ND_INT # stats for this period should be complete - only # snow flags and outliers are missing data else: # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_rgb_filename = rgb_filenames[mi_ndx] midday_ir_filename = ir_filenames[mi_ndx] midday_ndvi = ndvi_vals[mi_ndx] # get stats for this time interval image_count = img_cnt gcc_90 = quantile(gcc_vals, 0.9) ndvi_mean = np.nanmean(ndvi_vals) ndvi_std = np.nanstd(ndvi_vals) ndvi_50 = quantile(ndvi_vals, 0.5) ndvi_75 = quantile(ndvi_vals, 0.75) ndvi_90 = quantile(ndvi_vals, 0.9) max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_ndvi_mean = ND_INT outlierflag_ndvi_50 = ND_INT outlierflag_ndvi_75 = ND_INT outlierflag_ndvi_90 = ND_INT # append to NDVI timeseries year = ndvi_date.year ndvi_ts_row = ndvi_summary_ts.insert_row( ndvi_date, year, doy, image_count, midday_rgb_filename, midday_ir_filename, midday_ndvi, gcc_90, ndvi_mean, ndvi_std, ndvi_50, ndvi_75, ndvi_90, max_solar_elev, snow_flag, outlierflag_ndvi_mean, outlierflag_ndvi_50, outlierflag_ndvi_75, outlierflag_ndvi_90, ) # print(result if verbose) if verbose: csvstr = ndvi_summary_ts.format_csvrow(ndvi_ts_row) print(csvstr) if dryrun: nout = 0 else: nout = ndvi_summary_ts.writeCSV(outpath) print("Total: %d" % (nout, ))
def main(): # set up command line argument processing parser = argparse.ArgumentParser() # options parser.add_argument( "-v", "--verbose", help="increase output verbosity", action="store_true", default=False, ) parser.add_argument( "-n", "--dry-run", help="Process data but don't save results", action="store_true", default=False, ) parser.add_argument( "-p", "--aggregation-period", help="Number of Days to Aggregate", nargs="?", type=int, choices=range(1, 5, 2), default=1, ) # positional arguments parser.add_argument("site", help="PhenoCam site name") parser.add_argument("roiname", help="ROI name, e.g. canopy_0001") # get args args = parser.parse_args() sitename = args.site roiname = args.roiname verbose = args.verbose dryrun = args.dry_run ndays = args.aggregation_period if verbose: print("site: {0}".format(sitename)) print("roiname: {0}".format(roiname)) print("verbose: {0}".format(verbose)) print("dryrun: {0}".format(dryrun)) print("period: {0}".format(ndays)) # set up output filename outdir = os.path.join(vi.config.archive_dir, sitename, "ROI") outfile = "{0}_{1}_{2}day.csv".format(sitename, roiname, ndays) outpath = os.path.join(outdir, outfile) # since this is "update" output file should already exist # if not just bail out if not os.path.exists(outpath): sys.stderr.write( "Existing gcc90 file {0} not found.\n".format(outpath)) sys.exit(1) # read in existing CSV file gcc_ts = vi.GCCTimeSeries(site=sitename, ROIListID=roiname, nday=ndays) gcc_ts.readCSV(outpath) # read in config file for this site/roi if it exists config_file = "{0}_{1}.cfg".format(sitename, roiname) config_path = os.path.join(archive_dir, sitename, "ROI", config_file) if os.path.exists(config_path): cfgparser = ConfigParser( defaults={ "nimage_threshold": str(default_nimage_threshold), "time_min": str(default_time_min), "time_max": str(default_time_max), "sunelev_min": str(default_sunelev_min), "brt_min": str(default_brt_min), "brt_max": str(default_brt_max), }) cfgparser.read(config_path) if cfgparser.has_section("gcc90_calculation"): nimage_threshold = cfgparser.getint("gcc90_calculation", "nimage_threshold") time_max_str = cfgparser.get("gcc90_calculation", "time_max") print("time_max_str: {0}".format(time_max_str)) [tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) time_min_str = cfgparser.get("gcc90_calculation", "time_min") [tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = cfgparser.getfloat("gcc90_calculation", "sunelev_min") brt_min = cfgparser.getint("gcc90_calculation", "brt_min") brt_max = cfgparser.getint("gcc90_calculation", "brt_max") else: nimage_threshold = gcc_ts.nmin time_max = gcc_ts.tod_max time_min = gcc_ts.tod_min sunelev_min = gcc_ts.sunelev_min brt_min = default_brt_min brt_max = default_brt_max else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max # verify that config file matches CSV header! if nimage_threshold != gcc_ts.nmin: sys.stderr.write( "nimage_threshold from config doesn't match CSV header\n") sys.exit(1) if brt_min != gcc_ts.brt_min: sys.stderr.write("brt_min from config file doesn't match CSV header\n") sys.exit(1) if brt_max != gcc_ts.brt_max: sys.stderr.write("brt_max from config file doesn't match CSV header\n") sys.exit(1) if time_min != gcc_ts.tod_min: sys.stderr.write("tod_min from config file doesn't match CSV header\n") sys.exit(1) if time_min != gcc_ts.tod_min: sys.stderr.write("tod_min from config file doesn't match CSV header\n") sys.exit(1) if sunelev_min != gcc_ts.sunelev_min: sys.stderr.write( "sunelev_min from config file doesn't match CSV header\n") sys.exit(1) else: nimage_threshold = gcc_ts.nmin time_max = gcc_ts.tod_max time_min = gcc_ts.tod_min sunelev_min = gcc_ts.sunelev_min brt_min = default_brt_min brt_max = default_brt_max # grab remaining metadata gcc_ts_create_date = gcc_ts.created_at gcc_ts_update_date = gcc_ts.updated_at # print config values if verbose: print("") print("gcc config:") print("===========") print("roi_list: ", "{0}_{1}".format(sitename, roiname)) if os.path.exists(config_path): print("config file: {0}".format(config_file)) else: print("config file: None") print("nimage threshold: ", nimage_threshold) print("time of day min: ", time_min) print("time of day max: ", time_max) print("sun elev min: ", sunelev_min) print("aggregate days: ", ndays) print("minimum brightness: ", brt_min) print("maximum brightness: ", brt_max) print("creation date: ", gcc_ts_create_date) print("last update: ", gcc_ts_update_date) # get number of rows in existing/old CSV ngccrows = len(gcc_ts.rows) # get the next to last date in gcc90 CSV # NOTE: always redo last date since we may be adding images # (if ndays > 1 and we haven't finished interval) gcc90_date_last = gcc_ts.rows[ngccrows - 1]["date"] if verbose: print("last date in timeseries: ", gcc90_date_last) print("") # get roi timeseries for this site and roi roits = vi.get_roi_timeseries(sitename, roiname) if verbose: print("") print("ROI timeseries info:") print("====================") print("site: ", roits.site) print("ROI list id: ", roits.roilistid) print("create date: ", roits.created_at) print("update date: ", roits.updated_at) print("nrows: ", len(roits.rows)) # make list of rows which match image selection criteria roits_rows = roits.select_rows( tod_min=time_min, tod_max=time_max, sunelev_min=sunelev_min, brt_min=brt_min, brt_max=brt_max, ) # calculate offset for timeseries based on nday day_offset = ndays / 2 date_offset = timedelta(days=day_offset) # find rows that are in or more recent than beginning of last timeperiod # of GCC new_roits_rows = [] for row in roits_rows: if row["datetime"].date() >= (gcc90_date_last - date_offset): new_roits_rows.append(row) # check that some rows passed selection criteria nrows = len(new_roits_rows) if nrows == 0: print("No rows passed the selection criteria") sys.exit(0) if debug: print("New selected rows: {0}".format(nrows)) # make a list of dates for selected images img_date = [] for row in new_roits_rows: img_date.append(row["datetime"].date()) # list is ordered so find first and last dates dt_first = img_date[0] dt_last = img_date[nrows - 1] # set up a generator which yields dates for the start # of the next nday period covering the date range of # new images gcc_dr = vi.daterange2(dt_first, dt_last, ndays) # roits_ndx will be index into ROI timeseries roits_ndx = 0 # set up vars for accumulating stats img_cnt = 0 update_cnt = 0 filenames = [] r_dn_vals = [] rcc_vals = [] g_dn_vals = [] gcc_vals = [] b_dn_vals = [] bcc_vals = [] solar_elev_vals = [] midday_delta_vals = [] # loop over ndays time periods for gcc_ndx, start_date in enumerate(gcc_dr): end_date = start_date + timedelta(ndays) gcc_date = start_date + date_offset doy = gcc_date.timetuple().tm_yday midday_noon = datetime(gcc_date.year, gcc_date.month, gcc_date.day, 12, 0, 0) # get roits rows for this time period while (roits_ndx < nrows and img_date[roits_ndx] >= start_date and img_date[roits_ndx] < end_date): # skip this row if awbflag is 1 if roits_rows[roits_ndx]["awbflag"] == 1: if roits_ndx < nrows: roits_ndx += 1 continue else: break filenames.append(new_roits_rows[roits_ndx]["filename"]) r_dn = new_roits_rows[roits_ndx]["r_mean"] r_dn_vals.append(r_dn) g_dn = new_roits_rows[roits_ndx]["g_mean"] g_dn_vals.append(g_dn) b_dn = new_roits_rows[roits_ndx]["b_mean"] b_dn_vals.append(b_dn) dnsum = r_dn + g_dn + b_dn if dnsum <= 0: rcc = np.nan bcc = np.nan gcc = np.nan else: rcc = r_dn / dnsum bcc = b_dn / dnsum gcc = new_roits_rows[roits_ndx]["gcc"] solar_elev = new_roits_rows[roits_ndx]["solar_elev"] rcc_vals.append(rcc) gcc_vals.append(gcc) bcc_vals.append(bcc) solar_elev_vals.append(solar_elev) midday_td = new_roits_rows[roits_ndx]["datetime"] - midday_noon midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds) midday_delta_vals.append(midday_td_secs) img_cnt += 1 if roits_ndx < nrows: roits_ndx += 1 else: break # check to see if we got any images if img_cnt == 0: # nodata for this time period image_count = 0 midday_filename = ND_STRING midday_r = ND_FLOAT midday_g = ND_FLOAT midday_b = ND_FLOAT midday_gcc = ND_FLOAT midday_rcc = ND_FLOAT r_mean = ND_FLOAT r_std = ND_FLOAT g_mean = ND_FLOAT g_std = ND_FLOAT b_mean = ND_FLOAT b_std = ND_FLOAT gcc_mean = ND_FLOAT gcc_std = ND_FLOAT gcc_50 = ND_FLOAT gcc_75 = ND_FLOAT gcc_90 = ND_FLOAT rcc_mean = ND_FLOAT rcc_std = ND_FLOAT rcc_50 = ND_FLOAT rcc_75 = ND_FLOAT rcc_90 = ND_FLOAT max_solar_elev = ND_FLOAT snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT elif img_cnt < nimage_threshold: # not enough images image_count = img_cnt # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_filename = filenames[mi_ndx] if not midday_filename: midday_filename = ND_STRING midday_r = r_dn_vals[mi_ndx] midday_g = g_dn_vals[mi_ndx] midday_b = b_dn_vals[mi_ndx] midday_gcc = gcc_vals[mi_ndx] midday_rcc = rcc_vals[mi_ndx] # no stats for this time interval r_mean = ND_FLOAT r_std = ND_FLOAT g_mean = ND_FLOAT g_std = ND_FLOAT b_mean = ND_FLOAT b_std = ND_FLOAT gcc_mean = ND_FLOAT gcc_std = ND_FLOAT gcc_50 = ND_FLOAT gcc_75 = ND_FLOAT gcc_90 = ND_FLOAT rcc_mean = ND_FLOAT rcc_std = ND_FLOAT rcc_50 = ND_FLOAT rcc_75 = ND_FLOAT rcc_90 = ND_FLOAT max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT else: # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_filename = filenames[mi_ndx] midday_r = r_dn_vals[mi_ndx] midday_g = g_dn_vals[mi_ndx] midday_b = b_dn_vals[mi_ndx] midday_gcc = gcc_vals[mi_ndx] midday_rcc = rcc_vals[mi_ndx] # get stats for this time interval image_count = img_cnt r_mean = np.nanmean(r_dn_vals) r_std = np.nanstd(r_dn_vals) g_mean = np.nanmean(g_dn_vals) g_std = np.nanstd(g_dn_vals) b_mean = np.nanmean(b_dn_vals) b_std = np.nanstd(b_dn_vals) gcc_mean = np.nanmean(gcc_vals) gcc_std = np.nanstd(gcc_vals) gcc_50 = quantile(gcc_vals, 0.5) gcc_75 = quantile(gcc_vals, 0.75) gcc_90 = quantile(gcc_vals, 0.9) rcc_mean = np.mean(rcc_vals) rcc_std = np.std(rcc_vals) rcc_50 = quantile(rcc_vals, 0.5) rcc_75 = quantile(rcc_vals, 0.75) rcc_90 = quantile(rcc_vals, 0.9) max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # append to gcc timeseries gcc_ts_row = gcc_ts.insert_row( gcc_date, doy, image_count, midday_filename, midday_r, midday_g, midday_b, midday_gcc, midday_rcc, r_mean, r_std, g_mean, g_std, b_mean, b_std, gcc_mean, gcc_std, gcc_50, gcc_75, gcc_90, rcc_mean, rcc_std, rcc_50, rcc_75, rcc_90, max_solar_elev, snow_flag, outlierflag_gcc_mean, outlierflag_gcc_50, outlierflag_gcc_75, outlierflag_gcc_90, ) update_cnt += 1 # print result if verbose if verbose: csvstr = gcc_ts.format_csvrow(gcc_ts_row) print(csvstr) # reset accumulated values img_cnt = 0 filenames = [] r_dn_vals = [] rcc_vals = [] g_dn_vals = [] gcc_vals = [] b_dn_vals = [] bcc_vals = [] solar_elev_vals = [] midday_delta_vals = [] if dryrun: nout = 0 else: nout = gcc_ts.writeCSV(outpath) print("GCC90 Rows updated: 1 Rows added: {0}".format(update_cnt - 1)) print("Total: {0}".format(nout))
--------------- Tests for `vegindex.vegindex.daterange2` module """ import os from datetime import date import numpy as np from vegindex import vegindex as vi end_date = date(2008, 1, 31) start_date = date(2008, 1, 1) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 1)) start_date = date(2008, 1, 2) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 1)) start_date = date(2008, 1, 3) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 1)) start_date = date(2008, 1, 4) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 4)) start_date = date(2008, 1, 5)