def main():
# set up command line argument processing
parser = argparse.ArgumentParser()
# options
parser.add_argument(
"-v",
"--verbose",
help="increase output verbosity",
action="store_true",
default=False,
)
parser.add_argument(
"-n",
"--dry-run",
help="Process data but don't save results",
action="store_true",
default=False,
)
parser.add_argument(
"-p",
"--aggregation-period",
help="Number of Days to Aggregate (default=1)",
nargs="?",
type=int,
choices=range(1, 5, 2),
default=1,
)
# positional arguments
parser.add_argument("site", help="PhenoCam site name")
parser.add_argument("roiname", help="ROI name, e.g. canopy_0001")
# get args
args = parser.parse_args()
sitename = args.site
roiname = args.roiname
verbose = args.verbose
dryrun = args.dry_run
ndays = args.aggregation_period
if verbose:
print("site: {0}".format(sitename))
print("roiname: {0}".format(roiname))
print("verbose: {0}".format(verbose))
print("dryrun: {0}".format(dryrun))
print("period: {0}".format(ndays))
# read in config file for this ROI List if it exists
config_file = "{0}_{1}.cfg".format(sitename, roiname)
config_path = os.path.join(archive_dir, sitename, "ROI", config_file)
if os.path.exists(config_path):
# NOTE: should probably subclass safe config parser
# and add gettime() method which checks for time validity
cfgparser = configparser(
defaults={
"nimage_threshold": str(default_nimage_threshold),
"time_min": str(default_time_min),
"time_max": str(default_time_max),
"sunelev_min": str(default_sunelev_min),
"brt_min": str(default_brt_min),
"brt_max": str(default_brt_max),
}
)
cfgparser.read(config_path)
if cfgparser.has_section("gcc90_calculation"):
nimage_threshold = cfgparser.getint("gcc90_calculation", "nimage_threshold")
time_max_str = cfgparser.get("gcc90_calculation", "time_max")
[tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
time_min_str = cfgparser.get("gcc90_calculation", "time_min")
[tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = cfgparser.getfloat("gcc90_calculation", "sunelev_min")
brt_min = cfgparser.getint("gcc90_calculation", "brt_min")
brt_max = cfgparser.getint("gcc90_calculation", "brt_max")
else:
nimage_threshold = int(default_nimage_threshold)
[tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
[tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = default_sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
else:
nimage_threshold = int(default_nimage_threshold)
[tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
[tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = default_sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
# print config values
if verbose:
print("")
print("gcc config:")
print("===========")
print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname))
if os.path.exists(config_path):
print("config file: {0}".format(config_file))
else:
print("config file: None")
print("nimage threshold: ", nimage_threshold)
print("time of day min: ", time_min)
print("time of day max: ", time_max)
print("sun elev min: ", sunelev_min)
print("aggregate days: ", ndays)
print("minimum brightness: ", brt_min)
print("maximum brightness: ", brt_max)
# set up output filename
outdir = os.path.join(archive_dir, sitename, "ROI")
outfile = "{0}_{1}_{2}day.csv".format(sitename, roiname, ndays)
outpath = os.path.join(outdir, outfile)
if verbose:
print("output file: ", outfile)
# create gcc timeseries object as empty list
gcc_ts = GCCTimeSeries(
site=sitename,
ROIListID=roiname,
nday=ndays,
nmin=nimage_threshold,
tod_min=time_min,
tod_max=time_max,
sunelev_min=sunelev_min,
brt_min=brt_min,
brt_max=brt_max,
)
# get roi timeseries for this site and roi
roits = get_roi_timeseries(sitename, roiname)
if verbose:
print("")
print("ROI timeseries info:")
print("====================")
print("site: ", roits.site)
print("ROI list id: ", roits.roilistid)
print("create date: ", roits.created_at)
print("update date: ", roits.updated_at)
print("nrows: ", len(roits.rows))
# make list of rows which match image selection criteria
roits_rows = roits.select_rows(
tod_min=time_min,
tod_max=time_max,
sunelev_min=sunelev_min,
brt_min=brt_min,
brt_max=brt_max,
)
# check that some rows passed selection criteria
nrows = len(roits_rows)
if nrows == 0:
print("No rows passed the selection criteria")
return
if verbose:
print("Number of selected rows: {0}".format(nrows))
# make a list of dates for selected images
img_date = []
for row in roits_rows:
img_date.append(row["datetime"].date())
# list is ordered so find first and last dates
dt_first = img_date[0]
dt_last = img_date[nrows - 1]
if verbose:
print("date first: {}".format(dt_first))
print("date last: {}".format(dt_last))
# set up a generator which yields dates for the start
# of the next nday period covering the date range of image
gcc_dr = daterange2(dt_first, dt_last, ndays)
# calculate offset for timeseries based on nday
day_offset = ndays / 2
date_offset = timedelta(days=day_offset)
# roits_ndx will be index into ROI timeseries
roits_ndx = 0
# set up vars for accumulating stats
img_cnt = 0
filenames = []
r_dn_vals = []
rcc_vals = []
g_dn_vals = []
gcc_vals = []
b_dn_vals = []
bcc_vals = []
solar_elev_vals = []
midday_delta_vals = []
# loop over nday time periods
for gcc_ndx, start_date in enumerate(gcc_dr):
end_date = start_date + timedelta(ndays)
gcc_date = start_date + date_offset
doy = gcc_date.timetuple().tm_yday
midday_noon = datetime(gcc_date.year, gcc_date.month, gcc_date.day, 12, 0, 0)
# get roits rows for this time period
while (
roits_ndx < nrows
and img_date[roits_ndx] >= start_date
and img_date[roits_ndx] < end_date
):
# skip this row if awbflag is 1
if roits_rows[roits_ndx]["awbflag"] == 1:
if roits_ndx < nrows:
roits_ndx += 1
continue
else:
break
filenames.append(roits_rows[roits_ndx]["filename"])
r_dn = roits_rows[roits_ndx]["r_mean"]
r_dn_vals.append(r_dn)
g_dn = roits_rows[roits_ndx]["g_mean"]
g_dn_vals.append(g_dn)
b_dn = roits_rows[roits_ndx]["b_mean"]
b_dn_vals.append(b_dn)
dnsum = r_dn + g_dn + b_dn
# NOTE: I'm recomputing gcc, rcc, bcc from DN values rather
# than using value stored in roistats CSV
if dnsum <= 0:
rcc = np.nan
bcc = np.nan
gcc = np.nan
else:
img_cnt += 1
rcc = r_dn / dnsum
bcc = b_dn / dnsum
gcc = roits_rows[roits_ndx]["gcc"]
solar_elev = roits_rows[roits_ndx]["solar_elev"]
# note that rcc_vals can include NaN's
rcc_vals.append(rcc)
gcc_vals.append(gcc)
bcc_vals.append(bcc)
solar_elev_vals.append(solar_elev)
midday_td = roits_rows[roits_ndx]["datetime"] - midday_noon
midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds)
midday_delta_vals.append(midday_td_secs)
if roits_ndx < nrows:
roits_ndx += 1
else:
break
# check to see if we got any (good) images
if img_cnt == 0:
# nodata for this time period
image_count = 0
midday_filename = ND_STRING
midday_r = ND_FLOAT
midday_g = ND_FLOAT
midday_b = ND_FLOAT
midday_gcc = ND_FLOAT
midday_rcc = ND_FLOAT
r_mean = ND_FLOAT
r_std = ND_FLOAT
g_mean = ND_FLOAT
g_std = ND_FLOAT
b_mean = ND_FLOAT
b_std = ND_FLOAT
gcc_mean = ND_FLOAT
gcc_std = ND_FLOAT
gcc_50 = ND_FLOAT
gcc_75 = ND_FLOAT
gcc_90 = ND_FLOAT
rcc_mean = ND_FLOAT
rcc_std = ND_FLOAT
rcc_50 = ND_FLOAT
rcc_75 = ND_FLOAT
rcc_90 = ND_FLOAT
max_solar_elev = ND_FLOAT
snow_flag = ND_INT
outlierflag_gcc_mean = ND_INT
outlierflag_gcc_50 = ND_INT
outlierflag_gcc_75 = ND_INT
outlierflag_gcc_90 = ND_INT
# got some good images but not enough - probably there
# are cases where this will fail e.g. not images on the
# midday of a 3-day aggregation period.
elif img_cnt < nimage_threshold:
# not enough images
image_count = img_cnt
# find nearest image to midday (noon) on mid-interval date
mi_ndx = midday_delta_vals.index(min(midday_delta_vals))
midday_filename = filenames[mi_ndx]
midday_r = r_dn_vals[mi_ndx]
midday_g = g_dn_vals[mi_ndx]
midday_b = b_dn_vals[mi_ndx]
midday_gcc = gcc_vals[mi_ndx]
midday_rcc = rcc_vals[mi_ndx]
# no stats for this time interval
r_mean = ND_FLOAT
r_std = ND_FLOAT
g_mean = ND_FLOAT
g_std = ND_FLOAT
b_mean = ND_FLOAT
b_std = ND_FLOAT
gcc_mean = ND_FLOAT
gcc_std = ND_FLOAT
gcc_50 = ND_FLOAT
gcc_75 = ND_FLOAT
gcc_90 = ND_FLOAT
rcc_mean = ND_FLOAT
rcc_std = ND_FLOAT
rcc_50 = ND_FLOAT
rcc_75 = ND_FLOAT
rcc_90 = ND_FLOAT
max_solar_elev = max(solar_elev_vals)
snow_flag = ND_INT
outlierflag_gcc_mean = ND_INT
outlierflag_gcc_50 = ND_INT
outlierflag_gcc_75 = ND_INT
outlierflag_gcc_90 = ND_INT
# stats for this period should be complete - only
# snow flags are missing data
else:
# find nearest image to midday (noon) on mid-interval date
mi_ndx = midday_delta_vals.index(min(midday_delta_vals))
midday_filename = filenames[mi_ndx]
midday_r = r_dn_vals[mi_ndx]
midday_g = g_dn_vals[mi_ndx]
midday_b = b_dn_vals[mi_ndx]
midday_gcc = gcc_vals[mi_ndx]
midday_rcc = rcc_vals[mi_ndx]
# get stats for this time interval
image_count = img_cnt
r_mean = np.nanmean(r_dn_vals)
r_std = np.nanstd(r_dn_vals)
g_mean = np.nanmean(g_dn_vals)
g_std = np.nanstd(g_dn_vals)
b_mean = np.nanmean(b_dn_vals)
b_std = np.nanstd(b_dn_vals)
gcc_mean = np.nanmean(gcc_vals)
gcc_std = np.nanstd(gcc_vals)
gcc_50 = quantile(gcc_vals, 0.5)
gcc_75 = quantile(gcc_vals, 0.75)
gcc_90 = quantile(gcc_vals, 0.9)
rcc_mean = np.nanmean(rcc_vals)
rcc_std = np.nanstd(rcc_vals)
rcc_50 = quantile(rcc_vals, 0.5)
rcc_75 = quantile(rcc_vals, 0.75)
rcc_90 = quantile(rcc_vals, 0.9)
max_solar_elev = max(solar_elev_vals)
snow_flag = ND_INT
outlierflag_gcc_mean = ND_INT
outlierflag_gcc_50 = ND_INT
outlierflag_gcc_75 = ND_INT
outlierflag_gcc_90 = ND_INT
# append to gcc timeseries
gcc_ts_row = gcc_ts.insert_row(
gcc_date,
doy,
image_count,
midday_filename,
midday_r,
midday_g,
midday_b,
midday_gcc,
midday_rcc,
r_mean,
r_std,
g_mean,
g_std,
b_mean,
b_std,
gcc_mean,
gcc_std,
gcc_50,
gcc_75,
gcc_90,
rcc_mean,
rcc_std,
rcc_50,
rcc_75,
rcc_90,
max_solar_elev,
snow_flag,
outlierflag_gcc_mean,
outlierflag_gcc_50,
outlierflag_gcc_75,
outlierflag_gcc_90,
)
# print(result if verbose)
if verbose:
csvstr = gcc_ts.format_csvrow(gcc_ts_row)
print(csvstr)
# reset accumulated values
img_cnt = 0
filenames = []
r_dn_vals = []
rcc_vals = []
g_dn_vals = []
gcc_vals = []
b_dn_vals = []
bcc_vals = []
solar_elev_vals = []
midday_delta_vals = []
if dryrun:
nout = 0
else:
nout = gcc_ts.writeCSV(outpath)
print("Total: %d" % (nout,))
def main():
# set up command line argument processing
parser = argparse.ArgumentParser(
description="Generate a summary/aggregated NDVI file")
# options
parser.add_argument(
"-v",
"--verbose",
help="increase output verbosity",
action="store_true",
default=False,
)
parser.add_argument(
"-n",
"--dry-run",
help="Process data but don't save results",
action="store_true",
default=False,
)
parser.add_argument(
"-p",
"--aggregation-period",
help="Number of Days to Aggregate (default=1)",
nargs="?",
type=int,
choices=range(1, 5, 2),
default=1,
)
# positional arguments
parser.add_argument("site", help="PhenoCam site name")
parser.add_argument("roiname", help="ROI name, e.g. canopy_0001")
# get args
args = parser.parse_args()
sitename = args.site
roiname = args.roiname
verbose = args.verbose
dryrun = args.dry_run
ndays = args.aggregation_period
if verbose:
print("site: {0}".format(sitename))
print("roiname: {0}".format(roiname))
print("verbose: {0}".format(verbose))
print("dryrun: {0}".format(dryrun))
print("period: {0}".format(ndays))
# read in config file for this ROI List if it exists
config_file = "{0}_{1}.cfg".format(sitename, roiname)
config_path = os.path.join(archive_dir, sitename, "ROI", config_file)
if os.path.exists(config_path):
# NOTE: should probably subclass safe config parser
# and add gettime() method which checks for time validity
cfgparser = configparser(
defaults={
"nimage_threshold": str(default_nimage_threshold),
"time_min": str(default_time_min),
"time_max": str(default_time_max),
"sunelev_min": str(default_sunelev_min),
"brt_min": str(default_brt_min),
"brt_max": str(default_brt_max),
})
cfgparser.read(config_path)
if cfgparser.has_section("gcc90_calculation"):
nimage_threshold = cfgparser.getint("gcc90_calculation",
"nimage_threshold")
time_max_str = cfgparser.get("gcc90_calculation", "time_max")
[tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
time_min_str = cfgparser.get("gcc90_calculation", "time_min")
[tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = cfgparser.getfloat("gcc90_calculation",
"sunelev_min")
brt_min = cfgparser.getint("gcc90_calculation", "brt_min")
brt_max = cfgparser.getint("gcc90_calculation", "brt_max")
else:
nimage_threshold = int(default_nimage_threshold)
[tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
[tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = default_sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
else:
nimage_threshold = int(default_nimage_threshold)
[tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
[tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = default_sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
# print config values
if verbose:
print("")
print("gcc config:")
print("===========")
print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname))
if os.path.exists(config_path):
print("config file: {0}".format(config_file))
else:
print("config file: None")
print("nimage threshold: ", nimage_threshold)
print("time of day min: ", time_min)
print("time of day max: ", time_max)
print("sun elev min: ", sunelev_min)
print("aggregate days: ", ndays)
print("minimum brightness: ", brt_min)
print("maximum brightness: ", brt_max)
# set up output filename
outdir = os.path.join(archive_dir, sitename, "ROI")
outfile = "{0}_{1}_ndvi_{2}day.csv".format(sitename, roiname, ndays)
outpath = os.path.join(outdir, outfile)
if verbose:
print("output file: ", outfile)
# since this is "update" output file should already exist
# if not just bail out
if not os.path.exists(outpath):
sys.stderr.write(
"Existing NDVI summary file {0} not found.\n".format(outpath))
sys.exit(1)
# read in existing CSV file
ndvi_summary_ts = vi.NDVISummaryTimeSeries(site=sitename,
ROIListID=roiname,
ndays=ndays)
ndvi_summary_ts.readCSV(outpath)
nrows = len(ndvi_summary_ts.rows)
print("Read {} rows".format(nrows))
sys.exit(0)
# get NDVI timeseries for this site and roi
ndvits = get_ndvi_timeseries(sitename, roiname)
if verbose:
print("")
print("NDVI timeseries info:")
print("=====================")
print("site: ", ndvits.site)
print("ROI list id: ", ndvits.roilistid)
print("create date: ", ndvits.created_at)
print("update date: ", ndvits.updated_at)
print("nrows: ", len(ndvits.rows))
# make list of rows which match image selection criteria
ndvits_rows = ndvits.select_rows(
tod_min=time_min,
tod_max=time_max,
sunelev_min=sunelev_min,
brt_min=brt_min,
brt_max=brt_max,
)
# check that some rows passed selection criteria
nrows = len(ndvits_rows)
if nrows == 0:
print("No rows passed the selection criteria")
return
if verbose:
print("Number of selected rows: {0}".format(nrows))
# make a list of dates for selected images
img_date = []
for row in ndvits_rows:
img_date.append(row["datetime"].date())
# list is ordered so find first and last dates
dt_first = img_date[0]
dt_last = img_date[nrows - 1]
# set up a generator which yields dates for the start
# of the next nday period covering the date range of image
ndvi_dr = daterange2(dt_first, dt_last, ndays)
# calculate offset for timeseries based on nday
day_offset = ndays / 2
date_offset = timedelta(days=day_offset)
# ndvits_ndx will be index into ROI timeseries
ndvits_ndx = 0
# loop over nday time periods
for ndvi_ndx, start_date in enumerate(ndvi_dr):
# set up vars for accumulating stats for this period
img_cnt = 0
rgb_filenames = []
ir_filenames = []
r_mean_vals = []
g_mean_vals = []
b_mean_vals = []
ir_mean_vals = []
gcc_vals = []
ndvi_vals = []
solar_elev_vals = []
midday_delta_vals = []
end_date = start_date + timedelta(ndays)
ndvi_date = start_date + date_offset
doy = ndvi_date.timetuple().tm_yday
midday_noon = datetime(ndvi_date.year, ndvi_date.month, ndvi_date.day,
12, 0, 0)
# get ndvits rows for this time period
while (ndvits_ndx < nrows and img_date[ndvits_ndx] >= start_date
and img_date[ndvits_ndx] < end_date):
# # skip this row if awbflag is 1
# if ndvits_rows[ndvits_ndx]["awbflag"] == 1:
# if ndvits_ndx < nrows:
# ndvits_ndx += 1
# continue
# else:
# break
# # filter on exposures
# exposure_ir = ndvits_rows[ndvits_ndx]["exposure_ir"]
# exposure_rgb = ndvits_rows[ndvits_ndx]["exposure_rgb"]
# if exposure_ir/exposure_rgb > 0.8:
# ndvits_ndx += 1
# continue
# # filter on negative NDVI
# if ndvits_rows[ndvits_ndx]["NDVI_c"] < 0:
# ndvits_ndx += 1
# continue
rgb_filenames.append(ndvits_rows[ndvits_ndx]["filename_rgb"])
ir_filenames.append(ndvits_rows[ndvits_ndx]["filename_ir"])
r_dn = ndvits_rows[ndvits_ndx]["r_mean"]
r_mean_vals.append(r_dn)
g_dn = ndvits_rows[ndvits_ndx]["g_mean"]
g_mean_vals.append(g_dn)
b_dn = ndvits_rows[ndvits_ndx]["b_mean"]
b_mean_vals.append(b_dn)
ir_dn = ndvits_rows[ndvits_ndx]["ir_mean"]
ir_mean_vals.append(ir_dn)
dnsum = r_dn + g_dn + b_dn
# check that dnsum > 0 -- not sure why this is here!
if dnsum <= 0:
gcc = np.nan
else:
img_cnt += 1
gcc = ndvits_rows[ndvits_ndx]["gcc"]
gcc_vals.append(gcc)
ndvi = ndvits_rows[ndvits_ndx]["NDVI_c"]
ndvi_vals.append(ndvi)
solar_elev = ndvits_rows[ndvits_ndx]["solar_elev"]
solar_elev_vals.append(solar_elev)
midday_td = ndvits_rows[ndvits_ndx]["datetime"] - midday_noon
midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds)
midday_delta_vals.append(midday_td_secs)
if ndvits_ndx < nrows:
ndvits_ndx += 1
else:
break
# check to see if we got any (good) images
if img_cnt == 0:
# nodata for this time period
image_count = 0
midday_rgb_filename = ND_STRING
midday_ir_filename = ND_STRING
midday_ndvi = ND_FLOAT
gcc_90 = ND_FLOAT
ndvi_mean = ND_FLOAT
ndvi_std = ND_FLOAT
ndvi_50 = ND_FLOAT
ndvi_75 = ND_FLOAT
ndvi_90 = ND_FLOAT
max_solar_elev = ND_FLOAT
snow_flag = ND_INT
outlierflag_ndvi_mean = ND_INT
outlierflag_ndvi_50 = ND_INT
outlierflag_ndvi_75 = ND_INT
outlierflag_ndvi_90 = ND_INT
# got some good images but not enough - probably there
# are cases where this will fail e.g. no images on the
# midday of a 3-day aggregation period.
elif img_cnt < nimage_threshold:
# not enough images
image_count = img_cnt
# find nearest image to midday (noon) on mid-interval date
mi_ndx = midday_delta_vals.index(min(midday_delta_vals))
midday_rgb_filename = rgb_filenames[mi_ndx]
midday_ir_filename = ir_filenames[mi_ndx]
midday_ndvi = ndvi_vals[mi_ndx]
# no stats for this time interval
gcc_90 = ND_FLOAT
ndvi_mean = ND_FLOAT
ndvi_std = ND_FLOAT
ndvi_50 = ND_FLOAT
ndvi_75 = ND_FLOAT
ndvi_90 = ND_FLOAT
max_solar_elev = max(solar_elev_vals)
snow_flag = ND_INT
outlierflag_ndvi_mean = ND_INT
outlierflag_ndvi_50 = ND_INT
outlierflag_ndvi_75 = ND_INT
outlierflag_ndvi_90 = ND_INT
# stats for this period should be complete - only
# snow flags and outliers are missing data
else:
# find nearest image to midday (noon) on mid-interval date
mi_ndx = midday_delta_vals.index(min(midday_delta_vals))
midday_rgb_filename = rgb_filenames[mi_ndx]
midday_ir_filename = ir_filenames[mi_ndx]
midday_ndvi = ndvi_vals[mi_ndx]
# get stats for this time interval
image_count = img_cnt
gcc_90 = quantile(gcc_vals, 0.9)
ndvi_mean = np.nanmean(ndvi_vals)
ndvi_std = np.nanstd(ndvi_vals)
ndvi_50 = quantile(ndvi_vals, 0.5)
ndvi_75 = quantile(ndvi_vals, 0.75)
ndvi_90 = quantile(ndvi_vals, 0.9)
max_solar_elev = max(solar_elev_vals)
snow_flag = ND_INT
outlierflag_ndvi_mean = ND_INT
outlierflag_ndvi_50 = ND_INT
outlierflag_ndvi_75 = ND_INT
outlierflag_ndvi_90 = ND_INT
# append to NDVI timeseries
year = ndvi_date.year
ndvi_ts_row = ndvi_summary_ts.insert_row(
ndvi_date,
year,
doy,
image_count,
midday_rgb_filename,
midday_ir_filename,
midday_ndvi,
gcc_90,
ndvi_mean,
ndvi_std,
ndvi_50,
ndvi_75,
ndvi_90,
max_solar_elev,
snow_flag,
outlierflag_ndvi_mean,
outlierflag_ndvi_50,
outlierflag_ndvi_75,
outlierflag_ndvi_90,
)
# print(result if verbose)
if verbose:
csvstr = ndvi_summary_ts.format_csvrow(ndvi_ts_row)
print(csvstr)
if dryrun:
nout = 0
else:
nout = ndvi_summary_ts.writeCSV(outpath)
print("Total: %d" % (nout, ))
def main():
# set up command line argument processing
parser = argparse.ArgumentParser()
# options
parser.add_argument(
"-v",
"--verbose",
help="increase output verbosity",
action="store_true",
default=False,
)
parser.add_argument(
"-n",
"--dry-run",
help="Process data but don't save results",
action="store_true",
default=False,
)
parser.add_argument(
"-p",
"--aggregation-period",
help="Number of Days to Aggregate",
nargs="?",
type=int,
choices=range(1, 5, 2),
default=1,
)
# positional arguments
parser.add_argument("site", help="PhenoCam site name")
parser.add_argument("roiname", help="ROI name, e.g. canopy_0001")
# get args
args = parser.parse_args()
sitename = args.site
roiname = args.roiname
verbose = args.verbose
dryrun = args.dry_run
ndays = args.aggregation_period
if verbose:
print("site: {0}".format(sitename))
print("roiname: {0}".format(roiname))
print("verbose: {0}".format(verbose))
print("dryrun: {0}".format(dryrun))
print("period: {0}".format(ndays))
# set up output filename
outdir = os.path.join(vi.config.archive_dir, sitename, "ROI")
outfile = "{0}_{1}_{2}day.csv".format(sitename, roiname, ndays)
outpath = os.path.join(outdir, outfile)
# since this is "update" output file should already exist
# if not just bail out
if not os.path.exists(outpath):
sys.stderr.write(
"Existing gcc90 file {0} not found.\n".format(outpath))
sys.exit(1)
# read in existing CSV file
gcc_ts = vi.GCCTimeSeries(site=sitename, ROIListID=roiname, nday=ndays)
gcc_ts.readCSV(outpath)
# read in config file for this site/roi if it exists
config_file = "{0}_{1}.cfg".format(sitename, roiname)
config_path = os.path.join(archive_dir, sitename, "ROI", config_file)
if os.path.exists(config_path):
cfgparser = ConfigParser(
defaults={
"nimage_threshold": str(default_nimage_threshold),
"time_min": str(default_time_min),
"time_max": str(default_time_max),
"sunelev_min": str(default_sunelev_min),
"brt_min": str(default_brt_min),
"brt_max": str(default_brt_max),
})
cfgparser.read(config_path)
if cfgparser.has_section("gcc90_calculation"):
nimage_threshold = cfgparser.getint("gcc90_calculation",
"nimage_threshold")
time_max_str = cfgparser.get("gcc90_calculation", "time_max")
print("time_max_str: {0}".format(time_max_str))
[tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
time_min_str = cfgparser.get("gcc90_calculation", "time_min")
[tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = cfgparser.getfloat("gcc90_calculation",
"sunelev_min")
brt_min = cfgparser.getint("gcc90_calculation", "brt_min")
brt_max = cfgparser.getint("gcc90_calculation", "brt_max")
else:
nimage_threshold = gcc_ts.nmin
time_max = gcc_ts.tod_max
time_min = gcc_ts.tod_min
sunelev_min = gcc_ts.sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
else:
nimage_threshold = int(default_nimage_threshold)
[tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":")
time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc))
[tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":")
time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc))
sunelev_min = default_sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
# verify that config file matches CSV header!
if nimage_threshold != gcc_ts.nmin:
sys.stderr.write(
"nimage_threshold from config doesn't match CSV header\n")
sys.exit(1)
if brt_min != gcc_ts.brt_min:
sys.stderr.write("brt_min from config file doesn't match CSV header\n")
sys.exit(1)
if brt_max != gcc_ts.brt_max:
sys.stderr.write("brt_max from config file doesn't match CSV header\n")
sys.exit(1)
if time_min != gcc_ts.tod_min:
sys.stderr.write("tod_min from config file doesn't match CSV header\n")
sys.exit(1)
if time_min != gcc_ts.tod_min:
sys.stderr.write("tod_min from config file doesn't match CSV header\n")
sys.exit(1)
if sunelev_min != gcc_ts.sunelev_min:
sys.stderr.write(
"sunelev_min from config file doesn't match CSV header\n")
sys.exit(1)
else:
nimage_threshold = gcc_ts.nmin
time_max = gcc_ts.tod_max
time_min = gcc_ts.tod_min
sunelev_min = gcc_ts.sunelev_min
brt_min = default_brt_min
brt_max = default_brt_max
# grab remaining metadata
gcc_ts_create_date = gcc_ts.created_at
gcc_ts_update_date = gcc_ts.updated_at
# print config values
if verbose:
print("")
print("gcc config:")
print("===========")
print("roi_list: ", "{0}_{1}".format(sitename, roiname))
if os.path.exists(config_path):
print("config file: {0}".format(config_file))
else:
print("config file: None")
print("nimage threshold: ", nimage_threshold)
print("time of day min: ", time_min)
print("time of day max: ", time_max)
print("sun elev min: ", sunelev_min)
print("aggregate days: ", ndays)
print("minimum brightness: ", brt_min)
print("maximum brightness: ", brt_max)
print("creation date: ", gcc_ts_create_date)
print("last update: ", gcc_ts_update_date)
# get number of rows in existing/old CSV
ngccrows = len(gcc_ts.rows)
# get the next to last date in gcc90 CSV
# NOTE: always redo last date since we may be adding images
# (if ndays > 1 and we haven't finished interval)
gcc90_date_last = gcc_ts.rows[ngccrows - 1]["date"]
if verbose:
print("last date in timeseries: ", gcc90_date_last)
print("")
# get roi timeseries for this site and roi
roits = vi.get_roi_timeseries(sitename, roiname)
if verbose:
print("")
print("ROI timeseries info:")
print("====================")
print("site: ", roits.site)
print("ROI list id: ", roits.roilistid)
print("create date: ", roits.created_at)
print("update date: ", roits.updated_at)
print("nrows: ", len(roits.rows))
# make list of rows which match image selection criteria
roits_rows = roits.select_rows(
tod_min=time_min,
tod_max=time_max,
sunelev_min=sunelev_min,
brt_min=brt_min,
brt_max=brt_max,
)
# calculate offset for timeseries based on nday
day_offset = ndays / 2
date_offset = timedelta(days=day_offset)
# find rows that are in or more recent than beginning of last timeperiod
# of GCC
new_roits_rows = []
for row in roits_rows:
if row["datetime"].date() >= (gcc90_date_last - date_offset):
new_roits_rows.append(row)
# check that some rows passed selection criteria
nrows = len(new_roits_rows)
if nrows == 0:
print("No rows passed the selection criteria")
sys.exit(0)
if debug:
print("New selected rows: {0}".format(nrows))
# make a list of dates for selected images
img_date = []
for row in new_roits_rows:
img_date.append(row["datetime"].date())
# list is ordered so find first and last dates
dt_first = img_date[0]
dt_last = img_date[nrows - 1]
# set up a generator which yields dates for the start
# of the next nday period covering the date range of
# new images
gcc_dr = vi.daterange2(dt_first, dt_last, ndays)
# roits_ndx will be index into ROI timeseries
roits_ndx = 0
# set up vars for accumulating stats
img_cnt = 0
update_cnt = 0
filenames = []
r_dn_vals = []
rcc_vals = []
g_dn_vals = []
gcc_vals = []
b_dn_vals = []
bcc_vals = []
solar_elev_vals = []
midday_delta_vals = []
# loop over ndays time periods
for gcc_ndx, start_date in enumerate(gcc_dr):
end_date = start_date + timedelta(ndays)
gcc_date = start_date + date_offset
doy = gcc_date.timetuple().tm_yday
midday_noon = datetime(gcc_date.year, gcc_date.month, gcc_date.day, 12,
0, 0)
# get roits rows for this time period
while (roits_ndx < nrows and img_date[roits_ndx] >= start_date
and img_date[roits_ndx] < end_date):
# skip this row if awbflag is 1
if roits_rows[roits_ndx]["awbflag"] == 1:
if roits_ndx < nrows:
roits_ndx += 1
continue
else:
break
filenames.append(new_roits_rows[roits_ndx]["filename"])
r_dn = new_roits_rows[roits_ndx]["r_mean"]
r_dn_vals.append(r_dn)
g_dn = new_roits_rows[roits_ndx]["g_mean"]
g_dn_vals.append(g_dn)
b_dn = new_roits_rows[roits_ndx]["b_mean"]
b_dn_vals.append(b_dn)
dnsum = r_dn + g_dn + b_dn
if dnsum <= 0:
rcc = np.nan
bcc = np.nan
gcc = np.nan
else:
rcc = r_dn / dnsum
bcc = b_dn / dnsum
gcc = new_roits_rows[roits_ndx]["gcc"]
solar_elev = new_roits_rows[roits_ndx]["solar_elev"]
rcc_vals.append(rcc)
gcc_vals.append(gcc)
bcc_vals.append(bcc)
solar_elev_vals.append(solar_elev)
midday_td = new_roits_rows[roits_ndx]["datetime"] - midday_noon
midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds)
midday_delta_vals.append(midday_td_secs)
img_cnt += 1
if roits_ndx < nrows:
roits_ndx += 1
else:
break
# check to see if we got any images
if img_cnt == 0:
# nodata for this time period
image_count = 0
midday_filename = ND_STRING
midday_r = ND_FLOAT
midday_g = ND_FLOAT
midday_b = ND_FLOAT
midday_gcc = ND_FLOAT
midday_rcc = ND_FLOAT
r_mean = ND_FLOAT
r_std = ND_FLOAT
g_mean = ND_FLOAT
g_std = ND_FLOAT
b_mean = ND_FLOAT
b_std = ND_FLOAT
gcc_mean = ND_FLOAT
gcc_std = ND_FLOAT
gcc_50 = ND_FLOAT
gcc_75 = ND_FLOAT
gcc_90 = ND_FLOAT
rcc_mean = ND_FLOAT
rcc_std = ND_FLOAT
rcc_50 = ND_FLOAT
rcc_75 = ND_FLOAT
rcc_90 = ND_FLOAT
max_solar_elev = ND_FLOAT
snow_flag = ND_INT
outlierflag_gcc_mean = ND_INT
outlierflag_gcc_50 = ND_INT
outlierflag_gcc_75 = ND_INT
outlierflag_gcc_90 = ND_INT
elif img_cnt < nimage_threshold:
# not enough images
image_count = img_cnt
# find nearest image to midday (noon) on mid-interval date
mi_ndx = midday_delta_vals.index(min(midday_delta_vals))
midday_filename = filenames[mi_ndx]
if not midday_filename:
midday_filename = ND_STRING
midday_r = r_dn_vals[mi_ndx]
midday_g = g_dn_vals[mi_ndx]
midday_b = b_dn_vals[mi_ndx]
midday_gcc = gcc_vals[mi_ndx]
midday_rcc = rcc_vals[mi_ndx]
# no stats for this time interval
r_mean = ND_FLOAT
r_std = ND_FLOAT
g_mean = ND_FLOAT
g_std = ND_FLOAT
b_mean = ND_FLOAT
b_std = ND_FLOAT
gcc_mean = ND_FLOAT
gcc_std = ND_FLOAT
gcc_50 = ND_FLOAT
gcc_75 = ND_FLOAT
gcc_90 = ND_FLOAT
rcc_mean = ND_FLOAT
rcc_std = ND_FLOAT
rcc_50 = ND_FLOAT
rcc_75 = ND_FLOAT
rcc_90 = ND_FLOAT
max_solar_elev = max(solar_elev_vals)
snow_flag = ND_INT
outlierflag_gcc_mean = ND_INT
outlierflag_gcc_50 = ND_INT
outlierflag_gcc_75 = ND_INT
outlierflag_gcc_90 = ND_INT
else:
# find nearest image to midday (noon) on mid-interval date
mi_ndx = midday_delta_vals.index(min(midday_delta_vals))
midday_filename = filenames[mi_ndx]
midday_r = r_dn_vals[mi_ndx]
midday_g = g_dn_vals[mi_ndx]
midday_b = b_dn_vals[mi_ndx]
midday_gcc = gcc_vals[mi_ndx]
midday_rcc = rcc_vals[mi_ndx]
# get stats for this time interval
image_count = img_cnt
r_mean = np.nanmean(r_dn_vals)
r_std = np.nanstd(r_dn_vals)
g_mean = np.nanmean(g_dn_vals)
g_std = np.nanstd(g_dn_vals)
b_mean = np.nanmean(b_dn_vals)
b_std = np.nanstd(b_dn_vals)
gcc_mean = np.nanmean(gcc_vals)
gcc_std = np.nanstd(gcc_vals)
gcc_50 = quantile(gcc_vals, 0.5)
gcc_75 = quantile(gcc_vals, 0.75)
gcc_90 = quantile(gcc_vals, 0.9)
rcc_mean = np.mean(rcc_vals)
rcc_std = np.std(rcc_vals)
rcc_50 = quantile(rcc_vals, 0.5)
rcc_75 = quantile(rcc_vals, 0.75)
rcc_90 = quantile(rcc_vals, 0.9)
max_solar_elev = max(solar_elev_vals)
snow_flag = ND_INT
outlierflag_gcc_mean = ND_INT
outlierflag_gcc_50 = ND_INT
outlierflag_gcc_75 = ND_INT
outlierflag_gcc_90 = ND_INT
# append to gcc timeseries
gcc_ts_row = gcc_ts.insert_row(
gcc_date,
doy,
image_count,
midday_filename,
midday_r,
midday_g,
midday_b,
midday_gcc,
midday_rcc,
r_mean,
r_std,
g_mean,
g_std,
b_mean,
b_std,
gcc_mean,
gcc_std,
gcc_50,
gcc_75,
gcc_90,
rcc_mean,
rcc_std,
rcc_50,
rcc_75,
rcc_90,
max_solar_elev,
snow_flag,
outlierflag_gcc_mean,
outlierflag_gcc_50,
outlierflag_gcc_75,
outlierflag_gcc_90,
)
update_cnt += 1
# print result if verbose
if verbose:
csvstr = gcc_ts.format_csvrow(gcc_ts_row)
print(csvstr)
# reset accumulated values
img_cnt = 0
filenames = []
r_dn_vals = []
rcc_vals = []
g_dn_vals = []
gcc_vals = []
b_dn_vals = []
bcc_vals = []
solar_elev_vals = []
midday_delta_vals = []
if dryrun:
nout = 0
else:
nout = gcc_ts.writeCSV(outpath)
print("GCC90 Rows updated: 1 Rows added: {0}".format(update_cnt - 1))
print("Total: {0}".format(nout))
--------------- Tests for `vegindex.vegindex.daterange2` module """ import os from datetime import date import numpy as np from vegindex import vegindex as vi end_date = date(2008, 1, 31) start_date = date(2008, 1, 1) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 1)) start_date = date(2008, 1, 2) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 1)) start_date = date(2008, 1, 3) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 1)) start_date = date(2008, 1, 4) mydate = next(vi.daterange2(start_date, end_date, 3)) np.testing.assert_equal(mydate, date(2008, 1, 4)) start_date = date(2008, 1, 5)