def plotGraph(args, obs, data, sizeX=1, sizeY=1, dpi=80):
fig = plt.figure(figsize=(sizeX, sizeY), dpi=dpi, tight_layout=True)
ax = fig.add_subplot(111)
if args.plottype == PLOT_TYPE_STD or \
args.plottype == PLOT_TYPE_LOGY:
x = obs.index
elif args.plottype == PLOT_TYPE_CDF:
x = np.linspace(min_x, max_x, num=len(obs) )
# Plot observed values
# Standard or log plot
obs_y = obs
if args.plottype == PLOT_TYPE_CDF:
obs_ecdf = sm.distributions.ECDF(obs)
obs_y = obs_ecdf(x)
obs_plt = None
if not args.supressObs:
(obs_plt,) = ax.plot(x, obs_y, linewidth=2.0, color='black')
# Plot modeled values
data_plt = []
for (i, d) in enumerate(data):
# Standard or log plot
mod_y = d
if args.plottype == PLOT_TYPE_CDF:
mod_ecdf = sm.distributions.ECDF(d)
mod_y = mod_ecdf(x)
# Plot (we could move this outside of the for loop)
if args.linewidth:
linewidth = args.linewidth[i]
else:
linewidth = 1.0
if args.linestyle:
linestyle = LINE_TYPE_DICT[ args.linestyle[i] ]
else:
# Rotate styles
styleIdx = ( (i + 1) % NUM_LINE_TYPES ) - 1
linestyle = LINE_TYPE_DICT[ LINE_TYPES[styleIdx] ]
if args.color:
(mod_plt,) = ax.plot(x, mod_y, linewidth=linewidth, linestyle=linestyle,
color=args.color[i])
else:
(mod_plt,) = ax.plot(x, mod_y, linewidth=linewidth, linestyle=linestyle)
data_plt.append(mod_plt)
# Plot annotations
columnName = args.column.capitalize()
if args.title:
title = args.title
else:
if args.plottype == PLOT_TYPE_STD:
title = columnName
elif args.plottype == PLOT_TYPE_LOGY:
title = "log(%s)" % (columnName,)
elif args.plottype == PLOT_TYPE_CDF:
title = "Cummulative distribution - %s" % (columnName,)
fig.suptitle(title, y=0.99)
# X-axis
if args.plottype == PLOT_TYPE_STD or \
args.plottype == PLOT_TYPE_LOGY:
num_years = len(x) / 365
if num_years > 4:
if num_years > 10:
ax.xaxis.set_major_locator(matplotlib.dates.YearLocator())
else:
ax.xaxis.set_major_locator(matplotlib.dates.MonthLocator(interval=3))
else:
ax.xaxis.set_major_locator(matplotlib.dates.MonthLocator())
ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%b-%Y'))
# Rotate
plt.setp( ax.xaxis.get_majorticklabels(), rotation=45)
plt.setp( ax.xaxis.get_majorticklabels(), fontsize='x-small')
if args.plottype == PLOT_TYPE_CDF:
ax.set_xlim(min_x, max_x)
ax.set_xscale('log')
if args.xlabel:
ax.set_xlabel(args.xlabel)
else:
ax.set_xlabel( columnName )
elif args.xlabel:
ax.set_xlabel(args.xlabel)
# Y-axis
if args.plottype == PLOT_TYPE_LOGY:
ax.set_yscale('log')
if args.ylabel:
ax.set_ylabel(args.ylabel)
elif args.plottype != PLOT_TYPE_CDF:
y_label = columnName
if args.plottype == PLOT_TYPE_LOGY:
y_label = "log( %s )" % (columnName,)
ax.set_ylabel( y_label )
if args.supressObs:
legend_items = args.legend
else:
data_plt.insert(0, obs_plt)
legend_items = ['Observed'] + args.legend
# Plot secondary data (if specified)
if args.secondaryData and \
(args.plottype == PLOT_TYPE_STD or args.plottype == PLOT_TYPE_LOGY):
sec_file = open(args.secondaryData, 'r')
(sec_datetime, sec_data) = RHESSysOutput.readColumnFromFile(sec_file,
args.secondaryColumn,
startHour=0)
sec_file.close()
sec = pd.Series(sec_data, index=sec_datetime)
# Align timeseries
(sec_align, obs_align) = sec.align(obs, join='inner')
# Plot
ax2 = ax.twinx()
if args.secondaryPlotType == 'line':
(sec_plot,) = ax2.plot(x, sec_align)
elif args.secondaryPlotType == 'bar':
sec_plot = ax2.bar(x, sec_align, facecolor='blue', edgecolor='none', width=2.0)
secondaryLabel = args.secondaryColumn.capitalize()
if args.secondaryLabel:
secondaryLabel = args.secondaryLabel
ax2.invert_yaxis()
ax2.set_ylabel(args.secondaryLabel)
#ax.set_zorder(ax2.get_zorder()+1) # put ax in front of ax2
#ax.patch.set_visible(False) # hide the 'canvas'
# Plot legend last
num_cols = len(data)
if not args.supressObs:
num_cols += 1
if args.plottype == PLOT_TYPE_CDF:
fig.legend( data_plt, legend_items, 'lower center', fontsize='x-small',
bbox_to_anchor=(0.5, -0.015), ncol=num_cols, frameon=False )
else:
fig.legend( data_plt, legend_items, 'lower center', fontsize='x-small',
bbox_to_anchor=(0.5, -0.01), ncol=num_cols, frameon=False )
# Determine output variables
variables = ['patchID'] + args.outputVariables
sys.stdout.write('scenario,patchid')
for var in args.outputVariables:
sys.stdout.write(',sum_' + var)
sys.stdout.write('\n')
# For each rhessys output file ...
for (i, patchDailyFilepath) in enumerate(patchDailyFilepaths):
scenario = os.path.basename( os.path.dirname(patchDailyFilepath) )
f = open(patchDailyFilepath)
data = RHESSysOutput.readColumnsFromPatchDailyFile(f, variables)
f.close()
if len(data) < 1:
sys.exit("No data found for variable in RHESSys output file '%s'" % \
(patchDailyFilepath,) )
# For each day
variablesDict = {}
patchIDs = None
variable = None
for (i, key) in enumerate(data):
if startDate and key < startDate:
continue
if endDate and key > endDate:
break
#import pdb; pdb.set_trace()
# Zoom to map extent
result = grassLib.script.run_command('g.region', rast='MASK', zoom='MASK')
if result != 0:
sys.exit("Failed to set region to layer %s" % \
(args.mask,) )
# Set environment variables for GRASS PNG output driver
os.environ['GRASS_RENDER_IMMEDIATE'] = 'FALSE'
os.environ['GRASS_TRUECOLOR'] = 'TRUE'
os.environ['GRASS_WIDTH'] = '960'
os.environ['GRASS_HEIGHT'] = '720'
# 3. Open file ending in "patch.daily" in rhessys output dir
print("Reading RHESSys output data (this may take a while)...")
f = open(patchDailyFilepath)
data = RHESSysOutput.readColumnsFromPatchDailyFile(f, variables)
f.close()
if len(data) < 1:
sys.exit("No data found for variable in RHESSys output file '%s'" % \
(patchDailyFilepath,) )
# 4. For each day
for (i, key) in enumerate(data):
dateStr = "%d/%d/%d" % (key.month, key.day, key.year)
# Set filename env for PNG driver
imageFilename = "%s%04d.png" % (RECLASS_MAP_TMP, i + 1)
reclassImagePath = os.path.join(tmpDir, imageFilename)
os.environ['GRASS_PNGFILE'] = reclassImagePath
dataForDate = data[key]
# a. Write reclass rule to temp file
sys.exit('A secondary data file was specified, but the secondary column to use was not')
if args.data and ( len(args.data) != len(args.legend) ):
sys.exit('Number of legend items must equal the number of data files')
elif args.behavioralData and ( len(args.behavioralData) != len(args.legend) ):
sys.exit('Number of legend items must equal the number of data files')
# Open data and align to observed
obs_align = None
data = []
max_x = min_x = 0
if args.data:
# Open observed data
obs_file = open(args.obs, 'r')
(obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file,
readHour=False)
obs_file.close()
obs = pd.Series(obs_data, index=obs_datetime)
for d in args.data:
mod_file = open(d, 'r')
(tmp_datetime, tmp_data) = RHESSysOutput.readColumnFromFile(mod_file, args.column, startHour=0)
tmp_mod = pd.Series(tmp_data, index=tmp_datetime)
# Align timeseries
(mod_align, obs_align) = tmp_mod.align(obs, join='inner')
tmp_max_x = max(mod_align.max(), obs_align.max())
if tmp_max_x > max_x:
max_x = tmp_max_x
min_x = max(min_x, mod_align.min())
mod_file.close()
# help='A data file containing the varaible to plot on a secondary Y-axis')
parser.add_argument('--secondaryColumn', required=False,
help='Name of column to use from secondary data file')
parser.add_argument('--secondaryLabel', required=False,
help='Label to use for seconary Y-axis')
args = parser.parse_args()
# if args.secondaryData and not args.secondaryColumn:
# sys.exit('A secondary data file was specified, but the secondary column to use was not')
# if len(args.data) != len(args.legend):
# sys.exit('Number of legend items must equal the number of data files')
# Open observed data
obs_file = open(args.obs, 'r')
(obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file,
readHour=False)
obs_file.close()
#obs = pd.Series(obs_data, index=obs_datetime)
obs = pd.DataFrame(obs_data, index=obs_datetime, columns=['observed'])
# Open data and align to observed
cols = ['streamflow', 'evap', 'trans', 'precip']
obs_align = None
#data = []
max_x = min_x = 0
#for d in args.data:
#mod_file = open(d, 'r')
mod_file = open(args.data, 'r')
mod_df = RHESSysOutput.readColumnsFromFile(mod_file,
cols)
#tmp_mod = pd.Series(tmp_data, index=tmp_datetime)
default='bar',
help='Type of plot to use for secondary data.')
parser.add_argument('--secondaryLabel',
required=False,
help='Label to use for seconary Y-axis')
args = parser.parse_args()
# Open observed data
obs = pd.read_csv(args.obs, index_col=0, parse_dates=True)
# Open data and align to observed
cols = ['streamflow', 'evap', 'trans', 'precip']
obs_align = None
max_x = min_x = 0
mod_file = open(args.data, 'r')
mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols, readHour=False)
# Align timeseries
(mod_align, obs_align) = mod_df.align(obs, axis=0, join='inner')
tmp_max_x = max(mod_align['streamflow'].max(),
obs_align[OBS_HEADER_STREAMFLOW].max())
if tmp_max_x > max_x:
max_x = tmp_max_x
min_x = max(min_x, mod_align['streamflow'].min())
mod_file.close()
fig = plt.figure(figsize=(args.figureX, args.figureY),
dpi=80,
tight_layout=True)
ax_std = fig.add_subplot(221)
choices=["bar", "line"],
default="bar",
help="Type of plot to use for secondary data.",
)
parser.add_argument("--secondaryLabel", required=False, help="Label to use for seconary Y-axis")
args = parser.parse_args()
# Open observed data
obs = pd.read_csv(args.obs, index_col=0, parse_dates=True)
# Open data and align to observed
cols = ["streamflow", "evap", "trans", "precip"]
obs_align = None
max_x = min_x = 0
mod_file = open(args.data, "r")
mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols, readHour=False)
# Align timeseries
(mod_align, obs_align) = mod_df.align(obs, axis=0, join="inner")
tmp_max_x = max(mod_align["streamflow"].max(), obs_align[OBS_HEADER_STREAMFLOW].max())
if tmp_max_x > max_x:
max_x = tmp_max_x
min_x = max(min_x, mod_align["streamflow"].min())
mod_file.close()
fig = plt.figure(figsize=(args.figureX, args.figureY), dpi=80, tight_layout=True)
ax_std = fig.add_subplot(221)
ax_log = fig.add_subplot(222)
ax_cdf = fig.add_subplot(223)
ax_tab = fig.add_subplot(224)
def plotGraph(args, obs, data, sizeX=1, sizeY=1, dpi=80):
fig = plt.figure(figsize=(sizeX, sizeY), dpi=dpi, tight_layout=True)
ax = fig.add_subplot(111)
if args.plottype == PLOT_TYPE_STD or \
args.plottype == PLOT_TYPE_LOGY:
x = obs.index
elif args.plottype == PLOT_TYPE_CDF:
x = np.linspace(min_x, max_x, num=len(obs))
# Plot observed values
# Standard or log plot
obs_y = obs
if args.plottype == PLOT_TYPE_CDF:
obs_ecdf = sm.distributions.ECDF(obs)
obs_y = obs_ecdf(x)
obs_plt = None
if not args.supressObs:
(obs_plt, ) = ax.plot(x, obs_y, linewidth=2.0, color='black')
# Plot modeled values
data_plt = []
for (i, d) in enumerate(data):
# Standard or log plot
mod_y = d
if args.plottype == PLOT_TYPE_CDF:
mod_ecdf = sm.distributions.ECDF(d)
mod_y = mod_ecdf(x)
# Plot (we could move this outside of the for loop)
if args.linewidth:
linewidth = args.linewidth[i]
else:
linewidth = 1.0
if args.linestyle:
linestyle = LINE_TYPE_DICT[args.linestyle[i]]
else:
# Rotate styles
styleIdx = ((i + 1) % NUM_LINE_TYPES) - 1
linestyle = LINE_TYPE_DICT[LINE_TYPES[styleIdx]]
if args.color:
(mod_plt, ) = ax.plot(x,
mod_y,
linewidth=linewidth,
linestyle=linestyle,
color=args.color[i])
else:
(mod_plt, ) = ax.plot(x,
mod_y,
linewidth=linewidth,
linestyle=linestyle)
data_plt.append(mod_plt)
# Plot annotations
columnName = args.column.capitalize()
if args.title:
title = args.title
else:
if args.plottype == PLOT_TYPE_STD:
title = columnName
elif args.plottype == PLOT_TYPE_LOGY:
title = "log(%s)" % (columnName, )
elif args.plottype == PLOT_TYPE_CDF:
title = "Cummulative distribution - %s" % (columnName, )
fig.suptitle(title, y=0.99)
# X-axis
if args.plottype == PLOT_TYPE_STD or \
args.plottype == PLOT_TYPE_LOGY:
num_years = len(x) / 365
if num_years > 4:
if num_years > 10:
ax.xaxis.set_major_locator(matplotlib.dates.YearLocator())
else:
ax.xaxis.set_major_locator(
matplotlib.dates.MonthLocator(interval=3))
else:
ax.xaxis.set_major_locator(matplotlib.dates.MonthLocator())
ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%b-%Y'))
# Rotate
plt.setp(ax.xaxis.get_majorticklabels(), rotation=45)
plt.setp(ax.xaxis.get_majorticklabels(), fontsize='x-small')
if args.plottype == PLOT_TYPE_CDF:
ax.set_xlim(min_x, max_x)
ax.set_xscale('log')
if args.xlabel:
ax.set_xlabel(args.xlabel)
else:
ax.set_xlabel(columnName)
elif args.xlabel:
ax.set_xlabel(args.xlabel)
# Y-axis
if args.plottype == PLOT_TYPE_LOGY:
ax.set_yscale('log')
if args.ylabel:
ax.set_ylabel(args.ylabel)
elif args.plottype != PLOT_TYPE_CDF:
y_label = columnName
if args.plottype == PLOT_TYPE_LOGY:
y_label = "log( %s )" % (columnName, )
ax.set_ylabel(y_label)
if args.supressObs:
legend_items = args.legend
else:
data_plt.insert(0, obs_plt)
legend_items = ['Observed'] + args.legend
# Plot secondary data (if specified)
if args.secondaryData and \
(args.plottype == PLOT_TYPE_STD or args.plottype == PLOT_TYPE_LOGY):
sec_file = open(args.secondaryData, 'r')
(sec_datetime,
sec_data) = RHESSysOutput.readColumnFromFile(sec_file,
args.secondaryColumn,
startHour=0)
sec_file.close()
sec = pd.Series(sec_data, index=sec_datetime)
# Align timeseries
(sec_align, obs_align) = sec.align(obs, join='inner')
# Plot
ax2 = ax.twinx()
if args.secondaryPlotType == 'line':
(sec_plot, ) = ax2.plot(x, sec_align)
elif args.secondaryPlotType == 'bar':
sec_plot = ax2.bar(x,
sec_align,
facecolor='blue',
edgecolor='none',
width=2.0)
secondaryLabel = args.secondaryColumn.capitalize()
if args.secondaryLabel:
secondaryLabel = args.secondaryLabel
ax2.invert_yaxis()
ax2.set_ylabel(args.secondaryLabel)
#ax.set_zorder(ax2.get_zorder()+1) # put ax in front of ax2
#ax.patch.set_visible(False) # hide the 'canvas'
# Plot legend last
num_cols = len(data)
if not args.supressObs:
num_cols += 1
if args.plottype == PLOT_TYPE_CDF:
fig.legend(data_plt,
legend_items,
'lower center',
fontsize='x-small',
bbox_to_anchor=(0.5, -0.015),
ncol=num_cols,
frameon=False)
else:
fig.legend(data_plt,
legend_items,
'lower center',
fontsize='x-small',
bbox_to_anchor=(0.5, -0.01),
ncol=num_cols,
frameon=False)
if args.data and (len(args.data) != len(args.legend)):
sys.exit('Number of legend items must equal the number of data files')
elif args.behavioralData and (len(args.behavioralData) != len(
args.legend)):
sys.exit('Number of legend items must equal the number of data files')
# Open data and align to observed
obs_align = None
data = []
max_x = min_x = 0
if args.data:
# Open observed data
obs_file = open(args.obs, 'r')
(obs_datetime,
obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file,
readHour=False)
obs_file.close()
obs = pd.Series(obs_data, index=obs_datetime)
for d in args.data:
mod_file = open(d, 'r')
(tmp_datetime,
tmp_data) = RHESSysOutput.readColumnFromFile(mod_file,
args.column,
startHour=0)
tmp_mod = pd.Series(tmp_data, index=tmp_datetime)
# Align timeseries
(mod_align, obs_align) = tmp_mod.align(obs, join='inner')
tmp_max_x = max(mod_align.max(), obs_align.max())
if tmp_max_x > max_x:
max_x = tmp_max_x
def readBehavioralData(
self, basedir, session_id, variable="streamflow", observed_file=None, behavioral_filter=None, end_date=None
):
dbPath = RHESSysCalibrator.getDBPath(basedir)
if not os.access(dbPath, os.R_OK):
raise IOError(errno.EACCES, "The database at %s is not readable" % dbPath)
self.logger.debug("DB path: %s" % dbPath)
outputPath = RHESSysCalibrator.getOutputPath(basedir)
if not os.access(outputPath, os.R_OK):
raise IOError(errno.EACCES, "The output directory %s is not readable" % outputPath)
self.logger.debug("Output path: %s" % outputPath)
rhessysPath = RHESSysCalibrator.getRhessysPath(basedir)
calibratorDB = ModelRunnerDB(RHESSysCalibrator.getDBPath(basedir))
# Make sure the session exists
session = calibratorDB.getSession(session_id)
if None == session:
raise Exception("Session %d was not found in the calibration database %s" % (session_id, dbPath))
if session.status != "complete":
print "WARNING: session status is: %s. Some model runs may not have completed." % (session.status,)
else:
self.logger.debug("Session status is: %s" % (session.status,))
# Determine observation file path
if observed_file:
obs_file = observed_file
else:
# Get observered file from session
assert session.obs_filename != None
obs_file = session.obs_filename
obsPath = RHESSysCalibrator.getObsPath(basedir)
obsFilePath = os.path.join(obsPath, obs_file)
if not os.access(obsFilePath, os.R_OK):
raise IOError(errno.EACCES, "The observed data file %s is not readable" % obsFilePath)
self.logger.debug("Obs path: %s" % obsFilePath)
# Get runs in session
runs = calibratorDB.getRunsInSession(session.id, where_clause=behavioral_filter)
numRuns = len(runs)
if numRuns == 0:
raise Exception("No runs found for session %d" % (session.id,))
response = raw_input(
"%d runs selected for plotting from session %d in basedir '%s', continue? [yes | no] "
% (numRuns, session_id, os.path.basename(basedir))
)
response = response.lower()
if response != "y" and response != "yes":
# Exit normally
return 0
self.logger.debug("%d behavioral runs" % (numRuns,))
# Read observed data from file
obsFile = open(obsFilePath, "r")
(obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obsFile)
obsFile.close()
obs = pd.Series(obs_data, index=obs_datetime)
if end_date:
obs = obs[:end_date]
self.logger.debug("Observed data: %s" % obs_data)
likelihood = np.empty(numRuns)
ysim = None
x = None
runsProcessed = False
for (i, run) in enumerate(runs):
if "DONE" == run.status:
runOutput = os.path.join(rhessysPath, run.output_path)
self.logger.debug(">>>\nOutput dir of run %d is %s" % (run.id, runOutput))
tmpOutfile = RHESSysCalibrator.getRunOutputFilePath(runOutput)
if not os.access(tmpOutfile, os.R_OK):
print "Output file %s for run %d not found or not readable, unable to calculate fitness statistics for this run" % (
tmpOutfile,
run.id,
)
continue
tmpFile = open(tmpOutfile, "r")
(tmp_datetime, tmp_data) = RHESSysOutput.readColumnFromFile(tmpFile, "streamflow")
tmp_mod = pd.Series(tmp_data, index=tmp_datetime)
# Align timeseries to observed
(mod, obs) = tmp_mod.align(obs, join="inner")
# Stash date for X values (assume they are the same for all runs
if x == None:
x = [datetime.strptime(str(d), "%Y-%m-%d %H:%M:%S") for d in mod.index]
# Put data in matrix
dataLen = len(mod)
if ysim == None:
# Allocate matrix for results
ysim = np.empty((numRuns, dataLen))
assert np.shape(ysim)[1] == dataLen
ysim[i,] = mod
# Store fitness parameter
likelihood[i] = run.nse
tmpFile.close()
runsProcessed = True
return (runsProcessed, obs, x, ysim, likelihood)
default=False,
help="Plot in color")
parser.add_argument('--secondaryPlotType',
required=False,
choices=['bar', 'line'],
default='bar',
help='Type of plot to use for secondary data.')
parser.add_argument('--secondaryLabel',
required=False,
help='Label to use for seconary Y-axis')
args = parser.parse_args()
# Open observed data
obs_file = open(args.obs, 'r')
(obs_datetime,
obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file,
readHour=False)
obs_file.close()
obs = pd.DataFrame(obs_data, index=obs_datetime, columns=['observed'])
# Open data and align to observed
cols = ['streamflow', 'evap', 'trans', 'precip']
obs_align = None
max_x = min_x = 0
mod_file = open(args.data, 'r')
mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols, readHour=False)
# Align timeseries
(mod_align, obs_align) = mod_df.align(obs, axis=0, join='inner')
tmp_max_x = max(mod_align['streamflow'].max(), obs_align['observed'].max())
if tmp_max_x > max_x:
max_x = tmp_max_x
