def plotGraph(args, obs, data, sizeX=1, sizeY=1, dpi=80): fig = plt.figure(figsize=(sizeX, sizeY), dpi=dpi, tight_layout=True) ax = fig.add_subplot(111) if args.plottype == PLOT_TYPE_STD or \ args.plottype == PLOT_TYPE_LOGY: x = obs.index elif args.plottype == PLOT_TYPE_CDF: x = np.linspace(min_x, max_x, num=len(obs) ) # Plot observed values # Standard or log plot obs_y = obs if args.plottype == PLOT_TYPE_CDF: obs_ecdf = sm.distributions.ECDF(obs) obs_y = obs_ecdf(x) obs_plt = None if not args.supressObs: (obs_plt,) = ax.plot(x, obs_y, linewidth=2.0, color='black') # Plot modeled values data_plt = [] for (i, d) in enumerate(data): # Standard or log plot mod_y = d if args.plottype == PLOT_TYPE_CDF: mod_ecdf = sm.distributions.ECDF(d) mod_y = mod_ecdf(x) # Plot (we could move this outside of the for loop) if args.linewidth: linewidth = args.linewidth[i] else: linewidth = 1.0 if args.linestyle: linestyle = LINE_TYPE_DICT[ args.linestyle[i] ] else: # Rotate styles styleIdx = ( (i + 1) % NUM_LINE_TYPES ) - 1 linestyle = LINE_TYPE_DICT[ LINE_TYPES[styleIdx] ] if args.color: (mod_plt,) = ax.plot(x, mod_y, linewidth=linewidth, linestyle=linestyle, color=args.color[i]) else: (mod_plt,) = ax.plot(x, mod_y, linewidth=linewidth, linestyle=linestyle) data_plt.append(mod_plt) # Plot annotations columnName = args.column.capitalize() if args.title: title = args.title else: if args.plottype == PLOT_TYPE_STD: title = columnName elif args.plottype == PLOT_TYPE_LOGY: title = "log(%s)" % (columnName,) elif args.plottype == PLOT_TYPE_CDF: title = "Cummulative distribution - %s" % (columnName,) fig.suptitle(title, y=0.99) # X-axis if args.plottype == PLOT_TYPE_STD or \ args.plottype == PLOT_TYPE_LOGY: num_years = len(x) / 365 if num_years > 4: if num_years > 10: ax.xaxis.set_major_locator(matplotlib.dates.YearLocator()) else: ax.xaxis.set_major_locator(matplotlib.dates.MonthLocator(interval=3)) else: ax.xaxis.set_major_locator(matplotlib.dates.MonthLocator()) ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%b-%Y')) # Rotate plt.setp( ax.xaxis.get_majorticklabels(), rotation=45) plt.setp( ax.xaxis.get_majorticklabels(), fontsize='x-small') if args.plottype == PLOT_TYPE_CDF: ax.set_xlim(min_x, max_x) ax.set_xscale('log') if args.xlabel: ax.set_xlabel(args.xlabel) else: ax.set_xlabel( columnName ) elif args.xlabel: ax.set_xlabel(args.xlabel) # Y-axis if args.plottype == PLOT_TYPE_LOGY: ax.set_yscale('log') if args.ylabel: ax.set_ylabel(args.ylabel) elif args.plottype != PLOT_TYPE_CDF: y_label = columnName if args.plottype == PLOT_TYPE_LOGY: y_label = "log( %s )" % (columnName,) ax.set_ylabel( y_label ) if args.supressObs: legend_items = args.legend else: data_plt.insert(0, obs_plt) legend_items = ['Observed'] + args.legend # Plot secondary data (if specified) if args.secondaryData and \ (args.plottype == PLOT_TYPE_STD or args.plottype == PLOT_TYPE_LOGY): sec_file = open(args.secondaryData, 'r') (sec_datetime, sec_data) = RHESSysOutput.readColumnFromFile(sec_file, args.secondaryColumn, startHour=0) sec_file.close() sec = pd.Series(sec_data, index=sec_datetime) # Align timeseries (sec_align, obs_align) = sec.align(obs, join='inner') # Plot ax2 = ax.twinx() if args.secondaryPlotType == 'line': (sec_plot,) = ax2.plot(x, sec_align) elif args.secondaryPlotType == 'bar': sec_plot = ax2.bar(x, sec_align, facecolor='blue', edgecolor='none', width=2.0) secondaryLabel = args.secondaryColumn.capitalize() if args.secondaryLabel: secondaryLabel = args.secondaryLabel ax2.invert_yaxis() ax2.set_ylabel(args.secondaryLabel) #ax.set_zorder(ax2.get_zorder()+1) # put ax in front of ax2 #ax.patch.set_visible(False) # hide the 'canvas' # Plot legend last num_cols = len(data) if not args.supressObs: num_cols += 1 if args.plottype == PLOT_TYPE_CDF: fig.legend( data_plt, legend_items, 'lower center', fontsize='x-small', bbox_to_anchor=(0.5, -0.015), ncol=num_cols, frameon=False ) else: fig.legend( data_plt, legend_items, 'lower center', fontsize='x-small', bbox_to_anchor=(0.5, -0.01), ncol=num_cols, frameon=False )
# Determine output variables variables = ['patchID'] + args.outputVariables sys.stdout.write('scenario,patchid') for var in args.outputVariables: sys.stdout.write(',sum_' + var) sys.stdout.write('\n') # For each rhessys output file ... for (i, patchDailyFilepath) in enumerate(patchDailyFilepaths): scenario = os.path.basename( os.path.dirname(patchDailyFilepath) ) f = open(patchDailyFilepath) data = RHESSysOutput.readColumnsFromPatchDailyFile(f, variables) f.close() if len(data) < 1: sys.exit("No data found for variable in RHESSys output file '%s'" % \ (patchDailyFilepath,) ) # For each day variablesDict = {} patchIDs = None variable = None for (i, key) in enumerate(data): if startDate and key < startDate: continue if endDate and key > endDate: break #import pdb; pdb.set_trace()
# Zoom to map extent result = grassLib.script.run_command('g.region', rast='MASK', zoom='MASK') if result != 0: sys.exit("Failed to set region to layer %s" % \ (args.mask,) ) # Set environment variables for GRASS PNG output driver os.environ['GRASS_RENDER_IMMEDIATE'] = 'FALSE' os.environ['GRASS_TRUECOLOR'] = 'TRUE' os.environ['GRASS_WIDTH'] = '960' os.environ['GRASS_HEIGHT'] = '720' # 3. Open file ending in "patch.daily" in rhessys output dir print("Reading RHESSys output data (this may take a while)...") f = open(patchDailyFilepath) data = RHESSysOutput.readColumnsFromPatchDailyFile(f, variables) f.close() if len(data) < 1: sys.exit("No data found for variable in RHESSys output file '%s'" % \ (patchDailyFilepath,) ) # 4. For each day for (i, key) in enumerate(data): dateStr = "%d/%d/%d" % (key.month, key.day, key.year) # Set filename env for PNG driver imageFilename = "%s%04d.png" % (RECLASS_MAP_TMP, i + 1) reclassImagePath = os.path.join(tmpDir, imageFilename) os.environ['GRASS_PNGFILE'] = reclassImagePath dataForDate = data[key] # a. Write reclass rule to temp file
sys.exit('A secondary data file was specified, but the secondary column to use was not') if args.data and ( len(args.data) != len(args.legend) ): sys.exit('Number of legend items must equal the number of data files') elif args.behavioralData and ( len(args.behavioralData) != len(args.legend) ): sys.exit('Number of legend items must equal the number of data files') # Open data and align to observed obs_align = None data = [] max_x = min_x = 0 if args.data: # Open observed data obs_file = open(args.obs, 'r') (obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file, readHour=False) obs_file.close() obs = pd.Series(obs_data, index=obs_datetime) for d in args.data: mod_file = open(d, 'r') (tmp_datetime, tmp_data) = RHESSysOutput.readColumnFromFile(mod_file, args.column, startHour=0) tmp_mod = pd.Series(tmp_data, index=tmp_datetime) # Align timeseries (mod_align, obs_align) = tmp_mod.align(obs, join='inner') tmp_max_x = max(mod_align.max(), obs_align.max()) if tmp_max_x > max_x: max_x = tmp_max_x min_x = max(min_x, mod_align.min()) mod_file.close()
# help='A data file containing the varaible to plot on a secondary Y-axis') parser.add_argument('--secondaryColumn', required=False, help='Name of column to use from secondary data file') parser.add_argument('--secondaryLabel', required=False, help='Label to use for seconary Y-axis') args = parser.parse_args() # if args.secondaryData and not args.secondaryColumn: # sys.exit('A secondary data file was specified, but the secondary column to use was not') # if len(args.data) != len(args.legend): # sys.exit('Number of legend items must equal the number of data files') # Open observed data obs_file = open(args.obs, 'r') (obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file, readHour=False) obs_file.close() #obs = pd.Series(obs_data, index=obs_datetime) obs = pd.DataFrame(obs_data, index=obs_datetime, columns=['observed']) # Open data and align to observed cols = ['streamflow', 'evap', 'trans', 'precip'] obs_align = None #data = [] max_x = min_x = 0 #for d in args.data: #mod_file = open(d, 'r') mod_file = open(args.data, 'r') mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols) #tmp_mod = pd.Series(tmp_data, index=tmp_datetime)
default='bar', help='Type of plot to use for secondary data.') parser.add_argument('--secondaryLabel', required=False, help='Label to use for seconary Y-axis') args = parser.parse_args() # Open observed data obs = pd.read_csv(args.obs, index_col=0, parse_dates=True) # Open data and align to observed cols = ['streamflow', 'evap', 'trans', 'precip'] obs_align = None max_x = min_x = 0 mod_file = open(args.data, 'r') mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols, readHour=False) # Align timeseries (mod_align, obs_align) = mod_df.align(obs, axis=0, join='inner') tmp_max_x = max(mod_align['streamflow'].max(), obs_align[OBS_HEADER_STREAMFLOW].max()) if tmp_max_x > max_x: max_x = tmp_max_x min_x = max(min_x, mod_align['streamflow'].min()) mod_file.close() fig = plt.figure(figsize=(args.figureX, args.figureY), dpi=80, tight_layout=True) ax_std = fig.add_subplot(221)
choices=["bar", "line"], default="bar", help="Type of plot to use for secondary data.", ) parser.add_argument("--secondaryLabel", required=False, help="Label to use for seconary Y-axis") args = parser.parse_args() # Open observed data obs = pd.read_csv(args.obs, index_col=0, parse_dates=True) # Open data and align to observed cols = ["streamflow", "evap", "trans", "precip"] obs_align = None max_x = min_x = 0 mod_file = open(args.data, "r") mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols, readHour=False) # Align timeseries (mod_align, obs_align) = mod_df.align(obs, axis=0, join="inner") tmp_max_x = max(mod_align["streamflow"].max(), obs_align[OBS_HEADER_STREAMFLOW].max()) if tmp_max_x > max_x: max_x = tmp_max_x min_x = max(min_x, mod_align["streamflow"].min()) mod_file.close() fig = plt.figure(figsize=(args.figureX, args.figureY), dpi=80, tight_layout=True) ax_std = fig.add_subplot(221) ax_log = fig.add_subplot(222) ax_cdf = fig.add_subplot(223) ax_tab = fig.add_subplot(224)
def plotGraph(args, obs, data, sizeX=1, sizeY=1, dpi=80): fig = plt.figure(figsize=(sizeX, sizeY), dpi=dpi, tight_layout=True) ax = fig.add_subplot(111) if args.plottype == PLOT_TYPE_STD or \ args.plottype == PLOT_TYPE_LOGY: x = obs.index elif args.plottype == PLOT_TYPE_CDF: x = np.linspace(min_x, max_x, num=len(obs)) # Plot observed values # Standard or log plot obs_y = obs if args.plottype == PLOT_TYPE_CDF: obs_ecdf = sm.distributions.ECDF(obs) obs_y = obs_ecdf(x) obs_plt = None if not args.supressObs: (obs_plt, ) = ax.plot(x, obs_y, linewidth=2.0, color='black') # Plot modeled values data_plt = [] for (i, d) in enumerate(data): # Standard or log plot mod_y = d if args.plottype == PLOT_TYPE_CDF: mod_ecdf = sm.distributions.ECDF(d) mod_y = mod_ecdf(x) # Plot (we could move this outside of the for loop) if args.linewidth: linewidth = args.linewidth[i] else: linewidth = 1.0 if args.linestyle: linestyle = LINE_TYPE_DICT[args.linestyle[i]] else: # Rotate styles styleIdx = ((i + 1) % NUM_LINE_TYPES) - 1 linestyle = LINE_TYPE_DICT[LINE_TYPES[styleIdx]] if args.color: (mod_plt, ) = ax.plot(x, mod_y, linewidth=linewidth, linestyle=linestyle, color=args.color[i]) else: (mod_plt, ) = ax.plot(x, mod_y, linewidth=linewidth, linestyle=linestyle) data_plt.append(mod_plt) # Plot annotations columnName = args.column.capitalize() if args.title: title = args.title else: if args.plottype == PLOT_TYPE_STD: title = columnName elif args.plottype == PLOT_TYPE_LOGY: title = "log(%s)" % (columnName, ) elif args.plottype == PLOT_TYPE_CDF: title = "Cummulative distribution - %s" % (columnName, ) fig.suptitle(title, y=0.99) # X-axis if args.plottype == PLOT_TYPE_STD or \ args.plottype == PLOT_TYPE_LOGY: num_years = len(x) / 365 if num_years > 4: if num_years > 10: ax.xaxis.set_major_locator(matplotlib.dates.YearLocator()) else: ax.xaxis.set_major_locator( matplotlib.dates.MonthLocator(interval=3)) else: ax.xaxis.set_major_locator(matplotlib.dates.MonthLocator()) ax.xaxis.set_major_formatter(matplotlib.dates.DateFormatter('%b-%Y')) # Rotate plt.setp(ax.xaxis.get_majorticklabels(), rotation=45) plt.setp(ax.xaxis.get_majorticklabels(), fontsize='x-small') if args.plottype == PLOT_TYPE_CDF: ax.set_xlim(min_x, max_x) ax.set_xscale('log') if args.xlabel: ax.set_xlabel(args.xlabel) else: ax.set_xlabel(columnName) elif args.xlabel: ax.set_xlabel(args.xlabel) # Y-axis if args.plottype == PLOT_TYPE_LOGY: ax.set_yscale('log') if args.ylabel: ax.set_ylabel(args.ylabel) elif args.plottype != PLOT_TYPE_CDF: y_label = columnName if args.plottype == PLOT_TYPE_LOGY: y_label = "log( %s )" % (columnName, ) ax.set_ylabel(y_label) if args.supressObs: legend_items = args.legend else: data_plt.insert(0, obs_plt) legend_items = ['Observed'] + args.legend # Plot secondary data (if specified) if args.secondaryData and \ (args.plottype == PLOT_TYPE_STD or args.plottype == PLOT_TYPE_LOGY): sec_file = open(args.secondaryData, 'r') (sec_datetime, sec_data) = RHESSysOutput.readColumnFromFile(sec_file, args.secondaryColumn, startHour=0) sec_file.close() sec = pd.Series(sec_data, index=sec_datetime) # Align timeseries (sec_align, obs_align) = sec.align(obs, join='inner') # Plot ax2 = ax.twinx() if args.secondaryPlotType == 'line': (sec_plot, ) = ax2.plot(x, sec_align) elif args.secondaryPlotType == 'bar': sec_plot = ax2.bar(x, sec_align, facecolor='blue', edgecolor='none', width=2.0) secondaryLabel = args.secondaryColumn.capitalize() if args.secondaryLabel: secondaryLabel = args.secondaryLabel ax2.invert_yaxis() ax2.set_ylabel(args.secondaryLabel) #ax.set_zorder(ax2.get_zorder()+1) # put ax in front of ax2 #ax.patch.set_visible(False) # hide the 'canvas' # Plot legend last num_cols = len(data) if not args.supressObs: num_cols += 1 if args.plottype == PLOT_TYPE_CDF: fig.legend(data_plt, legend_items, 'lower center', fontsize='x-small', bbox_to_anchor=(0.5, -0.015), ncol=num_cols, frameon=False) else: fig.legend(data_plt, legend_items, 'lower center', fontsize='x-small', bbox_to_anchor=(0.5, -0.01), ncol=num_cols, frameon=False)
if args.data and (len(args.data) != len(args.legend)): sys.exit('Number of legend items must equal the number of data files') elif args.behavioralData and (len(args.behavioralData) != len( args.legend)): sys.exit('Number of legend items must equal the number of data files') # Open data and align to observed obs_align = None data = [] max_x = min_x = 0 if args.data: # Open observed data obs_file = open(args.obs, 'r') (obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file, readHour=False) obs_file.close() obs = pd.Series(obs_data, index=obs_datetime) for d in args.data: mod_file = open(d, 'r') (tmp_datetime, tmp_data) = RHESSysOutput.readColumnFromFile(mod_file, args.column, startHour=0) tmp_mod = pd.Series(tmp_data, index=tmp_datetime) # Align timeseries (mod_align, obs_align) = tmp_mod.align(obs, join='inner') tmp_max_x = max(mod_align.max(), obs_align.max()) if tmp_max_x > max_x: max_x = tmp_max_x
def readBehavioralData( self, basedir, session_id, variable="streamflow", observed_file=None, behavioral_filter=None, end_date=None ): dbPath = RHESSysCalibrator.getDBPath(basedir) if not os.access(dbPath, os.R_OK): raise IOError(errno.EACCES, "The database at %s is not readable" % dbPath) self.logger.debug("DB path: %s" % dbPath) outputPath = RHESSysCalibrator.getOutputPath(basedir) if not os.access(outputPath, os.R_OK): raise IOError(errno.EACCES, "The output directory %s is not readable" % outputPath) self.logger.debug("Output path: %s" % outputPath) rhessysPath = RHESSysCalibrator.getRhessysPath(basedir) calibratorDB = ModelRunnerDB(RHESSysCalibrator.getDBPath(basedir)) # Make sure the session exists session = calibratorDB.getSession(session_id) if None == session: raise Exception("Session %d was not found in the calibration database %s" % (session_id, dbPath)) if session.status != "complete": print "WARNING: session status is: %s. Some model runs may not have completed." % (session.status,) else: self.logger.debug("Session status is: %s" % (session.status,)) # Determine observation file path if observed_file: obs_file = observed_file else: # Get observered file from session assert session.obs_filename != None obs_file = session.obs_filename obsPath = RHESSysCalibrator.getObsPath(basedir) obsFilePath = os.path.join(obsPath, obs_file) if not os.access(obsFilePath, os.R_OK): raise IOError(errno.EACCES, "The observed data file %s is not readable" % obsFilePath) self.logger.debug("Obs path: %s" % obsFilePath) # Get runs in session runs = calibratorDB.getRunsInSession(session.id, where_clause=behavioral_filter) numRuns = len(runs) if numRuns == 0: raise Exception("No runs found for session %d" % (session.id,)) response = raw_input( "%d runs selected for plotting from session %d in basedir '%s', continue? [yes | no] " % (numRuns, session_id, os.path.basename(basedir)) ) response = response.lower() if response != "y" and response != "yes": # Exit normally return 0 self.logger.debug("%d behavioral runs" % (numRuns,)) # Read observed data from file obsFile = open(obsFilePath, "r") (obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obsFile) obsFile.close() obs = pd.Series(obs_data, index=obs_datetime) if end_date: obs = obs[:end_date] self.logger.debug("Observed data: %s" % obs_data) likelihood = np.empty(numRuns) ysim = None x = None runsProcessed = False for (i, run) in enumerate(runs): if "DONE" == run.status: runOutput = os.path.join(rhessysPath, run.output_path) self.logger.debug(">>>\nOutput dir of run %d is %s" % (run.id, runOutput)) tmpOutfile = RHESSysCalibrator.getRunOutputFilePath(runOutput) if not os.access(tmpOutfile, os.R_OK): print "Output file %s for run %d not found or not readable, unable to calculate fitness statistics for this run" % ( tmpOutfile, run.id, ) continue tmpFile = open(tmpOutfile, "r") (tmp_datetime, tmp_data) = RHESSysOutput.readColumnFromFile(tmpFile, "streamflow") tmp_mod = pd.Series(tmp_data, index=tmp_datetime) # Align timeseries to observed (mod, obs) = tmp_mod.align(obs, join="inner") # Stash date for X values (assume they are the same for all runs if x == None: x = [datetime.strptime(str(d), "%Y-%m-%d %H:%M:%S") for d in mod.index] # Put data in matrix dataLen = len(mod) if ysim == None: # Allocate matrix for results ysim = np.empty((numRuns, dataLen)) assert np.shape(ysim)[1] == dataLen ysim[i,] = mod # Store fitness parameter likelihood[i] = run.nse tmpFile.close() runsProcessed = True return (runsProcessed, obs, x, ysim, likelihood)
default=False, help="Plot in color") parser.add_argument('--secondaryPlotType', required=False, choices=['bar', 'line'], default='bar', help='Type of plot to use for secondary data.') parser.add_argument('--secondaryLabel', required=False, help='Label to use for seconary Y-axis') args = parser.parse_args() # Open observed data obs_file = open(args.obs, 'r') (obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obs_file, readHour=False) obs_file.close() obs = pd.DataFrame(obs_data, index=obs_datetime, columns=['observed']) # Open data and align to observed cols = ['streamflow', 'evap', 'trans', 'precip'] obs_align = None max_x = min_x = 0 mod_file = open(args.data, 'r') mod_df = RHESSysOutput.readColumnsFromFile(mod_file, cols, readHour=False) # Align timeseries (mod_align, obs_align) = mod_df.align(obs, axis=0, join='inner') tmp_max_x = max(mod_align['streamflow'].max(), obs_align['observed'].max()) if tmp_max_x > max_x: max_x = tmp_max_x