def setUpClass(cls):
# We zip the test RHESSys mode to be nice to GitHub, unzip it
cls.basedir = os.path.abspath('./rhessyscalibrator/tests/data/DR5')
basedirZip = "%s.zip" % (cls.basedir, )
if not os.access(basedirZip, os.R_OK):
raise IOError(
errno.EACCES,
"Unable to read test RHESSys model zip %s" % basedirZip)
basedirParent = os.path.split(basedirZip)[0]
if not os.access(basedirParent, os.W_OK):
raise IOError(
errno.EACCES,
"Unable to write to test RHESSys model parent dir %s" %
basedirParent)
zip = ZipFile(basedirZip, 'r')
extractDir = os.path.split(cls.basedir)[0]
zip.extractall(path=extractDir)
# Build RHESSys
currDir = os.getcwd()
os.chdir('./rhessyscalibrator/tests/data/DR5/rhessys/src/rhessys')
res = subprocess.call('make install RHESSYS_BIN=../../bin', shell=True)
# Make sure binary is executable else some tests will fail
os.chmod('../../bin/rhessys5.15',
stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
os.chdir(currDir)
cls.clusterCalibrator = RHESSysCalibrator()
def main(self, args):
# Set up command line options
parser = argparse.ArgumentParser(description="Tool for performing behavioral model runs for RHESSys")
parser.add_argument("-b", "--basedir", action="store",
dest="basedir", required=True,
help="Base directory for the calibration session")
parser.add_argument("-s", "--postprocess_session", action="store", type=int,
dest="postprocess_id", required=True,
help="Post-process session to use for behavioral runs.")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-st", dest="startDate", nargs=4, type=int,
help='Start date and time for behavioral model runs, of the form "YYYY M D H"')
group.add_argument("-c", "--cmdproto", dest="cmdproto",
help="Filename of cmd.proto to use for behavioral runs (relative to basedir)")
parser.add_argument('-ed', dest='endDate', required=False, nargs=4, type=int,
help='Date date and time for behavioral model runs, of the form "YYYY M D H"')
parser.add_argument("-f", "--behavioral_filter", action="store",
dest="behavioral_filter", required=False,
default="nse>0.5 and nse_log>0.5",
help="SQL where clause to use to determine which runs qualify as behavioral parameters. E.g. 'nse>0.5 AND nse_log>0.5' (use quotes)")
parser.add_argument("-u", "--user", action="store",
dest="user", required=False, default=os.getlogin(),
help="User to associate with the calibration session. If not supplied, the value of os.getlogin() will be used.")
parser.add_argument("-p", "--project", action="store",
dest="project", required=True,
help="Name of the project ot associate with the calibration session.")
parser.add_argument("-j", "--jobs", action="store", type=calibrator.num_jobs_type,
dest="processes", required=True,
help="The number of simultaneous jobs (runs) to run at any given time in the calibration session (e.g. --jobs 32). Maximum is %s." % (calibrator.MAX_PROCESSORS,) )
parser.add_argument("--simulator_path", action="store",
dest="simulator_path", required=False,
help="Set path for LSF simulator. When supplied, jobs will be submitted to the simulator, not via actual LSF commands. Must be the absolute path (e.g. /Users/joeuser/rhessys_calibrator/lsf-sim)")
parser.add_argument("-q", "--queue", action="store",
dest="queue_name", required=False,
help="Set queue name to submit jobs to using the underlying queue manager. " +
"Applies only to non-process-based calibration runners (specified by parallel_mode option).")
parser.add_argument("--parallel_mode", action="store",
dest="parallel_mode", required=False,
default='lsf', choices=calibrator.PARALLEL_MODES,
help="Set method to use for running jobs in parallel.")
parser.add_argument("--polling_delay", action="store", type=calibrator.polling_delay_type,
dest="polling_delay", required=False,
default=1,
help="[ADVANCED] Set multiplier for how long to wait in between successive pollings of job status. Default polling delay is 60 seconds, thus a multiplier of 5 will result in a delay of 5 minutes instead of 1 minute. Maximum is %d." % (calibrator.MAX_POLLING_DELAY_MULT,) )
parser.add_argument("--bsub_exclusive_mode", action="store_true",
dest="bsub_exclusive_mode", required=False,
help="[ADVANCED] For LSF parallel mode: run bsub with arguments \"-n 1 -R 'span[hosts=1]' -x\" to ensure jobs only run exclusively (i.e. the only job on a node). This can be useful for models that use a lot of memory.")
parser.add_argument("--mem_limit", action="store", type=int,
dest="mem_limit", required=False,
default=4,
help="For non-process based parallel modes: Specify memory limit for jobs. Unit: gigabytes Defaults to 4.")
parser.add_argument("--wall_time", action="store",
type=int, dest="wall_time",
help="For PBS- and SLURM-based parallel mode: Specify wall time in hours that jobs should take.")
parser.add_argument("-l", "--loglevel", action="store",
dest="loglevel", default="OFF", choices=['OFF', 'DEBUG', 'CRITICAL'], required=False,
help="Set logging level")
options = parser.parse_args()
# Handle command line parameters
if "DEBUG" == options.loglevel:
self._initLogger(logging.DEBUG)
elif "CRITICAL" == options.loglevel:
self._initLogger(logging.CRITICAL)
else:
self._initLogger(logging.NOTSET)
if options.parallel_mode != calibrator.PARALLEL_MODE_PROCESS and not options.queue_name:
sys.exit("""Please specify a queue/partition name that is valid for your system.""")
wall_time = None
if options.wall_time:
if options.parallel_mode == calibrator.PARALLEL_MODE_PBS or options.parallel_mode == calibrator.PARALLEL_MODE_SLURM:
if options.wall_time < 1 or options.wall_time > 168:
sys.exit("Wall time must be greater than 0 and less than 169 hours")
wall_time = options.wall_time
if not os.path.isdir(options.basedir) or not os.access(options.basedir, os.R_OK):
sys.exit("Unable to read project directory %s" % (options.basedir,) )
self.basedir = os.path.abspath(options.basedir)
self.logger.debug("parallel mode: %s" % options.parallel_mode)
self.logger.debug("basedir: %s" % self.basedir)
self.logger.debug("user: %s" % options.user)
self.logger.debug("project: %s" % options.project)
self.logger.debug("jobs: %d" % options.processes)
if options.startDate:
if not options.endDate:
sys.exit("You must specify a simulation end date")
startDate = datetime(options.startDate[0],
options.startDate[1],
options.startDate[2],
options.startDate[3])
endDate = datetime(options.endDate[0],
options.endDate[1],
options.endDate[2],
options.endDate[3])
# Make sure start date is before end date
if startDate >= endDate:
sys.exit("Start date %s is not before end date %s" % (str(startDate), str(endDate)) )
startDateStr = ' '.join([str(d) for d in options.startDate])
endDateStr = ' '.join([str(d) for d in options.endDate])
readCmdProtoFromRun = True
self.logger.debug("start date: %s" % (startDate,) )
self.logger.debug("end date: %s" % (endDate,) )
if options.cmdproto:
cmdProtoPath = os.path.join(self.basedir, options.cmdproto)
if not os.access(cmdProtoPath, os.R_OK):
sys.exit("Unable to read behavioral cmd.proto: %s" % (cmdProtoPath,) )
readCmdProtoFromRun = False
self.logger.debug("behavioral cmd.proto: %s" % (cmdProtoPath,) )
notes = "Behavioral run, using filter: %s" % (options.behavioral_filter,)
try:
dbPath = RHESSysCalibrator.getDBPath(self.basedir)
self.calibratorDB = ModelRunnerDB2(dbPath)
# Get post-process session
postproc = self.calibratorDB.getPostProcess(options.postprocess_id)
if None == postproc:
raise Exception("Post-process session %d was not found in the calibration database %s" % (options.postprocess_id, dbPath))
# Get session
calibSession = self.calibratorDB.getSession(postproc.session_id)
if None == calibSession:
raise Exception("Session %d was not found in the calibration database %s" % (postproc.session_id, dbPath))
calibItr = calibSession.iterations
# Get runs in calibration session
runs = self.calibratorDB.getRunsInPostProcess(postproc.id, where_clause=options.behavioral_filter)
numRuns = len(runs)
print("\n{0}\n".format(notes))
response = raw_input("%d runs selected of %d total runs (%.2f%%) in post process session %d, calibration session %d, continue? [yes | no] " % \
(numRuns, calibItr, (float(numRuns) / float(calibItr)) * 100, options.postprocess_id, postproc.session_id ) )
response = response.lower()
if response != 'y' and response != 'yes':
# Exit normally
return 0
self.logger.debug("%d runs selected" % (numRuns,) )
# Make sure we have everything we need to run behavioral runs
# Get list of worldfiles
self.worldfiles = self.getWorldfiles(self.basedir)
if len(self.worldfiles) < 1:
raise Exception("No worldfiles found")
self.logger.debug("worldfiles: %s" % self.worldfiles)
# Get tecfile name
(res, tecfilePath) = self.getTecfilePath(self.basedir)
if not res:
raise Exception("No tecfile found")
# Get RHESSys executable path
(rhessysExecFound, rhessysExec, rhessysExecPath) = \
self.getRHESSysExecPath(self.basedir)
if not rhessysExecFound:
raise Exception("RHESSys executable not found")
if readCmdProtoFromRun:
# Rewrite cmd_proto to use dates from command line
cmd_proto = re.sub("-st (\d{4} \d{1,2} \d{1,2} \d{1,2})",
"-st %s" % (startDateStr,),
calibSession.cmd_proto)
cmd_proto = re.sub("-ed (\d{4} \d{1,2} \d{1,2} \d{1,2})",
"-ed %s" % (endDateStr,),
cmd_proto)
self.logger.debug("Original cmd.proto: %s" % (calibSession.cmd_proto,) )
self.logger.debug("Behavioral cmd.proto: %s" % (cmd_proto,) )
else:
# Use cmd proto from file
fd = open(cmdProtoPath)
cmd_proto = fd.read()
fd.close()
# Strip any parameter ranges from cmd.proto
cmd_proto_noparam = self.stripParameterRangesFromCmdProto(cmd_proto)
# Pre-process cmd.proto to add rhessys exec and tecfile path
cmd_proto_pre = self.preProcessCmdProto(cmd_proto_noparam,
os.path.join(rhessysExecPath, rhessysExec),
tecfilePath)
# Check for explicit routing and surface flowtable in cmd_proto, get dicts of
# flowtables from basedir
(self.flowtablePath, self.surfaceFlowtablePath) = self.determineRouting(cmd_proto_noparam)
# Create behavioral session
self.session = self.createCalibrationSession(options.user,
options.project,
numRuns,
options.processes,
self.basedir,
notes,
cmd_proto_noparam)
# Create postprocess session
behave_postproc_id = self.calibratorDB.insertPostProcess(self.session.id, postproc.obs_filename, postproc.fitness_period,
exclude_date_ranges=postproc.exclude_date_ranges)
# Initialize CalibrationRunner consumers for executing jobs
(runQueue, consumers) = \
RHESSysCalibrator.initializeCalibrationRunnerConsumers(self.basedir, self.logger,
self.session.id, options.parallel_mode, options.processes, options.polling_delay,
options.queue_name,
mem_limit=options.mem_limit,
wall_time=wall_time,
bsub_exclusive_mode=options.bsub_exclusive_mode,
simulator_path=options.simulator_path)
# Dispatch runs to consumer
# Note: we're iterating over behavioral runs to get their paramter values
for (i, run) in enumerate(runs):
itr = i + 1
# Get parameters for run
parameterValues = run.getCalibrationParameters()
itr_cmd_proto = self.addParametersToCmdProto(cmd_proto_pre,
parameterValues)
# For each world file
for worldfile in self.worldfiles.keys():
self.logger.critical("Iteration %d, worldfile: %s" %
(itr, worldfile))
# Create new ModelRun2 object for this run
behavioralRun = ModelRun2()
behavioralRun.session_id = self.session.id
behavioralRun.worldfile = worldfile
behavioralRun.setCalibrationParameters(parameterValues)
# Create new Runfitness2 object for this run, copying fitness parameters so that we can
# draw undercertainty bounds later
behavioralRunfit = RunFitness2()
behavioralRunfit.postprocess_id = behave_postproc_id
behavioralRunfit.nse = run.run_fitness.nse
behavioralRunfit.nse_log = run.run_fitness.nse_log
behavioralRunfit.pbias = run.run_fitness.pbias
behavioralRunfit.rsr = run.run_fitness.rsr
behavioralRunfit.runoff_ratio = run.run_fitness.runoff_ratio
behavioralRunfit.userfitness = run.run_fitness.userfitness
behavioralRun.run_fitness = behavioralRunfit
# Add worldfile and flowtable paths to command
if self.explicitRouting:
if self.surfaceFlowtable:
cmd_raw_proto = self.addWorldfileAndFlowtableToCmdProto(\
itr_cmd_proto, self.worldfiles[worldfile],
self.flowtablePath[worldfile],
self.surfaceFlowtablePath[worldfile])
else:
cmd_raw_proto = self.addWorldfileAndFlowtableToCmdProto(\
itr_cmd_proto, self.worldfiles[worldfile],
self.flowtablePath[worldfile])
else:
cmd_raw_proto = self.addWorldfileToCmdProto(\
itr_cmd_proto, self.worldfiles[worldfile])
# Finally, create output_path and generate cmd_raw
behavioralRun.output_path = self.createOutputPath(self.basedir,
self.session.id,
worldfile,
itr)
behavioralRun.cmd_raw = self.getCmdRawForRun(cmd_raw_proto,
behavioralRun.output_path)
if "process" == options.parallel_mode:
# Set job ID if we are in process parallel mode
# (in lsf mode, we will use the LSF job number instead of itr)
behavioralRun.job_id = itr
# Dispatch to consumer
runQueue.put(behavioralRun)
time.sleep(5)
# Wait for all jobs to finish
self.logger.critical("calling runQueue.join() ...")
runQueue.join()
for consumerProcess in consumers:
consumerProcess.join()
# Update session endtime and status
self.calibratorDB.updateSessionEndtime(self.session.id,
datetime.utcnow(),
"complete")
print("\n\nBehavioral results saved to session {0}, post-process session {1}".format(self.session.id, behave_postproc_id))
except:
raise
else:
self.logger.debug("exiting normally")
return 0
finally:
self.calibratorDB = None
def __init__(self):
RHESSysCalibrator.__init__(self)
CalibrationRunner class and descendants
20110626: 1.0.3: refactored CalibrationRunner to ModelRunner for generalization
only update accounting of active jobs after verifying that
job returned by bjobs belongs to us (is in current session
record)
20110628: fixed bug where --create option required iterations and jobs options
to be specified
20110710: 1.0.4: Added subprocess parallelization method; Removed pipe-based IPC method.
20120105: 1.0.5: Added support for running RHESSys in TOPMODEL model (i.e. no flow tables);
Added support for using the same calibration parameters for both horizontal
and vertical m parameter
Changed LSF queues to reflect those on KillDevil
20120114: 1.0.6: Added support for running bsub with exclusive mode parameter
Removed old PIPE-based CalibrationRunner implementation
20120215: 1.0.7: Updated --use_horizontal_m_for_vertical to apply also to the K parameter
(new option is --use_horizontal_m_and_K_for_vertical)
20120327: 1.0.8: Added support for vgsen and svalt calibration parameters
20120328: 1.0.9: Fixed bug where third -s parameter was not set by CalibrationParameters
20120529: 1.0.10: Added filter for worldfile lookup that excludes redefine worldfiles.
"""
import sys
from rhessyscalibrator.calibrator import RHESSysCalibrator
if __name__ == "__main__":
RHESSysCalibrator = RHESSysCalibrator()
# main's return value will be the exit code
sys.exit(RHESSysCalibrator.main(sys.argv))
def readBehavioralData(
self, basedir, session_id, variable="streamflow", observed_file=None, behavioral_filter=None, end_date=None
):
dbPath = RHESSysCalibrator.getDBPath(basedir)
if not os.access(dbPath, os.R_OK):
raise IOError(errno.EACCES, "The database at %s is not readable" % dbPath)
self.logger.debug("DB path: %s" % dbPath)
outputPath = RHESSysCalibrator.getOutputPath(basedir)
if not os.access(outputPath, os.R_OK):
raise IOError(errno.EACCES, "The output directory %s is not readable" % outputPath)
self.logger.debug("Output path: %s" % outputPath)
rhessysPath = RHESSysCalibrator.getRhessysPath(basedir)
calibratorDB = ModelRunnerDB(RHESSysCalibrator.getDBPath(basedir))
# Make sure the session exists
session = calibratorDB.getSession(session_id)
if None == session:
raise Exception("Session %d was not found in the calibration database %s" % (session_id, dbPath))
if session.status != "complete":
print "WARNING: session status is: %s. Some model runs may not have completed." % (session.status,)
else:
self.logger.debug("Session status is: %s" % (session.status,))
# Determine observation file path
if observed_file:
obs_file = observed_file
else:
# Get observered file from session
assert session.obs_filename != None
obs_file = session.obs_filename
obsPath = RHESSysCalibrator.getObsPath(basedir)
obsFilePath = os.path.join(obsPath, obs_file)
if not os.access(obsFilePath, os.R_OK):
raise IOError(errno.EACCES, "The observed data file %s is not readable" % obsFilePath)
self.logger.debug("Obs path: %s" % obsFilePath)
# Get runs in session
runs = calibratorDB.getRunsInSession(session.id, where_clause=behavioral_filter)
numRuns = len(runs)
if numRuns == 0:
raise Exception("No runs found for session %d" % (session.id,))
response = raw_input(
"%d runs selected for plotting from session %d in basedir '%s', continue? [yes | no] "
% (numRuns, session_id, os.path.basename(basedir))
)
response = response.lower()
if response != "y" and response != "yes":
# Exit normally
return 0
self.logger.debug("%d behavioral runs" % (numRuns,))
# Read observed data from file
obsFile = open(obsFilePath, "r")
(obs_datetime, obs_data) = RHESSysOutput.readObservedDataFromFile(obsFile)
obsFile.close()
obs = pd.Series(obs_data, index=obs_datetime)
if end_date:
obs = obs[:end_date]
self.logger.debug("Observed data: %s" % obs_data)
likelihood = np.empty(numRuns)
ysim = None
x = None
runsProcessed = False
for (i, run) in enumerate(runs):
if "DONE" == run.status:
runOutput = os.path.join(rhessysPath, run.output_path)
self.logger.debug(">>>\nOutput dir of run %d is %s" % (run.id, runOutput))
tmpOutfile = RHESSysCalibrator.getRunOutputFilePath(runOutput)
if not os.access(tmpOutfile, os.R_OK):
print "Output file %s for run %d not found or not readable, unable to calculate fitness statistics for this run" % (
tmpOutfile,
run.id,
)
continue
tmpFile = open(tmpOutfile, "r")
(tmp_datetime, tmp_data) = RHESSysOutput.readColumnFromFile(tmpFile, "streamflow")
tmp_mod = pd.Series(tmp_data, index=tmp_datetime)
# Align timeseries to observed
(mod, obs) = tmp_mod.align(obs, join="inner")
# Stash date for X values (assume they are the same for all runs
if x == None:
x = [datetime.strptime(str(d), "%Y-%m-%d %H:%M:%S") for d in mod.index]
# Put data in matrix
dataLen = len(mod)
if ysim == None:
# Allocate matrix for results
ysim = np.empty((numRuns, dataLen))
assert np.shape(ysim)[1] == dataLen
ysim[i,] = mod
# Store fitness parameter
likelihood[i] = run.nse
tmpFile.close()
runsProcessed = True
return (runsProcessed, obs, x, ysim, likelihood)
def main(self, args):
# Set up command line options
parser = argparse.ArgumentParser(description="Tool for performing behavioral model runs for RHESSys")
parser.add_argument(
"-b",
"--basedir",
action="store",
dest="basedir",
required=True,
help="Base directory for the calibration session",
)
parser.add_argument(
"-s",
"--behavioral_session",
action="store",
type=int,
dest="session_id",
required=True,
help="Session to use for behavioral runs.",
)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
"-st",
dest="startDate",
nargs=4,
type=int,
help='Start date and time for behavioral model runs, of the form "YYYY M D H"',
)
group.add_argument(
"-c",
"--cmdproto",
dest="cmdproto",
help="Filename of cmd.proto to use for behavioral runs (relative to basedir)",
)
parser.add_argument(
"-ed",
dest="endDate",
required=False,
nargs=4,
type=int,
help='Date date and time for behavioral model runs, of the form "YYYY M D H"',
)
parser.add_argument(
"-f",
"--behavioral_filter",
action="store",
dest="behavioral_filter",
required=False,
default="nse>0.5 and nse_log>0.5",
help="SQL where clause to use to determine which runs qualify as behavioral parameters. E.g. 'nse>0.5 AND nse_log>0.5' (use quotes)",
)
parser.add_argument(
"-u",
"--user",
action="store",
dest="user",
required=False,
default=os.getlogin(),
help="User to associate with the calibration session. If not supplied, the value of os.getlogin() will be used.",
)
parser.add_argument(
"-p",
"--project",
action="store",
dest="project",
required=True,
help="Name of the project ot associate with the calibration session.",
)
parser.add_argument(
"-j",
"--jobs",
action="store",
type=calibrator.num_jobs_type,
dest="processes",
required=True,
help="The number of simultaneous jobs (runs) to run at any given time in the calibration session (e.g. --jobs 32). Maximum is %s."
% (calibrator.MAX_PROCESSORS,),
)
parser.add_argument(
"--simulator_path",
action="store",
dest="simulator_path",
required=False,
help="Set path for LSF simulator. When supplied, jobs will be submitted to the simulator, not via actual LSF commands. Must be the absolute path (e.g. /Users/joeuser/rhessys_calibrator/lsf-sim)",
)
parser.add_argument(
"-q",
"--queue",
action="store",
dest="lsf_queue",
required=False,
default="day",
choices=calibrator.LSF_QUEUES,
help="Set queue name to pass to LSF job submission command.",
)
parser.add_argument(
"--parallel_mode",
action="store",
dest="parallel_mode",
required=False,
default="lsf",
choices=calibrator.PARALLEL_MODES,
help="Set method to use for running jobs in parallel.",
)
parser.add_argument(
"--polling_delay",
action="store",
type=calibrator.polling_delay_type,
dest="polling_delay",
required=False,
default=1,
help="[ADVANCED] Set multiplier for how long to wait in between successive pollings of job status. Default polling delay is 60 seconds, thus a multiplier of 5 will result in a delay of 5 minutes instead of 1 minute. Maximum is %d."
% (calibrator.MAX_POLLING_DELAY_MULT,),
)
parser.add_argument(
"--bsub_exclusive_mode",
action="store_true",
dest="bsub_exclusive_mode",
required=False,
help="[ADVANCED] run bsub with arguments \"-n 1 -R 'span[hosts=1]' -x\" to ensure jobs only run exclusively (i.e. the only job on a node). This can be useful for models that use a lot of memory.",
)
parser.add_argument(
"--bsub_mem_limit",
action="store",
type=int,
dest="bsub_mem_limit",
required=False,
default=4,
help="[ADVANCED] run bsub with -M mem_limit option. Defaults to 4GB",
)
parser.add_argument(
"-l",
"--loglevel",
action="store",
dest="loglevel",
default="OFF",
choices=["OFF", "DEBUG", "CRITICAL"],
required=False,
help="Set logging level",
)
options = parser.parse_args()
# Handle command line parameters
if "DEBUG" == options.loglevel:
self._initLogger(logging.DEBUG)
elif "CRITICAL" == options.loglevel:
self._initLogger(logging.CRITICAL)
else:
self._initLogger(logging.NOTSET)
if not os.path.isdir(options.basedir) or not os.access(options.basedir, os.R_OK):
sys.exit("Unable to read project directory %s" % (options.basedir,))
self.basedir = os.path.abspath(options.basedir)
self.logger.critical("parallel mode: %s" % options.parallel_mode)
self.logger.debug("basedir: %s" % self.basedir)
self.logger.debug("user: %s" % options.user)
self.logger.debug("project: %s" % options.project)
self.logger.debug("jobs: %d" % options.processes)
if options.startDate:
if not options.endDate:
sys.exit("You must specify a simulation end date")
startDate = datetime(options.startDate[0], options.startDate[1], options.startDate[2], options.startDate[3])
endDate = datetime(options.endDate[0], options.endDate[1], options.endDate[2], options.endDate[3])
# Make sure start date is before end date
if startDate >= endDate:
sys.exit("Start date %s is not before end date %s" % (str(startDate), str(endDate)))
startDateStr = " ".join([str(d) for d in options.startDate])
endDateStr = " ".join([str(d) for d in options.endDate])
readCmdProtoFromRun = True
self.logger.debug("start date: %s" % (startDate,))
self.logger.debug("end date: %s" % (endDate,))
if options.cmdproto:
cmdProtoPath = os.path.join(self.basedir, options.cmdproto)
if not os.access(cmdProtoPath, os.R_OK):
sys.exit("Unable to read behavioral cmd.proto: %s" % (cmdProtoPath,))
readCmdProtoFromRun = False
self.logger.debug("behavioral cmd.proto: %s" % (cmdProtoPath,))
notes = "Behavioral run, using filter: %s" % (options.behavioral_filter,)
if options.simulator_path:
run_cmd = RHESSysCalibrator.getRunCmdSim(options.simulator_path)
run_status_cmd = RHESSysCalibrator.getRunStatusCmdSim(options.simulator_path)
elif "lsf" == options.parallel_mode:
run_cmd = RHESSysCalibrator.getRunCmd(options.bsub_mem_limit, options.bsub_exclusive_mode)
run_status_cmd = RHESSysCalibrator.getRunStatusCmd()
else:
run_cmd = run_status_cmd = None
try:
dbPath = RHESSysCalibrator.getDBPath(self.basedir)
self.calibratorDB = ModelRunnerDB(dbPath)
# Get calibration session
calibSession = self.calibratorDB.getSession(options.session_id)
if None == calibSession:
raise Exception(
"Session %d was not found in the calibration database %s" % (options.session_id, dbPath)
)
calibItr = calibSession.iterations
# Get runs in calibration session
runs = self.calibratorDB.getRunsInSession(calibSession.id, options.behavioral_filter)
numRuns = len(runs)
print(notes)
response = raw_input(
"%d runs selected of %d total runs (%.2f%%) in session %d, continue? [yes | no] "
% (numRuns, calibItr, (float(numRuns) / float(calibItr)) * 100, options.session_id)
)
response = response.lower()
if response != "y" and response != "yes":
# Exit normally
return 0
self.logger.debug("%d runs selected" % (numRuns,))
# Make sure we have everything we need to run behavioral runs
# Get list of worldfiles
self.worldfiles = self.getWorldfiles(self.basedir)
if len(self.worldfiles) < 1:
raise Exception("No worldfiles found")
self.logger.debug("worldfiles: %s" % self.worldfiles)
# Get tecfile name
(res, tecfilePath) = self.getTecfilePath(self.basedir)
if not res:
raise Exception("No tecfile found")
# Get RHESSys executable path
(rhessysExecFound, rhessysExec, rhessysExecPath) = self.getRHESSysExecPath(self.basedir)
if not rhessysExecFound:
raise Exception("RHESSys executable not found")
if readCmdProtoFromRun:
# Rewrite cmd_proto to use dates from command line
cmd_proto = re.sub(
"-st (\d{4} \d{1,2} \d{1,2} \d{1,2})", "-st %s" % (startDateStr,), calibSession.cmd_proto
)
cmd_proto = re.sub("-ed (\d{4} \d{1,2} \d{1,2} \d{1,2})", "-ed %s" % (endDateStr,), cmd_proto)
self.logger.debug("Original cmd.proto: %s" % (calibSession.cmd_proto,))
self.logger.debug("Behavioral cmd.proto: %s" % (cmd_proto,))
else:
# Use cmd proto from file
fd = open(cmdProtoPath)
cmd_proto = fd.read()
fd.close()
# Pre-process cmd.proto to add rhessys exec and tecfile path
cmd_proto_pre = self.preProcessCmdProto(cmd_proto, os.path.join(rhessysExecPath, rhessysExec), tecfilePath)
# Check for explicit routing and surface flowtable in cmd_proto, get dicts of
# flowtables from basedir
(self.flowtablePath, self.surfaceFlowtablePath) = self.determineRouting(cmd_proto)
# Create behavioral session
self.session = self.createCalibrationSession(
options.user, options.project, numRuns, options.processes, self.basedir, notes, cmd_proto
)
# Get observation file from calibrationSession
self.session.obs_filename = calibSession.obs_filename
self.calibratorDB.updateSessionObservationFilename(self.session.id, self.session.obs_filename)
# Initialize CalibrationRunner consumers for executing jobs
(runQueue, consumers) = RHESSysCalibrator.initializeCalibrationRunnerConsumers(
self.basedir,
self.logger,
self.session.id,
options.parallel_mode,
options.processes,
options.polling_delay,
options.lsf_queue,
run_cmd,
run_status_cmd,
)
# Dispatch runs to consumer
# Note: we're iterating over behavioral runs to get their paramter values
for (i, run) in enumerate(runs):
itr = i + 1
# Get parameters for run
parameterValues = run.getCalibrationParameters()
itr_cmd_proto = self.addParametersToCmdProto(cmd_proto_pre, parameterValues)
# For each world file
for worldfile in self.worldfiles.keys():
self.logger.critical("Iteration %d, worldfile: %s" % (itr, worldfile))
# Create new ModelRun object for this run
behavioralRun = ModelRun()
behavioralRun.session_id = self.session.id
behavioralRun.worldfile = worldfile
behavioralRun.setCalibrationParameters(parameterValues)
# Copy fitness parameters so that we can draw undercertainty bounds later
behavioralRun.nse = run.nse
behavioralRun.nse_log = run.nse_log
behavioralRun.pbias = run.pbias
behavioralRun.rsr = run.rsr
behavioralRun.user1 = run.user1
behavioralRun.user2 = run.user2
behavioralRun.user3 = run.user3
behavioralRun.fitness_period = run.fitness_period
# Add worldfile and flowtable paths to command
if self.explicitRouting:
if self.surfaceFlowtable:
cmd_raw_proto = self.addWorldfileAndFlowtableToCmdProto(
itr_cmd_proto,
self.worldfiles[worldfile],
self.flowtablePath[worldfile],
self.surfaceFlowtablePath[worldfile],
)
else:
cmd_raw_proto = self.addWorldfileAndFlowtableToCmdProto(
itr_cmd_proto, self.worldfiles[worldfile], self.flowtablePath[worldfile]
)
else:
cmd_raw_proto = self.addWorldfileToCmdProto(itr_cmd_proto, self.worldfiles[worldfile])
# Finally, create output_path and generate cmd_raw
behavioralRun.output_path = self.createOutputPath(self.basedir, self.session.id, worldfile, itr)
behavioralRun.cmd_raw = self.getCmdRawForRun(cmd_raw_proto, behavioralRun.output_path)
if "process" == options.parallel_mode:
# Set job ID if we are in process parallel mode
# (in lsf mode, we will use the LSF job number instead of itr)
behavioralRun.job_id = itr
# Dispatch to consumer
runQueue.put(behavioralRun)
time.sleep(5)
# Wait for all jobs to finish
self.logger.critical("calling runQueue.join() ...")
runQueue.join()
for consumerProcess in consumers:
consumerProcess.join()
# Update session endtime and status
self.calibratorDB.updateSessionEndtime(self.session.id, datetime.utcnow(), "complete")
except:
raise
else:
self.logger.debug("exiting normally")
return 0
finally:
self.calibratorDB = None
