def model_init(self, policy, kwargs):
"""
Init of the model, The provided implementation here assumes
that `self.modelFile` is set correctly. In case of using different
vensim models for different policies, it is recomended to extent
this method, extract the model file from the policy dict, set
`self.modelFile` to this file and then call this implementation through
calling `super`.
:param policy: a dict specifying the policy. In this
implementation, this argument is ignored.
:param kwargs: additional keyword arguments. In this implementation
this argument is ignored.
"""
load_model(self.workingDirectory + self.modelFile) # load the model
EMAlogging.debug("model initialized successfully")
be_quiet() # minimize the screens that are shown
try:
initialTime = get_val("INITIAL TIME")
finalTime = get_val("FINAL TIME")
timeStep = get_val("TIME STEP")
savePer = get_val("SAVEPER")
if savePer > 0:
timeStep = savePer
self.runLength = (finalTime - initialTime) / timeStep + 1
except VensimWarning:
raise EMAWarning(str(VensimWarning))
def run(self):
"""
read from the queue and write to the log handlers
The logging documentation says logging is thread safe, so there
shouldn't be contention between normal logging (from the main
process) and this thread.
Note that we're using the name of the original logger.
"""
while True:
try:
record = self.queue.get()
# get the logger for this record
if record is None:
EMAlogging.debug("none received")
break
logger = logging.getLogger(record.name)
logger.callHandlers(record)
except (KeyboardInterrupt, SystemExit):
raise
except EOFError:
break
except:
traceback.print_exc(file=sys.stderr)
def _terminate_pool(cls,
taskqueue,
inqueue,
outqueue,
pool,
task_handler,
result_handler,
cache,
workingDirectories):
EMAlogging.info("terminating pool")
# this is guaranteed to only be called once
debug('finalizing pool')
TERMINATE = 2
task_handler._state = TERMINATE
for p in pool:
taskqueue.put(None) # sentinel
time.sleep(1)
debug('helping task handler/workers to finish')
cls._help_stuff_finish(inqueue, task_handler, len(pool))
assert result_handler.is_alive() or len(cache) == 0
result_handler._state = TERMINATE
outqueue.put(None) # sentinel
if pool and hasattr(pool[0], 'terminate'):
debug('terminating workers')
for p in pool:
p.terminate()
debug('joining task handler')
task_handler.join(1e100)
debug('joining result handler')
result_handler.join(1e100)
if pool and hasattr(pool[0], 'terminate'):
debug('joining pool workers')
for p in pool:
p.join()
# cleaning up directories
# TODO investigate whether the multiprocessing.util tempdirectory
# functionality can be used instead
for directory in workingDirectories:
directory = os.path.dirname(directory)
EMAlogging.debug("deleting "+str(directory))
shutil.rmtree(directory)
def test_experiments_to_cases():
from examples.FLUvensimExample import FluModel
from expWorkbench.model import SimpleModelEnsemble
EMAlogging.log_to_stderr(EMAlogging.INFO)
data = load_results(r'../analysis/1000 flu cases.cPickle')
experiments, results = data
cases = experiments_to_cases(experiments)
model = FluModel(r'..\..\models\flu', "fluCase")
ensemble = SimpleModelEnsemble()
ensemble.set_model_structure(model)
ensemble.perform_experiments(cases)
def read_cin_file(file):
"""
read a .cin file
:param file: location of the .cin file.
:exception: raises a :class:`~EMAExceptions.VensimWarning` if the cin file
cannot be read.
"""
EMAlogging.debug("executing COMMAND: SIMULATE>READCIN|" + file)
try:
command(r"SIMULATE>READCIN|" + file)
except VensimWarning as w:
EMAlogging.debug(str(w))
raise w
def load_model(file):
"""
load the model
:param file: the location of the .vpm file to be loaded.
:exception: raises a :class:`~EMAExceptions.VensimError` if the model
cannot be loaded.
.. note: only works for .vpm files
"""
EMAlogging.debug("executing COMMAND: SIMULATE>SPECIAL>LOADMODEL|" + file)
try:
command(r"SPECIAL>LOADMODEL|" + file)
except VensimWarning as w:
EMAlogging.warning(str(w))
raise VensimError("vensim file not found")
def cleanup(self):
'''
cleaning up prior to finishing performing experiments. This
will close the workbook and close Excel.
'''
EMAlogging.debug("cleaning up")
if self.wb:
self.wb.Close(False)
del self.wb
if self.xl:
self.xl.DisplayAlerts = False
self.xl.Quit()
del self.xl
self.xl = None
self.wb = None
def get_data(filename, varname, step=1):
"""
Retrieves data from simulation runs or imported data sets.
:param filename: the name of the .vdf file that contains the data
:param varname: the name of the variable to retrieve data on
:param step: steps used in slicing. Defaults to 1, meaning the full
recored time series is returned.
:return: an array with the values for varname over the simulation
"""
vval = []
try:
vval, tval = vensimDLLwrapper.get_data(filename, varname)
except VensimWarning as w:
EMAlogging.warning(str(w))
return vval
def model_init(self, policy, kwargs):
'''
:param policy: policy to be run, in the default implementation, this
argument is ignored. Extent :meth:`model_init` to
specify how this argument should be used.
:param kwargs: keyword arguments to be used by :meth:`model_init`
'''
if not self.xl:
try:
EMAlogging.debug("trying to start Excel")
self.xl = win32com.client.Dispatch("Excel.Application")
EMAlogging.debug("Excel started")
EMAlogging.debug("trying to open workbook")
self.wb = self.xl.Workbooks.Open(self.workingDirectory + self.workbook)
EMAlogging.debug("workbook opened")
except com_error as e:
raise EMAError(str(e))
EMAlogging.debug(self.workingDirectory)
def set_value(variable, value):
"""
set the value of a variable to value
current implementation only works for lookups and normal values. In case
of a list, a lookup is assumed, else a normal value is assumed.
See the DSS reference supplement, p. 58 for details.
:param variable: name of the variable to set.
:param value: the value for the variable.
**note**: the value can be either a list, or an float/integer.
If it is a list, it is assumed the variable is a lookup.
"""
if type(value) == types.ListType:
command(r"SIMULATE>SETVAL|" + variable + "(" + str(value)[1:-1] + ")")
else:
try:
command(r"SIMULATE>SETVAL|" + variable + "=" + str(value))
except VensimWarning:
EMAlogging.warning("variable: '" + variable + "' not found")
def run_model(self, case):
"""
Method for running an instantiated model structures. This
implementation assumes that the names of the uncertainties correspond
to the name of the cells in Excel. See e.g. `this site <http://spreadsheets.about.com/od/exceltips/qt/named_range.htm>`_
for details or use Google and search on 'named range'. One of the
requirements on the names is that the cannot contains spaces.
For the extraction of results, the same approach is used. That is,
this implementation assumes that the name of a :class:`~outcomes.Outcome`
instance corresponds to the name of a cell, or set of cells.
:param case: dictionary with arguments for running the model
"""
#find right sheet
try:
sheet = self.wb.Sheets(self.sheet)
except Exception as e:
EMAlogging.warning("com error: sheet not found")
self.cleanup()
raise
#set values on sheet
for key, value in case.items():
try:
sheet.Range(key).Value = value
except com_error:
EMAlogging.warning("com error: no cell(s) named %s found" % key,)
#get results
results = {}
for outcome in self.outcomes:
try:
output = sheet.Range(outcome.name).Value
try:
output = [value[0] for value in output]
output = np.array(output)
except TypeError:
output = np.array(output)
results[outcome.name] = output
except com_error:
EMAlogging.warning("com error: no cell(s) named %s found" % outcome.name,)
self.output = results
def run_simulation(file):
"""
Convenient function to run a model and store the results of the run in
the specified .vdf file. The specified output file will be overwritten
by default
:param file: the location of the outputfile
:exception: raises a :class:`~EMAExceptions.VensimError` if running
the model failed in some way.
"""
try:
EMAlogging.debug(" executing COMMAND: SIMULATE>RUNNAME|" + file + "|O")
command("SIMULATE>RUNNAME|" + file + "|O")
EMAlogging.debug(r"MENU>RUN|o")
command(r"MENU>RUN|o")
except VensimWarning as w:
EMAlogging.warning((str(w)))
raise VensimError(str(w))
def run_model(self, case):
"""
Method for running an instantiated model structure.
the provided implementation assumes that the keys in the
case match the variable names in the Vensim model.
If lookups are to be set specify their transformation from
uncertainties to lookup values in the extension of this method,
then call this one using super with the updated case dict.
if you want to use cinFiles, set the cinFile, or cinFiles in
the extension of this method to `self.cinFile`.
:param case: the case to run
.. note:: setting parameters should always be done via run_model.
The model is reset to its initial values automatically after
each run.
"""
if self.cinFile:
try:
read_cin_file(self.workingDirectory + self.cinFile)
except VensimWarning as w:
EMAlogging.debug(str(w))
else:
EMAlogging.debug("cin file read successfully")
for key, value in case.items():
set_value(key, value)
EMAlogging.debug("model parameters set successfully")
EMAlogging.debug("run simulation, results stored in " + self.workingDirectory + self.resultFile)
try:
run_simulation(self.workingDirectory + self.resultFile)
except VensimError as e:
raise
results = {}
error = False
for output in self.outcomes:
EMAlogging.debug("getting data for %s" % output.name)
result = get_data(self.workingDirectory + self.resultFile, output.name)
EMAlogging.debug("successfully retrieved data for %s" % output.name)
if not result == []:
if result.shape[0] != self.runLength:
a = np.zeros((self.runLength))
a[0 : result.shape[0]] = result
result = a
error = True
if not output.time:
result = [-1]
else:
result = result[0 :: self.step]
try:
results[output.name] = result
except ValueError as e:
print "what"
self.output = results
if error:
raise CaseError("run not completed", case)
def worker(inqueue,
outqueue,
modelInterfaces,
modelInitKwargs=None):
#
# Code run by worker processes
#
debug("worker started")
put = outqueue.put
get = inqueue.get
if hasattr(inqueue, '_writer'):
inqueue._writer.close()
outqueue._reader.close()
def cleanup(modelInterfaces):
for msi in modelInterfaces:
msi.cleanup()
del msi
oldPolicy = {}
modelInitialized = False
while 1:
try:
task = get()
except (EOFError, IOError):
debug('worker got EOFError or IOError -- exiting')
break
if task is None:
debug('worker got sentinel -- exiting')
cleanup(modelInterfaces)
break
job, i, case, policy = task
for modelInterface in modelInterfaces:
if policy != oldPolicy:
modelInitialized = False
try:
debug("invoking model init")
modelInterface.model_init(policy, modelInitKwargs)
debug("model initialized successfully")
modelInitialized = True
except EMAError as e:
exception("init not implemented")
raise
except Exception:
exception("some exception occurred when invoking the init")
if modelInitialized:
try:
try:
debug("trying to run model")
modelInterface.run_model(copy.deepcopy(case))
except CaseError as e:
EMAlogging.warning(e)
debug("trying to retrieve output")
result = modelInterface.retrieve_output()
debug("trying to reset model")
modelInterface.reset_model()
result = (True, (case, policy, modelInterface.name, result))
except Exception as e:
result = (False, e)
else:
result = (False, EMAParallelError("failure to initialize"))
put((job, i, result))
oldPolicy = policy
def __init__(self,
modelStructureInterfaces,
processes=None,
callback = None,
kwargs=None):
'''
:param modelStructureInterface: modelInterface class
:param processes: nr. of processes to spawn, if none, it is
set to equal the nr. of cores
:param callback: callback function for handling the output
:param kwargs: kwargs to be pased to :meth:`model_init`
'''
self._setup_queues()
self._taskqueue = Queue.Queue()
self._cache = {}
self._state = RUN
self._callback = callback
if processes is None:
try:
processes = cpu_count()
except NotImplementedError:
processes = 1
info("nr of processes is "+str(processes))
self.Process = LoggingProcess
self.logQueue = multiprocessing.Queue()
h = EMAlogging.NullHandler()
logging.getLogger(EMAlogging.LOGGER_NAME).addHandler(h)
# This thread will read from the subprocesses and write to the
# main log's handlers.
log_queue_reader = LogQueueReader(self.logQueue)
log_queue_reader.start()
self._pool = []
workingDirectories = []
debug('generating workers')
workerRoot = None
for i in range(processes):
debug('generating worker '+str(i))
workerName = 'PoolWorker'+str(i)
def ignore_function(path, names):
if path.find('.svn') != -1:
return names
else:
return []
#setup working directories for parallelEMA
for msi in modelStructureInterfaces:
if msi.workingDirectory != None:
if workerRoot == None:
workerRoot = os.path.dirname(os.path.abspath(modelStructureInterfaces[0].workingDirectory))
workingDirectory = os.path.join(workerRoot, workerName, msi.name)
workingDirectories.append(workingDirectory)
shutil.copytree(msi.workingDirectory,
workingDirectory,
ignore = ignore_function)
msi.set_working_directory(workingDirectory)
w = self.Process(
self.logQueue,
level = logging.getLogger(EMAlogging.LOGGER_NAME)\
.getEffectiveLevel(),
target=worker,
args=(self._inqueue,
self._outqueue,
modelStructureInterfaces,
kwargs)
)
self._pool.append(w)
w.name = w.name.replace('Process', workerName)
w.daemon = True
w.start()
debug(' worker '+str(i) + ' generated')
self._task_handler = threading.Thread(
target=CalculatorPool._handle_tasks,
args=(self._taskqueue,
self._quick_put,
self._outqueue,
self._pool,
self.logQueue)
)
self._task_handler.daemon = True
self._task_handler._state = RUN
self._task_handler.start()
self._result_handler = threading.Thread(
target=CalculatorPool._handle_results,
args=(self._outqueue, self._quick_get, self._cache)
)
self._result_handler.daemon = True
self._result_handler._state = RUN
self._result_handler.start()
self._terminate = Finalize(self,
self._terminate_pool,
args=(self._taskqueue,
self._inqueue,
self._outqueue,
self._pool,
self._task_handler,
self._result_handler,
self._cache,
workingDirectories,
),
exitpriority=15
)
EMAlogging.info("pool has been set up")
# process handlers. For now, just set it from the global default.
logger.setLevel(self.level)
self.logger = logger
def run(self):
self._setupLogger()
p = multiprocessing.current_process()
debug('process %s with pid %s started' % (p.name, p.pid))
#call the run of the super, which in turn will call the worker function
super(LoggingProcess, self).run()
#==============================================================================
# test functions and classes
#==============================================================================
if __name__ == '__main__':
from examples.pythonExample import SimplePythonModel
from model import SimpleModelEnsemble
EMAlogging.log_to_stderr(logging.INFO)
modelInterface = SimplePythonModel(r'D:\jhkwakkel\workspace\EMA workbench\models\test', "testModel")
ensemble = SimpleModelEnsemble()
ensemble.setModelStructure(modelInterface)
ensemble.parallel = True
ensemble.performExperiments(10)
