def _make_box(x): ''' Make a box that encompasses all the data Parameters ---------- x : structured numpy array ''' box = np.zeros((2, ), x.dtype) names = recfunctions.get_names(x.dtype) for name in names: dtype = x.dtype.fields.get(name)[0] mask = np.ma.getmaskarray(x[name]) values = x[name][mask==False] if dtype == 'object': try: values = set(values) box[name][:] = values except TypeError as e: ema_logging.warning("{} has unhashable values".format(name)) raise e else: box[name][0] = np.min(values, axis=0) box[name][1] = np.max(values, axis=0) return box
def run_simulation(file_name): ''' Convenient function to run a model and store the results of the run in the specified .vdf file. The specified output file will be overwritten by default Parameters ---------- file_name : str the file name of the output file relative to the working directory Raises ------ VensimError if running the model failed in some way. ''' file_name = str(file_name) try: debug(" executing COMMAND: SIMULATE>RUNNAME|"+file_name+"|O") command("SIMULATE>RUNNAME|"+file_name+"|O") debug(r"MENU>RUN|o") command(r"MENU>RUN|o") except VensimWarning as w: warning((str(w))) raise VensimError(str(w))
def set_value(variable, value): ''' set the value of a variable to value current implementation only works for lookups and normal values. In case of a list, a lookup is assumed, else a normal value is assumed. See the DSS reference supplement, p. 58 for details. Parameters ---------- variable : str name of the variable to set. value : int, float, or list the value for the variable. **note**: the value can be either a list, or an float/integer. If it is a list, it is assumed the variable is a lookup. ''' variable = str(variable) if type(value) == types.ListType: value = [str(entry) for entry in value] command("SIMULATE>SETVAL|" + variable + "(" + str(value[1:-1]) + ")") else: try: command(r"SIMULATE>SETVAL|" + variable + "=" + str(value)) except VensimWarning: warning('variable: \'' + variable + '\' not found')
def run_simulation(file_name): ''' Convenient function to run a model and store the results of the run in the specified .vdf file. The specified output file will be overwritten by default Parameters ---------- file_name : str the file name of the output file relative to the working directory Raises ------ VensimError if running the model failed in some way. ''' file_name = str(file_name) try: debug(" executing COMMAND: SIMULATE>RUNNAME|" + file_name + "|O") command("SIMULATE>RUNNAME|" + file_name + "|O") debug(r"MENU>RUN|o") command(r"MENU>RUN|o") except VensimWarning as w: warning((str(w))) raise VensimError(str(w))
def get_data(filename, varname, step=1): ''' Retrieves data from simulation runs or imported data sets. Parameters ---------- filename : str the name of the .vdf file that contains the data varname : str the name of the variable to retrieve data on step : int (optional) steps used in slicing. Defaults to 1, meaning the full recored time series is returned. Returns ------- numpy array with the values for varname over the simulation ''' vval = [] try: vval, _ = vensimDLLwrapper.get_data(str(filename), str(varname)) except VensimWarning as w: warning(str(w)) return vval
def set_value(variable, value): ''' set the value of a variable to value current implementation only works for lookups and normal values. In case of a list, a lookup is assumed, else a normal value is assumed. See the DSS reference supplement, p. 58 for details. Parameters ---------- variable : str name of the variable to set. value : int, float, or list the value for the variable. **note**: the value can be either a list, or an float/integer. If it is a list, it is assumed the variable is a lookup. ''' variable = str(variable) if type(value) == types.ListType: value = [str(entry) for entry in value] command("SIMULATE>SETVAL|"+variable+"("+ str(value[1:-1]) + ")") else: try: command(r"SIMULATE>SETVAL|"+variable+"="+str(value)) except VensimWarning: warning('variable: \'' +variable+'\' not found')
def do_ylabels(ax, ylabels, outcome): ''' Helper function for setting the y labels on an ax Parameters ---------- ax : axes instance titles : dict a dict which maps outcome names to y labels outcome : str the outcome plotted in the ax. ''' if isinstance(ylabels, dict): if not ylabels: ax.set_ylabel(outcome) else: try: ax.set_ylabel(ylabels[outcome]) except KeyError: warning( "key error in do_ylabels, no ylabel provided for `%s`" % (outcome)) ax.set_ylabel(outcome)
def model_init(self, policy, kwargs): '''initializes the model''' try: self.model_file = policy['file'] except KeyError: ema_logging.warning("key 'file' not found in policy") super(FluModel, self).model_init(policy, kwargs)
def plot_violinplot(ax, value, log, group_labels=None): ''' helper function for plotting violin plots on axes Parameters ---------- ax : axes instance value : ndarray log : bool group_labels : list of str, optional ''' if log: warning("log option ignored for violin plot") pos = range(len(value)) dist = max(pos) - min(pos) _ = min(0.15 * max(dist, 1.0), 0.5) for data, p in zip(value, pos): if len(data) > 0: kde = gaussian_kde(data) #calculates the kernel density x = np.linspace(np.min(data), np.max(data), 250.) # support for violin v = kde.evaluate(x) #violin profile (density curve) scl = 1 / (v.max() / 0.4) v = v * scl #scaling the violin to the available space ax.fill_betweenx(x, p - v, p + v, facecolor=COLOR_LIST[p], alpha=0.6, lw=1.5) for percentile in [25, 75]: quant = scoreatpercentile(data.ravel(), percentile) q_x = kde.evaluate(quant) * scl q_x = [p - q_x, p + q_x] ax.plot(q_x, [quant, quant], linestyle=":", c='k') med = np.median(data) m_x = kde.evaluate(med) * scl m_x = [p - m_x, p + m_x] ax.plot(m_x, [med, med], linestyle="--", c='k', lw=1.5) if group_labels: labels = group_labels[:] labels.insert(0, '') ax.set_xticklabels(labels, rotation='vertical')
def group_by_envelopes(outcomes, outcome_to_plot, time, density, ax, ax_d, fill, group_labels, log): ''' Helper function, responsible for generating an envelope plot based on a grouping. Parameters ---------- outcomes : dict a dictonary containing the various outcomes to plot outcome_to_plot : str the specific outcome to plot time : str the name of the time dimension density : {None, HIST, KDE, VIOLIN, BOXPLOT} ax : Axes instance the ax on which to plot ax_d : Axes instance the ax on which to plot the density fill : bool group_by_labels : list of str order in which groups should be plotted log : bool ''' for j, key in enumerate(group_labels): value = outcomes[key] value = value[outcome_to_plot] try: plot_envelope(ax, j, time, value,fill) except ValueError: warning("value error when plotting for %s" % (key)) raise if density: group_density(ax_d, density, outcomes, outcome_to_plot, group_labels, log) ax_d.get_yaxis().set_view_interval( ax.get_yaxis().get_view_interval()[0], ax.get_yaxis().get_view_interval()[1])
def determine_kde(data, size_kde=1000, ymin=None, ymax=None): ''' Helper function responsible for performing a KDE Parameters ---------- data : ndarray size_kde : int, optional ymin : float, optional ymax : float, optional Returns ------- ndarray x values for kde ndarray y values for kde ..note:: x and y values are based on rotation as used in density plots for end states. ''' if not ymin: ymin = np.min(data) if not ymax: ymax = np.max(data) kde_y = np.linspace(ymin, ymax, size_kde) try: kde_x = kde.gaussian_kde(data) kde_x = kde_x.evaluate(kde_y) # grid = GridSearchCV(KernelDensity(kernel='gaussian'), # {'bandwidth': np.linspace(ymin, ymax, 20)}, # cv=20) # grid.fit(data[:, np.newaxis]) # best_kde = grid.best_estimator_ # kde_x = np.exp(best_kde.score_samples(kde_y[:, np.newaxis])) except Exception as e: warning(e) kde_x = np.zeros(kde_y.shape) return kde_x, kde_y
def plot_violinplot(ax, value, log, group_labels=None): ''' helper function for plotting violin plots on axes Parameters ---------- ax : axes instance value : ndarray log : bool group_labels : list of str, optional ''' if log: warning("log option ignored for violin plot") pos = range(len(value)) dist = max(pos)-min(pos) _ = min(0.15*max(dist,1.0),0.5) for data, p in zip(value,pos): if len(data)>0: kde = gaussian_kde(data) #calculates the kernel density x = np.linspace(np.min(data),np.max(data),250.) # support for violin v = kde.evaluate(x) #violin profile (density curve) scl = 1 / (v.max() / 0.4) v = v*scl #scaling the violin to the available space ax.fill_betweenx(x,p-v,p+v,facecolor=COLOR_LIST[p],alpha=0.6, lw=1.5) for percentile in [25, 75]: quant = scoreatpercentile(data.ravel(), percentile) q_x = kde.evaluate(quant) * scl q_x = [p - q_x, p + q_x] ax.plot(q_x, [quant, quant], linestyle=":", c='k') med = np.median(data) m_x = kde.evaluate(med) * scl m_x = [p - m_x, p + m_x] ax.plot(m_x, [med, med], linestyle="--", c='k', lw=1.5) if group_labels: labels = group_labels[:] labels.insert(0, '') ax.set_xticklabels(labels, rotation='vertical')
def plot_boxplots(ax, values, log, group_labels=None): ''' helper function for plotting a boxplot Parameters ---------- ax : axes instance value : ndarray log : bool group_labels : list of str, optional ''' if log: warning("log option ignored for boxplot") ax.boxplot(values) if group_labels: ax.set_xticklabels(group_labels, rotation='vertical')
def group_by_envelopes(outcomes, outcome_to_plot, time, density, ax, ax_d, fill, group_labels, log): ''' Helper function, responsible for generating an envelope plot based on a grouping. Parameters ---------- outcomes : dict a dictonary containing the various outcomes to plot outcome_to_plot : str the specific outcome to plot time : str the name of the time dimension density : {None, HIST, KDE, VIOLIN, BOXPLOT} ax : Axes instance the ax on which to plot ax_d : Axes instance the ax on which to plot the density fill : bool group_by_labels : list of str order in which groups should be plotted log : bool ''' for j, key in enumerate(group_labels): value = outcomes[key] value = value[outcome_to_plot] try: plot_envelope(ax, j, time, value, fill) except ValueError: warning("value error when plotting for %s" % (key)) raise if density: group_density(ax_d, density, outcomes, outcome_to_plot, group_labels, log) ax_d.get_yaxis().set_view_interval( ax.get_yaxis().get_view_interval()[0], ax.get_yaxis().get_view_interval()[1])
def load_model(file_name): ''' load the model Parameters ---------- file_name : str file name of model, relative to working directory Raises ------- VensimError if the model cannot be loaded. .. note: only works for .vpm files ''' debug("executing COMMAND: SIMULATE>SPECIAL>LOADMODEL|"+file_name) try: command("SPECIAL>LOADMODEL|"+str(file_name)) except VensimWarning as w: warning(str(w)) raise VensimError("vensim file not found")
def load_model(file_name): ''' load the model Parameters ---------- file_name : str file name of model, relative to working directory Raises ------- VensimError if the model cannot be loaded. .. note: only works for .vpm files ''' debug("executing COMMAND: SIMULATE>SPECIAL>LOADMODEL|" + file_name) try: command("SPECIAL>LOADMODEL|" + str(file_name)) except VensimWarning as w: warning(str(w)) raise VensimError("vensim file not found")
def test_log_messages(self): ema_logging.log_to_stderr(ema_logging.DEBUG) with mock.patch('util.ema_logging._logger') as mocked_logger: message = 'test message' ema_logging.debug(message) mocked_logger.debug.assert_called_with(message) ema_logging.info(message) mocked_logger.info.assert_called_with(message) ema_logging.warning(message) mocked_logger.warning.assert_called_with(message) ema_logging.error(message) mocked_logger.error.assert_called_with(message) ema_logging.exception(message) mocked_logger.exception.assert_called_with(message) ema_logging.critical(message) mocked_logger.critical.assert_called_with(message)
def do_ylabels(ax, ylabels, outcome): ''' Helper function for setting the y labels on an ax Parameters ---------- ax : axes instance titles : dict a dict which maps outcome names to y labels outcome : str the outcome plotted in the ax. ''' if isinstance(ylabels, dict): if not ylabels: ax.set_ylabel(outcome) else: try: ax.set_ylabel(ylabels[outcome]) except KeyError: warning("key error in do_ylabels, no ylabel provided for `%s`" % (outcome)) ax.set_ylabel(outcome)
def run_model(self, case): """ Method for running an instantiated model structures. This implementation assumes that the names of the uncertainties correspond to the name of the cells in Excel. See e.g. `this site <http://spreadsheets.about.com/od/exceltips/qt/named_range.htm>`_ for details or use Google and search on 'named range'. One of the requirements on the names is that the cannot contains spaces. For the extraction of results, the same approach is used. That is, this implementation assumes that the name of a :class:`~outcomes.Outcome` instance corresponds to the name of a cell, or set of cells. Parameters ---------- case : dict keyword arguments for running the model. The case is a dict with the names of the uncertainties as key, and the values to which to set these uncertainties. """ #find right sheet try: sheet = self.wb.Sheets(self.sheet) except Exception: ema_logging.warning("com error: sheet not found") self.cleanup() raise #set values on sheet for key, value in case.items(): try: sheet.Range(key).Value = value except com_error: ema_logging.warning( "com error: no cell(s) named %s found" % key, )
def run_model(self, case): """ Method for running an instantiated model structure. Parameters ---------- case : dict keyword arguments for running the model. The case is a dict with the names of the uncertainties as key, and the values to which to set these uncertainties. Raises ------ jpype.JavaException if there is any exception thrown by the netlogo model """ for key, value in case.iteritems(): try: self.netlogo.command(self.command_format.format(key, value)) except jpype.JavaException as e: warning('variable {} throws exception: {}'.format(key, str(e)))
def run_model(self, case): """ Method for running an instantiated model structures. This implementation assumes that the names of the uncertainties correspond to the name of the cells in Excel. See e.g. `this site <http://spreadsheets.about.com/od/exceltips/qt/named_range.htm>`_ for details or use Google and search on 'named range'. One of the requirements on the names is that the cannot contains spaces. For the extraction of results, the same approach is used. That is, this implementation assumes that the name of a :class:`~outcomes.Outcome` instance corresponds to the name of a cell, or set of cells. Parameters ---------- case : dict keyword arguments for running the model. The case is a dict with the names of the uncertainties as key, and the values to which to set these uncertainties. """ #find right sheet try: sheet = self.wb.Sheets(self.sheet) except Exception : ema_logging.warning("com error: sheet not found") self.cleanup() raise #set values on sheet for key, value in case.items(): try: sheet.Range(key).Value = value except com_error: ema_logging.warning("com error: no cell(s) named %s found" % key,) #get results results = {} for outcome in self.outcomes: try: output = sheet.Range(outcome.name).Value try: output = [value[0] for value in output] output = np.array(output) except TypeError: output = np.array(output) results[outcome.name] = output except com_error: ema_logging.warning("com error: no cell(s) named %s found" % outcome.name,) self.output = results
def starter(): logwatcher.start() try: logwatcher.loop.start() except (zmq.error.ZMQError, IOError): ema_logging.warning('shutting down log watcher')
# Created on 21 mrt. 2013 # # .. codeauthor:: jhkwakkel <j.h.kwakkel (at) tudelft (dot) nl> __all__ = ['NetLogoException', 'NetLogoLink'] if sys.platform == 'win32': NETLOGO_HOME = r'C:\Program Files (x86)\NetLogo 5.1.0' jar_separator = ";" # jars are separated by a ; on Windows elif sys.platform == 'darwin': jar_separator = ":" # jars are separated by a : on MacOS NETLOGO_HOME = r'/Applications/NetLogo 5.1.0' else: # TODO should raise and exception which is subsequently cached and # transformed into a a warning just like excel and vensim warning('netlogo support not available') PYNETLOGO_HOME = os.path.dirname(os.path.abspath(__file__)) class NetLogoException(Exception): pass class NetLogoLink(): def __init__(self, gui=False, thd=False): ''' Create a link with netlogo. Underneath, the netlogo jvm is started through jpype.
def run_experiment(self, experiment): '''The logic for running a single experiment. This code makes sure that model(s) are initialized correctly. Parameters ---------- experiment : dict Returns ------- experiment_id: int case : dict policy : str model_name : str result : dict Raises ------ EMAError if the model instance raises an EMA error, these are reraised. Exception Catch all for all other exceptions being raised by the model. These are reraised. ''' policy = experiment.pop('policy') model_name = experiment.pop('model') experiment_id = experiment.pop('experiment id') policy_name = policy['name'] ema_logging.debug("running policy {} for experiment {}".format(policy_name, experiment_id)) # check whether we already initialized the model for this # policy if not (policy_name, model_name) in self.msi_initialization.keys(): try: ema_logging.debug("invoking model init") msi = self.msis[model_name] msi.model_init(copy.deepcopy(policy), copy.deepcopy(self.model_kwargs)) except EMAError as inst: ema_logging.exception(inst) self.cleanup() raise inst except Exception as inst: ema_logging.exception("some exception occurred when invoking the init") self.cleanup() raise inst ema_logging.debug("initialized model %s with policy %s" % (model_name, policy_name)) self.msi_initialization = {(policy_name, model_name):self.msis[model_name]} msi = self.msis[model_name] case = copy.deepcopy(experiment) try: ema_logging.debug("trying to run model") msi.run_model(case) except CaseError as e: ema_logging.warning(str(e)) ema_logging.debug("trying to retrieve output") result = msi.retrieve_output() ema_logging.debug("trying to reset model") msi.reset_model() return experiment_id, case, policy, model_name, result
def run_model(self, case): """ Method for running an instantiated model structure. Parameters ---------- case : dict keyword arguments for running the model. The case is a dict with the names of the uncertainties as key, and the values to which to set these uncertainties. Raises ------ jpype.JavaException if there is any exception thrown by the netlogo model """ for key, value in case.iteritems(): try: self.netlogo.command(self.command_format.format(key, value)) except jpype.JavaException as e: warning('variable {} throws exception: {}'.format(key, str(e))) debug("model parameters set successfully") # finish setup and invoke run self.netlogo.command("setup") time_commands = [] end_commands = [] fns = {} for outcome in self.outcomes: name = outcome.name fn = r'{0}{3}{1}{2}'.format(self.working_directory, name, ".txt", os.sep) fns[name] = fn fn = '"{}"'.format(fn) fn = fn.replace(os.sep, '/') if self.netlogo.report('is-agentset? {}'.format(name)): # if name is name of an agentset, we # assume that we should count the total number of agents nc = r'{2} {0} {3} {4} {1}'.format(fn, name, "file-open", 'file-write', 'count') else: # it is not an agentset, so assume that it is # a reporter / global variable nc = r'{2} {0} {3} {1}'.format(fn, name, "file-open", 'file-write') if outcome.time: time_commands.append(nc) else: end_commands.append(nc) c_start = "repeat {} [".format(self.run_length) c_close = "go ]" c_middle = " ".join(time_commands) c_end = " ".join(end_commands) command = " ".join((c_start, c_middle, c_close)) debug(command) self.netlogo.command(command) # after the last go, we have not done a write for the outcomes # so we do that now self.netlogo.command(c_middle) # we also need to save the non time series outcomes self.netlogo.command(c_end) self.netlogo.command("file-close-all") self._handle_outcomes(fns)
class ExcelModelStructureInterface(ModelStructureInterface): ''' Base class for connecting the EMA workbench to models in Excel. To automate this connection as much as possible. This implementation relies on naming cells in Excel. These names can then be used here as names for the uncertainties and the outcomes. See e.g. `this site <http://spreadsheets.about.com/od/exceltips/qt/named_range.htm>`_ for details on naming cells and sets of cells. The provided implementation here does work with :mod:`parallel_ema`. ''' #: Reference to the Excel application. This attribute is `None` until #: model_init has been invoked. xl = None #: Reference to the workbook. This attribute is `None` until #: model_init has been invoked. wb = None #: Name of the sheet on which one want to set values sheet = None #: relative path to workbook workbook = None def model_init(self, policy, kwargs): ''' Method called to initialize the model. Parameters ---------- policy : dict policy to be run. kwargs : dict keyword arguments to be used by model_intit. This gives users to the ability to pass any additional arguments. ''' if not self.xl: try: ema_logging.debug("trying to start Excel") self.xl = win32com.client.Dispatch("Excel.Application") ema_logging.debug("Excel started") except com_error as e: raise EMAError(str(e)) ema_logging.debug("trying to open workbook") self.wb = self.xl.Workbooks.Open(self.working_directory + self.workbook) ema_logging.debug("workbook opened") ema_logging.debug(self.working_directory) def run_model(self, case): """ Method for running an instantiated model structures. This implementation assumes that the names of the uncertainties correspond to the name of the cells in Excel. See e.g. `this site <http://spreadsheets.about.com/od/exceltips/qt/named_range.htm>`_ for details or use Google and search on 'named range'. One of the requirements on the names is that the cannot contains spaces. For the extraction of results, the same approach is used. That is, this implementation assumes that the name of a :class:`~outcomes.Outcome` instance corresponds to the name of a cell, or set of cells. Parameters ---------- case : dict keyword arguments for running the model. The case is a dict with the names of the uncertainties as key, and the values to which to set these uncertainties. """ #find right sheet try: sheet = self.wb.Sheets(self.sheet) except Exception: ema_logging.warning("com error: sheet not found") self.cleanup() raise #set values on sheet for key, value in case.items(): try: sheet.Range(key).Value = value except com_error: ema_logging.warning( "com error: no cell(s) named %s found" % key, ) #get results results = {} for outcome in self.outcomes: try: output = sheet.Range(outcome.name).Value try: output = [value[0] for value in output] output = np.array(output) except TypeError: output = np.array(output) results[outcome.name] = output except com_error: ema_logging.warning( "com error: no cell(s) named %s found" % outcome.name, ) self.output = results
except WindowsError: vensim_double = None if vensim_single and vensim_double: vensim = vensim_single info("both single and double precision vensim available, using single") elif vensim_single: vensim = vensim_single info('using single precision vensim') elif vensim_double: vensim = vensim_double info('using single precision vensim') else: message = "vensim dll not found, vensim functionality not available" sys.stderr.write(message+"\n") warning(message) del sys def be_quiet(quietflag): ''' this allows you to turn off the work in progress dialog that Vensim displays during simulation and other activities, and also prevent the appearance of yes or no dialogs. use 0 for normal interaction, 1 to prevent the appearance of any work in progress windows, and 2 to also prevent the appearance of any interrogative dialogs' ''' if quietflag > 2: raise VensimError("incorrect value for quietflag")
# # .. codeauthor:: jhkwakkel <j.h.kwakkel (at) tudelft (dot) nl> __all__ = ['NetLogoException', 'NetLogoLink'] if sys.platform=='win32': NETLOGO_HOME = r'C:\Program Files (x86)\NetLogo 5.1.0' jar_separator = ";" # jars are separated by a ; on Windows elif sys.platform=='darwin': jar_separator = ":" # jars are separated by a : on MacOS NETLOGO_HOME = r'/Applications/NetLogo 5.1.0' else: # TODO should raise and exception which is subsequently cached and # transformed into a a warning just like excel and vensim warning('netlogo support not available') PYNETLOGO_HOME = os.path.dirname(os.path.abspath(__file__)) class NetLogoException(Exception): pass class NetLogoLink(): def __init__(self, gui=False, thd=False): ''' Create a link with netlogo. Underneath, the netlogo jvm is started through jpype.
def run_experiment(self, experiment): '''The logic for running a single experiment. This code makes sure that model(s) are initialized correctly. Parameters ---------- experiment : dict Returns ------- experiment_id: int case : dict policy : str model_name : str result : dict Raises ------ EMAError if the model instance raises an EMA error, these are reraised. Exception Catch all for all other exceptions being raised by the model. These are reraised. ''' policy = experiment.pop('policy') model_name = experiment.pop('model') experiment_id = experiment.pop('experiment id') policy_name = policy['name'] ema_logging.debug("running policy {} for experiment {}".format( policy_name, experiment_id)) # check whether we already initialized the model for this # policy if not (policy_name, model_name) in self.msi_initialization.keys(): try: ema_logging.debug("invoking model init") msi = self.msis[model_name] msi.model_init(copy.deepcopy(policy), copy.deepcopy(self.model_kwargs)) except EMAError as inst: ema_logging.exception(inst) self.cleanup() raise inst except Exception as inst: ema_logging.exception( "some exception occurred when invoking the init") self.cleanup() raise inst ema_logging.debug("initialized model %s with policy %s" % (model_name, policy_name)) self.msi_initialization = { (policy_name, model_name): self.msis[model_name] } msi = self.msis[model_name] case = copy.deepcopy(experiment) try: ema_logging.debug("trying to run model") msi.run_model(case) except CaseError as e: ema_logging.warning(str(e)) ema_logging.debug("trying to retrieve output") result = msi.retrieve_output() ema_logging.debug("trying to reset model") msi.reset_model() return experiment_id, case, policy, model_name, result