Ejemplo n.º 1
0
class Simulation(object):
    """
    A class for simulations.

    :param simfile: Filename of simulation configuration file.
    :type simfile: str
    :param settings: keyword name of simulation parameter to use for settings
    :type str:

    Simulation attributes can be passed directly as keyword arguments directly
    to :class:`~carousel.core.simulations.Simulation` or in a JSON file or as
    class attributes in a subclass or a combination of all 3 methods.

    To get a list of :class:`~carousel.core.simulations.Simulation` attributes
    and defaults get the :attr:`~carousel.core.simulations.Simulation.attrs`
    attribute.

    Any additional settings provided as keyword arguments will override settings
    from file.
    """

    __metaclass__ = SimBase
    attrs = {
        "ID": None,
        "path": os.path.join("~", "Carousel", "Simulations"),
        "commands": ["start", "pause"],
        "data": None,
        "thresholds": None,
        "interval": 1 * UREG.hour,
        "sim_length": 1 * UREG.year,
        "display_frequency": 1,
        "display_fields": None,
        "write_frequency": 8760,
        "write_fields": None,
    }
    deprecated = {"interval": "interval_length", "sim_length": "simulation_length"}

    def __init__(self, simfile=None, settings=None, **kwargs):
        # load simfile if it's an argument
        if simfile is not None:
            # read and load JSON parameter map file as "parameters"
            self.param_file = simfile
            with open(self.param_file, "r") as param_file:
                file_params = json.load(param_file)
                #: simulation parameters from file
                self.parameters = {settings: SimParameter(**params) for settings, params in file_params.iteritems()}
        # if not subclassed and metaclass skipped, then use kwargs
        if not hasattr(self, "parameters"):
            #: parameter file
            self.param_file = None
            #: simulation parameters from keyword arguments
            self.parameters = kwargs
        else:
            # use first settings
            if settings is None:
                self.settings, self.parameters = self.parameters.items()[0]
            else:
                #: name of sim settings used for parameters
                self.settings = settings
                self.parameters = self.parameters[settings]
            # use any keyword arguments instead of parameters
            self.parameters.update(kwargs)
        # make pycharm happy - attributes assigned in loop by attrs
        self.thresholds = {}
        self.display_frequency = 0
        self.display_fields = {}
        self.write_frequency = 0
        self.write_fields = {}
        # pop deprecated attribute names
        for k, v in self.deprecated.iteritems():
            val = self.parameters["extras"].pop(v, None)
            # update parameters if deprecated attr used and no new attr
            if val and k not in self.parameters:
                self.parameters[k] = val
        # Attributes
        for k, v in self.attrs.iteritems():
            setattr(self, k, self.parameters.get(k, v))
        # member docstrings are in documentation since attrs are generated
        if self.ID is None:
            # generate id from object class name and datetime in ISO format
            self.ID = id_maker(self)
        if self.path is not None:
            # expand environment variables, ~ and make absolute path
            self.path = os.path.expandvars(os.path.expanduser(self.path))
            self.path = os.path.abspath(self.path)
        # convert simulation interval to Pint Quantity
        if isinstance(self.interval, basestring):
            self.interval = UREG(self.interval)
        elif not isinstance(self.interval, Q_):
            self.interval = self.interval[0] * UREG[str(self.interval[1])]
        # convert simulation length to Pint Quantity
        if isinstance(self.sim_length, basestring):
            self.sim_length = UREG(self.sim_length)
        elif not isinstance(self.sim_length, Q_):
            self.sim_length = self.sim_length[0] * UREG[str(self.sim_length[1])]
        # convert simulation length to interval units to calc total intervals
        sim_to_interval_units = self.sim_length.to(self.interval.units)
        #: total number of intervals simulated
        self.number_intervals = np.ceil(sim_to_interval_units / self.interval)
        #: interval index, start at zero
        self.interval_idx = 0
        #: pause status
        self._ispaused = False
        #: finished status
        self._iscomplete = False
        #: initialized status
        self._isinitialized = False
        #: order of calculations
        self.calc_order = []
        #: command queue
        self.cmd_queue = Queue.Queue()
        #: index iterator
        self.idx_iter = self.index_iterator()
        #: data loaded status
        self._is_data_loaded = False

    @property
    def ispaused(self):
        """
        Pause property, read only. True if paused.
        """
        return self._ispaused

    @property
    def iscomplete(self):
        """
        Completion property, read only. True if finished.
        """
        return self._iscomplete

    @property
    def isinitialized(self):
        """
        Initialization property, read only. True if initialized.
        """
        return self._isinitialized

    @property
    def is_data_loaded(self):
        """
        Data loaded property, read only. True if data loaded.
        """
        return self._is_data_loaded

    def check_data(self, data):
        """
        Check if data loaded for all sources in data layer.

        :param data: data layer from model
        :type data: :class:`~carousel.core.layer.Data`
        :return: dictionary of data sources and objects or `None` if not loaded
        """
        data_objs = {data_src: data.objects.get(data_src) for data_src in data.layer}
        self._is_data_loaded = all(data_objs.values())
        return data_objs

    def initialize(self, calc_reg):
        """
        Initialize the simulation. Organize calculations by dependency.

        :param calc_reg: Calculation registry.
        :type calc_reg:
            :class:`~carousel.core.calculation.CalcRegistry`
        """
        self._isinitialized = True
        # TODO: if calculations are edited, loaded, added, etc. then reset
        self.calc_order = topological_sort(calc_reg.dependencies)

    def index_iterator(self):
        """
        Generator that resumes from same index, or restarts from sent index.
        """
        idx = 0  # index
        while idx < self.number_intervals:
            new_idx = yield idx
            idx += 1
            if new_idx:
                idx = new_idx - 1

    # TODO: change start to run

    def start(self, model, progress_hook=None):
        """
        Start the simulation from time zero.

        :param model: Model with layers and registries containing parameters
        :type: :class:`~carousel.core.models.Model`
        :param progress_hook: A function that receives either a string or a
            list containing the index followed by tuples of the data or outputs
            names and values specified by ``write_fields`` in the simfile.
        :type progress_hook: function


        The model registries should contain the following layer registries:
        * :class:`~carousel.core.data_sources.DataRegistry`,
        * :class:`~carousel.core.formulas.FormulaRegistry`,
        * :class:`~carousel.core.outputs.OutputRegistry`,
        * :class:`~carousel.core.calculation.CalcRegistry`
        """
        # check if data loaded
        data_objs = self.check_data(model.data)
        if not self.is_data_loaded:
            raise MissingDataError([ds for ds in data_objs if ds is None])
        # get layer registries
        data_reg = model.registries["data"]
        formula_reg = model.registries["formulas"]
        out_reg = model.registries["outputs"]
        calc_reg = model.registries["calculations"]
        # initialize
        if not self.isinitialized:
            self.initialize(calc_reg)
        # default progress hook
        if not progress_hook:
            progress_hook = functools.partial(sim_progress_hook, display_header=True)
        # start, resume or restart
        if self.ispaused:
            # if paused, then resume, do not resize outputs again.
            self._ispaused = False  # change pause state
            progress_hook("resume simulation")
        elif self.iscomplete:
            # if complete, then restart, do not resize outputs again.
            self._iscomplete = False  # change pause state
            progress_hook("restart simulation")
            self.idx_iter = self.index_iterator()
        else:
            # resize outputs
            # assumes that self.write_frequency is immutable
            # TODO: allow self.write_frequency to be changed
            # only resize outputs first time simulation is started
            # repeat output rows to self.write_frequency
            # put initial conditions of outputs last so it's copied when
            # idx == 0
            progress_hook("resize outputs")  # display progress
            for k in out_reg:
                if out_reg.isconstant[k]:
                    continue
                # repeat rows (axis=0)
                out_reg[k] = out_reg[k].repeat(self.write_frequency, 0)
                _initial_value = out_reg.initial_value[k]
                if not _initial_value:
                    continue
                if isinstance(_initial_value, basestring):
                    # initial value is from data registry
                    # assign in a scalar to a vector fills in the vector, yes!
                    out_reg[k][-1] = data_reg[_initial_value]
                else:
                    out_reg[k][-1] = _initial_value * out_reg[k].units
            progress_hook("start simulation")
        # check and/or make Carousel_Simulations and simulation ID folders
        mkdir_p(self.path)
        sim_id_path = os.path.join(self.path, self.ID)
        mkdir_p(sim_id_path)
        # header & units for save files
        data_fields = self.write_fields.get("data", [])  # any data fields
        out_fields = self.write_fields.get("outputs", [])  # any outputs fields
        save_header = tuple(data_fields + out_fields)  # concatenate fields
        # get units as strings from data & outputs
        data_units = [str(data_reg[f].dimensionality) for f in data_fields]
        out_units = [str(out_reg[f].dimensionality) for f in out_fields]
        save_units = tuple(data_units + out_units)  # concatenate units
        # string format for header & units
        save_str = ("%s" + ",%s" * (len(save_header) - 1)) + "\n"  # format
        save_header = (save_str * 2) % (save_header + save_units)  # header
        save_header = save_header[:-1]  # remove trailing new line
        # ===================
        # Static calculations
        # ===================
        progress_hook("static calcs")
        for calc in self.calc_order:
            if not calc_reg.is_dynamic[calc]:
                calc_reg.calculator[calc].calculate(calc_reg[calc], formula_reg, data_reg, out_reg)
        # ====================
        # Dynamic calculations
        # ====================
        progress_hook("dynamic calcs")
        # TODO: assumes that interval size and indices are same, but should
        # interpolate for any size interval or indices
        for idx_tot in self.idx_iter:
            self.interval_idx = idx_tot  # update simulation interval counter
            idx = idx_tot % self.write_frequency
            # update properties
            for k, v in out_reg.isproperty.iteritems():
                # set properties from previous interval at night
                if v:
                    out_reg[k][idx] = out_reg[k][idx - 1]
            # night if any threshold exceeded
            if self.thresholds:
                night = not all(
                    limits[0] < data_reg[data][idx] < limits[1] for data, limits in self.thresholds.iteritems()
                )
            else:
                night = None
            # daytime or always calculated outputs
            for calc in self.calc_order:
                # Determine if calculation is scheduled for this timestep
                # TODO: add ``start_at`` parameter combined with ``frequency``
                freq = calc_reg.frequency[calc]
                if not freq.dimensionality:
                    is_scheduled = (idx_tot % freq) == 0
                else:
                    # Frequency with units of time
                    is_scheduled = ((idx_tot * self.interval) % freq) == 0
                is_scheduled = is_scheduled and (not night or calc_reg.always_calc[calc])
                if calc_reg.is_dynamic[calc] and is_scheduled:
                    calc_reg.calculator[calc].calculate(
                        calc_reg[calc], formula_reg, data_reg, out_reg, timestep=self.interval, idx=idx
                    )
            # display progress
            if not (idx % self.display_frequency):
                progress_hook(self.format_progress(idx, data_reg, out_reg))
                # disp_head = False
            # create an index for the save file, 0 if not saving
            if not ((idx_tot + 1) % self.write_frequency):
                savenum = (idx_tot + 1) / self.write_frequency
            elif idx_tot == self.number_intervals - 1:
                # save file index should be integer!
                savenum = int(np.ceil((idx_tot + 1) / float(self.write_frequency)))
            else:
                savenum = 0  # not saving this iteration
            # save file to disk
            if savenum:
                savename = self.ID + "_" + str(savenum) + ".csv"  # filename
                savepath = os.path.join(sim_id_path, savename)  # path
                # create array of all data & outputs to save
                save_array = self.format_write(data_reg, out_reg, idx + 1)
                # save as csv using default format & turn comments off
                np.savetxt(savepath, save_array, delimiter=",", header=save_header, comments="")
            try:
                cmd = self.cmd_queue.get_nowait()
            except Queue.Empty:
                continue
            if cmd == "pause":
                self._ispaused = True
                return
        self._iscomplete = True  # change completion status

    def format_progress(self, idx, data_reg, out_reg):
        data_fields = self.display_fields.get("data", [])  # data fields
        data_args = [(f, data_reg[f][idx]) for f in data_fields]
        out_fields = self.display_fields.get("outputs", [])  # outputs fields
        out_args = [(f, out_reg[f][idx]) for f in out_fields]
        return [idx] + data_args + out_args

    def format_write(self, data_reg, out_reg, idx=None):
        data_fields = self.write_fields.get("data", [])  # any data fields
        data_args = [data_reg[f][:idx].reshape((-1, 1)) for f in data_fields]
        out_fields = self.write_fields.get("outputs", [])  # any outputs fields
        out_args = [out_reg[f][:idx] for f in out_fields]
        return np.concatenate(data_args + out_args, axis=1)

    def pause(self, progress_hook=None):
        """
        Pause the simulation. How is this different from stopping it? Maintain
        info sufficient to restart simulation. Sets ``is_paused`` to True.
        Will this state allow analysis? changing parameters? What can you do
        with a paused simulation?
        Should be capable of saving paused simulation for loading/resuming
        later, that is the main usage. EG: someone else need computer, or power
        goes out, so on battery backup quickly pause simulation, and save.
        Is save automatic? Should there be a parameter for auto save changed?
        """
        # default progress hook
        if progress_hook is None:
            progress_hook = sim_progress_hook
        progress_hook("simulation paused")
        self.cmd_queue.put("pause")
        self._ispaused = True

    def load(self, model, progress_hook=None, *args, **kwargs):
        # default progress hook
        if progress_hook is None:
            progress_hook = sim_progress_hook
        data = kwargs.get("data", {})
        if not data and args:
            data = args[0]
        for k, v in data.iteritems():
            progress_hook("loading simulation for %s" % k)
            model.data.open(k, **v)
        self.check_data(model.data)

    def run(self, model, progress_hook=None, *args, **kwargs):
        # default progress hook
        if progress_hook is None:
            progress_hook = sim_progress_hook
        progress_hook("running simulation")
        self.load(model, progress_hook, *args, **kwargs)
        self.start(model, progress_hook)
Ejemplo n.º 2
0
class Simulation(object):
    """
    A class for simulations.

    :param simfile: Filename of simulation configuration file.
    :type simfile: str

    Simulation attributes can be passed directly as keyword arguments directly
    to :class:`~carousel.core.simulations.Simulation` or in a JSON file or as
    class attributes in a subclass or a combination of all 3 methods.

    To get a list of :class:`~carousel.core.simulations.Simulation` attributes
    and defaults get the :attr:`~carousel.core.simulations.Simulation.attrs`
    attribute.
    """
    __metaclass__ = SimBase
    attrs = {
        'ID': None,
        'path': os.path.join('~', 'Carousel', 'Simulations'),
        'commands': ['start', 'pause'],
        'data': None,
        'thresholds': None,
        'interval': 1 * UREG.hour,
        'sim_length': 1 * UREG.year,
        'display_frequency': 1,
        'display_fields': None,
        'write_frequency': 8760,
        'write_fields': None
    }
    deprecated = {
        'interval': 'interval_length',
        'sim_length': 'simulation_length'
    }

    def __init__(self, simfile=None, **kwargs):
        # check if simulation file is first argument or is in keyword arguments
        simfile = simfile or kwargs.get('simfile')  # defaults to None
        # check if simulation file is still None or in parameters from metaclass
        simfile = simfile or getattr(self, 'param_file', None)
        #: parameter file
        self.param_file = simfile
        # read and load JSON parameter map file as "parameters"
        if self.param_file is not None:
            with open(self.param_file, 'r') as fp:
                #: parameters from file for simulation
                self.parameters = json.load(fp)
        # if not subclassed and metaclass skipped, then use kwargs
        if not hasattr(self, 'parameters'):
            self.parameters = kwargs
        else:
            # use any keyword arguments instead of parameters
            self.parameters.update(kwargs)
        # make pycharm happy - attributes assigned in loop by attrs
        self.thresholds = {}
        self.display_frequency = 0
        self.display_fields = {}
        self.write_frequency = 0
        self.write_fields = {}
        # pop deprecated attribute names
        for k, v in self.deprecated.iteritems():
            val = self.parameters.pop(v, None)
            # update parameters if deprecated attr used and no new attr
            if val and k not in self.parameters:
                self.parameters[k] = val
        # Attributes
        for k, v in self.attrs.iteritems():
            setattr(self, k, self.parameters.get(k, v))
        # member docstrings are in documentation since attrs are generated
        if self.ID is None:
            # generate id from object class name and datetime in ISO format
            self.ID = id_maker(self)
        if self.path is not None:
            # expand environment variables, ~ and make absolute path
            self.path = os.path.expandvars(os.path.expanduser(self.path))
            self.path = os.path.abspath(self.path)
        # convert simulation interval to Pint Quantity
        if isinstance(self.interval, basestring):
            self.interval = UREG(self.interval)
        elif not isinstance(self.interval, Q_):
            self.interval = self.interval[0] * UREG[str(self.interval[1])]
        # convert simulation length to Pint Quantity
        if isinstance(self.sim_length, basestring):
            self.sim_length = UREG(self.sim_length)
        elif not isinstance(self.sim_length, Q_):
            self.sim_length = self.sim_length[0] * UREG[str(
                self.sim_length[1])]
        # convert simulation length to interval units to calc total intervals
        sim_to_interval_units = self.sim_length.to(self.interval.units)
        #: total number of intervals simulated
        self.number_intervals = np.ceil(sim_to_interval_units / self.interval)
        #: interval index, start at zero
        self.interval_idx = 0
        #: pause status
        self._ispaused = False
        #: finished status
        self._iscomplete = False
        #: initialized status
        self._isinitialized = False
        #: order of calculations
        self.calc_order = []
        #: command queue
        self.cmd_queue = Queue.Queue()
        #: index iterator
        self.idx_iter = self.index_iterator()
        #: data loaded status
        self._is_data_loaded = False

    @property
    def ispaused(self):
        """
        Pause property, read only. True if paused.
        """
        return self._ispaused

    @property
    def iscomplete(self):
        """
        Completion property, read only. True if finished.
        """
        return self._iscomplete

    @property
    def isinitialized(self):
        """
        Initialization property, read only. True if initialized.
        """
        return self._isinitialized

    @property
    def is_data_loaded(self):
        """
        Data loaded property, read only. True if data loaded.
        """
        return self._is_data_loaded

    def check_data(self, data):
        """
        Check if data loaded for all sources in data layer.

        :param data: data layer from model
        :type data: :class:`~carousel.core.layer.Data`
        :return: dictionary of data sources and objects or `None` if not loaded
        """
        data_objs = {
            data_src: data.objects.get(data_src)
            for data_src in data.layer
        }
        self._is_data_loaded = all(data_objs.values())
        return data_objs

    def initialize(self, calc_reg):
        """
        Initialize the simulation. Organize calculations by dependency.

        :param calc_reg: Calculation registry.
        :type calc_reg:
            :class:`~carousel.core.calculation.CalcRegistry`
        """
        self._isinitialized = True
        # TODO: if calculations are edited, loaded, added, etc. then reset
        self.calc_order = topological_sort(calc_reg.dependencies)

    def index_iterator(self):
        """
        Generator that resumes from same index, or restarts from sent index.
        """
        idx = 0  # index
        while idx < self.number_intervals:
            new_idx = yield idx
            idx += 1
            if new_idx:
                idx = new_idx - 1

    # TODO: change start to run

    def start(self, model, progress_hook=None):
        """
        Start the simulation from time zero.

        :param model: Model with layers and registries containing parameters
        :type: :class:`~carousel.core.models.Model`
        :param progress_hook: A function that receives either a string or a
            list containing the index followed by tuples of the data or outputs
            names and values specified by ``write_fields`` in the simfile.
        :type progress_hook: function


        The model registries should contain the following layer registries:
        * :class:`~carousel.core.data_sources.DataRegistry`,
        * :class:`~carousel.core.formulas.FormulaRegistry`,
        * :class:`~carousel.core.outputs.OutputRegistry`,
        * :class:`~carousel.core.calculation.CalcRegistry`
        """
        # check if data loaded
        data_objs = self.check_data(model.data)
        if not self.is_data_loaded:
            raise MissingDataError([ds for ds in data_objs if ds is None])
        # get layer registries
        data_reg = model.registries['data']
        formula_reg = model.registries['formulas']
        out_reg = model.registries['outputs']
        calc_reg = model.registries['calculations']
        # initialize
        if not self.isinitialized:
            self.initialize(calc_reg)
        # default progress hook
        if not progress_hook:
            progress_hook = functools.partial(sim_progress_hook,
                                              display_header=True)
        # start, resume or restart
        if self.ispaused:
            # if paused, then resume, do not resize outputs again.
            self._ispaused = False  # change pause state
            progress_hook('resume simulation')
        elif self.iscomplete:
            # if complete, then restart, do not resize outputs again.
            self._iscomplete = False  # change pause state
            progress_hook('restart simulation')
            self.idx_iter = self.index_iterator()
        else:
            # resize outputs
            # assumes that self.write_frequency is immutable
            # TODO: allow self.write_frequency to be changed
            # only resize outputs first time simulation is started
            # repeat output rows to self.write_frequency
            # put initial conditions of outputs last so it's copied when
            # idx == 0
            progress_hook('resize outputs')  # display progress
            for k in out_reg:
                if out_reg.isconstant[k]:
                    continue
                # repeat rows (axis=0)
                out_reg[k] = out_reg[k].repeat(self.write_frequency, 0)
                _initial_value = out_reg.initial_value[k]
                if not _initial_value:
                    continue
                if isinstance(_initial_value, basestring):
                    # initial value is from data registry
                    # assign in a scalar to a vector fills in the vector, yes!
                    out_reg[k][-1] = data_reg[_initial_value]
                else:
                    out_reg[k][-1] = _initial_value * out_reg[k].units
            progress_hook('start simulation')
        # check and/or make Carousel_Simulations and simulation ID folders
        if not os.path.isdir(self.path):
            os.mkdir(self.path)
        sim_id_path = os.path.join(self.path, self.ID)
        if not os.path.isdir(sim_id_path):
            os.mkdir(sim_id_path)
        # header & units for save files
        data_fields = self.write_fields.get('data', [])  # any data fields
        out_fields = self.write_fields.get('outputs', [])  # any outputs fields
        save_header = tuple(data_fields + out_fields)  # concatenate fields
        # get units as strings from data & outputs
        data_units = [str(data_reg[f].dimensionality) for f in data_fields]
        out_units = [str(out_reg[f].dimensionality) for f in out_fields]
        save_units = tuple(data_units + out_units)  # concatenate units
        # string format for header & units
        save_str = ('%s' + ',%s' * (len(save_header) - 1)) + '\n'  # format
        save_header = (save_str * 2) % (save_header + save_units)  # header
        save_header = save_header[:-1]  # remove trailing new line
        # FIXME: static calcs may not have same topological order as dynamic
        # calcs, probably better to base sort on args instead of user definied
        # dependencies
        # Static calculations
        progress_hook('static calcs')
        for calc in self.calc_order:
            calc_reg[calc].calc_static(formula_reg,
                                       data_reg,
                                       out_reg,
                                       timestep=self.interval)
        # Dynamic calculations
        progress_hook('dynamic calcs')
        # TODO: assumes that interval size and indices are same, but should
        # interpolate for any size interval or indices
        for idx_tot in self.idx_iter:
            self.interval_idx = idx_tot  # update simulation interval counter
            idx = idx_tot % self.write_frequency
            # update properties
            for k, v in out_reg.isproperty.iteritems():
                # set properties from previous interval at night
                if v:
                    out_reg[k][idx] = out_reg[k][idx - 1]
            # night if any threshold exceeded
            if self.thresholds:
                night = not all(
                    limits[0] < data_reg[data][idx] < limits[1]
                    for data, limits in self.thresholds.iteritems())
            else:
                night = None
            # daytime or always calculated outputs
            for calc in self.calc_order:
                if not night or calc_reg.always_calc[calc]:
                    calc_reg[calc].calc_dynamic(idx,
                                                formula_reg,
                                                data_reg,
                                                out_reg,
                                                timestep=self.interval)
            # display progress
            if not (idx % self.display_frequency):
                progress_hook(self.format_progress(idx, data_reg, out_reg))
                # disp_head = False
            # create an index for the save file, 0 if not saving
            if not ((idx_tot + 1) % self.write_frequency):
                savenum = (idx_tot + 1) / self.write_frequency
            elif idx_tot == self.number_intervals - 1:
                # save file index should be integer!
                savenum = int(
                    np.ceil((idx_tot + 1) / float(self.write_frequency)))
            else:
                savenum = 0  # not saving this iteration
            # save file to disk
            if savenum:
                savename = self.ID + '_' + str(savenum) + '.csv'  # filename
                savepath = os.path.join(sim_id_path, savename)  # path
                # create array of all data & outputs to save
                save_array = self.format_write(data_reg, out_reg, idx + 1)
                # save as csv using default format & turn comments off
                np.savetxt(savepath,
                           save_array,
                           delimiter=',',
                           header=save_header,
                           comments='')
            try:
                cmd = self.cmd_queue.get_nowait()
            except Queue.Empty:
                continue
            if cmd == 'pause':
                self._ispaused = True
                return
        self._iscomplete = True  # change completion status

    def format_progress(self, idx, data_reg, out_reg):
        data_fields = self.display_fields.get('data', [])  # data fields
        data_args = [(f, data_reg[f][idx]) for f in data_fields]
        out_fields = self.display_fields.get('outputs', [])  # outputs fields
        out_args = [(f, out_reg[f][idx]) for f in out_fields]
        return [idx] + data_args + out_args

    def format_write(self, data_reg, out_reg, idx=None):
        data_fields = self.write_fields.get('data', [])  # any data fields
        data_args = [data_reg[f][:idx].reshape((-1, 1)) for f in data_fields]
        out_fields = self.write_fields.get('outputs', [])  # any outputs fields
        out_args = [out_reg[f][:idx] for f in out_fields]
        return np.concatenate(data_args + out_args, axis=1)

    def pause(self, progress_hook=None):
        """
        Pause the simulation. How is this different from stopping it? Maintain
        info sufficient to restart simulation. Sets ``is_paused`` to True.
        Will this state allow analysis? changing parameters? What can you do
        with a paused simulation?
        Should be capable of saving paused simulation for loading/resuming
        later, that is the main usage. EG: someone else need computer, or power
        goes out, so on battery backup quickly pause simulation, and save.
        Is save automatic? Should there be a parameter for auto save changed?
        """
        # default progress hook
        if progress_hook is None:
            progress_hook = sim_progress_hook
        progress_hook('simulation paused')
        self.cmd_queue.put('pause')
        self._ispaused = True

    def load(self, model, progress_hook=None, *args, **kwargs):
        # default progress hook
        if progress_hook is None:
            progress_hook = sim_progress_hook
        data = kwargs.get('data', {})
        if not data and args:
            data = args[0]
        for k, v in data.iteritems():
            progress_hook('loading simulation for %s' % k)
            model.data.open(k, **v)
        self.check_data(model.data)

    def run(self, model, progress_hook=None, *args, **kwargs):
        # default progress hook
        if progress_hook is None:
            progress_hook = sim_progress_hook
        progress_hook('running simulation')
        self.load(model, progress_hook, *args, **kwargs)
        self.start(model, progress_hook)