Exemplo n.º 1
0
def test_parallel() -> None:
    proton = _make_proton(parallel=True)
    experiment = Engine(**proton)

    log.debug(pf(experiment.state.get_config(True)))

    experiment.update(10.0)

    log.debug(pf(experiment.state.get_config(True)))
    log.debug(pf(experiment.state.divide_value()))

    experiment.end()
def test_profiler() -> None:
    engine = Engine(
        processes={
            'processA': ProcessA(),
            'processB': ProcessB(),
        },
        topology={
            'processA': {},
            'processB': {},
        },
        profile=True,
    )
    engine.update(3)
    engine.end()
    assert engine.stats is not None
    stats = engine.stats.strip_dirs()
    process_a_runtime = stats.stats[  # type: ignore
        ('test_profiler.py', 17, 'next_update')][3]
    process_b_runtime = stats.stats[  # type: ignore
        ('test_profiler.py', 32, 'next_update')][3]

    assert 0.6 <= process_a_runtime <= 0.7
    assert 0.3 <= process_b_runtime <= 0.4
Exemplo n.º 3
0
def simulate_experiment(experiment: Engine,
                        settings: Optional[Dict[str, Any]] = None) -> Dict:
    """Simulate an :term:`Engine`.

    Args:
        experiment: a configured experiment

    Returns:
        A timeseries of variables from all ports. If ``return_raw_data``
        is True, return the raw data instead.
    """
    settings = settings or {}
    total_time = settings.get('total_time', 10)
    return_raw_data = settings.get('return_raw_data', False)

    # run simulation
    experiment.update(total_time)
    experiment.end()

    # return data from emitter
    if return_raw_data:
        return experiment.emitter.get_data()
    return experiment.emitter.get_timeseries()
class ModelProfiler:
    """Profile Bioscrape-COBRA composites"""

    # model complexity
    n_agents = 1
    experiment_time = DEFAULT_EXPERIMENT_TIME
    parallel = False
    reuse_processes = False
    stochastic = False
    division = False
    spatial = False
    emit_step = 1

    # initialize
    composite = None
    experiment = None
    initial_state = None

    def set_parameters(
        self,
        n_agents=None,
        experiment_time=None,
        parallel=None,
        reuse_processes=None,
        emit_step=None,
        stochastic=None,
        division=None,
        spatial=None,
    ):
        self.n_agents = \
            n_agents if n_agents is not None else self.n_agents
        self.experiment_time = \
            experiment_time or self.experiment_time
        self.parallel = \
            parallel or self.parallel
        self.reuse_processes = \
            reuse_processes or self.reuse_processes
        self.emit_step = \
            emit_step or self.emit_step
        self.stochastic = \
            stochastic or self.stochastic
        self.division = \
            division or self.division
        self.spatial = \
            spatial or self.spatial

    def _generate_composite(self, **kwargs):
        initial_agent_states = [{
            'rates': {
                'k_leak': 0.005  # less leak -> less spontanteous expression
            }
        }]

        self.composite, _, self.initial_state = get_bioscrape_cobra_composite(
            n_agents=self.n_agents,
            initial_agent_states=initial_agent_states,
            stochastic=self.stochastic,
            division=self.division,
            spatial=self.spatial,
            initial_glucose=1e1,
            initial_lactose=5e1,
            depth=0.5,
            diffusion_rate=2e-2,
            jitter_force=1e-5,
            bounds=[30, 30],
            n_bins=[30, 30],
            sbml_file=STOCHASTIC_FILE
            if self.stochastic else DETERMINISTIC_FILE,
            parallel=self.parallel,
            reuse_processes=self.reuse_processes,
        )

    def _initialize_experiment(self, **kwargs):
        self.experiment = Engine(processes=self.composite['processes'],
                                 topology=self.composite['topology'],
                                 initial_state=self.initial_state,
                                 **kwargs)

    def _run_experiment(self, **kwargs):
        self.experiment.update(kwargs['experiment_time'])
        self.experiment.end()

    def _get_emitter_data(self, **kwargs):
        _ = kwargs
        data = self.experiment.emitter.get_data()
        return data

    def _get_emitter_timeseries(self, **kwargs):
        _ = kwargs
        timeseries = self.experiment.emitter.get_timeseries()
        return timeseries

    def _profile_method(self, method, **kwargs):
        """The main profiling method and of the simulation steps

        Args
            method: the simulation step. For example self._run_experiment
        """
        profiler = cProfile.Profile()
        profiler.enable()
        method(**kwargs)
        profiler.disable()
        stats = pstats.Stats(profiler)
        return stats

    def run_profile(self):
        print('GENERATE COMPOSITE')
        self._profile_method(self._generate_composite)

        print('INITIALIZE EXPERIMENT')
        self._profile_method(self._initialize_experiment)

        print('RUN EXPERIMENT')
        self._profile_method(self._run_experiment,
                             experiment_time=self.experiment_time)

        print('GET EMITTER DATA')
        self._profile_method(self._get_emitter_data)

    def profile_communication_latency(self):
        self._generate_composite()
        self._initialize_experiment(display_info=False)

        # profile the experiment
        stats = self._profile_method(
            self._run_experiment,
            experiment_time=self.experiment_time,
        )

        # get next_update runtime
        next_update_amount = ("next_update", )
        _, stats_list = stats.get_print_list(next_update_amount)

        process_update_time = 0
        for s in stats_list:
            process_update_time += stats.stats[s][3]

        # get total runtime
        experiment_time = stats.total_tt
        store_update_time = experiment_time - process_update_time

        # print_stats = stats.strip_dirs().sort_stats(-1).print_stats()
        # looping_stats = stats.sort_stats(SortKey.TIME).print_stats(20)

        return process_update_time, store_update_time
Exemplo n.º 5
0
def test_hyperdivision(profile: bool = True) -> None:
    total_time = 10
    n_agents = 100
    division_thresholds = [3, 4, 5, 6,
                           7]  # what values of x triggers division?

    # initialize agent composer
    agent_composer = ToyDivider()

    # make the composite
    composite = Composite()
    agent_ids = [str(agent_idx) for agent_idx in range(n_agents)]
    for agent_id in agent_ids:
        divider_config = {
            'divider': {
                'x_division_threshold': random.choice(division_thresholds),
            }
        }
        agent_composite = agent_composer.generate(config={
            'agent_id': agent_id,
            **divider_config,
        },
                                                  path=('agents', agent_id))
        composite.merge(agent_composite)

    # add an environment
    environment_process: Processes = {'environment': ToyEnvironment()}
    environment_topology: Topology = {
        'environment': {
            'agents': {
                '_path': ('agents', ),
                '*': {
                    'external': ('external', 'GLC')
                }
            },
        }
    }

    # combine the environment and agent
    composite.merge(
        processes=environment_process,
        topology=environment_topology,
    )

    # make the sim, run the sim, retrieve the data
    experiment = Engine(
        processes=composite.processes,
        steps=composite.steps,
        flow=composite.flow,
        topology=composite.topology,
        profile=profile,
    )
    experiment.update(total_time)
    experiment.end()
    data = experiment.emitter.get_data()

    print(f"n agents initial: {n_agents}")
    print(f"n agents final: {len(data[total_time]['agents'].keys())}")
    assert len(data[total_time]['agents'].keys()) > n_agents

    if profile:
        stats = experiment.stats
        stats.strip_dirs().sort_stats(  # type: ignore
            'cumulative', 'cumtime').print_stats(20)

        # make sure view_values is fast
        stats_view_values = stats.get_print_list(  # type: ignore
            ('view_values', ))[1]
        view_values_times = stats.stats[  # type: ignore
            stats_view_values[0]][3]
        total_runtime = stats.total_tt  # type: ignore
        assert view_values_times < 0.1 * total_runtime
class ComplexModelSim:
    """Profile Complex Models

    This class lets you initialize and profile the simulation of
    composite models with arbitrary numbers of processes, variables
    per process, and total stores.
    """

    # model complexity
    number_of_processes = DEFAULT_N_PROCESSES
    number_of_variables = DEFAULT_N_VARIABLES
    process_sleep = DEFAULT_PROCESS_SLEEP
    number_of_parallel_processes = 0
    number_of_stores = 10
    number_of_ports = 1
    hierarchy_depth = 1
    experiment_time = DEFAULT_EXPERIMENT_TIME

    # display
    print_top_stats = 4

    # initialize
    composite = None
    experiment = None

    def set_parameters(
        self,
        number_of_processes=None,
        number_of_parallel_processes=None,
        number_of_stores=None,
        number_of_ports=None,
        number_of_variables=None,
        hierarchy_depth=None,
        process_sleep=None,
        print_top_stats=None,
        experiment_time=None,
    ):
        self.number_of_processes = \
            number_of_processes or self.number_of_processes
        self.number_of_parallel_processes = \
            number_of_parallel_processes or self.number_of_parallel_processes
        self.number_of_ports = \
            number_of_ports or self.number_of_ports
        self.number_of_variables = \
            number_of_variables or self.number_of_variables
        self.number_of_stores = \
            number_of_stores or self.number_of_stores
        self.hierarchy_depth = \
            hierarchy_depth or self.hierarchy_depth
        self.process_sleep = \
            process_sleep or self.process_sleep
        self.print_top_stats = \
            print_top_stats or self.print_top_stats
        self.experiment_time = \
            experiment_time or self.experiment_time

    def _generate_composite(self, **kwargs):
        number_of_processes = kwargs.get('number_of_processes',
                                         self.number_of_processes)
        number_of_parallel_processes = kwargs.get(
            'number_of_parallel_processes', self.number_of_parallel_processes)
        number_of_stores = kwargs.get('number_of_stores',
                                      self.number_of_stores)
        number_of_ports = kwargs.get('number_of_ports', self.number_of_ports)
        number_of_variables = kwargs.get('number_of_variables',
                                         self.number_of_variables)
        hierarchy_depth = kwargs.get('hierarchy_depth', self.hierarchy_depth)
        process_sleep = kwargs.get('process_sleep', self.process_sleep)

        composer = ManyVariablesComposite({
            'number_of_processes': number_of_processes,
            'number_of_parallel_processes': number_of_parallel_processes,
            'number_of_stores': number_of_stores,
            'number_of_ports': number_of_ports,
            'number_of_variables': number_of_variables,
            'hierarchy_depth': hierarchy_depth,
            'process_sleep': process_sleep,
        })

        self.composite = composer.generate(**kwargs)

    def _initialize_experiment(self, **kwargs):
        self.experiment = Engine(processes=self.composite['processes'],
                                 topology=self.composite['topology'],
                                 **kwargs)

    def _run_experiment(self, **kwargs):
        self.experiment.update(kwargs['experiment_time'])
        self.experiment.end()

    def _get_emitter_data(self, **kwargs):
        _ = kwargs
        data = self.experiment.emitter.get_data()
        return data

    def _get_emitter_timeseries(self, **kwargs):
        _ = kwargs
        timeseries = self.experiment.emitter.get_timeseries()
        return timeseries

    def _profile_method(self, method, **kwargs):
        """The main profiling method and of the simulation steps

        Args
            method: the simulation step. For example self._run_experiment
        """
        print_top_stats = kwargs.get('print_top_stats', self.print_top_stats)
        profiler = cProfile.Profile()
        profiler.enable()
        method(**kwargs)
        profiler.disable()
        stats = pstats.Stats(profiler)
        if print_top_stats:
            stats.sort_stats('tottime').print_stats(print_top_stats)
        return stats

    def profile_communication_latency(self):

        self._generate_composite()
        self._initialize_experiment(display_info=False)

        # profile the experiment
        stats = self._profile_method(self._run_experiment,
                                     experiment_time=self.experiment_time,
                                     print_top_stats=None)

        # get next_update runtime
        next_update_amount = ("next_update", )
        _, stats_list = stats.get_print_list(next_update_amount)

        process_update_time = 0
        for s in stats_list:
            process_update_time += stats.stats[s][3]

        # get runtime
        experiment_time = stats.total_tt
        store_update_time = experiment_time - process_update_time

        return process_update_time, store_update_time
def simulate_bioscrape_cobra(
        division=False,
        stochastic=False,
        initial_glucose=1e1,
        initial_lactose=1e1,
        initial_agent_states=None,
        bounds=None,
        n_bins=None,
        depth=DEPTH,
        diffusion_rate=1e-1,
        jitter_force=1e-5,
        divide_threshold=2000 * units.fg,
        spatial=False,
        external_volume=None,
        n_agents=1,
        halt_threshold=100,
        total_time=100,
        sbml_file=None,
        emitter='timeseries',
        output_type=None,
        parallel=False,
):
    """ Main simulation function for BioscrapeCOBRA

    Args:
        * division (bool): sets whether the agents divides
        * stochastic (bool): load the stochastic lac operon model
        * initial_glucose (float): initial external glucose concentration
        * initial_lactose: (float): initial external initial_lactose concentration
        * initial_agent_states (dict): set initial state values
        * bounds (list): size of the environment [x, y] in microns
        * n_bins (list): number of bins in the [x, y] dimensions
        * depth (float): depth of the environment in microns
        * diffusion_rate (float): diffusion rate constant for all molecules, micron^s/sec.
        * divide_threshold (float): mass at which cells divide, in fg
        * spatial (bool): use spatial environment
        * external_volume (float): volume of external bin, if non-spatial environment
        * n_agents (int): number of initial agents in environment
        * halt_threshold (int): number of agents at which simulations will terminate
        * total_time (float): total simulation time, in seconds
        * sbml_file (str): the file for the Bioscrape process. Uses default if None.
        * emitter (str): type of emitter, 'timeseries' or 'database'.
        * output_type (str): 'timeseries' or 'unitless'. If None, return experiment instance
        * parallel (bool): run processes in parallel, useful for large compute machines
        * jitter_force (float): random force applied to cell bodies (in pN)
    """

    biocobra_composite, initial_composite, initial_state_full = get_bioscrape_cobra_composite(
        division=division,
        stochastic=stochastic,
        initial_glucose=initial_glucose,
        initial_lactose=initial_lactose,
        initial_agent_states=initial_agent_states,
        bounds=bounds,
        n_bins=n_bins,
        depth=depth,
        diffusion_rate=diffusion_rate,
        jitter_force=jitter_force,
        divide_threshold=divide_threshold,
        spatial=spatial,
        external_volume=external_volume,
        n_agents=n_agents,
        sbml_file=sbml_file,
        parallel=parallel)

    # make the experiment
    experiment_id = (f"{'stochastic' if stochastic else 'deterministic'}"
                     f"{'_division' if division else ''}"
                     f"{'_spatial' if spatial else ''}"
                     f"_{timestamp()}")
    experiment_config = {
        'processes': biocobra_composite.processes,
        'topology': biocobra_composite.topology,
        'initial_state': initial_state_full,
        'display_info': False,
        'experiment_id': experiment_id,
        'emit_step': max(BIOSCRAPE_TIMESTEP, COBRA_TIMESTEP),
        'emitter': {'type': emitter}}
    print(f'Initializing experiment {experiment_id}')
    biocobra_experiment = Engine(**experiment_config)

    # run the experiment
    clock_start = clock.time()
    if division:  # terminate upon reaching total_time or halt_threshold
        sim_step = max(BIOSCRAPE_TIMESTEP, COBRA_TIMESTEP) * 10
        for _ in tqdm(range(0, total_time, sim_step)):
            n_agents = len(biocobra_experiment.state.get_value()['agents'])
            if n_agents < halt_threshold:
                biocobra_experiment.update(sim_step)
    else:
        biocobra_experiment.update(total_time)

    # print runtime and finalize
    clock_finish = clock.time() - clock_start
    print(f'Completed in {clock_finish:.2f} seconds')
    biocobra_experiment.end()

    # retrieve the data
    if output_type == 'timeseries':
        return biocobra_experiment.emitter.get_timeseries(), initial_composite
    if output_type == 'unitless':
        return biocobra_experiment.emitter.get_data_unitless(), initial_composite
    return biocobra_experiment, initial_composite