def get_final_results(simulations, psus, eps, sessions, remote_threshold): flat_timesteps, tensor_fields = [], [] for sim_result, psu, ep in list(zip(simulations, psus, eps)): flat_timesteps.append(flatten(sim_result)) tensor_fields.append(create_tensor_field(psu, ep)) flat_simulations = flatten(flat_timesteps) if config_amt == 1: return simulations, tensor_fields, sessions elif config_amt > 1: return flat_simulations, tensor_fields, sessions
def get_final_results(simulations, psus, eps, sessions, remote_threshold): flat_timesteps, tensor_fields = [], [] for sim_result, psu, ep in tqdm(list(zip(simulations, psus, eps)), total=len(simulations), desc='Flattening results'): flat_timesteps.append(flatten(sim_result)) tensor_fields.append(create_tensor_field(psu, ep)) flat_simulations = flatten(flat_timesteps) if config_amt == 1: return simulations, tensor_fields, sessions elif (config_amt > 1): # and (config_amt < remote_threshold): return flat_simulations, tensor_fields, sessions
def single_proc_exec( simulation_execs: List[Callable], var_dict_list: List[VarDictType], states_lists: List[StatesListsType], configs_structs: List[ConfigsType], env_processes_list: List[EnvProcessesType], Ts: List[range], SimIDs, Ns: List[int], ExpIDs: List[int], SubsetIDs, SubsetWindows, configured_n ): print(f'Execution Mode: single_threaded') params = [ simulation_execs, states_lists, configs_structs, env_processes_list, Ts, SimIDs, Ns, SubsetIDs, SubsetWindows ] simulation_exec, states_list, config, env_processes, T, sim_id, N, subset_id, subset_window = list( map(lambda x: x.pop(), params) ) result = simulation_exec( var_dict_list, states_list, config, env_processes, T, sim_id, N, subset_id, subset_window, configured_n ) return flatten(result)
def simulation(self, sweep_dict: Dict[str, List[Any]], states_list: List[Dict[str, Any]], configs: List[Tuple[List[Callable], List[Callable]]], env_processes: Dict[str, Callable], time_seq: range, runs: int) -> List[List[Dict[str, Any]]]: def execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run) -> List[Dict[str, Any]]: run += 1 def generate_init_sys_metrics(genesis_states_list): for d in genesis_states_list: d['run'], d['substep'], d['timestep'] = run, 0, 0 yield d states_list_copy: List[Dict[str, Any]] = list( generate_init_sys_metrics(deepcopy(states_list))) first_timestep_per_run: List[Dict[str, Any]] = self.run_pipeline( sweep_dict, states_list_copy, configs, env_processes, time_seq, run) del states_list_copy return first_timestep_per_run tp = TPool(runs) pipe_run: List[List[Dict[str, Any]]] = flatten( tp.map( lambda run: execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run), list(range(runs)))) tp.clear() return pipe_run
def simulation( self, sweep_dict: Dict[str, List[Any]], states_list: List[Dict[str, Any]], configs, env_processes: Dict[str, Callable], time_seq: range, simulation_id: int, run: int, additional_objs=None ): def execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run) -> List[Dict[str, Any]]: run += 1 def generate_init_sys_metrics(genesis_states_list): # for d in genesis_states_list.asDict(): for d in genesis_states_list: d['simulation'], d['run'], d['substep'], d['timestep'] = simulation_id, run, 0, 0 yield d states_list_copy: List[Dict[str, Any]] = list(generate_init_sys_metrics(tuple(states_list))) first_timestep_per_run: List[Dict[str, Any]] = self.run_pipeline( sweep_dict, states_list_copy, configs, env_processes, time_seq, run, additional_objs ) del states_list_copy return first_timestep_per_run pipe_run = flatten( [execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run)] ) return pipe_run
def parallelize_simulations(simulation_execs: List[Callable], var_dict_list: List[VarDictType], states_lists: List[StatesListsType], configs_structs: List[ConfigsType], env_processes_list: List[EnvProcessesType], Ts: List[range], SimIDs, Ns: List[int], ExpIDs: List[int], SubsetIDs, SubsetWindows, configured_n): print(f'Execution Mode: parallelized') params = list( zip(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, SimIDs, Ns, SubsetIDs, SubsetWindows)) len_configs_structs = len(configs_structs) unique_runs = Counter(SimIDs) sim_count = max(unique_runs.values()) highest_divisor = int(len_configs_structs / sim_count) new_configs_structs, new_params = [], [] for count in range(sim_count): if count == 0: new_params.append(params[count:highest_divisor]) new_configs_structs.append(configs_structs[count:highest_divisor]) elif count > 0: new_params.append(params[count * highest_divisor:(count + 1) * highest_divisor]) new_configs_structs.append( configs_structs[count * highest_divisor:(count + 1) * highest_divisor]) def threaded_executor(params): tp = TPool() if len_configs_structs > 1: results = tp.map( lambda t: t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], configured_n), params) else: t = params[0] results = t[0](t[1], t[2], t[3], t[4], t[5], t[6], t[7], t[8], t[9], configured_n) tp.close() return results # pp = PPool() # results = flatten(list(pp.map(lambda params: threaded_executor(params), new_params))) results = flatten( list(map(lambda params: threaded_executor(params), new_params))) # pp.close() # pp.join() # pp.clear() # pp.restart() return results
def single_proc_exec( simulation_execs: List[Callable], var_dict_list: List[VarDictType], states_lists: List[StatesListsType], configs_structs: List[ConfigsType], env_processes_list: List[EnvProcessesType], Ts: List[range], Ns: List[int] ): l = [simulation_execs, states_lists, configs_structs, env_processes_list, Ts, Ns] simulation_exec, states_list, config, env_processes, T, N = list(map(lambda x: x.pop(), l)) result = simulation_exec(var_dict_list, states_list, config, env_processes, T, N) return flatten(result)
def execute(self) -> Tuple[List[Dict[str, Any]], DataFrame]: config_proc = Processor() create_tensor_field = TensorFieldReport(config_proc).create_tensor_field print(r''' __________ ____ ________ __ _____/ ____/ | / __ \ / ___/ __` / __ / / / /| | / / / / / /__/ /_/ / /_/ / /___/ ___ |/ /_/ / \___/\__,_/\__,_/\____/_/ |_/_____/ by BlockScience ''') print(f'Execution Mode: {self.exec_context + ": " + str(self.configs)}') print(f'Configurations: {self.configs}') var_dict_list, states_lists, Ts, Ns, eps, configs_structs, env_processes_list, partial_state_updates, simulation_execs = \ [], [], [], [], [], [], [], [], [] config_idx = 0 for x in self.configs: Ts.append(x.sim_config['T']) Ns.append(x.sim_config['N']) var_dict_list.append(x.sim_config['M']) states_lists.append([x.initial_state]) eps.append(list(x.exogenous_states.values())) configs_structs.append(config_proc.generate_config(x.initial_state, x.partial_state_updates, eps[config_idx])) # print(env_processes_list) env_processes_list.append(x.env_processes) partial_state_updates.append(x.partial_state_updates) simulation_execs.append(SimExecutor(x.policy_ops).simulation) config_idx += 1 final_result = None if self.exec_context == ExecutionMode.single_proc: tensor_field = create_tensor_field(partial_state_updates.pop(), eps.pop()) result = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns) final_result = result, tensor_field elif self.exec_context == ExecutionMode.multi_proc: # if len(self.configs) > 1: simulations = self.exec_method(simulation_execs, var_dict_list, states_lists, configs_structs, env_processes_list, Ts, Ns) results = [] for result, partial_state_updates, ep in list(zip(simulations, partial_state_updates, eps)): results.append((flatten(result), create_tensor_field(partial_state_updates, ep))) final_result = results return final_result
def simulation( self, sweep_dict: Dict[str, List[Any]], states_list: List[Dict[str, Any]], configs, env_processes: Dict[str, Callable], time_seq: range, simulation_id: int, run: int, subset_id, subset_window, configured_N, # remote_ind additional_objs=None): run += 1 subset_window.appendleft(subset_id) latest_subset_id, previous_subset_id = tuple(subset_window) if configured_N == 1 and latest_subset_id > previous_subset_id: run -= 1 def execute_run(sweep_dict, states_list, configs, env_processes, time_seq, _run) -> List[Dict[str, Any]]: def generate_init_sys_metrics(genesis_states_list, sim_id, _subset_id, _run, _subset_window): for D in genesis_states_list: d = D.copy() d['simulation'], d['subset'], d['run'], d['substep'], d['timestep'] = \ sim_id, _subset_id, _run, 0, 0 yield d states_list_copy: List[Dict[str, Any]] = list( generate_init_sys_metrics(tuple(states_list), simulation_id, subset_id, run, subset_window)) first_timestep_per_run: List[Dict[str, Any]] = self.run_pipeline( sweep_dict, states_list_copy, configs, env_processes, time_seq, run, additional_objs) del states_list_copy return first_timestep_per_run pipe_run = flatten([ execute_run(sweep_dict, states_list, configs, env_processes, time_seq, run) ]) return pipe_run